From 56a8f83e1880946a7c9c12cbad3d8016258d772a Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 26 Jun 2024 17:03:39 -0400 Subject: [PATCH 01/31] init --- src/registrar/models/domain.py | 5 + src/registrar/utility/csv_export.py | 1167 ++++++++++++++------------- src/registrar/views/admin_views.py | 24 +- 3 files changed, 615 insertions(+), 581 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 767227499..7fdc56971 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -151,6 +151,11 @@ class Domain(TimeStampedModel, DomainHelper): # previously existed but has been deleted from the registry DELETED = "deleted", "Deleted" + @classmethod + def get_state_label(cls, state: str): + """Returns the associated label for a given state value""" + return cls(state).label if state else None + @classmethod def get_help_text(cls, state) -> str: """Returns a help message for a desired state. If none is found, an empty string is returned""" diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 1a35c8164..345ed7be1 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,3 +1,4 @@ +from collections import defaultdict import csv import logging from datetime import datetime @@ -32,383 +33,6 @@ def write_header(writer, columns): writer.writerow(columns) -def get_domain_infos(filter_condition, sort_fields): - """ - Returns DomainInformation objects filtered and sorted based on the provided conditions. - filter_condition -> A dictionary of conditions to filter the objects. - sort_fields -> A list of fields to sort the resulting query set. - returns: A queryset of DomainInformation objects - """ - domain_infos = ( - DomainInformation.objects.select_related("domain", "authorizing_official") - .filter(**filter_condition) - .order_by(*sort_fields) - .distinct() - ) - - # Do a mass concat of the first and last name fields for authorizing_official. - # The old operation was computationally heavy for some reason, so if we precompute - # this here, it is vastly more efficient. - domain_infos_cleaned = domain_infos.annotate( - ao=Concat( - Coalesce(F("authorizing_official__first_name"), Value("")), - Value(" "), - Coalesce(F("authorizing_official__last_name"), Value("")), - output_field=CharField(), - ) - ) - return domain_infos_cleaned - - -def parse_row_for_domain( - columns, - domain_info: DomainInformation, - dict_security_emails=None, - should_get_domain_managers=False, - dict_domain_invitations_with_invited_status=None, - dict_user_domain_roles=None, -): - """Given a set of columns, generate a new row from cleaned column data""" - - # Domain should never be none when parsing this information - if domain_info.domain is None: - logger.error("Attemting to parse row for csv exports but Domain is none in a DomainInfo") - raise ValueError("Domain is none") - - domain = domain_info.domain # type: ignore - - # Grab the security email from a preset dictionary. - # If nothing exists in the dictionary, grab from .contacts. - if dict_security_emails is not None and domain.name in dict_security_emails: - _email = dict_security_emails.get(domain.name) - security_email = _email if _email is not None else " " - else: - # If the dictionary doesn't contain that data, lets filter for it manually. - # This is a last resort as this is a more expensive operation. - security_contacts = domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY) - _email = security_contacts[0].email if security_contacts else None - security_email = _email if _email is not None else " " - - # These are default emails that should not be displayed in the csv report - invalid_emails = {DefaultEmail.LEGACY_DEFAULT.value, DefaultEmail.PUBLIC_CONTACT_DEFAULT.value} - if security_email.lower() in invalid_emails: - security_email = "(blank)" - - if domain_info.federal_type and domain_info.organization_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: - domain_type = f"{domain_info.get_organization_type_display()} - {domain_info.get_federal_type_display()}" - else: - domain_type = domain_info.get_organization_type_display() - - # create a dictionary of fields which can be included in output - FIELDS = { - "Domain name": domain.name, - "Status": domain.get_state_display(), - "First ready on": domain.first_ready or "(blank)", - "Expiration date": domain.expiration_date or "(blank)", - "Domain type": domain_type, - "Agency": domain_info.federal_agency, - "Organization name": domain_info.organization_name, - "City": domain_info.city, - "State": domain_info.state_territory, - "AO": domain_info.ao, # type: ignore - "AO email": domain_info.authorizing_official.email if domain_info.authorizing_official else " ", - "Security contact email": security_email, - "Created at": domain.created_at, - "Deleted": domain.deleted, - } - - if should_get_domain_managers: - # Get lists of emails for active and invited domain managers - - dms_active_emails = dict_user_domain_roles.get(domain_info.domain.name, []) - dms_invited_emails = dict_domain_invitations_with_invited_status.get(domain_info.domain.name, []) - - # Set up the "matching headers" + row field data for email and status - i = 0 # Declare i outside of the loop to avoid a reference before assignment in the second loop - for i, dm_email in enumerate(dms_active_emails, start=1): - FIELDS[f"Domain manager {i}"] = dm_email - FIELDS[f"DM{i} status"] = "R" - - # Continue enumeration from where we left off and add data for invited domain managers - for j, dm_email in enumerate(dms_invited_emails, start=i + 1): - FIELDS[f"Domain manager {j}"] = dm_email - FIELDS[f"DM{j} status"] = "I" - - row = [FIELDS.get(column, "") for column in columns] - return row - - -def _get_security_emails(sec_contact_ids): - """ - Retrieve security contact emails for the given security contact IDs. - """ - dict_security_emails = {} - public_contacts = ( - PublicContact.objects.only("email", "domain__name") - .select_related("domain") - .filter(registry_id__in=sec_contact_ids) - ) - - # Populate a dictionary of domain names and their security contacts - for contact in public_contacts: - domain: Domain = contact.domain - if domain is not None and domain.name not in dict_security_emails: - dict_security_emails[domain.name] = contact.email - else: - logger.warning("csv_export -> Domain was none for PublicContact") - - return dict_security_emails - - -def count_domain_managers(domain_name, dict_domain_invitations_with_invited_status, dict_user_domain_roles): - """Count active and invited domain managers""" - dms_active = len(dict_user_domain_roles.get(domain_name, [])) - dms_invited = len(dict_domain_invitations_with_invited_status.get(domain_name, [])) - return dms_active, dms_invited - - -def update_columns(columns, dms_total, should_update_columns): - """Update columns if necessary""" - if should_update_columns: - for i in range(1, dms_total + 1): - email_column_header = f"Domain manager {i}" - status_column_header = f"DM{i} status" - if email_column_header not in columns: - columns.append(email_column_header) - columns.append(status_column_header) - should_update_columns = False - return columns, should_update_columns, dms_total - - -def update_columns_with_domain_managers( - columns, - domain_info, - should_update_columns, - dms_total, - dict_domain_invitations_with_invited_status, - dict_user_domain_roles, -): - """Helper function to update columns with domain manager information""" - - domain_name = domain_info.domain.name - - try: - dms_active, dms_invited = count_domain_managers( - domain_name, dict_domain_invitations_with_invited_status, dict_user_domain_roles - ) - - if dms_active + dms_invited > dms_total: - dms_total = dms_active + dms_invited - should_update_columns = True - - except Exception as err: - logger.error(f"Exception while parsing domain managers for reports: {err}") - - return update_columns(columns, dms_total, should_update_columns) - - -def build_dictionaries_for_domain_managers(dict_user_domain_roles, dict_domain_invitations_with_invited_status): - """Helper function that builds dicts for invited users and active domain - managers. We do so to avoid filtering within loops.""" - - user_domain_roles = UserDomainRole.objects.all() - - # Iterate through each user domain role and populate the dictionary - for user_domain_role in user_domain_roles: - domain_name = user_domain_role.domain.name - email = user_domain_role.user.email - if domain_name not in dict_user_domain_roles: - dict_user_domain_roles[domain_name] = [] - dict_user_domain_roles[domain_name].append(email) - - domain_invitations_with_invited_status = None - domain_invitations_with_invited_status = DomainInvitation.objects.filter( - status=DomainInvitation.DomainInvitationStatus.INVITED - ).select_related("domain") - - # Iterate through each domain invitation and populate the dictionary - for invite in domain_invitations_with_invited_status: - domain_name = invite.domain.name - email = invite.email - if domain_name not in dict_domain_invitations_with_invited_status: - dict_domain_invitations_with_invited_status[domain_name] = [] - dict_domain_invitations_with_invited_status[domain_name].append(email) - - return dict_user_domain_roles, dict_domain_invitations_with_invited_status - - -def write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, -): - """ - Receives params from the parent methods and outputs a CSV with filtered and sorted domains. - Works with write_header as long as the same writer object is passed. - should_get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv - should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice - """ - - # Retrieve domain information and all sec emails - all_domain_infos = get_domain_infos(filter_condition, sort_fields) - sec_contact_ids = all_domain_infos.values_list("domain__security_contact_registry_id", flat=True) - dict_security_emails = _get_security_emails(sec_contact_ids) - paginator = Paginator(all_domain_infos, 1000) - - # Initialize variables - dms_total = 0 - should_update_columns = False - total_body_rows = [] - dict_user_domain_roles = {} - dict_domain_invitations_with_invited_status = {} - - # Build dictionaries if necessary - if should_get_domain_managers: - dict_user_domain_roles, dict_domain_invitations_with_invited_status = build_dictionaries_for_domain_managers( - dict_user_domain_roles, dict_domain_invitations_with_invited_status - ) - - # Process domain information - for page_num in paginator.page_range: - rows = [] - page = paginator.page(page_num) - for domain_info in page.object_list: - if should_get_domain_managers: - columns, dms_total, should_update_columns = update_columns_with_domain_managers( - columns, - domain_info, - should_update_columns, - dms_total, - dict_domain_invitations_with_invited_status, - dict_user_domain_roles, - ) - - try: - row = parse_row_for_domain( - columns, - domain_info, - dict_security_emails, - should_get_domain_managers, - dict_domain_invitations_with_invited_status, - dict_user_domain_roles, - ) - rows.append(row) - except ValueError: - logger.error("csv_export -> Error when parsing row, domain was None") - continue - total_body_rows.extend(rows) - - if should_write_header: - write_header(writer, columns) - writer.writerows(total_body_rows) - - -def export_data_type_to_csv(csv_file): - """ - All domains report with extra columns. - This maps to the "All domain metadata" button. - Exports domains of all statuses. - """ - - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Status", - "First ready on", - "Expiration date", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - # For domain manager we are pass it in as a parameter below in write_body - ] - - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - write_csv_for_domains( - writer, columns, sort_fields, filter_condition={}, should_get_domain_managers=True, should_write_header=True - ) - - -def export_data_full_to_csv(csv_file): - """All domains report""" - - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True - ) - - -def export_data_federal_to_csv(csv_file): - """Federal domains report""" - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "organization_type__icontains": "federal", - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True - ) - - def get_default_start_date(): # Default to a date that's prior to our first deployment return timezone.make_aware(datetime(2023, 11, 1)) @@ -427,66 +51,6 @@ def format_end_date(end_date): return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() -def export_data_domain_growth_to_csv(csv_file, start_date, end_date): - """ - Growth report: - Receive start and end dates from the view, parse them. - Request from write_body READY domains that are created between - the start and end dates, as well as DELETED domains that are deleted between - the start and end dates. Specify sort params for both lists. - """ - - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Status", - "Expiration date", - "Created at", - "First ready", - "Deleted", - ] - sort_fields = [ - "domain__first_ready", - "domain__name", - ] - filter_condition = { - "domain__state__in": [Domain.State.READY], - "domain__first_ready__lte": end_date_formatted, - "domain__first_ready__gte": start_date_formatted, - } - - # We also want domains deleted between sar and end dates, sorted - sort_fields_for_deleted_domains = [ - "domain__deleted", - "domain__name", - ] - filter_condition_for_deleted_domains = { - "domain__state__in": [Domain.State.DELETED], - "domain__deleted__lte": end_date_formatted, - "domain__deleted__gte": start_date_formatted, - } - - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True - ) - write_csv_for_domains( - writer, - columns, - sort_fields_for_deleted_domains, - filter_condition_for_deleted_domains, - should_get_domain_managers=False, - should_write_header=False, - ) - - def get_sliced_domains(filter_condition): """Get filtered domains counts sliced by org type and election office. Pass distinct=True when filtering by permissions so we do not to count multiples @@ -558,150 +122,627 @@ def get_sliced_requests(filter_condition): election_board, ] +class DomainExport: + """ + A collection of functions which return csv files regarding the Domain model. + """ -def export_data_managed_domains_to_csv(csv_file, start_date, end_date): - """Get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date.""" - - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] - filter_managed_domains_start_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": start_date_formatted, - } - managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) - - writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", + @classmethod + def export_data_type_to_csv(cls, csv_file): + """ + All domain metadata: + Exports domains of all statuses plus domain managers. + """ + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Status", + "First ready on", + "Expiration date", + "Domain type", + "Agency", + "Organization name", "City", - "Special district", - "School district", - "Election office", + "State", + "AO", + "AO email", + "Security contact email", + "Domain managers", + "Invited domain managers", ] - ) - writer.writerow(managed_domains_sliced_at_start_date) - writer.writerow([]) - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date_formatted, - } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] - writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + + # Fetch all relevant Invite entries + domain_invitations = cls.get_all_domain_invitations() + + # Fetch all relevant ComainUserRole entries + user_domain_roles = cls.get_all_user_domain_roles() + + domain_infos = ( + DomainInformation.objects.select_related("domain", "authorizing_official") + .prefetch_related("permissions") + .order_by(*sort_fields) + .distinct() + ) + + annotations = cls._domain_metadata_annotations() + + # The .values returned from annotate_and_retrieve_fields can't go two levels deep + # (just returns the field id of say, "creator") - so we have to include this. + additional_values = [ + "domain__name", + "domain__state", + "domain__first_ready", + "domain__expiration_date", + "domain__created_at", + "domain__deleted", + "authorizing_official__email", + ] + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, domain_invitations, user_domain_roles, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + # Write the csv file + cls.write_csv_for_domains(writer, columns, requests_dict) + + @classmethod + def export_data_full_to_csv(cls, csv_file): + """Current full""" + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", "City", - "Special district", - "School district", - "Election office", + "State", + "Security contact email", ] - ) - writer.writerow(managed_domains_sliced_at_end_date) - writer.writerow([]) + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + filter_condition = { + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + } - write_csv_for_domains( + domain_infos = ( + DomainInformation.objects.select_related("domain") + .filter(**filter_condition) + .order_by(*sort_fields) + .distinct() + ) + + annotations = {} + additional_values = [ + "domain__name", + ] + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + # Write the csv file + cls.write_csv_for_domains(writer, columns, requests_dict) + + @classmethod + def export_data_federal_to_csv(cls, csv_file): + """Current federal""" + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Security contact email", + ] + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + filter_condition = { + "organization_type__icontains": "federal", + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + } + + domain_infos = ( + DomainInformation.objects.select_related("domain") + .filter(**filter_condition) + .order_by(*sort_fields) + .distinct() + ) + + annotations = {} + additional_values = [ + "domain__name", + ] + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + # Write the csv file + cls.write_csv_for_domains(writer, columns, requests_dict) + + @classmethod + def export_data_domain_growth_to_csv(cls, csv_file, start_date, end_date): + """ + Domain growth: + Receive start and end dates from the view, parse them. + Request from write_body READY domains that are created between + the start and end dates, as well as DELETED domains that are deleted between + the start and end dates. Specify sort params for both lists. + """ + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Status", + "Expiration date", + "Created at", + "First ready", + "Deleted", + ] + sort_fields = [ + "domain__first_ready", + "domain__name", + ] + filter_condition = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + "domain__first_ready__gte": start_date_formatted, + } + + # We also want domains deleted between sar and end dates, sorted + sort_fields_for_deleted_domains = [ + "domain__deleted", + "domain__name", + ] + filter_condition_for_deleted_domains = { + "domain__state__in": [Domain.State.DELETED], + "domain__deleted__lte": end_date_formatted, + "domain__deleted__gte": start_date_formatted, + } + + domain_infos = ( + DomainInformation.objects.select_related("domain") + .filter(**filter_condition) + .order_by(*sort_fields) + .distinct() + ) + deleted_domain_infos = ( + DomainInformation.objects.select_related("domain") + .filter(**filter_condition_for_deleted_domains) + .order_by(*sort_fields_for_deleted_domains) + .distinct() + ) + + annotations = {} + additional_values = [ + "domain__name", + "domain__state", + "domain__first_ready", + "domain__expiration_date", + "domain__created_at", + "domain__deleted", + ] + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + # Convert the domain request queryset to a dictionary (including annotated fields) + deleted_annotated_domains = cls.annotate_and_retrieve_fields(deleted_domain_infos, annotations, {}, {}, {}, additional_values) + deleted_requests_dict = convert_queryset_to_dict(deleted_annotated_domains, is_model=False) + + cls.write_csv_for_domains( + writer, columns, requests_dict + + ) + cls.write_csv_for_domains( + writer, + columns, + deleted_requests_dict, + should_write_header=False, + ) + + @classmethod + def export_data_managed_domains_to_csv(cls, csv_file, start_date, end_date): + """ + Managed domains: + Get counts for domains that have domain managers for two different dates, + get list of managed domains at end_date.""" + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + "Domain managers", + "Invited domain managers", + ] + sort_fields = [ + "domain__name", + ] + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date_formatted, + } + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + + writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_start_date) + writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + + writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_end_date) + writer.writerow([]) + + domain_invitations = cls.get_all_domain_invitations() + + # Fetch all relevant ComainUserRole entries + user_domain_roles = cls.get_all_user_domain_roles() + + annotations = {} + # The .values returned from annotate_and_retrieve_fields can't go two levels deep + # (just returns the field id of say, "creator") - so we have to include this. + additional_values = [ + "domain__name", + ] + + domain_infos = ( + DomainInformation.objects.select_related("domain") + .prefetch_related("permissions") + .filter(**filter_managed_domains_end_date) + .order_by(*sort_fields) + .distinct() + ) + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, domain_invitations, user_domain_roles, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + cls.write_csv_for_domains( + writer, + columns, + requests_dict + ) + + @classmethod + def export_data_unmanaged_domains_to_csv(cls, csv_file, start_date, end_date): + """ + Unmanaged domains: + Get counts for domains that have domain managers for two different dates, + get list of managed domains at end_date.""" + + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + + writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_start_date) + writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + + writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_end_date) + writer.writerow([]) + + annotations = {} + # The .values returned from annotate_and_retrieve_fields can't go two levels deep + # (just returns the field id of say, "creator") - so we have to include this. + additional_values = [ + "domain__name", + ] + domain_infos = ( + DomainInformation.objects.select_related("domain") + .filter(**filter_unmanaged_domains_end_date) + .order_by(*sort_fields) + .distinct() + ) + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + cls.write_csv_for_domains( + writer, + columns, + requests_dict + ) + + @classmethod + def _domain_metadata_annotations(cls, delimiter=", "): + """""" + return { + "ao_name": Concat( + Coalesce(F("authorizing_official__first_name"), Value("")), + Value(" "), + Coalesce(F("authorizing_official__last_name"), Value("")), + output_field=CharField(), + ), + } + + @classmethod + def annotate_and_retrieve_fields( + cls, domains, annotations, public_contacts={}, domain_invitations={}, user_domain_roles={}, additional_values=None, include_many_to_many=False + ) -> QuerySet: + """ + Applies annotations to a queryset and retrieves specified fields, + including class-defined and annotation-defined. + + Parameters: + requests (QuerySet): Initial queryset. + annotations (dict, optional): Fields to compute {field_name: expression}. + additional_values (list, optional): Extra fields to retrieve; defaults to annotation keys if None. + include_many_to_many (bool, optional): Determines if we should include many to many fields or not + + Returns: + QuerySet: Contains dictionaries with the specified fields for each record. + """ + + if additional_values is None: + additional_values = [] + + # We can infer that if we're passing in annotations, + # we want to grab the result of said annotation. + if annotations: + additional_values.extend(annotations.keys()) + + # Get prexisting fields on DomainRequest + domain_fields = set() + for field in DomainInformation._meta.get_fields(): + # Exclude many to many fields unless we specify + many_to_many = isinstance(field, ManyToManyField) and include_many_to_many + if many_to_many or not isinstance(field, ManyToManyField): + domain_fields.add(field.name) + + queryset = domains.annotate(**annotations).values(*domain_fields, *additional_values) + annotated_domains = [] + + # Create mapping of domain to a list of invited users and managers + invited_users_dict = defaultdict(list) + for domain, email in domain_invitations: + invited_users_dict[domain].append(email) + + managers_dict = defaultdict(list) + for domain, email in user_domain_roles: + managers_dict[domain].append(email) + + # Annotate with security_contact from public_contacts + for domain in queryset: + domain['security_contact_email'] = public_contacts.get(domain.get('domain__registry_id')) + domain['invited_users'] = ', '.join(invited_users_dict.get(domain.get('domain__name'), [])) + domain['managers'] = ', '.join(managers_dict.get(domain.get('domain__name'), [])) + annotated_domains.append(domain) + + if annotated_domains: + return annotated_domains + + return queryset + + @staticmethod + def parse_row_for_domains(columns, domain): + """ + Given a set of columns and a request dictionary, generate a new row from cleaned column data. + """ + + status = domain.get("domain__state") + human_readable_status = Domain.State.get_state_label(status) + + expiration_date = domain.get("domain__expiration_date") + if expiration_date is None: + expiration_date = "(blank)" + + first_ready_on = domain.get("domain__first_ready") + if first_ready_on is None: + first_ready_on = "(blank)" + + domain_org_type = domain.get("generic_org_type") + human_readable_domain_org_type = DomainRequest.OrganizationChoices.get_org_label(domain_org_type) + domain_federal_type = domain.get("federal_type") + human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) + domain_type = human_readable_domain_org_type + if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: + domain_type = f"{human_readable_domain_federal_type} - {human_readable_domain_org_type}" + + if domain.get("domain__name") == "18f.gov": + print(f'domain_type {domain_type}') + print(f'federal_agency {domain.get("federal_agency")}') + print(f'city {domain.get("city")}') + + # create a dictionary of fields which can be included in output. + # "extra_fields" are precomputed fields (generated in the DB or parsed). + FIELDS = { + + "Domain name": domain.get("domain__name"), + "Status": human_readable_status, + "First ready on": first_ready_on, + "Expiration date": expiration_date, + "Domain type": domain_type, + "Agency": domain.get("federal_agency"), + "Organization name": domain.get("organization_name"), + "City": domain.get("city"), + "State": domain.get("state_territory"), + "AO": domain.get("ao_name"), + "AO email": domain.get("authorizing_official__email"), + "Security contact email": domain.get("security_contact_email"), + "Created at": domain.get("domain__created_at"), + "Deleted": domain.get("domain__deleted"), + "Domain managers": domain.get("managers"), + "Invited domain managers": domain.get("invited_users"), + } + + row = [FIELDS.get(column, "") for column in columns] + return row + + @staticmethod + def write_csv_for_domains( writer, columns, - sort_fields, - filter_managed_domains_end_date, - should_get_domain_managers=True, + domains_dict, should_write_header=True, - ) + ): + """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. + Works with write_header as long as the same writer object is passed.""" + rows = [] + for domain in domains_dict.values(): + try: + row = DomainExport.parse_row_for_domains(columns, domain) + rows.append(row) + except ValueError as err: + logger.error(f"csv_export -> Error when parsing row: {err}") + continue -def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): - """Get counts for domains that do not have domain managers for two different dates, - get list of unmanaged domains at end_date.""" + if should_write_header: + write_header(writer, columns) - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] + writer.writerows(rows) - filter_unmanaged_domains_start_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": start_date_formatted, - } - unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + # ============================================================= # + # Helper functions for django ORM queries. # + # We are using these rather than pure python for speed reasons. # + # ============================================================= # - writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_start_date) - writer.writerow([]) + @classmethod + def get_all_security_emails(cls): + """ + Fetch all PublicContact entries and return a mapping of registry_id to email. + """ + public_contacts = PublicContact.objects.values_list('registry_id', 'email') + return {registry_id: email for registry_id, email in public_contacts} + + @classmethod + def get_all_domain_invitations(cls): + """ + Fetch all DomainInvitation entries and return a mapping of domain to email. + """ + domain_invitations = DomainInvitation.objects.filter(status="invited").values_list('domain__name', 'email') + return list(domain_invitations) - filter_unmanaged_domains_end_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": end_date_formatted, - } - unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + @classmethod + def get_all_user_domain_roles(cls): + """ + Fetch all UserDomainRole entries and return a mapping of domain to user__email. + """ + user_domain_roles = UserDomainRole.objects.select_related('user').values_list('domain__name', 'user__email') + return list(user_domain_roles) - writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_end_date) - writer.writerow([]) - - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_unmanaged_domains_end_date, - should_get_domain_managers=False, - should_write_header=True, - ) class DomainRequestExport: diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index f1baa72bd..9b013e36c 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -142,7 +142,7 @@ class ExportDataType(View): # match the CSV example with all the fields response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - csv_export.export_data_type_to_csv(response) + csv_export.DomainExport.export_data_type_to_csv(response) return response @@ -151,7 +151,7 @@ class ExportDataFull(View): # Smaller export based on 1 response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - csv_export.export_data_full_to_csv(response) + csv_export.DomainExport.export_data_full_to_csv(response) return response @@ -160,7 +160,7 @@ class ExportDataFederal(View): # Federal only response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - csv_export.export_data_federal_to_csv(response) + csv_export.DomainExport.export_data_federal_to_csv(response) return response @@ -177,31 +177,23 @@ class ExportDomainRequestDataFull(View): class ExportDataDomainsGrowth(View): def get(self, request, *args, **kwargs): - # Get start_date and end_date from the request's GET parameters - # #999: not needed if we switch to django forms start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"' - # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use - # in context to display this data in the template. - csv_export.export_data_domain_growth_to_csv(response, start_date, end_date) + csv_export.DomainExport.export_data_domain_growth_to_csv(response, start_date, end_date) return response class ExportDataRequestsGrowth(View): def get(self, request, *args, **kwargs): - # Get start_date and end_date from the request's GET parameters - # #999: not needed if we switch to django forms start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="requests-{start_date}-to-{end_date}.csv"' - # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use - # in context to display this data in the template. csv_export.DomainRequestExport.export_data_requests_growth_to_csv(response, start_date, end_date) return response @@ -209,25 +201,21 @@ class ExportDataRequestsGrowth(View): class ExportDataManagedDomains(View): def get(self, request, *args, **kwargs): - # Get start_date and end_date from the request's GET parameters - # #999: not needed if we switch to django forms start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' - csv_export.export_data_managed_domains_to_csv(response, start_date, end_date) + csv_export.DomainExport.export_data_managed_domains_to_csv(response, start_date, end_date) return response class ExportDataUnmanagedDomains(View): def get(self, request, *args, **kwargs): - # Get start_date and end_date from the request's GET parameters - # #999: not needed if we switch to django forms start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' - csv_export.export_data_unmanaged_domains_to_csv(response, start_date, end_date) + csv_export.DomainExport.export_data_unmanaged_domains_to_csv(response, start_date, end_date) return response From 4b40b95bac9efeb5ea144a3009f8d41d91f37220 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 26 Jun 2024 17:41:33 -0400 Subject: [PATCH 02/31] fix fed agency and org type displays --- src/registrar/utility/csv_export.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 345ed7be1..2c88f59d2 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -187,6 +187,7 @@ class DomainExport: "domain__created_at", "domain__deleted", "authorizing_official__email", + "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) @@ -234,6 +235,7 @@ class DomainExport: annotations = {} additional_values = [ "domain__name", + "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) @@ -282,6 +284,7 @@ class DomainExport: annotations = {} additional_values = [ "domain__name", + "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) @@ -359,6 +362,7 @@ class DomainExport: "domain__expiration_date", "domain__created_at", "domain__deleted", + "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) @@ -658,13 +662,17 @@ class DomainExport: human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) domain_type = human_readable_domain_org_type if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: - domain_type = f"{human_readable_domain_federal_type} - {human_readable_domain_org_type}" + domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" if domain.get("domain__name") == "18f.gov": print(f'domain_type {domain_type}') print(f'federal_agency {domain.get("federal_agency")}') print(f'city {domain.get("city")}') + print(f'agency {domain.get("agency")}') + + print(f'federal_agency__agency {domain.get("federal_agency__agency")}') + # create a dictionary of fields which can be included in output. # "extra_fields" are precomputed fields (generated in the DB or parsed). FIELDS = { @@ -674,7 +682,7 @@ class DomainExport: "First ready on": first_ready_on, "Expiration date": expiration_date, "Domain type": domain_type, - "Agency": domain.get("federal_agency"), + "Agency": domain.get("federal_agency__agency"), "Organization name": domain.get("organization_name"), "City": domain.get("city"), "State": domain.get("state_territory"), From c0d513c28f36bb219ed8fa80309883072bbaf977 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 26 Jun 2024 18:31:21 -0400 Subject: [PATCH 03/31] security contact email working --- src/registrar/utility/csv_export.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 2c88f59d2..e51c67f1f 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -186,6 +186,7 @@ class DomainExport: "domain__expiration_date", "domain__created_at", "domain__deleted", + "domain__security_contact_registry_id", "authorizing_official__email", "federal_agency__agency", ] @@ -225,6 +226,9 @@ class DomainExport: ], } + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + domain_infos = ( DomainInformation.objects.select_related("domain") .filter(**filter_condition) @@ -235,11 +239,12 @@ class DomainExport: annotations = {} additional_values = [ "domain__name", + "domain__security_contact_registry_id", "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, {}, {}, additional_values) requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) # Write the csv file @@ -274,6 +279,9 @@ class DomainExport: ], } + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + domain_infos = ( DomainInformation.objects.select_related("domain") .filter(**filter_condition) @@ -284,11 +292,12 @@ class DomainExport: annotations = {} additional_values = [ "domain__name", + "domain__security_contact_registry_id", "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, {}, {}, additional_values) requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) # Write the csv file @@ -629,7 +638,7 @@ class DomainExport: # Annotate with security_contact from public_contacts for domain in queryset: - domain['security_contact_email'] = public_contacts.get(domain.get('domain__registry_id')) + domain['security_contact_email'] = public_contacts.get(domain.get('domain__security_contact_registry_id')) domain['invited_users'] = ', '.join(invited_users_dict.get(domain.get('domain__name'), [])) domain['managers'] = ', '.join(managers_dict.get(domain.get('domain__name'), [])) annotated_domains.append(domain) From cd279468e515622b61ba73b2a16a387453d1710f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 27 Jun 2024 16:15:25 -0400 Subject: [PATCH 04/31] wip; working --- src/registrar/utility/csv_export.py | 996 +++++++++++++++++++++++++++- src/registrar/views/admin_views.py | 36 +- 2 files changed, 1016 insertions(+), 16 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index e51c67f1f..55b0bf37f 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,3 +1,4 @@ +from abc import ABC, abstractmethod from collections import defaultdict import csv import logging @@ -122,11 +123,1004 @@ def get_sliced_requests(filter_condition): election_board, ] -class DomainExport: +class BaseExport(ABC): + """ + A generic class for exporting data which returns a csv file for the given model. + """ + + @classmethod + @abstractmethod + def model(self): + """ + Property to specify the model that the export class will handle. + Must be implemented by subclasses. + """ + pass + + @classmethod + def get_columns(cls): + """ + Returns the columns for CSV export. Override in subclasses as needed. + """ + return [] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields for the CSV export. Override in subclasses as needed. + """ + return [] + + @classmethod + def get_additional_args(cls): + """ + Returns additional keyword arguments as an empty dictionary. + Override in subclasses to provide specific arguments. + """ + return {} + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [] + + @classmethod + def get_prefetch_related(cls): + """ + Get a list of tables to pass to prefetch_related when building queryset. + """ + return [] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + return Q() + + @classmethod + def get_computed_fields(cls): + """ + Get a dict of computed fields. + """ + return {} + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [] + + @classmethod + def update_queryset(cls, queryset, **kwargs): + """ + Returns an updated queryset. Override in subclass to update queryset. + """ + return queryset + + @classmethod + def write_csv_before(cls, csv_writer, start_date=None, end_date=None): + """ + Write to csv file before the write_csv method. + Override in subclasses where needed. + """ + pass + + @classmethod + def annotate_and_retrieve_fields( + cls, initial_queryset, computed_fields, related_table_fields=None, include_many_to_many=False, **kwargs + ) -> QuerySet: + """ + Applies annotations to a queryset and retrieves specified fields, + including class-defined and annotation-defined. + + Parameters: + initial_queryset (QuerySet): Initial queryset. + computed_fields (dict, optional): Fields to compute {field_name: expression}. + related_table_fields (list, optional): Extra fields to retrieve; defaults to annotation keys if None. + include_many_to_many (bool, optional): Determines if we should include many to many fields or not + **kwargs: Additional keyword arguments for specific parameters (e.g., public_contacts, domain_invitations, + user_domain_roles). + + Returns: + QuerySet: Contains dictionaries with the specified fields for each record. + """ + if related_table_fields is None: + related_table_fields = [] + + # We can infer that if we're passing in annotations, + # we want to grab the result of said annotation. + if computed_fields: + related_table_fields.extend(computed_fields.keys()) + + # Get prexisting fields on the model + model_fields = set() + for field in cls.model()._meta.get_fields(): + # Exclude many to many fields unless we specify + many_to_many = isinstance(field, ManyToManyField) and include_many_to_many + if many_to_many or not isinstance(field, ManyToManyField): + model_fields.add(field.name) + + queryset = initial_queryset.annotate(**computed_fields).values(*model_fields, *related_table_fields) + + return cls.update_queryset(queryset, **kwargs) + + @classmethod + def export_data_to_csv(cls, csv_file, start_date=None, end_date=None): + """ + All domain metadata: + Exports domains of all statuses plus domain managers. + """ + writer = csv.writer(csv_file) + columns = cls.get_columns() + sort_fields = cls.get_sort_fields() + kwargs = cls.get_additional_args() + select_related = cls.get_select_related() + prefetch_related = cls.get_prefetch_related() + filter_conditions = cls.get_filter_conditions(start_date, end_date) + computed_fields = cls.get_computed_fields() + related_table_fields = cls.get_related_table_fields() + + model_queryset = ( + cls.model().objects.select_related(*select_related) + .prefetch_related(*prefetch_related) + .filter(filter_conditions) + .order_by(*sort_fields) + .distinct() + ) + + # Convert the queryset to a dictionary (including annotated fields) + annotated_queryset = cls.annotate_and_retrieve_fields(model_queryset, computed_fields, related_table_fields, **kwargs) + models_dict = convert_queryset_to_dict(annotated_queryset, is_model=False) + + # Write to csv file before the write_csv + cls.write_csv_before(writer, start_date, end_date) + + # Write the csv file + cls.write_csv(writer, columns, models_dict) + + + @classmethod + def write_csv( + cls, + writer, + columns, + models_dict, + should_write_header=True, + ): + """Receives params from the parent methods and outputs a CSV with filtered and sorted objects. + Works with write_header as long as the same writer object is passed.""" + + rows = [] + for object in models_dict.values(): + try: + row = cls.parse_row(columns, object) + rows.append(row) + except ValueError as err: + logger.error(f"csv_export -> Error when parsing row: {err}") + continue + + if should_write_header: + write_header(writer, columns) + + writer.writerows(rows) + + @classmethod + def parse_row(cls, columns, model): + """ + Given a set of columns and a model dictionary, generate a new row from cleaned column data. + Must be implemented by subclasses + """ + pass + + +class NewDomainExport(BaseExport): """ A collection of functions which return csv files regarding the Domain model. """ + @classmethod + def model(cls): + # Return the model class that this export handles + return DomainInformation + + @classmethod + def update_queryset(cls, queryset, **kwargs): + """ + Returns an updated queryset. + + Add security_contact_email, invited_users, and managers to the queryset, + based on public_contacts, domain_invitations and user_domain_roles + passed through kwargs. + """ + public_contacts = kwargs.get('public_contacts', {}) + domain_invitations = kwargs.get('domain_invitations', {}) + user_domain_roles = kwargs.get('user_domain_roles', {}) + + annotated_domain_infos = [] + + # Create mapping of domain to a list of invited users and managers + invited_users_dict = defaultdict(list) + for domain, email in domain_invitations: + invited_users_dict[domain].append(email) + + managers_dict = defaultdict(list) + for domain, email in user_domain_roles: + managers_dict[domain].append(email) + + # Annotate with security_contact from public_contacts + for domain_info in queryset: + domain_info['security_contact_email'] = public_contacts.get(domain_info.get('domain__security_contact_registry_id')) + domain_info['invited_users'] = ', '.join(invited_users_dict.get(domain_info.get('domain__name'), [])) + domain_info['managers'] = ', '.join(managers_dict.get(domain_info.get('domain__name'), [])) + annotated_domain_infos.append(domain_info) + + if annotated_domain_infos: + return annotated_domain_infos + + return queryset + + # ============================================================= # + # Helper functions for django ORM queries. # + # We are using these rather than pure python for speed reasons. # + # ============================================================= # + + @classmethod + def get_all_security_emails(cls): + """ + Fetch all PublicContact entries and return a mapping of registry_id to email. + """ + public_contacts = PublicContact.objects.values_list('registry_id', 'email') + return {registry_id: email for registry_id, email in public_contacts} + + @classmethod + def get_all_domain_invitations(cls): + """ + Fetch all DomainInvitation entries and return a mapping of domain to email. + """ + domain_invitations = DomainInvitation.objects.filter(status="invited").values_list('domain__name', 'email') + return list(domain_invitations) + + @classmethod + def get_all_user_domain_roles(cls): + """ + Fetch all UserDomainRole entries and return a mapping of domain to user__email. + """ + user_domain_roles = UserDomainRole.objects.select_related('user').values_list('domain__name', 'user__email') + return list(user_domain_roles) + + @classmethod + def parse_row(cls, columns, model): + """ + Given a set of columns and a model dictionary, generate a new row from cleaned column data. + """ + + status = model.get("domain__state") + human_readable_status = Domain.State.get_state_label(status) + + expiration_date = model.get("domain__expiration_date") + if expiration_date is None: + expiration_date = "(blank)" + + first_ready_on = model.get("domain__first_ready") + if first_ready_on is None: + first_ready_on = "(blank)" + + domain_org_type = model.get("generic_org_type") + human_readable_domain_org_type = DomainRequest.OrganizationChoices.get_org_label(domain_org_type) + domain_federal_type = model.get("federal_type") + human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) + domain_type = human_readable_domain_org_type + if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: + domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" + + if model.get("domain__name") == "18f.gov": + print(f'domain_type {domain_type}') + print(f'federal_agency {model.get("federal_agency")}') + print(f'city {model.get("city")}') + + print(f'agency {model.get("agency")}') + + print(f'federal_agency__agency {model.get("federal_agency__agency")}') + + # create a dictionary of fields which can be included in output. + # "extra_fields" are precomputed fields (generated in the DB or parsed). + FIELDS = { + + "Domain name": model.get("domain__name"), + "Status": human_readable_status, + "First ready on": first_ready_on, + "Expiration date": expiration_date, + "Domain type": domain_type, + "Agency": model.get("federal_agency__agency"), + "Organization name": model.get("organization_name"), + "City": model.get("city"), + "State": model.get("state_territory"), + "AO": model.get("ao_name"), + "AO email": model.get("authorizing_official__email"), + "Security contact email": model.get("security_contact_email"), + "Created at": model.get("domain__created_at"), + "Deleted": model.get("domain__deleted"), + "Domain managers": model.get("managers"), + "Invited domain managers": model.get("invited_users"), + } + + row = [FIELDS.get(column, "") for column in columns] + return row + + @classmethod + def get_sliced_domains(cls, filter_condition): + """Get filtered domains counts sliced by org type and election office. + Pass distinct=True when filtering by permissions so we do not to count multiples + when a domain has more that one manager. + """ + + domains = DomainInformation.objects.all().filter(**filter_condition).distinct() + domains_count = domains.count() + federal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() + interstate = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).count() + state_or_territory = ( + domains.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + ) + tribal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() + county = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() + city = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() + special_district = ( + domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + ) + school_district = ( + domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + ) + election_board = domains.filter(is_election_board=True).distinct().count() + + return [ + domains_count, + federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board, + ] + + +class DomainDataType(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Status", + "First ready on", + "Expiration date", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "AO", + "AO email", + "Security contact email", + "Domain managers", + "Invited domain managers", + ] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + # Coalesce is used to replace federal_type of None with ZZZZZ + return [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + + @classmethod + def get_additional_args(cls): + """ + Returns additional keyword arguments specific to DomainExport. + + Returns: + dict: Dictionary containing public_contacts, domain_invitations, and user_domain_roles. + """ + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + + # Fetch all relevant Invite entries + domain_invitations = cls.get_all_domain_invitations() + + # Fetch all relevant UserDomainRole entries + user_domain_roles = cls.get_all_user_domain_roles() + + return { + 'public_contacts': public_contacts, + 'domain_invitations': domain_invitations, + 'user_domain_roles': user_domain_roles, + } + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain", + "authorizing_official" + ] + + @classmethod + def get_prefetch_related(cls): + """ + Get a list of tables to pass to prefetch_related when building queryset. + """ + return [ + "permissions" + ] + + @classmethod + def get_computed_fields(cls, delimiter=", "): + """ + Get a dict of computed fields. + """ + return { + "ao_name": Concat( + Coalesce(F("authorizing_official__first_name"), Value("")), + Value(" "), + Coalesce(F("authorizing_official__last_name"), Value("")), + output_field=CharField(), + ), + } + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + print("DomainDataType::get_related_table_fields") + return [ + "domain__name", + "domain__state", + "domain__first_ready", + "domain__expiration_date", + "domain__created_at", + "domain__deleted", + "domain__security_contact_registry_id", + "authorizing_official__email", + "federal_agency__agency", + ] + + +class DomainDataFull(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Security contact email", + ] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + # Coalesce is used to replace federal_type of None with ZZZZZ + return [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + + @classmethod + def get_additional_args(cls): + """ + Returns additional keyword arguments specific to DomainExport. + + Returns: + dict: Dictionary containing public_contacts, domain_invitations, and user_domain_roles. + """ + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + + return { + 'public_contacts': public_contacts, + } + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain" + ] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + return Q( + domain__state__in = [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + ) + + @classmethod + def get_computed_fields(cls, delimiter=", "): + """ + Get a dict of computed fields. + """ + return { + "ao_name": Concat( + Coalesce(F("authorizing_official__first_name"), Value("")), + Value(" "), + Coalesce(F("authorizing_official__last_name"), Value("")), + output_field=CharField(), + ), + } + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "domain__name", + "domain__security_contact_registry_id", + "federal_agency__agency", + ] + + +class DomainDataFederal(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Security contact email", + ] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + # Coalesce is used to replace federal_type of None with ZZZZZ + return [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + + @classmethod + def get_additional_args(cls): + """ + Returns additional keyword arguments specific to DomainExport. + + Returns: + dict: Dictionary containing public_contacts, domain_invitations, and user_domain_roles. + """ + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + + return { + 'public_contacts': public_contacts, + } + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain" + ] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + return Q( + organization_type__icontains="federal", + domain__state__in=[ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ] + ) + + @classmethod + def get_computed_fields(cls, delimiter=", "): + """ + Get a dict of computed fields. + """ + return { + "ao_name": Concat( + Coalesce(F("authorizing_official__first_name"), Value("")), + Value(" "), + Coalesce(F("authorizing_official__last_name"), Value("")), + output_field=CharField(), + ), + } + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "domain__name", + "domain__security_contact_registry_id", + "federal_agency__agency", + ] + + +class DomainGrowth(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Status", + "Expiration date", + "Created at", + "First ready", + "Deleted", + ] + + # TODO: The below sort is not working properly + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + return [ + '-domain__state', + 'domain__first_ready', + 'domain__deleted', + 'domain__name', + ] + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain" + ] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + filter_ready = Q( + domain__state__in=[Domain.State.READY], + domain__first_ready__gte=start_date, + domain__first_ready__lte=end_date + ) + filter_deleted = Q( + domain__state__in=[Domain.State.DELETED], + domain__deleted__gte=start_date, + domain__deleted__lte=end_date + ) + return filter_ready | filter_deleted + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "domain__name", + "domain__state", + "domain__first_ready", + "domain__expiration_date", + "domain__created_at", + "domain__deleted", + "federal_agency__agency", + ] + + +class DomainManaged(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Domain type", + "Domain managers", + "Invited domain managers", + ] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + return [ + 'domain__name', + ] + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain" + ] + + @classmethod + def get_prefetch_related(cls): + """ + Get a list of tables to pass to prefetch_related when building queryset. + """ + return [ + "permissions" + ] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + end_date_formatted = format_end_date(end_date) + return Q( + domain__permissions__isnull=False, + domain__first_ready__lte=end_date_formatted, + ) + + + @classmethod + def get_additional_args(cls): + """ + Returns additional keyword arguments specific to DomainExport. + + Returns: + dict: Dictionary containing public_contacts, domain_invitations, and user_domain_roles. + """ + + # Fetch all relevant Invite entries + domain_invitations = cls.get_all_domain_invitations() + + # Fetch all relevant UserDomainRole entries + user_domain_roles = cls.get_all_user_domain_roles() + + return { + 'domain_invitations': domain_invitations, + 'user_domain_roles': user_domain_roles, + } + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "domain__name", + ] + + @classmethod + def write_csv_before(cls, csv_writer, start_date=None, end_date=None): + """ + Write to csv file before the write_csv method. + """ + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date_formatted, + } + managed_domains_sliced_at_start_date = cls.get_sliced_domains(filter_managed_domains_start_date) + + csv_writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) + csv_writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + csv_writer.writerow(managed_domains_sliced_at_start_date) + csv_writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_end_date = cls.get_sliced_domains(filter_managed_domains_end_date) + + csv_writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) + csv_writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + csv_writer.writerow(managed_domains_sliced_at_end_date) + csv_writer.writerow([]) + + +class DomainUnmanaged(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Domain type", + ] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + return [ + 'domain__name', + ] + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain" + ] + + @classmethod + def get_prefetch_related(cls): + """ + Get a list of tables to pass to prefetch_related when building queryset. + """ + return [ + "permissions" + ] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + end_date_formatted = format_end_date(end_date) + return Q( + domain__permissions__isnull=True, + domain__first_ready__lte=end_date_formatted, + ) + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "domain__name", + ] + + @classmethod + def write_csv_before(cls, csv_writer, start_date=None, end_date=None): + """ + Write to csv file before the write_csv method. + + """ + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + unmanaged_domains_sliced_at_start_date = cls.get_sliced_domains(filter_unmanaged_domains_start_date) + + csv_writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) + csv_writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + csv_writer.writerow(unmanaged_domains_sliced_at_start_date) + csv_writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_end_date = cls.get_sliced_domains(filter_unmanaged_domains_end_date) + + csv_writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) + csv_writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + csv_writer.writerow(unmanaged_domains_sliced_at_end_date) + csv_writer.writerow([]) + + +class DomainExport: @classmethod def export_data_type_to_csv(cls, csv_file): """ diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 9b013e36c..ba57737cb 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -49,8 +49,8 @@ class AnalyticsView(View): "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } - managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) - managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) + managed_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, @@ -60,8 +60,8 @@ class AnalyticsView(View): "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } - unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) - unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_unmanaged_domains_start_date) + unmanaged_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_unmanaged_domains_end_date) filter_ready_domains_start_date = { "domain__state__in": [models.Domain.State.READY], @@ -71,8 +71,8 @@ class AnalyticsView(View): "domain__state__in": [models.Domain.State.READY], "domain__first_ready__lte": end_date_formatted, } - ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) - ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) + ready_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_ready_domains_start_date) + ready_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_ready_domains_end_date) filter_deleted_domains_start_date = { "domain__state__in": [models.Domain.State.DELETED], @@ -82,8 +82,8 @@ class AnalyticsView(View): "domain__state__in": [models.Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) - deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) + deleted_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_deleted_domains_end_date) filter_requests_start_date = { "created_at__lte": start_date_formatted, @@ -142,7 +142,8 @@ class ExportDataType(View): # match the CSV example with all the fields response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - csv_export.DomainExport.export_data_type_to_csv(response) + #csv_export.DomainExport.export_data_type_to_csv(response) + csv_export.DomainDataType.export_data_to_csv(response) return response @@ -151,7 +152,8 @@ class ExportDataFull(View): # Smaller export based on 1 response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - csv_export.DomainExport.export_data_full_to_csv(response) + #csv_export.DomainExport.export_data_full_to_csv(response) + csv_export.DomainDataFull.export_data_to_csv(response) return response @@ -160,7 +162,8 @@ class ExportDataFederal(View): # Federal only response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - csv_export.DomainExport.export_data_federal_to_csv(response) + #csv_export.DomainExport.export_data_federal_to_csv(response) + csv_export.DomainDataFederal.export_data_to_csv(response) return response @@ -182,7 +185,8 @@ class ExportDataDomainsGrowth(View): response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"' - csv_export.DomainExport.export_data_domain_growth_to_csv(response, start_date, end_date) + #csv_export.DomainExport.export_data_domain_growth_to_csv(response, start_date, end_date) + csv_export.DomainGrowth.export_data_to_csv(response, start_date, end_date) return response @@ -205,7 +209,8 @@ class ExportDataManagedDomains(View): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' - csv_export.DomainExport.export_data_managed_domains_to_csv(response, start_date, end_date) + #csv_export.DomainExport.export_data_managed_domains_to_csv(response, start_date, end_date) + csv_export.DomainManaged.export_data_to_csv(response, start_date, end_date) return response @@ -215,7 +220,8 @@ class ExportDataUnmanagedDomains(View): start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' - csv_export.DomainExport.export_data_unmanaged_domains_to_csv(response, start_date, end_date) + response["Content-Disposition"] = f'attachment; filename="unmanaged-domains-{start_date}-to-{end_date}.csv"' + #csv_export.DomainExport.export_data_unmanaged_domains_to_csv(response, start_date, end_date) + csv_export.DomainUnmanaged.export_data_to_csv(response, start_date, end_date) return response From 10529e40946c1c053ce07deaa8afcfea091450d8 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 27 Jun 2024 16:24:50 -0400 Subject: [PATCH 05/31] refactored domain exports --- src/registrar/utility/csv_export.py | 650 +--------------------------- src/registrar/views/admin_views.py | 16 +- 2 files changed, 15 insertions(+), 651 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 55b0bf37f..5a8e88431 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -317,7 +317,7 @@ class BaseExport(ABC): pass -class NewDomainExport(BaseExport): +class DomainExport(BaseExport): """ A collection of functions which return csv files regarding the Domain model. """ @@ -490,7 +490,7 @@ class NewDomainExport(BaseExport): ] -class DomainDataType(NewDomainExport): +class DomainDataType(DomainExport): @classmethod def get_columns(cls): @@ -602,7 +602,7 @@ class DomainDataType(NewDomainExport): ] -class DomainDataFull(NewDomainExport): +class DomainDataFull(DomainExport): @classmethod def get_columns(cls): @@ -695,7 +695,7 @@ class DomainDataFull(NewDomainExport): ] -class DomainDataFederal(NewDomainExport): +class DomainDataFederal(DomainExport): @classmethod def get_columns(cls): @@ -789,7 +789,7 @@ class DomainDataFederal(NewDomainExport): ] -class DomainGrowth(NewDomainExport): +class DomainGrowth(DomainExport): @classmethod def get_columns(cls): @@ -866,7 +866,7 @@ class DomainGrowth(NewDomainExport): ] -class DomainManaged(NewDomainExport): +class DomainManaged(DomainExport): @classmethod def get_columns(cls): @@ -1004,7 +1004,7 @@ class DomainManaged(NewDomainExport): csv_writer.writerow([]) -class DomainUnmanaged(NewDomainExport): +class DomainUnmanaged(DomainExport): @classmethod def get_columns(cls): @@ -1120,642 +1120,6 @@ class DomainUnmanaged(NewDomainExport): csv_writer.writerow([]) -class DomainExport: - @classmethod - def export_data_type_to_csv(cls, csv_file): - """ - All domain metadata: - Exports domains of all statuses plus domain managers. - """ - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Status", - "First ready on", - "Expiration date", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - "Domain managers", - "Invited domain managers", - ] - - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - - # Fetch all relevant PublicContact entries - public_contacts = cls.get_all_security_emails() - - # Fetch all relevant Invite entries - domain_invitations = cls.get_all_domain_invitations() - - # Fetch all relevant ComainUserRole entries - user_domain_roles = cls.get_all_user_domain_roles() - - domain_infos = ( - DomainInformation.objects.select_related("domain", "authorizing_official") - .prefetch_related("permissions") - .order_by(*sort_fields) - .distinct() - ) - - annotations = cls._domain_metadata_annotations() - - # The .values returned from annotate_and_retrieve_fields can't go two levels deep - # (just returns the field id of say, "creator") - so we have to include this. - additional_values = [ - "domain__name", - "domain__state", - "domain__first_ready", - "domain__expiration_date", - "domain__created_at", - "domain__deleted", - "domain__security_contact_registry_id", - "authorizing_official__email", - "federal_agency__agency", - ] - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, domain_invitations, user_domain_roles, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - # Write the csv file - cls.write_csv_for_domains(writer, columns, requests_dict) - - @classmethod - def export_data_full_to_csv(cls, csv_file): - """Current full""" - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - - # Fetch all relevant PublicContact entries - public_contacts = cls.get_all_security_emails() - - domain_infos = ( - DomainInformation.objects.select_related("domain") - .filter(**filter_condition) - .order_by(*sort_fields) - .distinct() - ) - - annotations = {} - additional_values = [ - "domain__name", - "domain__security_contact_registry_id", - "federal_agency__agency", - ] - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, {}, {}, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - # Write the csv file - cls.write_csv_for_domains(writer, columns, requests_dict) - - @classmethod - def export_data_federal_to_csv(cls, csv_file): - """Current federal""" - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "organization_type__icontains": "federal", - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - - # Fetch all relevant PublicContact entries - public_contacts = cls.get_all_security_emails() - - domain_infos = ( - DomainInformation.objects.select_related("domain") - .filter(**filter_condition) - .order_by(*sort_fields) - .distinct() - ) - - annotations = {} - additional_values = [ - "domain__name", - "domain__security_contact_registry_id", - "federal_agency__agency", - ] - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, {}, {}, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - # Write the csv file - cls.write_csv_for_domains(writer, columns, requests_dict) - - @classmethod - def export_data_domain_growth_to_csv(cls, csv_file, start_date, end_date): - """ - Domain growth: - Receive start and end dates from the view, parse them. - Request from write_body READY domains that are created between - the start and end dates, as well as DELETED domains that are deleted between - the start and end dates. Specify sort params for both lists. - """ - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Status", - "Expiration date", - "Created at", - "First ready", - "Deleted", - ] - sort_fields = [ - "domain__first_ready", - "domain__name", - ] - filter_condition = { - "domain__state__in": [Domain.State.READY], - "domain__first_ready__lte": end_date_formatted, - "domain__first_ready__gte": start_date_formatted, - } - - # We also want domains deleted between sar and end dates, sorted - sort_fields_for_deleted_domains = [ - "domain__deleted", - "domain__name", - ] - filter_condition_for_deleted_domains = { - "domain__state__in": [Domain.State.DELETED], - "domain__deleted__lte": end_date_formatted, - "domain__deleted__gte": start_date_formatted, - } - - domain_infos = ( - DomainInformation.objects.select_related("domain") - .filter(**filter_condition) - .order_by(*sort_fields) - .distinct() - ) - deleted_domain_infos = ( - DomainInformation.objects.select_related("domain") - .filter(**filter_condition_for_deleted_domains) - .order_by(*sort_fields_for_deleted_domains) - .distinct() - ) - - annotations = {} - additional_values = [ - "domain__name", - "domain__state", - "domain__first_ready", - "domain__expiration_date", - "domain__created_at", - "domain__deleted", - "federal_agency__agency", - ] - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - # Convert the domain request queryset to a dictionary (including annotated fields) - deleted_annotated_domains = cls.annotate_and_retrieve_fields(deleted_domain_infos, annotations, {}, {}, {}, additional_values) - deleted_requests_dict = convert_queryset_to_dict(deleted_annotated_domains, is_model=False) - - cls.write_csv_for_domains( - writer, columns, requests_dict - - ) - cls.write_csv_for_domains( - writer, - columns, - deleted_requests_dict, - should_write_header=False, - ) - - @classmethod - def export_data_managed_domains_to_csv(cls, csv_file, start_date, end_date): - """ - Managed domains: - Get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date.""" - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - "Domain managers", - "Invited domain managers", - ] - sort_fields = [ - "domain__name", - ] - filter_managed_domains_start_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": start_date_formatted, - } - managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) - - writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(managed_domains_sliced_at_start_date) - writer.writerow([]) - - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date_formatted, - } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) - - writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(managed_domains_sliced_at_end_date) - writer.writerow([]) - - domain_invitations = cls.get_all_domain_invitations() - - # Fetch all relevant ComainUserRole entries - user_domain_roles = cls.get_all_user_domain_roles() - - annotations = {} - # The .values returned from annotate_and_retrieve_fields can't go two levels deep - # (just returns the field id of say, "creator") - so we have to include this. - additional_values = [ - "domain__name", - ] - - domain_infos = ( - DomainInformation.objects.select_related("domain") - .prefetch_related("permissions") - .filter(**filter_managed_domains_end_date) - .order_by(*sort_fields) - .distinct() - ) - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, domain_invitations, user_domain_roles, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - cls.write_csv_for_domains( - writer, - columns, - requests_dict - ) - - @classmethod - def export_data_unmanaged_domains_to_csv(cls, csv_file, start_date, end_date): - """ - Unmanaged domains: - Get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date.""" - - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] - - filter_unmanaged_domains_start_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": start_date_formatted, - } - unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) - - writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_start_date) - writer.writerow([]) - - filter_unmanaged_domains_end_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": end_date_formatted, - } - unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) - - writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_end_date) - writer.writerow([]) - - annotations = {} - # The .values returned from annotate_and_retrieve_fields can't go two levels deep - # (just returns the field id of say, "creator") - so we have to include this. - additional_values = [ - "domain__name", - ] - domain_infos = ( - DomainInformation.objects.select_related("domain") - .filter(**filter_unmanaged_domains_end_date) - .order_by(*sort_fields) - .distinct() - ) - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - cls.write_csv_for_domains( - writer, - columns, - requests_dict - ) - - @classmethod - def _domain_metadata_annotations(cls, delimiter=", "): - """""" - return { - "ao_name": Concat( - Coalesce(F("authorizing_official__first_name"), Value("")), - Value(" "), - Coalesce(F("authorizing_official__last_name"), Value("")), - output_field=CharField(), - ), - } - - @classmethod - def annotate_and_retrieve_fields( - cls, domains, annotations, public_contacts={}, domain_invitations={}, user_domain_roles={}, additional_values=None, include_many_to_many=False - ) -> QuerySet: - """ - Applies annotations to a queryset and retrieves specified fields, - including class-defined and annotation-defined. - - Parameters: - requests (QuerySet): Initial queryset. - annotations (dict, optional): Fields to compute {field_name: expression}. - additional_values (list, optional): Extra fields to retrieve; defaults to annotation keys if None. - include_many_to_many (bool, optional): Determines if we should include many to many fields or not - - Returns: - QuerySet: Contains dictionaries with the specified fields for each record. - """ - - if additional_values is None: - additional_values = [] - - # We can infer that if we're passing in annotations, - # we want to grab the result of said annotation. - if annotations: - additional_values.extend(annotations.keys()) - - # Get prexisting fields on DomainRequest - domain_fields = set() - for field in DomainInformation._meta.get_fields(): - # Exclude many to many fields unless we specify - many_to_many = isinstance(field, ManyToManyField) and include_many_to_many - if many_to_many or not isinstance(field, ManyToManyField): - domain_fields.add(field.name) - - queryset = domains.annotate(**annotations).values(*domain_fields, *additional_values) - annotated_domains = [] - - # Create mapping of domain to a list of invited users and managers - invited_users_dict = defaultdict(list) - for domain, email in domain_invitations: - invited_users_dict[domain].append(email) - - managers_dict = defaultdict(list) - for domain, email in user_domain_roles: - managers_dict[domain].append(email) - - # Annotate with security_contact from public_contacts - for domain in queryset: - domain['security_contact_email'] = public_contacts.get(domain.get('domain__security_contact_registry_id')) - domain['invited_users'] = ', '.join(invited_users_dict.get(domain.get('domain__name'), [])) - domain['managers'] = ', '.join(managers_dict.get(domain.get('domain__name'), [])) - annotated_domains.append(domain) - - if annotated_domains: - return annotated_domains - - return queryset - - @staticmethod - def parse_row_for_domains(columns, domain): - """ - Given a set of columns and a request dictionary, generate a new row from cleaned column data. - """ - - status = domain.get("domain__state") - human_readable_status = Domain.State.get_state_label(status) - - expiration_date = domain.get("domain__expiration_date") - if expiration_date is None: - expiration_date = "(blank)" - - first_ready_on = domain.get("domain__first_ready") - if first_ready_on is None: - first_ready_on = "(blank)" - - domain_org_type = domain.get("generic_org_type") - human_readable_domain_org_type = DomainRequest.OrganizationChoices.get_org_label(domain_org_type) - domain_federal_type = domain.get("federal_type") - human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) - domain_type = human_readable_domain_org_type - if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: - domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" - - if domain.get("domain__name") == "18f.gov": - print(f'domain_type {domain_type}') - print(f'federal_agency {domain.get("federal_agency")}') - print(f'city {domain.get("city")}') - - print(f'agency {domain.get("agency")}') - - print(f'federal_agency__agency {domain.get("federal_agency__agency")}') - - # create a dictionary of fields which can be included in output. - # "extra_fields" are precomputed fields (generated in the DB or parsed). - FIELDS = { - - "Domain name": domain.get("domain__name"), - "Status": human_readable_status, - "First ready on": first_ready_on, - "Expiration date": expiration_date, - "Domain type": domain_type, - "Agency": domain.get("federal_agency__agency"), - "Organization name": domain.get("organization_name"), - "City": domain.get("city"), - "State": domain.get("state_territory"), - "AO": domain.get("ao_name"), - "AO email": domain.get("authorizing_official__email"), - "Security contact email": domain.get("security_contact_email"), - "Created at": domain.get("domain__created_at"), - "Deleted": domain.get("domain__deleted"), - "Domain managers": domain.get("managers"), - "Invited domain managers": domain.get("invited_users"), - } - - row = [FIELDS.get(column, "") for column in columns] - return row - - @staticmethod - def write_csv_for_domains( - writer, - columns, - domains_dict, - should_write_header=True, - ): - """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. - Works with write_header as long as the same writer object is passed.""" - - rows = [] - for domain in domains_dict.values(): - try: - row = DomainExport.parse_row_for_domains(columns, domain) - rows.append(row) - except ValueError as err: - logger.error(f"csv_export -> Error when parsing row: {err}") - continue - - if should_write_header: - write_header(writer, columns) - - writer.writerows(rows) - - # ============================================================= # - # Helper functions for django ORM queries. # - # We are using these rather than pure python for speed reasons. # - # ============================================================= # - - @classmethod - def get_all_security_emails(cls): - """ - Fetch all PublicContact entries and return a mapping of registry_id to email. - """ - public_contacts = PublicContact.objects.values_list('registry_id', 'email') - return {registry_id: email for registry_id, email in public_contacts} - - @classmethod - def get_all_domain_invitations(cls): - """ - Fetch all DomainInvitation entries and return a mapping of domain to email. - """ - domain_invitations = DomainInvitation.objects.filter(status="invited").values_list('domain__name', 'email') - return list(domain_invitations) - - @classmethod - def get_all_user_domain_roles(cls): - """ - Fetch all UserDomainRole entries and return a mapping of domain to user__email. - """ - user_domain_roles = UserDomainRole.objects.select_related('user').values_list('domain__name', 'user__email') - return list(user_domain_roles) - - - class DomainRequestExport: """ A collection of functions which return csv files regarding the DomainRequest model. diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index ba57737cb..9f89c9bd5 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -49,8 +49,8 @@ class AnalyticsView(View): "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } - managed_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_managed_domains_start_date) - managed_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_managed_domains_end_date) + managed_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, @@ -60,8 +60,8 @@ class AnalyticsView(View): "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } - unmanaged_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_unmanaged_domains_start_date) - unmanaged_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_start_date) + unmanaged_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_end_date) filter_ready_domains_start_date = { "domain__state__in": [models.Domain.State.READY], @@ -71,8 +71,8 @@ class AnalyticsView(View): "domain__state__in": [models.Domain.State.READY], "domain__first_ready__lte": end_date_formatted, } - ready_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_ready_domains_start_date) - ready_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_ready_domains_end_date) + ready_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_ready_domains_start_date) + ready_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_ready_domains_end_date) filter_deleted_domains_start_date = { "domain__state__in": [models.Domain.State.DELETED], @@ -82,8 +82,8 @@ class AnalyticsView(View): "domain__state__in": [models.Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_deleted_domains_start_date) - deleted_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_deleted_domains_end_date) + deleted_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_end_date) filter_requests_start_date = { "created_at__lte": start_date_formatted, From 38125ea97f0b937160ff9e7041469b8dd6b97884 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 28 Jun 2024 00:24:25 -0400 Subject: [PATCH 06/31] refactored domain request exports --- src/registrar/utility/csv_export.py | 593 ++++++++++++---------------- src/registrar/views/admin_views.py | 18 +- 2 files changed, 268 insertions(+), 343 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 5a8e88431..0940e087f 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -33,17 +33,14 @@ def write_header(writer, columns): """ writer.writerow(columns) - def get_default_start_date(): # Default to a date that's prior to our first deployment return timezone.make_aware(datetime(2023, 11, 1)) - def get_default_end_date(): # Default to now() return timezone.now() - def format_start_date(start_date): return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() @@ -51,78 +48,6 @@ def format_start_date(start_date): def format_end_date(end_date): return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() - -def get_sliced_domains(filter_condition): - """Get filtered domains counts sliced by org type and election office. - Pass distinct=True when filtering by permissions so we do not to count multiples - when a domain has more that one manager. - """ - - domains = DomainInformation.objects.all().filter(**filter_condition).distinct() - domains_count = domains.count() - federal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).count() - state_or_territory = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() - ) - tribal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() - special_district = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - ) - school_district = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() - ) - election_board = domains.filter(is_election_board=True).distinct().count() - - return [ - domains_count, - federal, - interstate, - state_or_territory, - tribal, - county, - city, - special_district, - school_district, - election_board, - ] - - -def get_sliced_requests(filter_condition): - """Get filtered requests counts sliced by org type and election office.""" - requests = DomainRequest.objects.all().filter(**filter_condition).distinct() - requests_count = requests.count() - federal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() - state_or_territory = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() - ) - tribal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() - special_district = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - ) - school_district = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() - ) - election_board = requests.filter(is_election_board=True).distinct().count() - - return [ - requests_count, - federal, - interstate, - state_or_territory, - tribal, - county, - city, - special_district, - school_district, - election_board, - ] - class BaseExport(ABC): """ A generic class for exporting data which returns a csv file for the given model. @@ -173,6 +98,13 @@ class BaseExport(ABC): """ return [] + @classmethod + def get_exclusions(cls): + """ + Get a Q object of exclusion conditions to use when building queryset. + """ + return Q() + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ @@ -260,6 +192,7 @@ class BaseExport(ABC): kwargs = cls.get_additional_args() select_related = cls.get_select_related() prefetch_related = cls.get_prefetch_related() + exclusions = cls.get_exclusions() filter_conditions = cls.get_filter_conditions(start_date, end_date) computed_fields = cls.get_computed_fields() related_table_fields = cls.get_related_table_fields() @@ -268,6 +201,7 @@ class BaseExport(ABC): cls.model().objects.select_related(*select_related) .prefetch_related(*prefetch_related) .filter(filter_conditions) + .exclude(exclusions) .order_by(*sort_fields) .distinct() ) @@ -282,7 +216,6 @@ class BaseExport(ABC): # Write the csv file cls.write_csv(writer, columns, models_dict) - @classmethod def write_csv( cls, @@ -588,7 +521,6 @@ class DomainDataType(DomainExport): """ Get a list of fields from related tables. """ - print("DomainDataType::get_related_table_fields") return [ "domain__name", "domain__state", @@ -1120,142 +1052,263 @@ class DomainUnmanaged(DomainExport): csv_writer.writerow([]) -class DomainRequestExport: - """ - A collection of functions which return csv files regarding the DomainRequest model. - """ - - # Get all columns on the full metadata report - all_columns = [ - "Domain request", - "Submitted at", - "Status", - "Domain type", - "Federal type", - "Federal agency", - "Organization name", - "Election office", - "City", - "State/territory", - "Region", - "Creator first name", - "Creator last name", - "Creator email", - "Creator approved domains count", - "Creator active requests count", - "Alternative domains", - "AO first name", - "AO last name", - "AO email", - "AO title/role", - "Request purpose", - "Request additional details", - "Other contacts", - "CISA regional representative", - "Current websites", - "Investigator", - ] +class DomainRequestExport(BaseExport): @classmethod - def export_data_requests_growth_to_csv(cls, csv_file, start_date, end_date): + def model(cls): + # Return the model class that this export handles + return DomainRequest + + @classmethod + def get_sliced_requests(cls, filter_condition): + """Get filtered requests counts sliced by org type and election office.""" + requests = DomainRequest.objects.all().filter(**filter_condition).distinct() + requests_count = requests.count() + federal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() + interstate = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() + state_or_territory = ( + requests.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + ) + tribal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() + county = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() + city = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() + special_district = ( + requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + ) + school_district = ( + requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + ) + election_board = requests.filter(is_election_board=True).distinct().count() + + return [ + requests_count, + federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board, + ] + + @classmethod + def parse_row(cls, columns, model): """ - Growth report: - Receive start and end dates from the view, parse them. - Request from write_requests_body SUBMITTED requests that are created between - the start and end dates. Specify sort params. + Given a set of columns and a model dictionary, generate a new row from cleaned column data. """ - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ + # Handle the federal_type field. Defaults to the wrong format. + federal_type = model.get("federal_type") + human_readable_federal_type = BranchChoices.get_branch_label(federal_type) if federal_type else None + + # Handle the org_type field + org_type = model.get("generic_org_type") or model.get("organization_type") + human_readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type) if org_type else None + + # Handle the status field. Defaults to the wrong format. + status = model.get("status") + status_display = DomainRequest.DomainRequestStatus.get_status_label(status) if status else None + + # Handle the region field. + state_territory = model.get("state_territory") + region = get_region(state_territory) if state_territory else None + + # Handle the requested_domain field (add a default if None) + requested_domain = model.get("requested_domain__name") + requested_domain_name = requested_domain if requested_domain else "No requested domain" + + # Handle the election field. N/A if None, "Yes"/"No" if boolean + human_readable_election_board = "N/A" + is_election_board = model.get("is_election_board") + if is_election_board is not None: + human_readable_election_board = "Yes" if is_election_board else "No" + + # Handle the additional details field. Pipe seperated. + cisa_rep_first = model.get("cisa_representative_first_name") + cisa_rep_last = model.get("cisa_representative_last_name") + name = [n for n in [cisa_rep_first, cisa_rep_last] if n] + + cisa_rep = " ".join(name) if name else None + details = [cisa_rep, model.get("anything_else")] + additional_details = " | ".join([field for field in details if field]) + + # create a dictionary of fields which can be included in output. + # "extra_fields" are precomputed fields (generated in the DB or parsed). + FIELDS = { + # Parsed fields - defined above. + "Domain request": requested_domain_name, + "Region": region, + "Status": status_display, + "Election office": human_readable_election_board, + "Federal type": human_readable_federal_type, + "Domain type": human_readable_org_type, + "Request additional details": additional_details, + # Annotated fields - passed into the request dict. + "Creator approved domains count": model.get("creator_approved_domains_count", 0), + "Creator active requests count": model.get("creator_active_requests_count", 0), + "Alternative domains": model.get("all_alternative_domains"), + "Other contacts": model.get("all_other_contacts"), + "Current websites": model.get("all_current_websites"), + # Untouched FK fields - passed into the request dict. + "Federal agency": model.get("federal_agency__agency"), + "AO first name": model.get("authorizing_official__first_name"), + "AO last name": model.get("authorizing_official__last_name"), + "AO email": model.get("authorizing_official__email"), + "AO title/role": model.get("authorizing_official__title"), + "Creator first name": model.get("creator__first_name"), + "Creator last name": model.get("creator__last_name"), + "Creator email": model.get("creator__email"), + "Investigator": model.get("investigator__email"), + # Untouched fields + "Organization name": model.get("organization_name"), + "City": model.get("city"), + "State/territory": model.get("state_territory"), + "Request purpose": model.get("purpose"), + "CISA regional representative": model.get("cisa_representative_email"), + "Submitted at": model.get("submission_date"), + } + + row = [FIELDS.get(column, "") for column in columns] + return row + + +class DomainRequestGrowth(DomainRequestExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainRequestGrowth. + """ + return [ "Domain request", "Domain type", "Federal type", "Submitted at", ] - sort_fields = [ + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + return [ "requested_domain__name", ] - filter_condition = { - "status": DomainRequest.DomainRequestStatus.SUBMITTED, - "submission_date__lte": end_date_formatted, - "submission_date__gte": start_date_formatted, - } - - # We don't want to annotate anything, but we do want to access the requested domain name - annotations = {} - additional_values = ["requested_domain__name"] - - all_requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct() - - annotated_requests = cls.annotate_and_retrieve_fields(all_requests, annotations, additional_values) - requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False) - - cls.write_csv_for_requests(writer, columns, requests_dict) - + @classmethod - def export_full_domain_request_report(cls, csv_file): + def get_filter_conditions(cls, start_date=None, end_date=None): """ - Generates a detailed domain request report to a CSV file. - - Retrieves and annotates DomainRequest objects, excluding 'STARTED' status, - with related data optimizations via select/prefetch and annotation. - - Annotated with counts and aggregates of related entities. - Converts to dict and writes to CSV using predefined columns. - - Parameters: - csv_file (file-like object): Target CSV file. + Get a Q object of filter conditions to filter when building queryset. """ - writer = csv.writer(csv_file) - requests = ( - DomainRequest.objects.select_related( - "creator", "authorizing_official", "federal_agency", "investigator", "requested_domain" - ) - .prefetch_related("current_websites", "other_contacts", "alternative_domains") - .exclude(status__in=[DomainRequest.DomainRequestStatus.STARTED]) - .order_by( - "status", - "requested_domain__name", - ) - .distinct() + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + return Q( + status=DomainRequest.DomainRequestStatus.SUBMITTED, + submission_date__lte=end_date_formatted, + submission_date__gte=start_date_formatted, ) - # Annotations are custom columns returned to the queryset (AKA: computed in the DB). - annotations = cls._full_domain_request_annotations() - - # The .values returned from annotate_and_retrieve_fields can't go two levels deep - # (just returns the field id of say, "creator") - so we have to include this. - additional_values = [ - "requested_domain__name", - "federal_agency__agency", - "authorizing_official__first_name", - "authorizing_official__last_name", - "authorizing_official__email", - "authorizing_official__title", - "creator__first_name", - "creator__last_name", - "creator__email", - "investigator__email", + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "requested_domain__name" ] + - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_requests = cls.annotate_and_retrieve_fields(requests, annotations, additional_values) - requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False) - - # Write the csv file - cls.write_csv_for_requests(writer, cls.all_columns, requests_dict) +class DomainRequestDataFull(DomainRequestExport): @classmethod - def _full_domain_request_annotations(cls, delimiter=" | "): - """Returns the annotations for the full domain request report""" + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainRequestGrowth. + """ + return [ + "Domain request", + "Submitted at", + "Status", + "Domain type", + "Federal type", + "Federal agency", + "Organization name", + "Election office", + "City", + "State/territory", + "Region", + "Creator first name", + "Creator last name", + "Creator email", + "Creator approved domains count", + "Creator active requests count", + "Alternative domains", + "AO first name", + "AO last name", + "AO email", + "AO title/role", + "Request purpose", + "Request additional details", + "Other contacts", + "CISA regional representative", + "Current websites", + "Investigator", + ] + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "creator", + "authorizing_official", + "federal_agency", + "investigator", + "requested_domain" + ] + + @classmethod + def get_prefetch_related(cls): + """ + Get a list of tables to pass to prefetch_related when building queryset. + """ + return [ + "current_websites", + "other_contacts", + "alternative_domains" + ] + + @classmethod + def get_exclusions(cls): + """ + Get a Q object of exclusion conditions to use when building queryset. + """ + return Q( + status__in=[DomainRequest.DomainRequestStatus.STARTED] + ) + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + return [ + "status", + "requested_domain__name", + ] + + @classmethod + def get_computed_fields(cls, delimiter=", "): + """ + Get a dict of computed fields. + """ return { - "creator_approved_domains_count": DomainRequestExport.get_creator_approved_domains_count_query(), - "creator_active_requests_count": DomainRequestExport.get_creator_active_requests_count_query(), + "creator_approved_domains_count": cls.get_creator_approved_domains_count_query(), + "creator_active_requests_count": cls.get_creator_active_requests_count_query(), "all_current_websites": StringAgg("current_websites__website", delimiter=delimiter, distinct=True), "all_alternative_domains": StringAgg("alternative_domains__website", delimiter=delimiter, distinct=True), # Coerce the other contacts object to "{first_name} {last_name} {email}" @@ -1271,155 +1324,33 @@ class DomainRequestExport: distinct=True, ), } - - @staticmethod - def write_csv_for_requests( - writer, - columns, - requests_dict, - should_write_header=True, - ): - """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. - Works with write_header as long as the same writer object is passed.""" - - rows = [] - for request in requests_dict.values(): - try: - row = DomainRequestExport.parse_row_for_requests(columns, request) - rows.append(row) - except ValueError as err: - logger.error(f"csv_export -> Error when parsing row: {err}") - continue - - if should_write_header: - write_header(writer, columns) - - writer.writerows(rows) - - @staticmethod - def parse_row_for_requests(columns, request): - """ - Given a set of columns and a request dictionary, generate a new row from cleaned column data. - """ - - # Handle the federal_type field. Defaults to the wrong format. - federal_type = request.get("federal_type") - human_readable_federal_type = BranchChoices.get_branch_label(federal_type) if federal_type else None - - # Handle the org_type field - org_type = request.get("generic_org_type") or request.get("organization_type") - human_readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type) if org_type else None - - # Handle the status field. Defaults to the wrong format. - status = request.get("status") - status_display = DomainRequest.DomainRequestStatus.get_status_label(status) if status else None - - # Handle the region field. - state_territory = request.get("state_territory") - region = get_region(state_territory) if state_territory else None - - # Handle the requested_domain field (add a default if None) - requested_domain = request.get("requested_domain__name") - requested_domain_name = requested_domain if requested_domain else "No requested domain" - - # Handle the election field. N/A if None, "Yes"/"No" if boolean - human_readable_election_board = "N/A" - is_election_board = request.get("is_election_board") - if is_election_board is not None: - human_readable_election_board = "Yes" if is_election_board else "No" - - # Handle the additional details field. Pipe seperated. - cisa_rep_first = request.get("cisa_representative_first_name") - cisa_rep_last = request.get("cisa_representative_last_name") - name = [n for n in [cisa_rep_first, cisa_rep_last] if n] - - cisa_rep = " ".join(name) if name else None - details = [cisa_rep, request.get("anything_else")] - additional_details = " | ".join([field for field in details if field]) - - # create a dictionary of fields which can be included in output. - # "extra_fields" are precomputed fields (generated in the DB or parsed). - FIELDS = { - # Parsed fields - defined above. - "Domain request": requested_domain_name, - "Region": region, - "Status": status_display, - "Election office": human_readable_election_board, - "Federal type": human_readable_federal_type, - "Domain type": human_readable_org_type, - "Request additional details": additional_details, - # Annotated fields - passed into the request dict. - "Creator approved domains count": request.get("creator_approved_domains_count", 0), - "Creator active requests count": request.get("creator_active_requests_count", 0), - "Alternative domains": request.get("all_alternative_domains"), - "Other contacts": request.get("all_other_contacts"), - "Current websites": request.get("all_current_websites"), - # Untouched FK fields - passed into the request dict. - "Federal agency": request.get("federal_agency__agency"), - "AO first name": request.get("authorizing_official__first_name"), - "AO last name": request.get("authorizing_official__last_name"), - "AO email": request.get("authorizing_official__email"), - "AO title/role": request.get("authorizing_official__title"), - "Creator first name": request.get("creator__first_name"), - "Creator last name": request.get("creator__last_name"), - "Creator email": request.get("creator__email"), - "Investigator": request.get("investigator__email"), - # Untouched fields - "Organization name": request.get("organization_name"), - "City": request.get("city"), - "State/territory": request.get("state_territory"), - "Request purpose": request.get("purpose"), - "CISA regional representative": request.get("cisa_representative_email"), - "Submitted at": request.get("submission_date"), - } - - row = [FIELDS.get(column, "") for column in columns] - return row - + @classmethod - def annotate_and_retrieve_fields( - cls, requests, annotations, additional_values=None, include_many_to_many=False - ) -> QuerySet: + def get_related_table_fields(cls): """ - Applies annotations to a queryset and retrieves specified fields, - including class-defined and annotation-defined. - - Parameters: - requests (QuerySet): Initial queryset. - annotations (dict, optional): Fields to compute {field_name: expression}. - additional_values (list, optional): Extra fields to retrieve; defaults to annotation keys if None. - include_many_to_many (bool, optional): Determines if we should include many to many fields or not - - Returns: - QuerySet: Contains dictionaries with the specified fields for each record. + Get a list of fields from related tables. """ - - if additional_values is None: - additional_values = [] - - # We can infer that if we're passing in annotations, - # we want to grab the result of said annotation. - if annotations: - additional_values.extend(annotations.keys()) - - # Get prexisting fields on DomainRequest - domain_request_fields = set() - for field in DomainRequest._meta.get_fields(): - # Exclude many to many fields unless we specify - many_to_many = isinstance(field, ManyToManyField) and include_many_to_many - if many_to_many or not isinstance(field, ManyToManyField): - domain_request_fields.add(field.name) - - queryset = requests.annotate(**annotations).values(*domain_request_fields, *additional_values) - return queryset + return [ + "requested_domain__name", + "federal_agency__agency", + "authorizing_official__first_name", + "authorizing_official__last_name", + "authorizing_official__email", + "authorizing_official__title", + "creator__first_name", + "creator__last_name", + "creator__email", + "investigator__email", + ] + # ============================================================= # # Helper functions for django ORM queries. # # We are using these rather than pure python for speed reasons. # # ============================================================= # - @staticmethod - def get_creator_approved_domains_count_query(): + @classmethod + def get_creator_approved_domains_count_query(cls): """ Generates a Count query for distinct approved domain requests per creator. @@ -1434,8 +1365,8 @@ class DomainRequestExport: ) return query - @staticmethod - def get_creator_active_requests_count_query(): + @classmethod + def get_creator_active_requests_count_query(cls): """ Generates a Count query for distinct approved domain requests per creator. diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 9f89c9bd5..63e8492b5 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -91,8 +91,8 @@ class AnalyticsView(View): filter_requests_end_date = { "created_at__lte": end_date_formatted, } - requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) - requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) + requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests(filter_requests_start_date) + requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests(filter_requests_end_date) filter_submitted_requests_start_date = { "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, @@ -102,8 +102,8 @@ class AnalyticsView(View): "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": end_date_formatted, } - submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) - submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) + submitted_requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_start_date) + submitted_requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_end_date) context = dict( # Generate a dictionary of context variables that are common across all admin templates @@ -142,7 +142,6 @@ class ExportDataType(View): # match the CSV example with all the fields response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - #csv_export.DomainExport.export_data_type_to_csv(response) csv_export.DomainDataType.export_data_to_csv(response) return response @@ -152,7 +151,6 @@ class ExportDataFull(View): # Smaller export based on 1 response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - #csv_export.DomainExport.export_data_full_to_csv(response) csv_export.DomainDataFull.export_data_to_csv(response) return response @@ -162,7 +160,6 @@ class ExportDataFederal(View): # Federal only response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - #csv_export.DomainExport.export_data_federal_to_csv(response) csv_export.DomainDataFederal.export_data_to_csv(response) return response @@ -174,7 +171,7 @@ class ExportDomainRequestDataFull(View): """Returns a content disposition response for current-full-domain-request.csv""" response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-full-domain-request.csv"' - csv_export.DomainRequestExport.export_full_domain_request_report(response) + csv_export.DomainRequestDataFull.export_data_to_csv(response) return response @@ -185,7 +182,6 @@ class ExportDataDomainsGrowth(View): response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"' - #csv_export.DomainExport.export_data_domain_growth_to_csv(response, start_date, end_date) csv_export.DomainGrowth.export_data_to_csv(response, start_date, end_date) return response @@ -198,7 +194,7 @@ class ExportDataRequestsGrowth(View): response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="requests-{start_date}-to-{end_date}.csv"' - csv_export.DomainRequestExport.export_data_requests_growth_to_csv(response, start_date, end_date) + csv_export.DomainRequestGrowth.export_data_to_csv(response, start_date, end_date) return response @@ -209,7 +205,6 @@ class ExportDataManagedDomains(View): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' - #csv_export.DomainExport.export_data_managed_domains_to_csv(response, start_date, end_date) csv_export.DomainManaged.export_data_to_csv(response, start_date, end_date) return response @@ -221,7 +216,6 @@ class ExportDataUnmanagedDomains(View): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="unmanaged-domains-{start_date}-to-{end_date}.csv"' - #csv_export.DomainExport.export_data_unmanaged_domains_to_csv(response, start_date, end_date) csv_export.DomainUnmanaged.export_data_to_csv(response, start_date, end_date) return response From 59e912b69a8fa3e437d8249c50a24bcc1e4c51e8 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 28 Jun 2024 00:55:42 -0400 Subject: [PATCH 07/31] sorting in domain growth report --- src/registrar/utility/csv_export.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 0940e087f..6875f0e3d 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -12,6 +12,7 @@ from registrar.models import ( UserDomainRole, ) from django.db.models import QuerySet, Value, CharField, Count, Q, F +from django.db.models import Case, When, DateField from django.db.models import ManyToManyField from django.utils import timezone from django.core.paginator import Paginator @@ -119,6 +120,13 @@ class BaseExport(ABC): """ return {} + @classmethod + def get_annotations_for_sort(cls): + """ + Get a dict of annotations to make available for order_by clause. + """ + return {} + @classmethod def get_related_table_fields(cls): """ @@ -193,6 +201,7 @@ class BaseExport(ABC): select_related = cls.get_select_related() prefetch_related = cls.get_prefetch_related() exclusions = cls.get_exclusions() + annotations_for_sort = cls.get_annotations_for_sort() filter_conditions = cls.get_filter_conditions(start_date, end_date) computed_fields = cls.get_computed_fields() related_table_fields = cls.get_related_table_fields() @@ -202,6 +211,7 @@ class BaseExport(ABC): .prefetch_related(*prefetch_related) .filter(filter_conditions) .exclude(exclusions) + .annotate(**annotations_for_sort) .order_by(*sort_fields) .distinct() ) @@ -742,7 +752,20 @@ class DomainGrowth(DomainExport): "Deleted", ] - # TODO: The below sort is not working properly + @classmethod + def get_annotations_for_sort(cls, delimiter=", "): + """ + Get a dict of annotations to make available for sorting. + """ + today = timezone.now().date() + return { + "custom_sort": Case( + When(domain__state=Domain.State.READY, then='domain__first_ready'), + When(domain__state=Domain.State.DELETED, then='domain__deleted'), + default=Value(today), # Default value if no conditions match + output_field=DateField() + ) + } @classmethod def get_sort_fields(cls): @@ -751,8 +774,7 @@ class DomainGrowth(DomainExport): """ return [ '-domain__state', - 'domain__first_ready', - 'domain__deleted', + 'custom_sort', 'domain__name', ] From b600a26eb86afd3ae3f1c930dfb195db2e1728b2 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 28 Jun 2024 16:34:39 -0400 Subject: [PATCH 08/31] Unit tests --- .../generate_current_federal_report.py | 2 +- .../commands/generate_current_full_report.py | 2 +- src/registrar/tests/test_reports.py | 926 +++++++----------- src/registrar/utility/csv_export.py | 284 +++--- src/registrar/views/admin_views.py | 24 +- 5 files changed, 534 insertions(+), 704 deletions(-) diff --git a/src/registrar/management/commands/generate_current_federal_report.py b/src/registrar/management/commands/generate_current_federal_report.py index 6516bf99b..97d4fd7e4 100644 --- a/src/registrar/management/commands/generate_current_federal_report.py +++ b/src/registrar/management/commands/generate_current_federal_report.py @@ -50,7 +50,7 @@ class Command(BaseCommand): # Generate a file locally for upload with open(file_path, "w") as file: - csv_export.export_data_federal_to_csv(file) + csv_export.DomainDataFederal.export_data_to_csv(file) if check_path and not os.path.exists(file_path): raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") diff --git a/src/registrar/management/commands/generate_current_full_report.py b/src/registrar/management/commands/generate_current_full_report.py index be810ee10..4bcb9f502 100644 --- a/src/registrar/management/commands/generate_current_full_report.py +++ b/src/registrar/management/commands/generate_current_full_report.py @@ -49,7 +49,7 @@ class Command(BaseCommand): # Generate a file locally for upload with open(file_path, "w") as file: - csv_export.export_data_full_to_csv(file) + csv_export.DomainDataFull.export_data_to_csv(file) if check_path and not os.path.exists(file_path): raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 0028034fb..0015ae84f 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -1,21 +1,23 @@ -import csv import io from django.test import Client, RequestFactory from io import StringIO from registrar.models.domain_request import DomainRequest from registrar.models.domain import Domain -from registrar.models.utility.generic_helper import convert_queryset_to_dict from registrar.utility.csv_export import ( - export_data_managed_domains_to_csv, - export_data_unmanaged_domains_to_csv, - get_sliced_domains, - get_sliced_requests, - write_csv_for_domains, + DomainDataFull, + DomainDataType, + DomainDataFederal, + DomainGrowth, + DomainManaged, + DomainUnmanaged, + DomainExport, + DomainRequestExport, + DomainRequestGrowth, + DomainRequestDataFull, get_default_start_date, get_default_end_date, - DomainRequestExport, ) - +from django.db.models import Case, When from django.core.management import call_command from unittest.mock import MagicMock, call, mock_open, patch from api.views import get_current_federal, get_current_full @@ -45,10 +47,10 @@ class CsvReportsTest(MockDb): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), - call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), + call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), + call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), ] # We don't actually want to write anything for a test case, # we just want to verify what is being written. @@ -67,11 +69,12 @@ class CsvReportsTest(MockDb): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), - call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), - call("adomain2.gov,Interstate,,,,, \r\n"), + call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), + call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), + call("adomain2.gov,Interstate,,,,,\r\n"), + call("zdomain12.gov,Interstate,,,,,\r\n"), ] # We don't actually want to write anything for a test case, # we just want to verify what is being written. @@ -202,494 +205,299 @@ class ExportDataTest(MockDb, MockEppLib): def tearDown(self): super().tearDown() - def test_export_domains_to_writer_security_emails_and_first_ready(self): - """Test that export_domains_to_writer returns the - expected security email and first_ready value""" + @less_console_noise_decorator + def test_domain_data_type(self): + """Shows security contacts, domain managers, ao""" + # Add security email information + self.domain_1.name = "defaultsecurity.gov" + self.domain_1.save() + # Invoke setter + self.domain_1.security_contact + # Invoke setter + self.domain_2.security_contact + # Invoke setter + self.domain_3.security_contact + # Add a first ready date on the first domain. Leaving the others blank. + self.domain_1.first_ready = get_default_start_date() + self.domain_1.save() + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainDataType.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains, + # sorted alphabetially by domain name + expected_content = ( + "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,AO," + "AO email,Security contact email,Domain managers,Invited domain managers\n" + "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,, ,,," + "meoward@rocks.com,\n" + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,," + ', ,,dotgov@cisa.dhs.gov,"meoward@rocks.com, info@example.com, big_lebowski@dude.co",' + "woofwardthethird@rocks.com\n" + "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,, ,,,," + "squeaker@rocks.com\n" + "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "ddomain3.gov,On hold,(blank),2023-11-15,Federal,Armed Forces Retirement Home,,,, ,," + "security@mail.gov,,\n" + "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,, ,,registrar@dotgov.gov," + "meoward@rocks.com,squeaker@rocks.com\n" + "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,, ,,,meoward@rocks.com,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - with less_console_noise(): - # Add security email information - self.domain_1.name = "defaultsecurity.gov" - self.domain_1.save() - # Invoke setter - self.domain_1.security_contact - # Invoke setter - self.domain_2.security_contact - # Invoke setter - self.domain_3.security_contact + @less_console_noise_decorator + def test_domain_data_full(self): + """Shows security contacts, filtered by state""" + # Add security email information + self.domain_1.name = "defaultsecurity.gov" + self.domain_1.save() + # Invoke setter + self.domain_1.security_contact + # Invoke setter + self.domain_2.security_contact + # Invoke setter + self.domain_3.security_contact + # Add a first ready date on the first domain. Leaving the others blank. + self.domain_1.first_ready = get_default_start_date() + self.domain_1.save() + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainDataFull.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains, + # sorted alphabetially by domain name + expected_content = ( + "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,dotgov@cisa.dhs.gov\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + "adomain2.gov,Interstate,,,,,registrar@dotgov.gov\n" + "zdomain12.gov,Interstate,,,,,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - # Add a first ready date on the first domain. Leaving the others blank. - self.domain_1.first_ready = get_default_start_date() - self.domain_1.save() + @less_console_noise_decorator + def test_domain_data_federal(self): + """Shows security contacts, filtered by state and org type""" + # Add security email information + self.domain_1.name = "defaultsecurity.gov" + self.domain_1.save() + # Invoke setter + self.domain_1.security_contact + # Invoke setter + self.domain_2.security_contact + # Invoke setter + self.domain_3.security_contact + # Add a first ready date on the first domain. Leaving the others blank. + self.domain_1.first_ready = get_default_start_date() + self.domain_1.save() + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainDataFederal.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains, + # sorted alphabetially by domain name + expected_content = ( + "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,dotgov@cisa.dhs.gov\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - "Status", - "Expiration date", - "First ready on", - ] - sort_fields = ["domain__name"] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, + @less_console_noise_decorator + def test_domain_growth(self): + """Shows ready and deleted domains within a date range, sorted""" + # Remove "Created at" and "First ready" because we can't guess this immutable, dynamically generated test data + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Status", + "Expiration date", + # "Created at", + # "First ready", + "Deleted", + ] + sort = { + "custom_sort": Case( + When(domain__state=Domain.State.READY, then="domain__created_at"), + When(domain__state=Domain.State.DELETED, then="domain__deleted"), ) + } + with patch("registrar.utility.csv_export.DomainGrowth.get_columns", return_value=columns): + with patch("registrar.utility.csv_export.DomainGrowth.get_annotations_for_sort", return_value=sort): + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainGrowth.export_data_to_csv( + csv_file, + self.start_date.strftime("%Y-%m-%d"), + self.end_date.strftime("%Y-%m-%d"), + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains first, created between day-2 and day+2, sorted by created_at then name + # and DELETED domains deleted between day-2 and day+2, sorted by deleted then name + expected_content = ( + "Domain name,Domain type,Agency,Organization name,City," + "State,Status,Expiration date, Deleted\n" + "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" + "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" + "zdomain12.govInterstateReady(blank)\n" + "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-01\n" + "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank),2024-04-02\n" + "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = ( + csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + ) + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains, - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City,State,AO," - "AO email,Security contact email,Status,Expiration date, First ready on\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,Ready,(blank),2024-04-03\n" - "adomain2.gov,Interstate,(blank),Dns needed,(blank),(blank)\n" - "cdomain11.gov,Federal-Executive,WorldWarICentennialCommission,Ready,(blank),2024-04-02\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,security@mail.gov,On hold,2023-11-15,(blank)\n" - "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission," - "(blank),Ready,(blank),2023-11-01\n" - "zdomain12.govInterstateReady,(blank),2024-04-02\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_write_csv_for_domains(self): - """Test that write_body returns the - existing domain, test that sort by domain name works, - test that filter works""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Submitter", - "Submitter title", - "Submitter email", - "Submitter phone", - "Security contact email", - "Status", - ] - sort_fields = ["domain__name"] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, - ) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains, - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City,State,AO," - "AO email,Submitter,Submitter title,Submitter email,Submitter phone," - "Security contact email,Status\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,Ready\n" - "adomain2.gov,Interstate,Dns needed\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady\n" - "cdomain1.gov,Federal - Executive,World War I Centennial Commission,Ready\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,On hold\n" - "zdomain12.govInterstateReady\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_write_domains_body_additional(self): - """An additional test for filters and multi-column sort""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - sort_fields = ["domain__name", "federal_agency", "generic_org_type"] - filter_condition = { - "generic_org_type__icontains": "federal", - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, - ) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains, - # federal only - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City," - "State,Security contact email\n" - "adomain10.gov,Federal,Armed Forces Retirement Home\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommission\n" - "cdomain1.gov,Federal - Executive,World War I Centennial Commission\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): - """Test that domains that are - 1. READY and their first_ready dates are in range - 2. DELETED and their deleted dates are in range - are pulled when the growth report conditions are applied to export_domains_to_writed. - Test that ready domains are sorted by first_ready/deleted dates first, names second. - - We considered testing export_data_domain_growth_to_csv which calls write_body - and would have been easy to set up, but expected_content would contain created_at dates - which are hard to mock. - - TODO: Simplify if created_at is not needed for the report.""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Status", - "Expiration date", - ] - sort_fields = [ - "created_at", - "domain__name", - ] - sort_fields_for_deleted_domains = [ - "domain__deleted", - "domain__name", - ] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - ], - "domain__first_ready__lte": self.end_date, - "domain__first_ready__gte": self.start_date, - } - filter_conditions_for_deleted_domains = { - "domain__state__in": [ - Domain.State.DELETED, - ], - "domain__deleted__lte": self.end_date, - "domain__deleted__gte": self.start_date, - } - - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, - ) - write_csv_for_domains( - writer, - columns, - sort_fields_for_deleted_domains, - filter_conditions_for_deleted_domains, - should_get_domain_managers=False, - should_write_header=False, - ) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - - # Read the content into a variable - csv_content = csv_file.read() - - # We expect READY domains first, created between day-2 and day+2, sorted by created_at then name - # and DELETED domains deleted between day-2 and day+2, sorted by deleted then name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City," - "State,Status,Expiration date\n" - "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" - "zdomain12.govInterstateReady(blank)\n" - "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank)\n" - "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank)\n" - "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank)\n" - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) - - def test_export_domains_to_writer_domain_managers(self): - """Test that export_domains_to_writer returns the - expected domain managers. + @less_console_noise_decorator + def test_domain_managed(self): + """Shows ready and deleted domains by an end date, sorted An invited user, woofwardthethird, should also be pulled into this report. squeaker@rocks.com is invited to domain2 (DNS_NEEDED) and domain10 (No managers). - She should show twice in this report but not in test_export_data_managed_domains_to_csv.""" + She should show twice in this report but not in test_DomainManaged.""" + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainManaged.export_data_to_csv( + csv_file, + self.start_date.strftime("%Y-%m-%d"), + self.end_date.strftime("%Y-%m-%d"), + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "MANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "MANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City," + "Special district,School district,Election office\n" + "3,2,1,0,0,0,0,0,0,0\n" + "\n" + "Domain name,Domain type,Domain managers,Invited domain managers\n" + "cdomain11.gov,Federal - Executive,meoward@rocks.com,\n" + 'cdomain1.gov,Federal - Executive,"meoward@rocks.com, info@example.com, big_lebowski@dude.co",' + "woofwardthethird@rocks.com\n" + "zdomain12.gov,Interstate,meoward@rocks.com,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - with less_console_noise(): + @less_console_noise_decorator + def test_domain_unmanaged(self): + """Shows unmanaged domains by an end date, sorted""" + # Create a CSV file in memory + csv_file = StringIO() + DomainUnmanaged.export_data_to_csv( + csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") + ) + + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "UNMANAGED DOMAINS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "UNMANAGED DOMAINS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "1,1,0,0,0,0,0,0,0,0\n" + "\n" + "Domain name,Domain type\n" + "adomain10.gov,Federal\n" + ) + + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + + self.assertEqual(csv_content, expected_content) + + @less_console_noise_decorator + def test_domain_request_growth(self): + """Shows submitted requests within a date range, sorted""" + # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data + columns = [ + "Domain request", + "Domain type", + "Federal type", + # "Submitted at", + ] + with patch("registrar.utility.csv_export.DomainRequestGrowth.get_columns", return_value=columns): # Create a CSV file in memory csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Status", - "Expiration date", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - ] - sort_fields = ["domain__name"] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=True, - should_write_header=True, + DomainRequestGrowth.export_data_to_csv( + csv_file, + self.start_date.strftime("%Y-%m-%d"), + self.end_date.strftime("%Y-%m-%d"), ) - # Reset the CSV file's position to the beginning csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - # We expect READY domains, - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Status,Expiration date,Domain type,Agency," - "Organization name,City,State,AO,AO email," - "Security contact email,Domain manager 1,DM1 status,Domain manager 2,DM2 status," - "Domain manager 3,DM3 status,Domain manager 4,DM4 status\n" - "adomain10.gov,Ready,(blank),Federal,Armed Forces Retirement Home,,,, , ,squeaker@rocks.com, I\n" - "adomain2.gov,Dns needed,(blank),Interstate,,,,, , , ,meoward@rocks.com, R,squeaker@rocks.com, I\n" - "cdomain11.govReady,(blank),Federal-ExecutiveWorldWarICentennialCommissionmeoward@rocks.comR\n" - "cdomain1.gov,Ready,(blank),Federal - Executive,World War I Centennial Commission,,," - ", , , ,meoward@rocks.com,R,info@example.com,R,big_lebowski@dude.co,R," - "woofwardthethird@rocks.com,I\n" - "ddomain3.gov,On hold,(blank),Federal,Armed Forces Retirement Home,,,, , , ,,\n" - "zdomain12.gov,Ready,(blank),Interstate,meoward@rocks.com,R\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_export_data_managed_domains_to_csv(self): - """Test get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date. - - An invited user, woofwardthethird, should also be pulled into this report.""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - export_data_managed_domains_to_csv( - csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") - ) - - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. - expected_content = ( - "MANAGED DOMAINS COUNTS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," - "School district,Election office\n" - "0,0,0,0,0,0,0,0,0,0\n" - "\n" - "MANAGED DOMAINS COUNTS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City," - "Special district,School district,Election office\n" - "3,2,1,0,0,0,0,0,0,0\n" - "\n" - "Domain name,Domain type,Domain manager 1,DM1 status,Domain manager 2,DM2 status," - "Domain manager 3,DM3 status,Domain manager 4,DM4 status\n" - "cdomain11.govFederal-Executivemeoward@rocks.com, R\n" - "cdomain1.gov,Federal - Executive,meoward@rocks.com,R,info@example.com,R," - "big_lebowski@dude.co,R,woofwardthethird@rocks.com,I\n" - "zdomain12.govInterstatemeoward@rocks.com,R\n" - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) - - def test_export_data_unmanaged_domains_to_csv(self): - """Test get counts for domains that do not have domain managers for two different dates, - get list of unmanaged domains at end_date.""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - export_data_unmanaged_domains_to_csv( - csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") - ) - - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. - expected_content = ( - "UNMANAGED DOMAINS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," - "School district,Election office\n" - "0,0,0,0,0,0,0,0,0,0\n" - "\n" - "UNMANAGED DOMAINS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," - "School district,Election office\n" - "1,1,0,0,0,0,0,0,0,0\n" - "\n" - "Domain name,Domain type\n" - "adomain10.gov,Federal\n" - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) - - def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): - """Test that requests that are - 1. SUBMITTED and their submission_date are in range - are pulled when the growth report conditions are applied to export_requests_to_writed. - Test that requests are sorted by requested domain name. - """ - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - # We'll skip submission date because it's dynamic and therefore - # impossible to set in expected_content - columns = ["Domain request", "Domain type", "Federal type"] - sort_fields = [ - "requested_domain__name", - ] - filter_condition = { - "status": DomainRequest.DomainRequestStatus.SUBMITTED, - "submission_date__lte": self.end_date, - "submission_date__gte": self.start_date, - } - - additional_values = ["requested_domain__name"] - all_requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct() - annotated_requests = DomainRequestExport.annotate_and_retrieve_fields(all_requests, {}, additional_values) - requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False) - DomainRequestExport.write_csv_for_requests(writer, columns, requests_dict) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name - # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name expected_content = ( "Domain request,Domain type,Federal type\n" "city3.gov,Federal,Executive\n" @@ -705,70 +513,82 @@ class ExportDataTest(MockDb, MockEppLib): self.assertEqual(csv_content, expected_content) @less_console_noise_decorator - def test_full_domain_request_report(self): + def test_domain_request_data_full(self): """Tests the full domain request report.""" - - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - - # Call the report. Get existing fields from the report itself. - annotations = DomainRequestExport._full_domain_request_annotations() - additional_values = [ - "requested_domain__name", - "federal_agency__agency", - "authorizing_official__first_name", - "authorizing_official__last_name", - "authorizing_official__email", - "authorizing_official__title", - "creator__first_name", - "creator__last_name", - "creator__email", - "investigator__email", + # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data + columns = [ + "Domain request", + # "Submitted at", + "Status", + "Domain type", + "Federal type", + "Federal agency", + "Organization name", + "Election office", + "City", + "State/territory", + "Region", + "Creator first name", + "Creator last name", + "Creator email", + "Creator approved domains count", + "Creator active requests count", + "Alternative domains", + "AO first name", + "AO last name", + "AO email", + "AO title/role", + "Request purpose", + "Request additional details", + "Other contacts", + "CISA regional representative", + "Current websites", + "Investigator", ] - requests = DomainRequest.objects.exclude(status=DomainRequest.DomainRequestStatus.STARTED) - annotated_requests = DomainRequestExport.annotate_and_retrieve_fields(requests, annotations, additional_values) - requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False) - DomainRequestExport.write_csv_for_requests(writer, DomainRequestExport.all_columns, requests_dict) - - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - print(csv_content) - self.maxDiff = None - expected_content = ( - # Header - "Domain request,Submitted at,Status,Domain type,Federal type," - "Federal agency,Organization name,Election office,City,State/territory," - "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," - "Creator active requests count,Alternative domains,AO first name,AO last name,AO email," - "AO title/role,Request purpose,Request additional details,Other contacts," - "CISA regional representative,Current websites,Investigator\n" - # Content - "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city3.gov,2024-04-02,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1," - "cheeseville.gov | city1.gov | igorville.gov,Testy,Tester,testy@town.com,Chief Tester," - "Purpose of the site,CISA-first-name CISA-last-name | There is more,Meow Tester24 te2@town.com | " - "Testy1232 Tester24 te2@town.com | Testy Tester testy2@town.com,test@igorville.com," - "city.com | https://www.example2.com | https://www.example.com,\n" - "city4.gov,2024-04-02,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester," - "testy@town.com,Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," - "Testy Tester testy2@town.com,cisaRep@igorville.gov,city.com,\n" - "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city6.gov,2024-04-02,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," - "testy@town.com,Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," - "Testy Tester testy2@town.com,cisaRep@igorville.gov,city.com," - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) + with patch("registrar.utility.csv_export.DomainRequestDataFull.get_columns", return_value=columns): + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainRequestDataFull.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + print(csv_content) + expected_content = ( + # Header + "Domain request,Status,Domain type,Federal type," + "Federal agency,Organization name,Election office,City,State/territory," + "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," + "Creator active requests count,Alternative domains,AO first name,AO last name,AO email," + "AO title/role,Request purpose,Request additional details,Other contacts," + "CISA regional representative,Current websites,Investigator\n" + # Content + "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," + "testy@town.com," + "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + 'city3.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,"cheeseville.gov, city1.gov,' + 'igorville.gov",Testy,Tester,testy@town.com,Chief Tester,Purpose of the site,CISA-first-name ' + "CISA-last-name " + '| There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, Testy Tester ' + 'testy2@town.com"' + ',test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' + "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " + "testy2@town.com" + ",cisaRep@igorville.gov,city.com,\n" + "city6.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " + "testy2@town.com," + "cisaRep@igorville.gov,city.com,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) class HelperFunctions(MockDb): @@ -794,12 +614,12 @@ class HelperFunctions(MockDb): "domain__first_ready__lte": self.end_date, } # Test with distinct - managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + managed_domains_sliced_at_end_date = DomainExport.get_sliced_domains(filter_condition) expected_content = [3, 2, 1, 0, 0, 0, 0, 0, 0, 0] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) # Test without distinct - managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + managed_domains_sliced_at_end_date = DomainExport.get_sliced_domains(filter_condition) expected_content = [3, 2, 1, 0, 0, 0, 0, 0, 0, 0] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) @@ -811,6 +631,6 @@ class HelperFunctions(MockDb): "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": self.end_date, } - submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) + submitted_requests_sliced_at_end_date = DomainRequestExport.get_sliced_requests(filter_condition) expected_content = [3, 2, 0, 0, 0, 0, 1, 0, 0, 1] self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 6875f0e3d..f2786c2cd 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -15,12 +15,10 @@ from django.db.models import QuerySet, Value, CharField, Count, Q, F from django.db.models import Case, When, DateField from django.db.models import ManyToManyField from django.utils import timezone -from django.core.paginator import Paginator from django.db.models.functions import Concat, Coalesce from django.contrib.postgres.aggregates import StringAgg from registrar.models.utility.generic_helper import convert_queryset_to_dict from registrar.templatetags.custom_filters import get_region -from registrar.utility.enums import DefaultEmail from registrar.utility.constants import BranchChoices @@ -34,14 +32,17 @@ def write_header(writer, columns): """ writer.writerow(columns) + def get_default_start_date(): - # Default to a date that's prior to our first deployment + """Default to a date that's prior to our first deployment""" return timezone.make_aware(datetime(2023, 11, 1)) + def get_default_end_date(): - # Default to now() + """Default to now()""" return timezone.now() + def format_start_date(start_date): return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() @@ -49,9 +50,11 @@ def format_start_date(start_date): def format_end_date(end_date): return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() + class BaseExport(ABC): """ A generic class for exporting data which returns a csv file for the given model. + Base class in an inheritance tree of 3. """ @classmethod @@ -69,14 +72,14 @@ class BaseExport(ABC): Returns the columns for CSV export. Override in subclasses as needed. """ return [] - + @classmethod def get_sort_fields(cls): """ Returns the sort fields for the CSV export. Override in subclasses as needed. """ return [] - + @classmethod def get_additional_args(cls): """ @@ -84,63 +87,63 @@ class BaseExport(ABC): Override in subclasses to provide specific arguments. """ return {} - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ return [] - + @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ return [] - + @classmethod def get_exclusions(cls): """ Get a Q object of exclusion conditions to use when building queryset. """ return Q() - + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ Get a Q object of filter conditions to filter when building queryset. """ return Q() - + @classmethod def get_computed_fields(cls): """ Get a dict of computed fields. """ return {} - + @classmethod def get_annotations_for_sort(cls): """ Get a dict of annotations to make available for order_by clause. """ return {} - + @classmethod def get_related_table_fields(cls): """ Get a list of fields from related tables. """ return [] - + @classmethod def update_queryset(cls, queryset, **kwargs): """ Returns an updated queryset. Override in subclass to update queryset. """ return queryset - + @classmethod def write_csv_before(cls, csv_writer, start_date=None, end_date=None): """ @@ -187,7 +190,7 @@ class BaseExport(ABC): queryset = initial_queryset.annotate(**computed_fields).values(*model_fields, *related_table_fields) return cls.update_queryset(queryset, **kwargs) - + @classmethod def export_data_to_csv(cls, csv_file, start_date=None, end_date=None): """ @@ -207,7 +210,8 @@ class BaseExport(ABC): related_table_fields = cls.get_related_table_fields() model_queryset = ( - cls.model().objects.select_related(*select_related) + cls.model() + .objects.select_related(*select_related) .prefetch_related(*prefetch_related) .filter(filter_conditions) .exclude(exclusions) @@ -217,7 +221,9 @@ class BaseExport(ABC): ) # Convert the queryset to a dictionary (including annotated fields) - annotated_queryset = cls.annotate_and_retrieve_fields(model_queryset, computed_fields, related_table_fields, **kwargs) + annotated_queryset = cls.annotate_and_retrieve_fields( + model_queryset, computed_fields, related_table_fields, **kwargs + ) models_dict = convert_queryset_to_dict(annotated_queryset, is_model=False) # Write to csv file before the write_csv @@ -259,10 +265,11 @@ class BaseExport(ABC): """ pass - + class DomainExport(BaseExport): """ A collection of functions which return csv files regarding the Domain model. + Second class in an inheritance tree of 3. """ @classmethod @@ -279,9 +286,9 @@ class DomainExport(BaseExport): based on public_contacts, domain_invitations and user_domain_roles passed through kwargs. """ - public_contacts = kwargs.get('public_contacts', {}) - domain_invitations = kwargs.get('domain_invitations', {}) - user_domain_roles = kwargs.get('user_domain_roles', {}) + public_contacts = kwargs.get("public_contacts", {}) + domain_invitations = kwargs.get("domain_invitations", {}) + user_domain_roles = kwargs.get("user_domain_roles", {}) annotated_domain_infos = [] @@ -296,16 +303,18 @@ class DomainExport(BaseExport): # Annotate with security_contact from public_contacts for domain_info in queryset: - domain_info['security_contact_email'] = public_contacts.get(domain_info.get('domain__security_contact_registry_id')) - domain_info['invited_users'] = ', '.join(invited_users_dict.get(domain_info.get('domain__name'), [])) - domain_info['managers'] = ', '.join(managers_dict.get(domain_info.get('domain__name'), [])) + domain_info["security_contact_email"] = public_contacts.get( + domain_info.get("domain__security_contact_registry_id") + ) + domain_info["invited_users"] = ", ".join(invited_users_dict.get(domain_info.get("domain__name"), [])) + domain_info["managers"] = ", ".join(managers_dict.get(domain_info.get("domain__name"), [])) annotated_domain_infos.append(domain_info) if annotated_domain_infos: return annotated_domain_infos - + return queryset - + # ============================================================= # # Helper functions for django ORM queries. # # We are using these rather than pure python for speed reasons. # @@ -316,15 +325,15 @@ class DomainExport(BaseExport): """ Fetch all PublicContact entries and return a mapping of registry_id to email. """ - public_contacts = PublicContact.objects.values_list('registry_id', 'email') + public_contacts = PublicContact.objects.values_list("registry_id", "email") return {registry_id: email for registry_id, email in public_contacts} - + @classmethod def get_all_domain_invitations(cls): """ Fetch all DomainInvitation entries and return a mapping of domain to email. """ - domain_invitations = DomainInvitation.objects.filter(status="invited").values_list('domain__name', 'email') + domain_invitations = DomainInvitation.objects.filter(status="invited").values_list("domain__name", "email") return list(domain_invitations) @classmethod @@ -332,7 +341,7 @@ class DomainExport(BaseExport): """ Fetch all UserDomainRole entries and return a mapping of domain to user__email. """ - user_domain_roles = UserDomainRole.objects.select_related('user').values_list('domain__name', 'user__email') + user_domain_roles = UserDomainRole.objects.select_related("user").values_list("domain__name", "user__email") return list(user_domain_roles) @classmethod @@ -360,19 +369,9 @@ class DomainExport(BaseExport): if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" - if model.get("domain__name") == "18f.gov": - print(f'domain_type {domain_type}') - print(f'federal_agency {model.get("federal_agency")}') - print(f'city {model.get("city")}') - - print(f'agency {model.get("agency")}') - - print(f'federal_agency__agency {model.get("federal_agency__agency")}') - # create a dictionary of fields which can be included in output. # "extra_fields" are precomputed fields (generated in the DB or parsed). FIELDS = { - "Domain name": model.get("domain__name"), "Status": human_readable_status, "First ready on": first_ready_on, @@ -434,6 +433,10 @@ class DomainExport(BaseExport): class DomainDataType(DomainExport): + """ + Shows security contacts, domain managers, ao + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -456,7 +459,7 @@ class DomainDataType(DomainExport): "Domain managers", "Invited domain managers", ] - + @classmethod def get_sort_fields(cls): """ @@ -488,29 +491,24 @@ class DomainDataType(DomainExport): user_domain_roles = cls.get_all_user_domain_roles() return { - 'public_contacts': public_contacts, - 'domain_invitations': domain_invitations, - 'user_domain_roles': user_domain_roles, + "public_contacts": public_contacts, + "domain_invitations": domain_invitations, + "user_domain_roles": user_domain_roles, } - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain", - "authorizing_official" - ] - + return ["domain", "authorizing_official"] + @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "permissions" - ] + return ["permissions"] @classmethod def get_computed_fields(cls, delimiter=", "): @@ -525,7 +523,7 @@ class DomainDataType(DomainExport): output_field=CharField(), ), } - + @classmethod def get_related_table_fields(cls): """ @@ -542,9 +540,13 @@ class DomainDataType(DomainExport): "authorizing_official__email", "federal_agency__agency", ] - + class DomainDataFull(DomainExport): + """ + Shows security contacts, filtered by state + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -560,7 +562,7 @@ class DomainDataFull(DomainExport): "State", "Security contact email", ] - + @classmethod def get_sort_fields(cls): """ @@ -586,17 +588,15 @@ class DomainDataFull(DomainExport): public_contacts = cls.get_all_security_emails() return { - 'public_contacts': public_contacts, + "public_contacts": public_contacts, } - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): @@ -604,13 +604,13 @@ class DomainDataFull(DomainExport): Get a Q object of filter conditions to filter when building queryset. """ return Q( - domain__state__in = [ + domain__state__in=[ Domain.State.READY, Domain.State.DNS_NEEDED, Domain.State.ON_HOLD, ], ) - + @classmethod def get_computed_fields(cls, delimiter=", "): """ @@ -624,7 +624,7 @@ class DomainDataFull(DomainExport): output_field=CharField(), ), } - + @classmethod def get_related_table_fields(cls): """ @@ -638,6 +638,10 @@ class DomainDataFull(DomainExport): class DomainDataFederal(DomainExport): + """ + Shows security contacts, filtered by state and org type + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -653,7 +657,7 @@ class DomainDataFederal(DomainExport): "State", "Security contact email", ] - + @classmethod def get_sort_fields(cls): """ @@ -679,17 +683,15 @@ class DomainDataFederal(DomainExport): public_contacts = cls.get_all_security_emails() return { - 'public_contacts': public_contacts, + "public_contacts": public_contacts, } - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): @@ -702,9 +704,9 @@ class DomainDataFederal(DomainExport): Domain.State.READY, Domain.State.DNS_NEEDED, Domain.State.ON_HOLD, - ] + ], ) - + @classmethod def get_computed_fields(cls, delimiter=", "): """ @@ -718,7 +720,7 @@ class DomainDataFederal(DomainExport): output_field=CharField(), ), } - + @classmethod def get_related_table_fields(cls): """ @@ -732,6 +734,10 @@ class DomainDataFederal(DomainExport): class DomainGrowth(DomainExport): + """ + Shows ready and deleted domains within a date range, sorted + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -751,7 +757,7 @@ class DomainGrowth(DomainExport): "First ready", "Deleted", ] - + @classmethod def get_annotations_for_sort(cls, delimiter=", "): """ @@ -760,10 +766,10 @@ class DomainGrowth(DomainExport): today = timezone.now().date() return { "custom_sort": Case( - When(domain__state=Domain.State.READY, then='domain__first_ready'), - When(domain__state=Domain.State.DELETED, then='domain__deleted'), + When(domain__state=Domain.State.READY, then="domain__first_ready"), + When(domain__state=Domain.State.DELETED, then="domain__deleted"), default=Value(today), # Default value if no conditions match - output_field=DateField() + output_field=DateField(), ) } @@ -773,19 +779,17 @@ class DomainGrowth(DomainExport): Returns the sort fields. """ return [ - '-domain__state', - 'custom_sort', - 'domain__name', + "-domain__state", + "custom_sort", + "domain__name", ] - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): @@ -795,15 +799,13 @@ class DomainGrowth(DomainExport): filter_ready = Q( domain__state__in=[Domain.State.READY], domain__first_ready__gte=start_date, - domain__first_ready__lte=end_date + domain__first_ready__lte=end_date, ) filter_deleted = Q( - domain__state__in=[Domain.State.DELETED], - domain__deleted__gte=start_date, - domain__deleted__lte=end_date + domain__state__in=[Domain.State.DELETED], domain__deleted__gte=start_date, domain__deleted__lte=end_date ) return filter_ready | filter_deleted - + @classmethod def get_related_table_fields(cls): """ @@ -821,6 +823,10 @@ class DomainGrowth(DomainExport): class DomainManaged(DomainExport): + """ + Shows managed domains by an end date, sorted + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -833,34 +839,30 @@ class DomainManaged(DomainExport): "Domain managers", "Invited domain managers", ] - + @classmethod def get_sort_fields(cls): """ Returns the sort fields. """ return [ - 'domain__name', + "domain__name", ] - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "permissions" - ] - + return ["permissions"] + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ @@ -871,7 +873,6 @@ class DomainManaged(DomainExport): domain__permissions__isnull=False, domain__first_ready__lte=end_date_formatted, ) - @classmethod def get_additional_args(cls): @@ -889,10 +890,10 @@ class DomainManaged(DomainExport): user_domain_roles = cls.get_all_user_domain_roles() return { - 'domain_invitations': domain_invitations, - 'user_domain_roles': user_domain_roles, + "domain_invitations": domain_invitations, + "user_domain_roles": user_domain_roles, } - + @classmethod def get_related_table_fields(cls): """ @@ -901,7 +902,7 @@ class DomainManaged(DomainExport): return [ "domain__name", ] - + @classmethod def write_csv_before(cls, csv_writer, start_date=None, end_date=None): """ @@ -959,6 +960,10 @@ class DomainManaged(DomainExport): class DomainUnmanaged(DomainExport): + """ + Shows unmanaged domains by an end date, sorted + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -969,34 +974,30 @@ class DomainUnmanaged(DomainExport): "Domain name", "Domain type", ] - + @classmethod def get_sort_fields(cls): """ Returns the sort fields. """ return [ - 'domain__name', + "domain__name", ] - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "permissions" - ] - + return ["permissions"] + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ @@ -1007,7 +1008,7 @@ class DomainUnmanaged(DomainExport): domain__permissions__isnull=True, domain__first_ready__lte=end_date_formatted, ) - + @classmethod def get_related_table_fields(cls): """ @@ -1016,12 +1017,12 @@ class DomainUnmanaged(DomainExport): return [ "domain__name", ] - + @classmethod def write_csv_before(cls, csv_writer, start_date=None, end_date=None): """ Write to csv file before the write_csv method. - + """ start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -1075,6 +1076,10 @@ class DomainUnmanaged(DomainExport): class DomainRequestExport(BaseExport): + """ + A collection of functions which return csv files regarding the DomainRequest model. + Second class in an inheritance tree of 3. + """ @classmethod def model(cls): @@ -1197,6 +1202,10 @@ class DomainRequestExport(BaseExport): class DomainRequestGrowth(DomainRequestExport): + """ + Shows submitted requests within a date range, sorted + Inherits from BaseExport -> DomainRequestExport + """ @classmethod def get_columns(cls): @@ -1218,7 +1227,7 @@ class DomainRequestGrowth(DomainRequestExport): return [ "requested_domain__name", ] - + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ @@ -1238,12 +1247,14 @@ class DomainRequestGrowth(DomainRequestExport): """ Get a list of fields from related tables. """ - return [ - "requested_domain__name" - ] - + return ["requested_domain__name"] + class DomainRequestDataFull(DomainRequestExport): + """ + Shows all but STARTED requests + Inherits from BaseExport -> DomainRequestExport + """ @classmethod def get_columns(cls): @@ -1285,34 +1296,22 @@ class DomainRequestDataFull(DomainRequestExport): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "creator", - "authorizing_official", - "federal_agency", - "investigator", - "requested_domain" - ] + return ["creator", "authorizing_official", "federal_agency", "investigator", "requested_domain"] @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "current_websites", - "other_contacts", - "alternative_domains" - ] - + return ["current_websites", "other_contacts", "alternative_domains"] + @classmethod def get_exclusions(cls): """ Get a Q object of exclusion conditions to use when building queryset. """ - return Q( - status__in=[DomainRequest.DomainRequestStatus.STARTED] - ) - + return Q(status__in=[DomainRequest.DomainRequestStatus.STARTED]) + @classmethod def get_sort_fields(cls): """ @@ -1322,7 +1321,7 @@ class DomainRequestDataFull(DomainRequestExport): "status", "requested_domain__name", ] - + @classmethod def get_computed_fields(cls, delimiter=", "): """ @@ -1346,7 +1345,7 @@ class DomainRequestDataFull(DomainRequestExport): distinct=True, ), } - + @classmethod def get_related_table_fields(cls): """ @@ -1364,7 +1363,6 @@ class DomainRequestDataFull(DomainRequestExport): "creator__email", "investigator__email", ] - # ============================================================= # # Helper functions for django ORM queries. # diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 63e8492b5..4d015ab37 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -49,7 +49,9 @@ class AnalyticsView(View): "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } - managed_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains( + filter_managed_domains_start_date + ) managed_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_start_date = { @@ -60,8 +62,12 @@ class AnalyticsView(View): "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } - unmanaged_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_start_date) - unmanaged_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains( + filter_unmanaged_domains_start_date + ) + unmanaged_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains( + filter_unmanaged_domains_end_date + ) filter_ready_domains_start_date = { "domain__state__in": [models.Domain.State.READY], @@ -82,7 +88,9 @@ class AnalyticsView(View): "domain__state__in": [models.Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains( + filter_deleted_domains_start_date + ) deleted_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_end_date) filter_requests_start_date = { @@ -102,8 +110,12 @@ class AnalyticsView(View): "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": end_date_formatted, } - submitted_requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_start_date) - submitted_requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_end_date) + submitted_requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests( + filter_submitted_requests_start_date + ) + submitted_requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests( + filter_submitted_requests_end_date + ) context = dict( # Generate a dictionary of context variables that are common across all admin templates From 6361369953c1a4cf2b0d92ed3eb18876a943fae1 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 15:10:49 -0400 Subject: [PATCH 09/31] removed user-contact connection --- docs/developer/README.md | 43 -------- src/registrar/apps.py | 9 -- src/registrar/forms/__init__.py | 2 +- src/registrar/forms/domain.py | 59 ++++++++++- src/registrar/forms/user_profile.py | 4 +- .../management/commands/import_tables.py | 7 -- src/registrar/models/contact.py | 7 -- src/registrar/models/user.py | 23 ++-- src/registrar/signals.py | 59 ----------- .../admin/includes/contact_detail_list.html | 27 ++--- src/registrar/templates/domain_detail.html | 2 +- .../includes/finish_profile_form.html | 4 +- .../includes/profile_information.html | 6 +- src/registrar/tests/common.py | 4 + src/registrar/tests/test_admin.py | 47 +++----- src/registrar/tests/test_models.py | 98 +---------------- src/registrar/tests/test_signals.py | 100 ------------------ src/registrar/tests/test_views.py | 14 ++- src/registrar/tests/test_views_domain.py | 4 +- src/registrar/views/domain.py | 6 +- src/registrar/views/user_profile.py | 16 ++- .../views/utility/permission_views.py | 6 +- 22 files changed, 129 insertions(+), 418 deletions(-) delete mode 100644 src/registrar/signals.py delete mode 100644 src/registrar/tests/test_signals.py diff --git a/docs/developer/README.md b/docs/developer/README.md index 72f6b9f20..f63f01938 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -353,49 +353,6 @@ cf env getgov-{app name} Then, copy the variables under the section labled `s3`. -## Signals -The application uses [Django signals](https://docs.djangoproject.com/en/5.0/topics/signals/). In particular, it uses a subset of prebuilt signals called [model signals](https://docs.djangoproject.com/en/5.0/ref/signals/#module-django.db.models.signals). - -Per Django, signals "[...allow certain senders to notify a set of receivers that some action has taken place.](https://docs.djangoproject.com/en/5.0/topics/signals/#module-django.dispatch)" - -In other words, signals are a mechanism that allows different parts of an application to communicate with each other by sending and receiving notifications when events occur. When an event occurs (such as creating, updating, or deleting a record), signals can automatically trigger specific actions in response. This allows different parts of an application to stay synchronized without tightly coupling the component. - -### Rules of use -When using signals, try to adhere to these guidelines: -1. Don't use signals when you can use another method, such as an override of `save()` or `__init__`. -2. Document its usage in this readme (or another centralized location), as well as briefly on the underlying class it is associated with. For instance, since the `handle_profile` directly affects the class `Contact`, the class description notes this and links to [signals.py](../../src/registrar/signals.py). -3. Where possible, avoid chaining signals together (i.e. a signal that calls a signal). If this has to be done, clearly document the flow. -4. Minimize logic complexity within the signal as much as possible. - -### When should you use signals? -Generally, you would use signals when you want an event to be synchronized across multiple areas of code at once (such as with two models or more models at once) in a way that would otherwise be difficult to achieve by overriding functions. - -However, in most scenarios, if you can get away with avoiding signals - you should. The reasoning for this is that [signals give the appearance of loose coupling, but they can quickly lead to code that is hard to understand, adjust and debug](https://docs.djangoproject.com/en/5.0/topics/signals/#module-django.dispatch). - -Consider using signals when: -1. Synchronizing events across multiple models or areas of code. -2. Performing logic before or after saving a model to the database (when otherwise difficult through `save()`). -3. Encountering an import loop when overriding functions such as `save()`. -4. You are otherwise unable to achieve the intended behavior by overrides or other means. -5. (Rare) Offloading tasks when multi-threading. - -For the vast majority of use cases, the [pre_save](https://docs.djangoproject.com/en/5.0/ref/signals/#pre-save) and [post_save](https://docs.djangoproject.com/en/5.0/ref/signals/#post-save) signals are sufficient in terms of model-to-model management. - -### Where should you use them? -This project compiles signals in a unified location to maintain readability. If you are adding a signal or otherwise utilizing one, you should always define them in [signals.py](../../src/registrar/signals.py). Except under rare circumstances, this should be adhered to for the reasons mentioned above. - -### How are we currently using signals? -At the time of writing, we currently only use signals for the Contact and User objects when synchronizing data returned from Login.gov. This is because the `Contact` object holds information that the user specified in our system, whereas the `User` object holds information that was specified in Login.gov. - -To keep our signal usage coherent and well-documented, add to this document when a new function is added for ease of reference and use. - -#### handle_profile -This function is triggered by the post_save event on the User model, designed to manage the synchronization between User and Contact entities. It operates under the following conditions: - -1. For New Users: Upon the creation of a new user, it checks for an existing `Contact` by email. If no matching contact is found, it creates a new Contact using the user's details from Login.gov. If a matching contact is found, it associates this contact with the user. In cases where multiple contacts with the same email exist, it logs a warning and associates the first contact found. - -2. For Existing Users: For users logging in subsequent times, the function ensures that any updates from Login.gov are applied to the associated User record. However, it does not alter any existing Contact records. - ## Disable email sending (toggling the disable_email_sending flag) 1. On the app, navigate to `\admin`. 2. Under models, click `Waffle flags`. diff --git a/src/registrar/apps.py b/src/registrar/apps.py index fcb5c17fd..b5952208b 100644 --- a/src/registrar/apps.py +++ b/src/registrar/apps.py @@ -5,12 +5,3 @@ class RegistrarConfig(AppConfig): """Configure signal handling for our registrar Django application.""" name = "registrar" - - def ready(self): - """Runs when all Django applications have been loaded. - - We use it here to load signals that connect related models. - """ - # noqa here because we are importing something to make the signals - # get registered, but not using what we import - from . import signals # noqa diff --git a/src/registrar/forms/__init__.py b/src/registrar/forms/__init__.py index be3b634f6..8f4f8ea14 100644 --- a/src/registrar/forms/__init__.py +++ b/src/registrar/forms/__init__.py @@ -4,7 +4,7 @@ from .domain import ( NameserverFormset, DomainSecurityEmailForm, DomainOrgNameAddressForm, - ContactForm, + UserForm, AuthorizingOfficialContactForm, DomainDnssecForm, DomainDsdataFormset, diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 0e9fbb693..6f6c9077e 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -16,7 +16,7 @@ from registrar.utility.errors import ( SecurityEmailErrorCodes, ) -from ..models import Contact, DomainInformation, Domain +from ..models import Contact, DomainInformation, Domain, User from .common import ( ALGORITHM_CHOICES, DIGEST_TYPE_CHOICES, @@ -203,6 +203,63 @@ NameserverFormset = formset_factory( ) +class UserForm(forms.ModelForm): + """Form for updating contacts.""" + + email = forms.EmailField(max_length=None) + + class Meta: + model = User + fields = ["first_name", "middle_name", "last_name", "title", "email", "phone"] + widgets = { + "first_name": forms.TextInput, + "middle_name": forms.TextInput, + "last_name": forms.TextInput, + "title": forms.TextInput, + "email": forms.EmailInput, + "phone": RegionalPhoneNumberWidget, + } + + # the database fields have blank=True so ModelForm doesn't create + # required fields by default. Use this list in __init__ to mark each + # of these fields as required + required = ["first_name", "last_name", "title", "email", "phone"] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # take off maxlength attribute for the phone number field + # which interferes with out input_with_errors template tag + self.fields["phone"].widget.attrs.pop("maxlength", None) + + # Define a custom validator for the email field with a custom error message + email_max_length_validator = MaxLengthValidator(320, message="Response must be less than 320 characters.") + self.fields["email"].validators.append(email_max_length_validator) + + for field_name in self.required: + self.fields[field_name].required = True + + # Set custom form label + self.fields["middle_name"].label = "Middle name (optional)" + + # Set custom error messages + self.fields["first_name"].error_messages = {"required": "Enter your first name / given name."} + self.fields["last_name"].error_messages = {"required": "Enter your last name / family name."} + self.fields["title"].error_messages = { + "required": "Enter your title or role in your organization (e.g., Chief Information Officer)" + } + self.fields["email"].error_messages = { + "required": "Enter your email address in the required format, like name@example.com." + } + self.fields["phone"].error_messages["required"] = "Enter your phone number." + self.domainInfo = None + + def set_domain_info(self, domainInfo): + """Set the domain information for the form. + The form instance is associated with the contact itself. In order to access the associated + domain information object, this needs to be set in the form by the view.""" + self.domainInfo = domainInfo + + class ContactForm(forms.ModelForm): """Form for updating contacts.""" diff --git a/src/registrar/forms/user_profile.py b/src/registrar/forms/user_profile.py index 682e1a5df..bfdcd0da8 100644 --- a/src/registrar/forms/user_profile.py +++ b/src/registrar/forms/user_profile.py @@ -1,6 +1,6 @@ from django import forms -from registrar.models.contact import Contact +from registrar.models.user import User from django.core.validators import MaxLengthValidator from phonenumber_field.widgets import RegionalPhoneNumberWidget @@ -13,7 +13,7 @@ class UserProfileForm(forms.ModelForm): redirect = forms.CharField(widget=forms.HiddenInput(), required=False) class Meta: - model = Contact + model = User fields = ["first_name", "middle_name", "last_name", "title", "email", "phone"] widgets = { "first_name": forms.TextInput, diff --git a/src/registrar/management/commands/import_tables.py b/src/registrar/management/commands/import_tables.py index cb78e13bd..62562e7f7 100644 --- a/src/registrar/management/commands/import_tables.py +++ b/src/registrar/management/commands/import_tables.py @@ -65,13 +65,6 @@ class Command(BaseCommand): resourcename = f"{table_name}Resource" - # if table_name is Contact, clean the table first - # User table is loaded before Contact, and signals create - # rows in Contact table which break the import, so need - # to be cleaned again before running import on Contact table - if table_name == "Contact": - self.clean_table(table_name) - # Define the directory and the pattern for csv filenames tmp_dir = "tmp" pattern = f"{table_name}_" diff --git a/src/registrar/models/contact.py b/src/registrar/models/contact.py index f94938dd1..f7bae3491 100644 --- a/src/registrar/models/contact.py +++ b/src/registrar/models/contact.py @@ -8,13 +8,6 @@ from phonenumber_field.modelfields import PhoneNumberField # type: ignore class Contact(TimeStampedModel): """ Contact information follows a similar pattern for each contact. - - This model uses signals [as defined in [signals.py](../../src/registrar/signals.py)]. - When a new user is created through Login.gov, a contact object will be created and - associated on the `user` field. - - If the `user` object already exists, the underlying user object - will be updated if any updates are made to it through Login.gov. """ class Meta: diff --git a/src/registrar/models/user.py b/src/registrar/models/user.py index bb0276607..87b7799d3 100644 --- a/src/registrar/models/user.py +++ b/src/registrar/models/user.py @@ -23,10 +23,6 @@ class User(AbstractUser): A custom user model that performs identically to the default user model but can be customized later. - This model uses signals [as defined in [signals.py](../../src/registrar/signals.py)]. - When a new user is created through Login.gov, a contact object will be created and - associated on the contacts `user` field. - If the `user` object already exists, said user object will be updated if any updates are made to it through Login.gov. """ @@ -113,15 +109,11 @@ class User(AbstractUser): Tracks if the user finished their profile setup or not. This is so we can globally enforce that new users provide additional account information before proceeding. """ - - # Change this to self once the user and contact objects are merged. - # For now, since they are linked, lets test on the underlying contact object. - user_info = self.contact # noqa user_values = [ - user_info.first_name, - user_info.last_name, - user_info.title, - user_info.phone, + self.first_name, + self.last_name, + self.title, + self.phone, ] return None not in user_values @@ -169,8 +161,13 @@ class User(AbstractUser): """Return count of ineligible requests""" return self.domain_requests_created.filter(status=DomainRequest.DomainRequestStatus.INELIGIBLE).count() + def get_formatted_name(self): + """Returns the contact's name in Western order.""" + names = [n for n in [self.first_name, self.middle_name, self.last_name] if n] + return " ".join(names) if names else "Unknown" + def has_contact_info(self): - return bool(self.contact.title or self.contact.email or self.contact.phone) + return bool(self.title or self.email or self.phone) @classmethod def needs_identity_verification(cls, email, uuid): diff --git a/src/registrar/signals.py b/src/registrar/signals.py deleted file mode 100644 index bc0480b2a..000000000 --- a/src/registrar/signals.py +++ /dev/null @@ -1,59 +0,0 @@ -import logging - -from django.db.models.signals import post_save -from django.dispatch import receiver - -from .models import User, Contact - - -logger = logging.getLogger(__name__) - - -@receiver(post_save, sender=User) -def handle_profile(sender, instance, **kwargs): - """Method for when a User is saved. - - A first time registrant may have been invited, so we'll search for a matching - Contact record, by email address, and associate them, if possible. - - A first time registrant may not have a matching Contact, so we'll create one, - copying the contact values we received from Login.gov in order to initialize it. - - During subsequent login, a User record may be updated with new data from Login.gov, - but in no case will we update contact values on an existing Contact record. - """ - - first_name = getattr(instance, "first_name", "") - middle_name = getattr(instance, "middle_name", "") - last_name = getattr(instance, "last_name", "") - email = getattr(instance, "email", "") - phone = getattr(instance, "phone", "") - title = getattr(instance, "title", "") - - is_new_user = kwargs.get("created", False) - - if is_new_user: - contacts = Contact.objects.filter(email=email) - else: - contacts = Contact.objects.filter(user=instance) - - if len(contacts) == 0: # no matching contact - Contact.objects.create( - user=instance, - first_name=first_name, - middle_name=middle_name, - last_name=last_name, - email=email, - phone=phone, - title=title, - ) - - if len(contacts) >= 1 and is_new_user: # a matching contact - contacts[0].user = instance - contacts[0].save() - - if len(contacts) > 1: # multiple matches - logger.warning( - "There are multiple Contacts with the same email address." - f" Picking #{contacts[0].id} for User #{instance.id}." - ) diff --git a/src/registrar/templates/django/admin/includes/contact_detail_list.html b/src/registrar/templates/django/admin/includes/contact_detail_list.html index 3b49e62a4..2ee490d76 100644 --- a/src/registrar/templates/django/admin/includes/contact_detail_list.html +++ b/src/registrar/templates/django/admin/includes/contact_detail_list.html @@ -12,37 +12,24 @@ {% if user.has_contact_info %} {# Title #} - {% if user.title or user.contact.title %} - {% if user.contact.title %} - {{ user.contact.title }} - {% else %} - {{ user.title }} - {% endif %} + {% if user.title %} + {{ user.title }}
{% else %} None
{% endif %} {# Email #} - {% if user.email or user.contact.email %} - {% if user.contact.email %} - {{ user.contact.email }} - {% include "admin/input_with_clipboard.html" with field=user invisible_input_field=True %} - {% else %} - {{ user.email }} - {% include "admin/input_with_clipboard.html" with field=user invisible_input_field=True %} - {% endif %} + {% if user.email %} + {{ user.email }} + {% include "admin/input_with_clipboard.html" with field=user invisible_input_field=True %}
{% else %} None
{% endif %} {# Phone #} - {% if user.phone or user.contact.phone %} - {% if user.contact.phone %} - {{ user.contact.phone }} - {% else %} - {{ user.phone }} - {% endif %} + {% if user.phone %} + {{ user.phone }}
{% else %} None
diff --git a/src/registrar/templates/domain_detail.html b/src/registrar/templates/domain_detail.html index 815df4942..d201831cc 100644 --- a/src/registrar/templates/domain_detail.html +++ b/src/registrar/templates/domain_detail.html @@ -62,7 +62,7 @@ {# Conditionally display profile #} {% if not has_profile_feature_flag %} {% url 'domain-your-contact-information' pk=domain.id as url %} - {% include "includes/summary_item.html" with title='Your contact information' value=request.user.contact contact='true' edit_link=url editable=domain.is_editable %} + {% include "includes/summary_item.html" with title='Your contact information' value=request.user contact='true' edit_link=url editable=domain.is_editable %} {% endif %} {% url 'domain-security-email' pk=domain.id as url %} diff --git a/src/registrar/templates/includes/finish_profile_form.html b/src/registrar/templates/includes/finish_profile_form.html index d43ff812c..88f7a73af 100644 --- a/src/registrar/templates/includes/finish_profile_form.html +++ b/src/registrar/templates/includes/finish_profile_form.html @@ -77,11 +77,11 @@
- {% if user_finished_setup and going_to_specific_page %} - {% endif %} diff --git a/src/registrar/templates/includes/profile_information.html b/src/registrar/templates/includes/profile_information.html index 2922fd3f7..3e7c827f1 100644 --- a/src/registrar/templates/includes/profile_information.html +++ b/src/registrar/templates/includes/profile_information.html @@ -13,10 +13,10 @@
    -
  • Full name: {{ user.contact.get_formatted_name }}
    • +
  • Full name: {{ user.get_formatted_name }}
  • Organization email: {{ user.email }}
    • -
  • Title or role in your organization: {{ user.contact.title }}
    • -
  • Phone: {{ user.contact.phone.as_national }}
    • +
  • Title or role in your organization: {{ user.title }}
    • +
  • Phone: {{ user.phone.as_national }}
diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 017964299..a8bdfec29 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -810,6 +810,8 @@ def create_superuser(): user = User.objects.create_user( username="superuser", email="admin@example.com", + first_name="first", + last_name="last", is_staff=True, password=p, ) @@ -826,6 +828,8 @@ def create_user(): user = User.objects.create_user( username="staffuser", email="staff@example.com", + first_name="first", + last_name="last", is_staff=True, password=p, ) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 07e97d159..ff68e5a7c 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -242,15 +242,11 @@ class TestDomainAdmin(MockEppLib, WebTest): username="MrMeoward", first_name="Meoward", last_name="Jones", + email="meoward.jones@igorville.gov", + phone="(555) 123 12345", + title="Treat inspector", ) - # Due to the relation between User <==> Contact, - # the underlying contact has to be modified this way. - _creator.contact.email = "meoward.jones@igorville.gov" - _creator.contact.phone = "(555) 123 12345" - _creator.contact.title = "Treat inspector" - _creator.contact.save() - # Create a fake domain request domain_request = completed_domain_request(status=DomainRequest.DomainRequestStatus.IN_REVIEW, user=_creator) domain_request.approve() @@ -2067,15 +2063,11 @@ class TestDomainRequestAdmin(MockEppLib): username="MrMeoward", first_name="Meoward", last_name="Jones", + email="meoward.jones@igorville.gov", + phone="(555) 123 12345", + title="Treat inspector", ) - # Due to the relation between User <==> Contact, - # the underlying contact has to be modified this way. - _creator.contact.email = "meoward.jones@igorville.gov" - _creator.contact.phone = "(555) 123 12345" - _creator.contact.title = "Treat inspector" - _creator.contact.save() - # Create a fake domain request domain_request = completed_domain_request(status=DomainRequest.DomainRequestStatus.IN_REVIEW, user=_creator) @@ -2092,11 +2084,11 @@ class TestDomainRequestAdmin(MockEppLib): # == Check for the creator == # - # Check for the right title, email, and phone number in the response. + # Check for the right title and phone number in the response. + # Email will appear more than once expected_creator_fields = [ # Field, expected value ("title", "Treat inspector"), - ("email", "meoward.jones@igorville.gov"), ("phone", "(555) 123 12345"), ] self.test_helper.assert_response_contains_distinct_values(response, expected_creator_fields) @@ -3103,15 +3095,11 @@ class TestDomainInformationAdmin(TestCase): username="MrMeoward", first_name="Meoward", last_name="Jones", + email="meoward.jones@igorville.gov", + phone="(555) 123 12345", + title="Treat inspector", ) - # Due to the relation between User <==> Contact, - # the underlying contact has to be modified this way. - _creator.contact.email = "meoward.jones@igorville.gov" - _creator.contact.phone = "(555) 123 12345" - _creator.contact.title = "Treat inspector" - _creator.contact.save() - # Create a fake domain request domain_request = completed_domain_request(status=DomainRequest.DomainRequestStatus.IN_REVIEW, user=_creator) domain_request.approve() @@ -3133,13 +3121,12 @@ class TestDomainInformationAdmin(TestCase): # == Check for the creator == # - # Check for the right title, email, and phone number in the response. + # Check for the right title and phone number in the response. # We only need to check for the end tag # (Otherwise this test will fail if we change classes, etc) expected_creator_fields = [ # Field, expected value ("title", "Treat inspector"), - ("email", "meoward.jones@igorville.gov"), ("phone", "(555) 123 12345"), ] self.test_helper.assert_response_contains_distinct_values(response, expected_creator_fields) @@ -4067,8 +4054,8 @@ class TestContactAdmin(TestCase): self.assertEqual(readonly_fields, expected_fields) def test_change_view_for_joined_contact_five_or_less(self): - """Create a contact, join it to 4 domain requests. The 5th join will be a user. - Assert that the warning on the contact form lists 5 joins.""" + """Create a contact, join it to 4 domain requests. 5th join is user. + Assert that the warning on the contact form lists 4 joins.""" with less_console_noise(): self.client.force_login(self.superuser) @@ -4099,17 +4086,17 @@ class TestContactAdmin(TestCase): "
  • Joined to DomainRequest: city4.gov
    • " "
  • Joined to User: staff@example.com
    • " + f"user/{self.staffuser.pk}/change/'>first last staff@example.com" "", ) def test_change_view_for_joined_contact_five_or_more(self): - """Create a contact, join it to 5 domain requests. The 6th join will be a user. + """Create a contact, join it to 5 domain requests. 6th join is user. Assert that the warning on the contact form lists 5 joins and a '1 more' ellispsis.""" with less_console_noise(): self.client.force_login(self.superuser) # Create an instance of the model - # join it to 5 domain requests. The 6th join will be a user. + # join it to 6 domain requests. contact, _ = Contact.objects.get_or_create(user=self.staffuser) domain_request1 = completed_domain_request(submitter=contact, name="city1.gov") domain_request2 = completed_domain_request(submitter=contact, name="city2.gov") diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index 37fdc5354..d7a8dc6f7 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -1209,24 +1209,14 @@ class TestUser(TestCase): # test with a user with contact info defined self.assertTrue(self.user.has_contact_info()) # test with a user without contact info defined - self.user.contact.title = None - self.user.contact.email = None - self.user.contact.phone = None + self.user.title = None + self.user.email = None + self.user.phone = None self.assertFalse(self.user.has_contact_info()) class TestContact(TestCase): def setUp(self): - self.email_for_invalid = "intern@igorville.gov" - self.invalid_user, _ = User.objects.get_or_create( - username=self.email_for_invalid, - email=self.email_for_invalid, - first_name="", - last_name="", - phone="", - ) - self.invalid_contact, _ = Contact.objects.get_or_create(user=self.invalid_user) - self.email = "mayor@igorville.gov" self.user, _ = User.objects.get_or_create( email=self.email, first_name="Jeff", last_name="Lebowski", phone="123456789" @@ -1242,87 +1232,6 @@ class TestContact(TestCase): Contact.objects.all().delete() User.objects.all().delete() - def test_saving_contact_updates_user_first_last_names_and_phone(self): - """When a contact is updated, we propagate the changes to the linked user if it exists.""" - - # User and Contact are created and linked as expected. - # An empty User object should create an empty contact. - self.assertEqual(self.invalid_contact.first_name, "") - self.assertEqual(self.invalid_contact.last_name, "") - self.assertEqual(self.invalid_contact.phone, "") - self.assertEqual(self.invalid_user.first_name, "") - self.assertEqual(self.invalid_user.last_name, "") - self.assertEqual(self.invalid_user.phone, "") - - # Manually update the contact - mimicking production (pre-existing data) - self.invalid_contact.first_name = "Joey" - self.invalid_contact.last_name = "Baloney" - self.invalid_contact.phone = "123456789" - self.invalid_contact.save() - - # Refresh the user object to reflect the changes made in the database - self.invalid_user.refresh_from_db() - - # Updating the contact's first and last names propagate to the user - self.assertEqual(self.invalid_contact.first_name, "Joey") - self.assertEqual(self.invalid_contact.last_name, "Baloney") - self.assertEqual(self.invalid_contact.phone, "123456789") - self.assertEqual(self.invalid_user.first_name, "Joey") - self.assertEqual(self.invalid_user.last_name, "Baloney") - self.assertEqual(self.invalid_user.phone, "123456789") - - def test_saving_contact_does_not_update_user_first_last_names_and_phone(self): - """When a contact is updated, we avoid propagating the changes to the linked user if it already has a value""" - - # User and Contact are created and linked as expected - self.assertEqual(self.contact.first_name, "Jeff") - self.assertEqual(self.contact.last_name, "Lebowski") - self.assertEqual(self.contact.phone, "123456789") - self.assertEqual(self.user.first_name, "Jeff") - self.assertEqual(self.user.last_name, "Lebowski") - self.assertEqual(self.user.phone, "123456789") - - self.contact.first_name = "Joey" - self.contact.last_name = "Baloney" - self.contact.phone = "987654321" - self.contact.save() - - # Refresh the user object to reflect the changes made in the database - self.user.refresh_from_db() - - # Updating the contact's first and last names propagate to the user - self.assertEqual(self.contact.first_name, "Joey") - self.assertEqual(self.contact.last_name, "Baloney") - self.assertEqual(self.contact.phone, "987654321") - self.assertEqual(self.user.first_name, "Jeff") - self.assertEqual(self.user.last_name, "Lebowski") - self.assertEqual(self.user.phone, "123456789") - - def test_saving_contact_does_not_update_user_email(self): - """When a contact's email is updated, the change is not propagated to the user.""" - self.contact.email = "joey.baloney@diaperville.com" - self.contact.save() - - # Refresh the user object to reflect the changes made in the database - self.user.refresh_from_db() - - # Updating the contact's email does not propagate - self.assertEqual(self.contact.email, "joey.baloney@diaperville.com") - self.assertEqual(self.user.email, "mayor@igorville.gov") - - def test_saving_contact_does_not_update_user_email_when_none(self): - """When a contact's email is updated, and the first/last name is none, - the change is not propagated to the user.""" - self.invalid_contact.email = "joey.baloney@diaperville.com" - self.invalid_contact.save() - - # Refresh the user object to reflect the changes made in the database - self.invalid_user.refresh_from_db() - - # Updating the contact's email does not propagate - self.assertEqual(self.invalid_contact.email, "joey.baloney@diaperville.com") - self.assertEqual(self.invalid_user.email, "intern@igorville.gov") - def test_has_more_than_one_join(self): """Test the Contact model method, has_more_than_one_join""" # test for a contact which has one user defined @@ -1334,6 +1243,7 @@ class TestContact(TestCase): def test_has_contact_info(self): """Test that has_contact_info properly returns""" + self.contact.title = "Title" # test with a contact with contact info defined self.assertTrue(self.contact.has_contact_info()) # test with a contact without contact info defined diff --git a/src/registrar/tests/test_signals.py b/src/registrar/tests/test_signals.py deleted file mode 100644 index e796bd12a..000000000 --- a/src/registrar/tests/test_signals.py +++ /dev/null @@ -1,100 +0,0 @@ -from django.test import TestCase -from django.contrib.auth import get_user_model -from registrar.models import Contact - - -class TestUserPostSave(TestCase): - def setUp(self): - self.username = "test_user" - self.first_name = "First" - self.last_name = "Last" - self.email = "info@example.com" - self.phone = "202-555-0133" - - self.preferred_first_name = "One" - self.preferred_last_name = "Two" - self.preferred_email = "front_desk@example.com" - self.preferred_phone = "202-555-0134" - - def test_user_created_without_matching_contact(self): - """Expect 1 Contact containing data copied from User.""" - self.assertEqual(len(Contact.objects.all()), 0) - user = get_user_model().objects.create( - username=self.username, - first_name=self.first_name, - last_name=self.last_name, - email=self.email, - phone=self.phone, - ) - actual = Contact.objects.get(user=user) - self.assertEqual(actual.first_name, self.first_name) - self.assertEqual(actual.last_name, self.last_name) - self.assertEqual(actual.email, self.email) - self.assertEqual(actual.phone, self.phone) - - def test_user_created_with_matching_contact(self): - """Expect 1 Contact associated, but with no data copied from User.""" - self.assertEqual(len(Contact.objects.all()), 0) - Contact.objects.create( - first_name=self.preferred_first_name, - last_name=self.preferred_last_name, - email=self.email, # must be the same, to find the match! - phone=self.preferred_phone, - ) - user = get_user_model().objects.create( - username=self.username, - first_name=self.first_name, - last_name=self.last_name, - email=self.email, - ) - actual = Contact.objects.get(user=user) - self.assertEqual(actual.first_name, self.preferred_first_name) - self.assertEqual(actual.last_name, self.preferred_last_name) - self.assertEqual(actual.email, self.email) - self.assertEqual(actual.phone, self.preferred_phone) - - def test_user_updated_without_matching_contact(self): - """Expect 1 Contact containing data copied from User.""" - # create the user - self.assertEqual(len(Contact.objects.all()), 0) - user = get_user_model().objects.create(username=self.username, first_name="", last_name="", email="", phone="") - # delete the contact - Contact.objects.all().delete() - self.assertEqual(len(Contact.objects.all()), 0) - # modify the user - user.username = self.username - user.first_name = self.first_name - user.last_name = self.last_name - user.email = self.email - user.phone = self.phone - user.save() - # test - actual = Contact.objects.get(user=user) - self.assertEqual(actual.first_name, self.first_name) - self.assertEqual(actual.last_name, self.last_name) - self.assertEqual(actual.email, self.email) - self.assertEqual(actual.phone, self.phone) - - def test_user_updated_with_matching_contact(self): - """Expect 1 Contact associated, but with no data copied from User.""" - # create the user - self.assertEqual(len(Contact.objects.all()), 0) - user = get_user_model().objects.create( - username=self.username, - first_name=self.first_name, - last_name=self.last_name, - email=self.email, - phone=self.phone, - ) - # modify the user - user.first_name = self.preferred_first_name - user.last_name = self.preferred_last_name - user.email = self.preferred_email - user.phone = self.preferred_phone - user.save() - # test - actual = Contact.objects.get(user=user) - self.assertEqual(actual.first_name, self.first_name) - self.assertEqual(actual.last_name, self.last_name) - self.assertEqual(actual.email, self.email) - self.assertEqual(actual.phone, self.phone) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 2ac0caf2c..54581ad65 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -57,12 +57,10 @@ class TestWithUser(MockEppLib): last_name = "Last" email = "info@example.com" phone = "8003111234" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email, phone=phone - ) title = "test title" - self.user.contact.title = title - self.user.contact.save() + self.user = get_user_model().objects.create( + username=username, first_name=first_name, last_name=last_name, title=title, email=email, phone=phone + ) username_regular_incomplete = "test_regular_user_incomplete" username_other_incomplete = "test_other_user_incomplete" @@ -560,7 +558,7 @@ class FinishUserProfileTests(TestWithUser, WebTest): self.assertContains(finish_setup_page, "Enter your phone number.") # Check for the name of the save button - self.assertContains(finish_setup_page, "contact_setup_save_button") + self.assertContains(finish_setup_page, "user_setup_save_button") # Add a phone number finish_setup_form = finish_setup_page.form @@ -598,7 +596,7 @@ class FinishUserProfileTests(TestWithUser, WebTest): self.assertContains(finish_setup_page, "Enter your phone number.") # Check for the name of the save button - self.assertContains(finish_setup_page, "contact_setup_save_button") + self.assertContains(finish_setup_page, "user_setup_save_button") # Add a phone number finish_setup_form = finish_setup_page.form @@ -613,7 +611,7 @@ class FinishUserProfileTests(TestWithUser, WebTest): # Submit the form using the specific submit button to execute the redirect completed_setup_page = self._submit_form_webtest( - finish_setup_form, follow=True, name="contact_setup_submit_button" + finish_setup_form, follow=True, name="user_setup_submit_button" ) self.assertEqual(completed_setup_page.status_code, 200) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index c3bcb02db..f62605d69 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -1478,8 +1478,8 @@ class TestDomainContactInformation(TestDomainOverview): def test_domain_your_contact_information_content(self): """Logged-in user's contact information appears on the page.""" - self.user.contact.first_name = "Testy" - self.user.contact.save() + self.user.first_name = "Testy" + self.user.save() page = self.app.get(reverse("domain-your-contact-information", kwargs={"pk": self.domain.id})) self.assertContains(page, "Testy") diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index b7ce06cce..d6436b058 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -40,7 +40,7 @@ from registrar.models.utility.contact_error import ContactError from registrar.views.utility.permission_views import UserDomainRolePermissionDeleteView from ..forms import ( - ContactForm, + UserForm, AuthorizingOfficialContactForm, DomainOrgNameAddressForm, DomainAddUserForm, @@ -573,7 +573,7 @@ class DomainYourContactInformationView(DomainFormBaseView): """Domain your contact information editing view.""" template_name = "domain_your_contact_information.html" - form_class = ContactForm + form_class = UserForm @waffle_flag("!profile_feature") # type: ignore def dispatch(self, request, *args, **kwargs): # type: ignore @@ -582,7 +582,7 @@ class DomainYourContactInformationView(DomainFormBaseView): def get_form_kwargs(self, *args, **kwargs): """Add domain_info.submitter instance to make a bound form.""" form_kwargs = super().get_form_kwargs(*args, **kwargs) - form_kwargs["instance"] = self.request.user.contact + form_kwargs["instance"] = self.request.user return form_kwargs def get_success_url(self): diff --git a/src/registrar/views/user_profile.py b/src/registrar/views/user_profile.py index 3f9aeb79f..e378b9180 100644 --- a/src/registrar/views/user_profile.py +++ b/src/registrar/views/user_profile.py @@ -9,9 +9,6 @@ from django.http import QueryDict from django.views.generic.edit import FormMixin from registrar.forms.user_profile import UserProfileForm, FinishSetupProfileForm from django.urls import NoReverseMatch, reverse -from registrar.models import ( - Contact, -) from registrar.models.user import User from registrar.models.utility.generic_helper import replace_url_queryparams from registrar.views.utility.permission_views import UserProfilePermissionView @@ -25,7 +22,7 @@ class UserProfileView(UserProfilePermissionView, FormMixin): Base View for the User Profile. Handles getting and setting the User Profile """ - model = Contact + model = User template_name = "profile.html" form_class = UserProfileForm base_view_name = "user-profile" @@ -57,6 +54,7 @@ class UserProfileView(UserProfilePermissionView, FormMixin): def get_context_data(self, **kwargs): """Extend get_context_data to include has_profile_feature_flag""" context = super().get_context_data(**kwargs) + logger.info("UserProfileView::get_context_data") # This is a django waffle flag which toggles features based off of the "flag" table context["has_profile_feature_flag"] = flag_is_active(self.request, "profile_feature") @@ -123,9 +121,7 @@ class UserProfileView(UserProfilePermissionView, FormMixin): def get_object(self, queryset=None): """Override get_object to return the logged-in user's contact""" self.user = self.request.user # get the logged in user - if hasattr(self.user, "contact"): # Check if the user has a contact instance - return self.user.contact - return None + return self.user class FinishProfileSetupView(UserProfileView): @@ -134,7 +130,7 @@ class FinishProfileSetupView(UserProfileView): template_name = "finish_profile_setup.html" form_class = FinishSetupProfileForm - model = Contact + model = User base_view_name = "finish-user-profile-setup" @@ -160,11 +156,11 @@ class FinishProfileSetupView(UserProfileView): # Get the current form and validate it if form.is_valid(): self.redirect_page = False - if "contact_setup_save_button" in request.POST: + if "user_setup_save_button" in request.POST: # Logic for when the 'Save' button is clicked, which indicates # user should stay on this page self.redirect_page = False - elif "contact_setup_submit_button" in request.POST: + elif "user_setup_submit_button" in request.POST: # Logic for when the other button is clicked, which indicates # the user should be taken to the redirect page self.redirect_page = True diff --git a/src/registrar/views/utility/permission_views.py b/src/registrar/views/utility/permission_views.py index d35647af2..db727c26e 100644 --- a/src/registrar/views/utility/permission_views.py +++ b/src/registrar/views/utility/permission_views.py @@ -4,7 +4,7 @@ import abc # abstract base class from django.views.generic import DetailView, DeleteView, TemplateView from registrar.models import Domain, DomainRequest, DomainInvitation -from registrar.models.contact import Contact +from registrar.models.user import User from registrar.models.user_domain_role import UserDomainRole from .mixins import ( @@ -154,9 +154,9 @@ class UserProfilePermissionView(UserProfilePermission, DetailView, abc.ABC): """ # DetailView property for what model this is viewing - model = Contact + model = User # variable name in template context for the model object - context_object_name = "contact" + context_object_name = "user" # Abstract property enforces NotImplementedError on an attribute. @property From 7c1b3ee698dc8fec69d15357759b59113f3a3212 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 15:20:29 -0400 Subject: [PATCH 10/31] fixed test_management_scripts for import_tables script --- src/registrar/tests/test_management_scripts.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index cebee9994..cfe19b091 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -1077,11 +1077,6 @@ class TestImportTables(TestCase): for table_name in table_names: mock_path_exists.assert_any_call(f"{table_name}_1.csv") - # Check that clean_tables is called for Contact - mock_get_model.assert_any_call("registrar", "Contact") - model_mock = mock_get_model.return_value - model_mock.objects.all().delete.assert_called() - # Check that logger.info was called for each successful import for table_name in table_names: mock_logger.info.assert_any_call(f"Successfully imported {table_name}_1.csv into {table_name}") From 80af4a0e6b5a2c15fd7f732e7acbbe775efff3e4 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:41:05 -0600 Subject: [PATCH 11/31] Steps for gpg keys --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 4d231a039..72e951bff 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -70,6 +70,7 @@ when setting up your key in Github. Now test commit signing is working by checking out a branch (`yourname/test-commit-signing`) and making some small change to a file. Commit the change (it should prompt you for your GPG credential) and push it to Github. Look on Github at your branch and ensure the commit is `verified`. +### MacOS **Note:** if you are on a mac and not able to successfully create a signed commit, getting the following error: ```zsh error: gpg failed to sign the data @@ -90,6 +91,11 @@ or source ~/.zshrc ``` +### Windows +If you are using windows, it may be helpful to use [gpg4win](https://www.gpg4win.org/get-gpg4win.html). From there, you should be able to access gpg through the terminal. + +When installing, consider a gpg key manager like Kleopatra if you run into issues with environment variables or with the gpg service not running on startup. + ## Setting up developer sandbox We have three types of environments: stable, staging, and sandbox. Stable (production)and staging (pre-prod) get deployed via tagged release, and developer sandboxes are given to get.gov developers to mess around in a production-like environment without disrupting stable or staging. Each sandbox is namespaced and will automatically be deployed too when the appropriate branch syntax is used for that space in an open pull request. There are several things you need to setup to make the sandbox work for a developer. From 3580d80b35ad2f30657ae3846ff6e17ff584ee52 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:45:51 -0600 Subject: [PATCH 12/31] Add steps for the cf cli --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 72e951bff..6418713a7 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -16,6 +16,8 @@ assignees: abroddrick There are several tools we use locally that you will need to have. - [ ] [Install the cf CLI v7](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html#pkg-mac) for the ability to deploy + - If you are using Windows, installation information can be found [here](https://github.com/cloudfoundry/cli/wiki/V8-CLI-Installation-Guide#installers-and-compressed-binaries) + - Alternatively, for Windows, [consider using chocolately](https://community.chocolatey.org/packages/cloudfoundry-cli/7.2.0) - [ ] Make sure you have `gpg` >2.1.7. Run `gpg --version` to check. If not, [install gnupg](https://formulae.brew.sh/formula/gnupg) - [ ] Install the [Github CLI](https://cli.github.com/) From 45c7f1aaa614b78838b8b9eb38ff041bd9a94e96 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 16:59:18 -0400 Subject: [PATCH 13/31] removed user from contact model and all associated logic --- src/registrar/admin.py | 26 +- .../copy_names_from_contacts_to_users.py | 242 ------------------ ...registrar_c_user_id_4059c4_idx_and_more.py | 21 ++ src/registrar/models/contact.py | 40 --- src/registrar/tests/test_admin.py | 23 +- .../test_copy_names_from_contacts_to_users.py | 124 --------- src/registrar/tests/test_models.py | 8 +- src/registrar/tests/test_views.py | 31 ++- src/registrar/tests/test_views_request.py | 5 +- src/registrar/views/domain_request.py | 6 +- 10 files changed, 67 insertions(+), 459 deletions(-) delete mode 100644 src/registrar/management/commands/copy_names_from_contacts_to_users.py create mode 100644 src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py delete mode 100644 src/registrar/tests/test_copy_names_from_contacts_to_users.py diff --git a/src/registrar/admin.py b/src/registrar/admin.py index a92e4c695..6f455fc89 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -593,12 +593,6 @@ class ListHeaderAdmin(AuditedAdmin, OrderableFieldsMixin): return filters -class UserContactInline(admin.StackedInline): - """Edit a user's profile on the user page.""" - - model = models.Contact - - class MyUserAdmin(BaseUserAdmin, ImportExportModelAdmin): """Custom user admin class to use our inlines.""" @@ -615,8 +609,6 @@ class MyUserAdmin(BaseUserAdmin, ImportExportModelAdmin): _meta = Meta() - inlines = [UserContactInline] - list_display = ( "username", "overridden_email_field", @@ -894,30 +886,20 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): list_display = [ "name", "email", - "user_exists", ] # this ordering effects the ordering of results - # in autocomplete_fields for user + # in autocomplete_fields ordering = ["first_name", "last_name", "email"] fieldsets = [ ( None, - {"fields": ["user", "first_name", "middle_name", "last_name", "title", "email", "phone"]}, + {"fields": ["first_name", "middle_name", "last_name", "title", "email", "phone"]}, ) ] - autocomplete_fields = ["user"] - change_form_template = "django/admin/email_clipboard_change_form.html" - def user_exists(self, obj): - """Check if the Contact has a related User""" - return "Yes" if obj.user is not None else "No" - - user_exists.short_description = "Is user" # type: ignore - user_exists.admin_order_field = "user" # type: ignore - # We name the custom prop 'contact' because linter # is not allowing a short_description attr on it # This gets around the linter limitation, for now. @@ -935,9 +917,7 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): name.admin_order_field = "first_name" # type: ignore # Read only that we'll leverage for CISA Analysts - analyst_readonly_fields = [ - "user", - ] + analyst_readonly_fields = [] def get_readonly_fields(self, request, obj=None): """Set the read-only state on form elements. diff --git a/src/registrar/management/commands/copy_names_from_contacts_to_users.py b/src/registrar/management/commands/copy_names_from_contacts_to_users.py deleted file mode 100644 index 384029400..000000000 --- a/src/registrar/management/commands/copy_names_from_contacts_to_users.py +++ /dev/null @@ -1,242 +0,0 @@ -import logging -import argparse -import sys - -from django.core.management import BaseCommand - -from registrar.management.commands.utility.terminal_helper import ( - TerminalColors, - TerminalHelper, -) -from registrar.models.contact import Contact -from registrar.models.user import User -from registrar.models.utility.domain_helper import DomainHelper - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - help = """Copy first and last names from a contact to - a related user if it exists and if its first and last name - properties are null or blank strings.""" - - # ====================================================== - # ===================== ARGUMENTS ===================== - # ====================================================== - def add_arguments(self, parser): - parser.add_argument("--debug", action=argparse.BooleanOptionalAction) - - # ====================================================== - # ===================== PRINTING ====================== - # ====================================================== - def print_debug_mode_statements(self, debug_on: bool): - """Prints additional terminal statements to indicate if --debug - or --limitParse are in use""" - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.OKCYAN} - ----------DEBUG MODE ON---------- - Detailed print statements activated. - {TerminalColors.ENDC} - """, - ) - - def print_summary_of_findings( - self, - skipped_contacts, - eligible_users, - processed_users, - debug_on, - ): - """Prints to terminal a summary of findings from - copying first and last names from contacts to users""" - - total_eligible_users = len(eligible_users) - total_skipped_contacts = len(skipped_contacts) - total_processed_users = len(processed_users) - - logger.info( - f"""{TerminalColors.OKGREEN} - ============= FINISHED =============== - Skipped {total_skipped_contacts} contacts - Found {total_eligible_users} users linked to contacts - Processed {total_processed_users} users - {TerminalColors.ENDC} - """ # noqa - ) - - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW} - ======= DEBUG OUTPUT ======= - Users who have a linked contact: - {eligible_users} - - Processed users (users who have a linked contact and a missing first or last name): - {processed_users} - - ===== SKIPPED CONTACTS ===== - {skipped_contacts} - - {TerminalColors.ENDC} - """, - ) - - # ====================================================== - # =================== USER ===================== - # ====================================================== - def update_user(self, contact: Contact, debug_on: bool): - """Given a contact with a first_name and last_name, find & update an existing - corresponding user if her first_name and last_name are null. - - Returns tuple of eligible (is linked to the contact) and processed - (first and last are blank) users. - """ - - user_exists = User.objects.filter(contact=contact).exists() - if user_exists: - try: - # ----------------------- UPDATE USER ----------------------- - # ---- GET THE USER - eligible_user = User.objects.get(contact=contact) - processed_user = None - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW} - > Found linked user for contact: - {contact} {contact.email} {contact.first_name} {contact.last_name} - > The linked user is {eligible_user} {eligible_user.username} - {TerminalColors.ENDC}""", # noqa - ) - - # Get the fields that exist on both User and Contact. Excludes id. - common_fields = DomainHelper.get_common_fields(User, Contact) - if "email" in common_fields: - # Don't change the email field. - common_fields.remove("email") - - for field in common_fields: - # Grab the value that contact has stored for this field - new_value = getattr(contact, field) - - # Set it on the user field - setattr(eligible_user, field, new_value) - - eligible_user.save() - processed_user = eligible_user - - return ( - eligible_user, - processed_user, - ) - - except Exception as error: - logger.warning( - f""" - {TerminalColors.FAIL} - !!! ERROR: An exception occured in the - User table for the following user: - {contact.email} {contact.first_name} {contact.last_name} - - Exception is: {error} - ----------TERMINATING----------""" - ) - sys.exit() - else: - return None, None - - # ====================================================== - # ================= PROCESS CONTACTS ================== - # ====================================================== - - def process_contacts( - self, - debug_on, - skipped_contacts=[], - eligible_users=[], - processed_users=[], - ): - for contact in Contact.objects.all(): - TerminalHelper.print_conditional( - debug_on, - f"{TerminalColors.OKCYAN}" - "Processing Contact: " - f"{contact.email}," - f" {contact.first_name}," - f" {contact.last_name}" - f"{TerminalColors.ENDC}", - ) - - # ====================================================== - # ====================== USER ======================= - (eligible_user, processed_user) = self.update_user(contact, debug_on) - - debug_string = "" - if eligible_user: - # ---------------- UPDATED ---------------- - eligible_users.append(contact.email) - debug_string = f"eligible user: {eligible_user}" - if processed_user: - processed_users.append(contact.email) - debug_string = f"processed user: {processed_user}" - else: - skipped_contacts.append(contact.email) - debug_string = f"skipped user: {contact.email}" - - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - (f"{TerminalColors.OKCYAN} {debug_string} {TerminalColors.ENDC}"), - ) - - return ( - skipped_contacts, - eligible_users, - processed_users, - ) - - # ====================================================== - # ===================== HANDLE ======================== - # ====================================================== - def handle( - self, - **options, - ): - """Parse entries in Contact table - and update valid corresponding entries in the - User table.""" - - # grab command line arguments and store locally... - debug_on = options.get("debug") - - self.print_debug_mode_statements(debug_on) - - logger.info( - f"""{TerminalColors.OKCYAN} - ========================== - Beginning Data Transfer - ========================== - {TerminalColors.ENDC}""" - ) - - logger.info( - f"""{TerminalColors.OKCYAN} - ========= Adding Domains and Domain Invitations ========= - {TerminalColors.ENDC}""" - ) - ( - skipped_contacts, - eligible_users, - processed_users, - ) = self.process_contacts( - debug_on, - ) - - self.print_summary_of_findings( - skipped_contacts, - eligible_users, - processed_users, - debug_on, - ) diff --git a/src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py b/src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py new file mode 100644 index 000000000..858210be7 --- /dev/null +++ b/src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.10 on 2024-07-02 19:52 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("registrar", "0109_domaininformation_sub_organization_and_more"), + ] + + operations = [ + migrations.RemoveIndex( + model_name="contact", + name="registrar_c_user_id_4059c4_idx", + ), + migrations.RemoveField( + model_name="contact", + name="user", + ), + ] diff --git a/src/registrar/models/contact.py b/src/registrar/models/contact.py index f7bae3491..903633749 100644 --- a/src/registrar/models/contact.py +++ b/src/registrar/models/contact.py @@ -14,17 +14,9 @@ class Contact(TimeStampedModel): """Contains meta information about this class""" indexes = [ - models.Index(fields=["user"]), models.Index(fields=["email"]), ] - user = models.OneToOneField( - "registrar.User", - null=True, - blank=True, - on_delete=models.SET_NULL, - ) - first_name = models.CharField( null=True, blank=True, @@ -103,38 +95,6 @@ class Contact(TimeStampedModel): def has_contact_info(self): return bool(self.title or self.email or self.phone) - def save(self, *args, **kwargs): - # Call the parent class's save method to perform the actual save - super().save(*args, **kwargs) - - if self.user: - updated = False - - # Update first name and last name if necessary - if not self.user.first_name or not self.user.last_name: - self.user.first_name = self.first_name - self.user.last_name = self.last_name - updated = True - - # Update middle_name if necessary - if not self.user.middle_name: - self.user.middle_name = self.middle_name - updated = True - - # Update phone if necessary - if not self.user.phone: - self.user.phone = self.phone - updated = True - - # Update title if necessary - if not self.user.title: - self.user.title = self.title - updated = True - - # Save user if any updates were made - if updated: - self.user.save() - def __str__(self): if self.first_name or self.last_name: return self.get_formatted_name() diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 65427bb65..a1532cd39 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -4036,9 +4036,7 @@ class TestContactAdmin(TestCase): readonly_fields = self.admin.get_readonly_fields(request) - expected_fields = [ - "user", - ] + expected_fields = [] self.assertEqual(readonly_fields, expected_fields) @@ -4054,15 +4052,18 @@ class TestContactAdmin(TestCase): self.assertEqual(readonly_fields, expected_fields) def test_change_view_for_joined_contact_five_or_less(self): - """Create a contact, join it to 4 domain requests. 5th join is user. + """Create a contact, join it to 4 domain requests. Assert that the warning on the contact form lists 4 joins.""" with less_console_noise(): self.client.force_login(self.superuser) # Create an instance of the model - contact, _ = Contact.objects.get_or_create(user=self.staffuser) + contact, _ = Contact.objects.get_or_create( + first_name="Henry", + last_name="McFakerson", + ) - # join it to 4 domain requests. The 5th join will be a user. + # join it to 4 domain requests. domain_request1 = completed_domain_request(submitter=contact, name="city1.gov") domain_request2 = completed_domain_request(submitter=contact, name="city2.gov") domain_request3 = completed_domain_request(submitter=contact, name="city3.gov") @@ -4085,24 +4086,26 @@ class TestContactAdmin(TestCase): f"domainrequest/{domain_request3.pk}/change/'>city3.gov" "
  • Joined to DomainRequest: city4.gov
    • " - "
  • Joined to User: first last staff@example.com
    • " "", ) def test_change_view_for_joined_contact_five_or_more(self): - """Create a contact, join it to 5 domain requests. 6th join is user. + """Create a contact, join it to 6 domain requests. Assert that the warning on the contact form lists 5 joins and a '1 more' ellispsis.""" with less_console_noise(): self.client.force_login(self.superuser) # Create an instance of the model # join it to 6 domain requests. - contact, _ = Contact.objects.get_or_create(user=self.staffuser) + contact, _ = Contact.objects.get_or_create( + first_name="Henry", + last_name="McFakerson", + ) domain_request1 = completed_domain_request(submitter=contact, name="city1.gov") domain_request2 = completed_domain_request(submitter=contact, name="city2.gov") domain_request3 = completed_domain_request(submitter=contact, name="city3.gov") domain_request4 = completed_domain_request(submitter=contact, name="city4.gov") domain_request5 = completed_domain_request(submitter=contact, name="city5.gov") + domain_request6 = completed_domain_request(submitter=contact, name="city6.gov") with patch("django.contrib.messages.warning") as mock_warning: # Use the test client to simulate the request response = self.client.get(reverse("admin:registrar_contact_change", args=[contact.pk])) diff --git a/src/registrar/tests/test_copy_names_from_contacts_to_users.py b/src/registrar/tests/test_copy_names_from_contacts_to_users.py deleted file mode 100644 index 7fcbede1e..000000000 --- a/src/registrar/tests/test_copy_names_from_contacts_to_users.py +++ /dev/null @@ -1,124 +0,0 @@ -from django.test import TestCase - -from registrar.models import ( - User, - Contact, -) - -from registrar.management.commands.copy_names_from_contacts_to_users import Command - - -class TestDataUpdates(TestCase): - def setUp(self): - """We cannot setup the user details because contacts will override the first and last names in its save method - so we will initiate the users, setup the contacts and link them, and leave the rest of the setup to the test(s). - """ - - self.user1 = User.objects.create(username="user1") - self.user2 = User.objects.create(username="user2") - self.user3 = User.objects.create(username="user3") - self.user4 = User.objects.create(username="user4") - # The last user created triggers the creation of a contact and attaches itself to it. See signals. - # This bs_user defuses that situation. - self.bs_user = User.objects.create() - - self.contact1 = Contact.objects.create( - user=self.user1, - email="email1@igorville.gov", - first_name="first1", - last_name="last1", - middle_name="middle1", - title="title1", - ) - self.contact2 = Contact.objects.create( - user=self.user2, - email="email2@igorville.gov", - first_name="first2", - last_name="last2", - middle_name="middle2", - title="title2", - ) - self.contact3 = Contact.objects.create( - user=self.user3, - email="email3@igorville.gov", - first_name="first3", - last_name="last3", - middle_name="middle3", - title="title3", - ) - self.contact4 = Contact.objects.create( - email="email4@igorville.gov", first_name="first4", last_name="last4", middle_name="middle4", title="title4" - ) - - self.command = Command() - - def tearDown(self): - """Clean up""" - # Delete users and contacts - User.objects.all().delete() - Contact.objects.all().delete() - - def test_script_updates_linked_users(self): - """Test the script that copies contact information to the user object""" - - # Set up the users' first and last names here so - # they that they don't get overwritten by Contact's save() - # User with no first or last names - self.user1.first_name = "" - self.user1.last_name = "" - self.user1.title = "dummytitle" - self.user1.middle_name = "dummymiddle" - self.user1.save() - - # User with a first name but no last name - self.user2.first_name = "First name but no last name" - self.user2.last_name = "" - self.user2.save() - - # User with a first and last name - self.user3.first_name = "An existing first name" - self.user3.last_name = "An existing last name" - self.user3.save() - - # Call the parent method the same way we do it in the script - skipped_contacts = [] - eligible_users = [] - processed_users = [] - ( - skipped_contacts, - eligible_users, - processed_users, - ) = self.command.process_contacts( - # Set debugging to False - False, - skipped_contacts, - eligible_users, - processed_users, - ) - - # Trigger DB refresh - self.user1.refresh_from_db() - self.user2.refresh_from_db() - self.user3.refresh_from_db() - - # Asserts - # The user that has no first and last names will get them from the contact - self.assertEqual(self.user1.first_name, "first1") - self.assertEqual(self.user1.last_name, "last1") - self.assertEqual(self.user1.middle_name, "middle1") - self.assertEqual(self.user1.title, "title1") - # The user that has a first but no last will be updated - self.assertEqual(self.user2.first_name, "first2") - self.assertEqual(self.user2.last_name, "last2") - self.assertEqual(self.user2.middle_name, "middle2") - self.assertEqual(self.user2.title, "title2") - # The user that has a first and a last will be updated - self.assertEqual(self.user3.first_name, "first3") - self.assertEqual(self.user3.last_name, "last3") - self.assertEqual(self.user3.middle_name, "middle3") - self.assertEqual(self.user3.title, "title3") - # The unlinked user will be left alone - self.assertEqual(self.user4.first_name, "") - self.assertEqual(self.user4.last_name, "") - self.assertEqual(self.user4.middle_name, None) - self.assertEqual(self.user4.title, None) diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index 9a29f5428..0c5ca9193 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -1221,7 +1221,10 @@ class TestContact(TestCase): self.user, _ = User.objects.get_or_create( email=self.email, first_name="Jeff", last_name="Lebowski", phone="123456789" ) - self.contact, _ = Contact.objects.get_or_create(user=self.user) + self.contact, _ = Contact.objects.get_or_create( + first_name="Jeff", + last_name="Lebowski", + ) self.contact_as_so, _ = Contact.objects.get_or_create(email="newguy@igorville.gov") self.domain_request = DomainRequest.objects.create(creator=self.user, senior_official=self.contact_as_so) @@ -1234,9 +1237,6 @@ class TestContact(TestCase): def test_has_more_than_one_join(self): """Test the Contact model method, has_more_than_one_join""" - # test for a contact which has one user defined - self.assertFalse(self.contact.has_more_than_one_join("user")) - self.assertTrue(self.contact.has_more_than_one_join("senior_official")) # test for a contact which is assigned as a senior official on a domain request self.assertFalse(self.contact_as_so.has_more_than_one_join("senior_official")) self.assertTrue(self.contact_as_so.has_more_than_one_join("submitted_domain_requests")) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 42c7b2d38..b98a30e79 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -372,7 +372,10 @@ class HomeTests(TestWithUser): ) # Attach a user object to a contact (should not be deleted) - contact_user, _ = Contact.objects.get_or_create(user=self.user) + contact_user, _ = Contact.objects.get_or_create( + first_name="Hank", + last_name="McFakey", + ) site = DraftDomain.objects.create(name="igorville.gov") domain_request = DomainRequest.objects.create( @@ -405,17 +408,12 @@ class HomeTests(TestWithUser): igorville = DomainRequest.objects.filter(requested_domain__name="igorville.gov") self.assertFalse(igorville.exists()) - # Check if the orphaned contact was deleted + # Check if the orphaned contacts were deleted orphan = Contact.objects.filter(id=contact.id) self.assertFalse(orphan.exists()) + orphan = Contact.objects.filter(id=contact_user.id) + self.assertFalse(orphan.exists()) - # All non-orphan contacts should still exist and are unaltered - try: - current_user = Contact.objects.filter(id=contact_user.id).get() - except Contact.DoesNotExist: - self.fail("contact_user (a non-orphaned contact) was deleted") - - self.assertEqual(current_user, contact_user) try: edge_case = Contact.objects.filter(id=contact_2.id).get() except Contact.DoesNotExist: @@ -444,7 +442,10 @@ class HomeTests(TestWithUser): ) # Attach a user object to a contact (should not be deleted) - contact_user, _ = Contact.objects.get_or_create(user=self.user) + contact_user, _ = Contact.objects.get_or_create( + first_name="Hank", + last_name="McFakey", + ) site = DraftDomain.objects.create(name="igorville.gov") domain_request = DomainRequest.objects.create( @@ -863,7 +864,10 @@ class UserProfileTests(TestWithUser, WebTest): def test_request_when_profile_feature_on(self): """test that Your profile is in request page when profile feature is on""" - contact_user, _ = Contact.objects.get_or_create(user=self.user) + contact_user, _ = Contact.objects.get_or_create( + first_name="Hank", + last_name="McFakerson", + ) site = DraftDomain.objects.create(name="igorville.gov") domain_request = DomainRequest.objects.create( creator=self.user, @@ -882,7 +886,10 @@ class UserProfileTests(TestWithUser, WebTest): def test_request_when_profile_feature_off(self): """test that Your profile is not in request page when profile feature is off""" - contact_user, _ = Contact.objects.get_or_create(user=self.user) + contact_user, _ = Contact.objects.get_or_create( + first_name="Hank", + last_name="McFakerson", + ) site = DraftDomain.objects.create(name="igorville.gov") domain_request = DomainRequest.objects.create( creator=self.user, diff --git a/src/registrar/tests/test_views_request.py b/src/registrar/tests/test_views_request.py index e78dfe860..de924576b 100644 --- a/src/registrar/tests/test_views_request.py +++ b/src/registrar/tests/test_views_request.py @@ -2974,7 +2974,10 @@ class TestWizardUnlockingSteps(TestWithUser, WebTest): ) # Attach a user object to a contact (should not be deleted) - contact_user, _ = Contact.objects.get_or_create(user=self.user) + contact_user, _ = Contact.objects.get_or_create( + first_name="Hank", + last_name="McFakey", + ) site = DraftDomain.objects.create(name="igorville.gov") domain_request = DomainRequest.objects.create( diff --git a/src/registrar/views/domain_request.py b/src/registrar/views/domain_request.py index e8e82500e..a7d6aa6ae 100644 --- a/src/registrar/views/domain_request.py +++ b/src/registrar/views/domain_request.py @@ -812,15 +812,15 @@ class DomainRequestDeleteView(DomainRequestPermissionDeleteView): self.object = self.get_object() self.object.delete() - # Delete orphaned contacts - but only for if they are not associated with a user - Contact.objects.filter(id__in=contacts_to_delete, user=None).delete() + # Delete orphaned contacts + Contact.objects.filter(id__in=contacts_to_delete).delete() # After a delete occurs, do a second sweep on any returned duplicates. # This determines if any of these three fields share a contact, which is used for # the edge case where the same user may be an SO, and a submitter, for example. if len(duplicates) > 0: duplicates_to_delete, _ = self._get_orphaned_contacts(domain_request, check_db=True) - Contact.objects.filter(id__in=duplicates_to_delete, user=None).delete() + Contact.objects.filter(id__in=duplicates_to_delete).delete() # Return a 200 response with an empty body return HttpResponse(status=200) From 1dc3bf883e299cd89df521b8ccad2163bcfc6976 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 17:10:52 -0400 Subject: [PATCH 14/31] updated for linter --- src/registrar/admin.py | 2 +- src/registrar/tests/test_admin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 6f455fc89..967c36494 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -917,7 +917,7 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): name.admin_order_field = "first_name" # type: ignore # Read only that we'll leverage for CISA Analysts - analyst_readonly_fields = [] + analyst_readonly_fields: list[str] = [] def get_readonly_fields(self, request, obj=None): """Set the read-only state on form elements. diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index a1532cd39..889a2c0e0 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -4105,7 +4105,7 @@ class TestContactAdmin(TestCase): domain_request3 = completed_domain_request(submitter=contact, name="city3.gov") domain_request4 = completed_domain_request(submitter=contact, name="city4.gov") domain_request5 = completed_domain_request(submitter=contact, name="city5.gov") - domain_request6 = completed_domain_request(submitter=contact, name="city6.gov") + completed_domain_request(submitter=contact, name="city6.gov") with patch("django.contrib.messages.warning") as mock_warning: # Use the test client to simulate the request response = self.client.get(reverse("admin:registrar_contact_change", args=[contact.pk])) From 6076400e7cf719ac606a4ae69ae489cdcdfb469f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 18:43:46 -0400 Subject: [PATCH 15/31] a number of small refactors and cleanup of comments --- src/registrar/utility/csv_export.py | 36 +++++++++++++++-------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index f2786c2cd..cfde68a32 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -11,9 +11,7 @@ from registrar.models import ( PublicContact, UserDomainRole, ) -from django.db.models import QuerySet, Value, CharField, Count, Q, F -from django.db.models import Case, When, DateField -from django.db.models import ManyToManyField +from django.db.models import Case, CharField, Count, DateField, F, ManyToManyField, Q, QuerySet, Value, When from django.utils import timezone from django.db.models.functions import Concat, Coalesce from django.contrib.postgres.aggregates import StringAgg @@ -105,7 +103,7 @@ class BaseExport(ABC): @classmethod def get_exclusions(cls): """ - Get a Q object of exclusion conditions to use when building queryset. + Get a Q object of exclusion conditions to pass to .exclude() when building queryset. """ return Q() @@ -119,7 +117,8 @@ class BaseExport(ABC): @classmethod def get_computed_fields(cls): """ - Get a dict of computed fields. + Get a dict of computed fields. These are fields that do not exist on the model normally + and will be passed to .annotate() when building a queryset. """ return {} @@ -258,6 +257,7 @@ class BaseExport(ABC): writer.writerows(rows) @classmethod + @abstractmethod def parse_row(cls, columns, model): """ Given a set of columns and a model dictionary, generate a new row from cleaned column data. @@ -268,7 +268,8 @@ class BaseExport(ABC): class DomainExport(BaseExport): """ - A collection of functions which return csv files regarding the Domain model. + A collection of functions which return csv files regarding Domains. Although class is + named DomainExport, the base model for the export is DomainInformation. Second class in an inheritance tree of 3. """ @@ -293,21 +294,22 @@ class DomainExport(BaseExport): annotated_domain_infos = [] # Create mapping of domain to a list of invited users and managers - invited_users_dict = defaultdict(list) - for domain, email in domain_invitations: - invited_users_dict[domain].append(email) + # invited_users_dict = defaultdict(list) + # for domain, email in domain_invitations: + # invited_users_dict[domain].append(email) - managers_dict = defaultdict(list) - for domain, email in user_domain_roles: - managers_dict[domain].append(email) + # managers_dict = defaultdict(list) + # for domain, email in user_domain_roles: + # managers_dict[domain].append(email) - # Annotate with security_contact from public_contacts + # Annotate with security_contact from public_contacts, invited users + # from domain_invitations, and managers from user_domain_roles for domain_info in queryset: domain_info["security_contact_email"] = public_contacts.get( domain_info.get("domain__security_contact_registry_id") ) - domain_info["invited_users"] = ", ".join(invited_users_dict.get(domain_info.get("domain__name"), [])) - domain_info["managers"] = ", ".join(managers_dict.get(domain_info.get("domain__name"), [])) + domain_info["invited_users"] = domain_invitations.get(domain_info.get("domain__name")) + domain_info["managers"] = user_domain_roles.get(domain_info.get("domain__name")) annotated_domain_infos.append(domain_info) if annotated_domain_infos: @@ -334,7 +336,7 @@ class DomainExport(BaseExport): Fetch all DomainInvitation entries and return a mapping of domain to email. """ domain_invitations = DomainInvitation.objects.filter(status="invited").values_list("domain__name", "email") - return list(domain_invitations) + return {domain__name: email for domain__name, email in domain_invitations} @classmethod def get_all_user_domain_roles(cls): @@ -342,7 +344,7 @@ class DomainExport(BaseExport): Fetch all UserDomainRole entries and return a mapping of domain to user__email. """ user_domain_roles = UserDomainRole.objects.select_related("user").values_list("domain__name", "user__email") - return list(user_domain_roles) + return {domain__name: user__email for domain__name, user__email in user_domain_roles} @classmethod def parse_row(cls, columns, model): From e0053fa319c7074ec8dfdda8bcbf1f8443f7671e Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 18:52:31 -0400 Subject: [PATCH 16/31] linter --- src/registrar/utility/csv_export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index cfde68a32..7f77db5fd 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,5 +1,4 @@ from abc import ABC, abstractmethod -from collections import defaultdict import csv import logging from datetime import datetime From 5cfdda3e8d847b07281d9f1ca2e33890d180bc8b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 3 Jul 2024 08:39:23 -0600 Subject: [PATCH 17/31] Add additional instructions --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 6418713a7..eb547d5ab 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -94,9 +94,13 @@ source ~/.zshrc ``` ### Windows -If you are using windows, it may be helpful to use [gpg4win](https://www.gpg4win.org/get-gpg4win.html). From there, you should be able to access gpg through the terminal. +If GPG doesn't work out of the box with git for you: +- You can [download the GPG binary directly](https://gnupg.org/download/). +- It may be helpful to use [gpg4win](https://www.gpg4win.org/get-gpg4win.html). -When installing, consider a gpg key manager like Kleopatra if you run into issues with environment variables or with the gpg service not running on startup. +From there, you should be able to access gpg through the terminal. + +Additionally, consider a gpg key manager like Kleopatra if you run into issues with environment variables or with the gpg service not running on startup. ## Setting up developer sandbox From d6d178e4e156f3cd7ad30c90eba074aca4016095 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 4 Jul 2024 08:52:15 -0400 Subject: [PATCH 18/31] updated authorizing official to senior official --- src/registrar/tests/test_reports.py | 20 +++++----- src/registrar/utility/csv_export.py | 58 ++++++++++++++--------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 0015ae84f..265876987 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -207,7 +207,7 @@ class ExportDataTest(MockDb, MockEppLib): @less_console_noise_decorator def test_domain_data_type(self): - """Shows security contacts, domain managers, ao""" + """Shows security contacts, domain managers, so""" # Add security email information self.domain_1.name = "defaultsecurity.gov" self.domain_1.save() @@ -231,8 +231,8 @@ class ExportDataTest(MockDb, MockEppLib): # We expect READY domains, # sorted alphabetially by domain name expected_content = ( - "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,AO," - "AO email,Security contact email,Domain managers,Invited domain managers\n" + "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,SO," + "SO email,Security contact email,Domain managers,Invited domain managers\n" "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,, ,,," "meoward@rocks.com,\n" "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,," @@ -534,10 +534,10 @@ class ExportDataTest(MockDb, MockEppLib): "Creator approved domains count", "Creator active requests count", "Alternative domains", - "AO first name", - "AO last name", - "AO email", - "AO title/role", + "SO first name", + "SO last name", + "SO email", + "SO title/role", "Request purpose", "Request additional details", "Other contacts", @@ -560,8 +560,8 @@ class ExportDataTest(MockDb, MockEppLib): "Domain request,Status,Domain type,Federal type," "Federal agency,Organization name,Election office,City,State/territory," "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," - "Creator active requests count,Alternative domains,AO first name,AO last name,AO email," - "AO title/role,Request purpose,Request additional details,Other contacts," + "Creator active requests count,Alternative domains,SO first name,SO last name,SO email," + "SO title/role,Request purpose,Request additional details,Other contacts," "CISA regional representative,Current websites,Investigator\n" # Content "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," @@ -633,4 +633,4 @@ class HelperFunctions(MockDb): } submitted_requests_sliced_at_end_date = DomainRequestExport.get_sliced_requests(filter_condition) expected_content = [3, 2, 0, 0, 0, 0, 1, 0, 0, 1] - self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) + self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 7f77db5fd..44f8e6662 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -382,8 +382,8 @@ class DomainExport(BaseExport): "Organization name": model.get("organization_name"), "City": model.get("city"), "State": model.get("state_territory"), - "AO": model.get("ao_name"), - "AO email": model.get("authorizing_official__email"), + "SO": model.get("so_name"), + "SO email": model.get("senior_official__email"), "Security contact email": model.get("security_contact_email"), "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), @@ -435,7 +435,7 @@ class DomainExport(BaseExport): class DomainDataType(DomainExport): """ - Shows security contacts, domain managers, ao + Shows security contacts, domain managers, so Inherits from BaseExport -> DomainExport """ @@ -454,8 +454,8 @@ class DomainDataType(DomainExport): "Organization name", "City", "State", - "AO", - "AO email", + "SO", + "SO email", "Security contact email", "Domain managers", "Invited domain managers", @@ -502,7 +502,7 @@ class DomainDataType(DomainExport): """ Get a list of tables to pass to select_related when building queryset. """ - return ["domain", "authorizing_official"] + return ["domain", "senior_official"] @classmethod def get_prefetch_related(cls): @@ -517,10 +517,10 @@ class DomainDataType(DomainExport): Get a dict of computed fields. """ return { - "ao_name": Concat( - Coalesce(F("authorizing_official__first_name"), Value("")), + "so_name": Concat( + Coalesce(F("senior_official__first_name"), Value("")), Value(" "), - Coalesce(F("authorizing_official__last_name"), Value("")), + Coalesce(F("senior_official__last_name"), Value("")), output_field=CharField(), ), } @@ -538,7 +538,7 @@ class DomainDataType(DomainExport): "domain__created_at", "domain__deleted", "domain__security_contact_registry_id", - "authorizing_official__email", + "senior_official__email", "federal_agency__agency", ] @@ -618,10 +618,10 @@ class DomainDataFull(DomainExport): Get a dict of computed fields. """ return { - "ao_name": Concat( - Coalesce(F("authorizing_official__first_name"), Value("")), + "so_name": Concat( + Coalesce(F("senior_official__first_name"), Value("")), Value(" "), - Coalesce(F("authorizing_official__last_name"), Value("")), + Coalesce(F("senior_official__last_name"), Value("")), output_field=CharField(), ), } @@ -714,10 +714,10 @@ class DomainDataFederal(DomainExport): Get a dict of computed fields. """ return { - "ao_name": Concat( - Coalesce(F("authorizing_official__first_name"), Value("")), + "so_name": Concat( + Coalesce(F("senior_official__first_name"), Value("")), Value(" "), - Coalesce(F("authorizing_official__last_name"), Value("")), + Coalesce(F("senior_official__last_name"), Value("")), output_field=CharField(), ), } @@ -1181,10 +1181,10 @@ class DomainRequestExport(BaseExport): "Current websites": model.get("all_current_websites"), # Untouched FK fields - passed into the request dict. "Federal agency": model.get("federal_agency__agency"), - "AO first name": model.get("authorizing_official__first_name"), - "AO last name": model.get("authorizing_official__last_name"), - "AO email": model.get("authorizing_official__email"), - "AO title/role": model.get("authorizing_official__title"), + "SO first name": model.get("senior_official__first_name"), + "SO last name": model.get("senior_official__last_name"), + "SO email": model.get("senior_official__email"), + "SO title/role": model.get("senior_official__title"), "Creator first name": model.get("creator__first_name"), "Creator last name": model.get("creator__last_name"), "Creator email": model.get("creator__email"), @@ -1280,10 +1280,10 @@ class DomainRequestDataFull(DomainRequestExport): "Creator approved domains count", "Creator active requests count", "Alternative domains", - "AO first name", - "AO last name", - "AO email", - "AO title/role", + "SO first name", + "SO last name", + "SO email", + "SO title/role", "Request purpose", "Request additional details", "Other contacts", @@ -1297,7 +1297,7 @@ class DomainRequestDataFull(DomainRequestExport): """ Get a list of tables to pass to select_related when building queryset. """ - return ["creator", "authorizing_official", "federal_agency", "investigator", "requested_domain"] + return ["creator", "senior_official", "federal_agency", "investigator", "requested_domain"] @classmethod def get_prefetch_related(cls): @@ -1355,10 +1355,10 @@ class DomainRequestDataFull(DomainRequestExport): return [ "requested_domain__name", "federal_agency__agency", - "authorizing_official__first_name", - "authorizing_official__last_name", - "authorizing_official__email", - "authorizing_official__title", + "senior_official__first_name", + "senior_official__last_name", + "senior_official__email", + "senior_official__title", "creator__first_name", "creator__last_name", "creator__email", From fecaa44cc175dfce9f9eaeab0daf3222368dc9b8 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 4 Jul 2024 09:33:28 -0400 Subject: [PATCH 19/31] fixed domain invitations and domain managers --- src/registrar/utility/csv_export.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 44f8e6662..334742d17 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from collections import defaultdict import csv import logging from datetime import datetime @@ -293,13 +294,13 @@ class DomainExport(BaseExport): annotated_domain_infos = [] # Create mapping of domain to a list of invited users and managers - # invited_users_dict = defaultdict(list) - # for domain, email in domain_invitations: - # invited_users_dict[domain].append(email) + invited_users_dict = defaultdict(list) + for domain, email in domain_invitations: + invited_users_dict[domain].append(email) - # managers_dict = defaultdict(list) - # for domain, email in user_domain_roles: - # managers_dict[domain].append(email) + managers_dict = defaultdict(list) + for domain, email in user_domain_roles: + managers_dict[domain].append(email) # Annotate with security_contact from public_contacts, invited users # from domain_invitations, and managers from user_domain_roles @@ -307,8 +308,8 @@ class DomainExport(BaseExport): domain_info["security_contact_email"] = public_contacts.get( domain_info.get("domain__security_contact_registry_id") ) - domain_info["invited_users"] = domain_invitations.get(domain_info.get("domain__name")) - domain_info["managers"] = user_domain_roles.get(domain_info.get("domain__name")) + domain_info["invited_users"] = ", ".join(invited_users_dict.get(domain_info.get("domain__name"), [])) + domain_info["managers"] = ", ".join(managers_dict.get(domain_info.get("domain__name"), [])) annotated_domain_infos.append(domain_info) if annotated_domain_infos: @@ -335,7 +336,7 @@ class DomainExport(BaseExport): Fetch all DomainInvitation entries and return a mapping of domain to email. """ domain_invitations = DomainInvitation.objects.filter(status="invited").values_list("domain__name", "email") - return {domain__name: email for domain__name, email in domain_invitations} + return list(domain_invitations) @classmethod def get_all_user_domain_roles(cls): @@ -343,7 +344,7 @@ class DomainExport(BaseExport): Fetch all UserDomainRole entries and return a mapping of domain to user__email. """ user_domain_roles = UserDomainRole.objects.select_related("user").values_list("domain__name", "user__email") - return {domain__name: user__email for domain__name, user__email in user_domain_roles} + return list(user_domain_roles) @classmethod def parse_row(cls, columns, model): From 6275de19b5f25651639bb5793504867436a4c1b9 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 4 Jul 2024 09:37:31 -0400 Subject: [PATCH 20/31] fixed invited managers and domain managers --- src/registrar/tests/test_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 265876987..ded04e31b 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -633,4 +633,4 @@ class HelperFunctions(MockDb): } submitted_requests_sliced_at_end_date = DomainRequestExport.get_sliced_requests(filter_condition) expected_content = [3, 2, 0, 0, 0, 0, 1, 0, 0, 1] - self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file + self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) From e31fefcef11a10f5e8aa2ede5347a01787d591b2 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 9 Jul 2024 14:53:56 -0400 Subject: [PATCH 21/31] New sandbox ms --- .github/workflows/migrate.yaml | 1 + .github/workflows/reset-db.yaml | 1 + ops/manifests/manifest-ms.yaml | 32 ++++++++++++++++++++++++++++++++ src/registrar/config/settings.py | 1 + 4 files changed, 35 insertions(+) create mode 100644 ops/manifests/manifest-ms.yaml diff --git a/.github/workflows/migrate.yaml b/.github/workflows/migrate.yaml index 81368f6e9..3ebee59f9 100644 --- a/.github/workflows/migrate.yaml +++ b/.github/workflows/migrate.yaml @@ -16,6 +16,7 @@ on: - stable - staging - development + - ms - ag - litterbox - hotgov diff --git a/.github/workflows/reset-db.yaml b/.github/workflows/reset-db.yaml index ad325c50a..49e4b5e5f 100644 --- a/.github/workflows/reset-db.yaml +++ b/.github/workflows/reset-db.yaml @@ -16,6 +16,7 @@ on: options: - staging - development + - ms - ag - litterbox - hotgov diff --git a/ops/manifests/manifest-ms.yaml b/ops/manifests/manifest-ms.yaml new file mode 100644 index 000000000..153ee5f08 --- /dev/null +++ b/ops/manifests/manifest-ms.yaml @@ -0,0 +1,32 @@ +--- +applications: +- name: getgov-ms + buildpacks: + - python_buildpack + path: ../../src + instances: 1 + memory: 512M + stack: cflinuxfs4 + timeout: 180 + command: ./run.sh + health-check-type: http + health-check-http-endpoint: /health + health-check-invocation-timeout: 40 + env: + # Send stdout and stderr straight to the terminal without buffering + PYTHONUNBUFFERED: yup + # Tell Django where to find its configuration + DJANGO_SETTINGS_MODULE: registrar.config.settings + # Tell Django where it is being hosted + DJANGO_BASE_URL: https://getgov-ms.app.cloud.gov + # Tell Django how much stuff to log + DJANGO_LOG_LEVEL: INFO + # default public site location + GETGOV_PUBLIC_SITE_URL: https://get.gov + # Flag to disable/enable features in prod environments + IS_PRODUCTION: False + routes: + - route: getgov-ms.app.cloud.gov + services: + - getgov-credentials + - getgov-ms-database diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 688a3e8ca..1b8a5005f 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -660,6 +660,7 @@ ALLOWED_HOSTS = [ "getgov-stable.app.cloud.gov", "getgov-staging.app.cloud.gov", "getgov-development.app.cloud.gov", + "getgov-ms.app.cloud.gov", "getgov-ag.app.cloud.gov", "getgov-litterbox.app.cloud.gov", "getgov-hotgov.app.cloud.gov", From 9209856b0c18e2995d11db58a931b8813e194cce Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 9 Jul 2024 14:55:59 -0400 Subject: [PATCH 22/31] edit deploy-sandbox --- .github/workflows/deploy-sandbox.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-sandbox.yaml b/.github/workflows/deploy-sandbox.yaml index 84f228893..4bd7f99dd 100644 --- a/.github/workflows/deploy-sandbox.yaml +++ b/.github/workflows/deploy-sandbox.yaml @@ -28,6 +28,7 @@ jobs: || startsWith(github.head_ref, 'hotgov/') || startsWith(github.head_ref, 'litterbox/') || startsWith(github.head_ref, 'ag/') + || startsWith(github.head_ref, 'ms/') outputs: environment: ${{ steps.var.outputs.environment}} runs-on: "ubuntu-latest" From a7abfa9cdb16ac85fad271a01c665e0a7f0e276b Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 9 Jul 2024 16:48:03 -0400 Subject: [PATCH 23/31] made email readonly and fixed some unit tests --- src/registrar/admin.py | 2 +- src/registrar/forms/domain.py | 2 +- src/registrar/tests/test_admin.py | 4 +++- src/registrar/views/user_profile.py | 1 - 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 7ddff76c0..325fc53dd 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -917,7 +917,7 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): name.admin_order_field = "first_name" # type: ignore # Read only that we'll leverage for CISA Analysts - analyst_readonly_fields: list[str] = [] + analyst_readonly_fields: list[str] = ["email"] def get_readonly_fields(self, request, obj=None): """Set the read-only state on form elements. diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 06163dbce..9b8f1b7fc 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -204,7 +204,7 @@ NameserverFormset = formset_factory( class UserForm(forms.ModelForm): - """Form for updating contacts.""" + """Form for updating users.""" email = forms.EmailField(max_length=None) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 87fd9b5b8..c8fe64c42 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -2105,6 +2105,7 @@ class TestDomainRequestAdmin(MockEppLib): ] self.test_helper.assert_response_contains_distinct_values(response, expected_submitter_fields) self.assertContains(response, "Testy2 Tester2") + self.assertContains(response, "meoward.jones@igorville.gov") # == Check for the senior_official == # self.assertContains(response, "testy@town.com", count=2) @@ -3130,6 +3131,7 @@ class TestDomainInformationAdmin(TestCase): ("phone", "(555) 123 12345"), ] self.test_helper.assert_response_contains_distinct_values(response, expected_creator_fields) + self.assertContains(response, "meoward.jones@igorville.gov") # Check for the field itself self.assertContains(response, "Meoward Jones") @@ -4036,7 +4038,7 @@ class TestContactAdmin(TestCase): readonly_fields = self.admin.get_readonly_fields(request) - expected_fields = [] + expected_fields = ["email"] self.assertEqual(readonly_fields, expected_fields) diff --git a/src/registrar/views/user_profile.py b/src/registrar/views/user_profile.py index e378b9180..dec1d1af1 100644 --- a/src/registrar/views/user_profile.py +++ b/src/registrar/views/user_profile.py @@ -54,7 +54,6 @@ class UserProfileView(UserProfilePermissionView, FormMixin): def get_context_data(self, **kwargs): """Extend get_context_data to include has_profile_feature_flag""" context = super().get_context_data(**kwargs) - logger.info("UserProfileView::get_context_data") # This is a django waffle flag which toggles features based off of the "flag" table context["has_profile_feature_flag"] = flag_is_active(self.request, "profile_feature") From d393350a7f132eb079acf5762404656a20af9f5c Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 9 Jul 2024 17:05:37 -0400 Subject: [PATCH 24/31] fixed migrations --- ...2_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/registrar/migrations/{0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py => 0112_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py} (84%) diff --git a/src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py b/src/registrar/migrations/0112_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py similarity index 84% rename from src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py rename to src/registrar/migrations/0112_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py index 858210be7..db9b7970a 100644 --- a/src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py +++ b/src/registrar/migrations/0112_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ("registrar", "0109_domaininformation_sub_organization_and_more"), + ("registrar", "0111_create_groups_v15"), ] operations = [ From f0dfe96e974e4313b1316e6d6717b088b9e86639 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 9 Jul 2024 17:24:35 -0400 Subject: [PATCH 25/31] fixed tests --- src/registrar/tests/test_admin.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 4fa443105..06fb27237 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -939,7 +939,10 @@ class TestDomainRequestAdmin(MockEppLib): SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") - contact, _ = Contact.objects.get_or_create(user=self.staffuser) + contact, _ = Contact.objects.get_or_create( + first_name="Henry", + last_name="McFakerson" + ) domain_request = completed_domain_request(submitter=contact, name="city1.gov") request = self.factory.post("/admin/registrar/domainrequest/{}/change/".format(domain_request.pk)) model_admin = AuditedAdmin(DomainRequest, self.site) @@ -2943,7 +2946,10 @@ class TestDomainInformationAdmin(TestCase): SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") - contact, _ = Contact.objects.get_or_create(user=self.staffuser) + contact, _ = Contact.objects.get_or_create( + first_name="Henry", + last_name="McFakerson" + ) domain_request = completed_domain_request( submitter=contact, name="city1244.gov", status=DomainRequest.DomainRequestStatus.IN_REVIEW ) From d981c410e827854e0321681451c7288a1b4e960d Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 9 Jul 2024 17:30:45 -0400 Subject: [PATCH 26/31] linter --- src/registrar/tests/test_admin.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 06fb27237..b77a1faf4 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -939,10 +939,7 @@ class TestDomainRequestAdmin(MockEppLib): SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") - contact, _ = Contact.objects.get_or_create( - first_name="Henry", - last_name="McFakerson" - ) + contact, _ = Contact.objects.get_or_create(first_name="Henry", last_name="McFakerson") domain_request = completed_domain_request(submitter=contact, name="city1.gov") request = self.factory.post("/admin/registrar/domainrequest/{}/change/".format(domain_request.pk)) model_admin = AuditedAdmin(DomainRequest, self.site) @@ -2946,10 +2943,7 @@ class TestDomainInformationAdmin(TestCase): SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") - contact, _ = Contact.objects.get_or_create( - first_name="Henry", - last_name="McFakerson" - ) + contact, _ = Contact.objects.get_or_create(first_name="Henry", last_name="McFakerson") domain_request = completed_domain_request( submitter=contact, name="city1244.gov", status=DomainRequest.DomainRequestStatus.IN_REVIEW ) From 1d5d30873f665811bbc74359a1f33be01d73c4f5 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Wed, 10 Jul 2024 09:24:38 -0500 Subject: [PATCH 27/31] Add fixtures for Matthew Spence --- src/registrar/fixtures_users.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/registrar/fixtures_users.py b/src/registrar/fixtures_users.py index 2aa9d224b..74fd4d15d 100644 --- a/src/registrar/fixtures_users.py +++ b/src/registrar/fixtures_users.py @@ -22,6 +22,11 @@ class UserFixture: """ ADMINS = [ + { + "username": "be17c826-e200-4999-9389-2ded48c43691", + "first_name": "Matthew", + "last_name": "Spence", + }, { "username": "5f283494-31bd-49b5-b024-a7e7cae00848", "first_name": "Rachid", @@ -115,6 +120,11 @@ class UserFixture: ] STAFF = [ + { + "username": "d6bf296b-fac5-47ff-9c12-f88ccc5c1b99", + "first_name": "Matthew-Analyst", + "last_name": "Spence-Analyst", + }, { "username": "319c490d-453b-43d9-bc4d-7d6cd8ff6844", "first_name": "Rachid-Analyst", From 72b1631facf7e51aec33a17d80a417ea05c9c35c Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 10 Jul 2024 22:00:50 -0400 Subject: [PATCH 28/31] senior official hotfix scripts --- ...late_domain_information_senior_official.py | 76 +++++++++++++++++ ...opulate_domain_request_senior_official.py | 81 +++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 src/registrar/management/commands/repopulate_domain_information_senior_official.py create mode 100644 src/registrar/management/commands/repopulate_domain_request_senior_official.py diff --git a/src/registrar/management/commands/repopulate_domain_information_senior_official.py b/src/registrar/management/commands/repopulate_domain_information_senior_official.py new file mode 100644 index 000000000..540f88154 --- /dev/null +++ b/src/registrar/management/commands/repopulate_domain_information_senior_official.py @@ -0,0 +1,76 @@ +import argparse +import csv +import logging +import os +from django.core.management import BaseCommand +from registrar.management.commands.utility.terminal_helper import PopulateScriptTemplate, TerminalColors +from registrar.models import DomainInformation + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand, PopulateScriptTemplate): + """ + This command uses the PopulateScriptTemplate, + which provides reusable logging and bulk updating functions for mass-updating fields. + """ + + help = "Loops through each valid DomainInformation object and updates its Senior Official" + prompt_title = "Do you wish to update all Senior Officials for Domain Information?" + + def handle(self, domain_info_csv_path, **kwargs): + """Loops through each valid DomainInformation object and updates its senior official field""" + + # Check if the provided file path is valid. + if not os.path.isfile(domain_info_csv_path): + raise argparse.ArgumentTypeError(f"Invalid file path '{domain_info_csv_path}'") + + # Simple check to make sure we don't accidentally pass in the wrong file. Crude but it works. + if "information" not in domain_info_csv_path.lower(): + raise argparse.ArgumentTypeError(f"Invalid file for domain information: '{domain_info_csv_path}'") + + # Get all ao data. + self.ao_dict = {} + self.ao_dict = self.read_csv_file_and_get_contacts(domain_info_csv_path) + + self.mass_update_records( + DomainInformation, filter_conditions={"senior_official__isnull": True}, fields_to_update=["senior_official"] + ) + + def add_arguments(self, parser): + """Add command line arguments.""" + parser.add_argument( + "--domain_info_csv_path", help="A csv containing the domain information id and the contact id" + ) + + def read_csv_file_and_get_contacts(self, file): + dict_data = {} + with open(file, "r") as requested_file: + reader = csv.DictReader(requested_file) + for row in reader: + domain_info_id = row.get("id") + ao_id = row.get("authorizing_official") + if ao_id: + ao_id = int(ao_id) + if domain_info_id and ao_id: + dict_data[int(domain_info_id)] = ao_id + + return dict_data + + def update_record(self, record: DomainInformation): + """Defines how we update the senior official field on each record.""" + record.senior_official_id = self.ao_dict.get(record.id) + logger.info(f"{TerminalColors.OKCYAN}Updating {str(record)} => {record.senior_official}{TerminalColors.ENDC}") + + def should_skip_record(self, record) -> bool: # noqa + """Defines the conditions in which we should skip updating a record.""" + # Don't update this record if there isn't ao data to pull from + if self.ao_dict.get(record.id) is None: + logger.info( + f"{TerminalColors.YELLOW}Skipping update for {str(record)} => " + f"Missing authorizing_official data.{TerminalColors.ENDC}" + ) + return True + else: + return False diff --git a/src/registrar/management/commands/repopulate_domain_request_senior_official.py b/src/registrar/management/commands/repopulate_domain_request_senior_official.py new file mode 100644 index 000000000..37fcea03e --- /dev/null +++ b/src/registrar/management/commands/repopulate_domain_request_senior_official.py @@ -0,0 +1,81 @@ +import argparse +import csv +import logging +import os +from django.core.management import BaseCommand +from registrar.management.commands.utility.terminal_helper import PopulateScriptTemplate, TerminalColors +from registrar.models import DomainRequest + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand, PopulateScriptTemplate): + """ + This command uses the PopulateScriptTemplate, + which provides reusable logging and bulk updating functions for mass-updating fields. + """ + + help = """Loops through each valid DomainRequest object and updates its senior official field""" + prompt_title = "Do you wish to update all Senior Officials for Domain Requests?" + + def handle(self, domain_request_csv_path, **kwargs): + """Loops through each valid DomainRequest object and updates its senior official field""" + + # Check if the provided file path is valid. + if not os.path.isfile(domain_request_csv_path): + raise argparse.ArgumentTypeError(f"Invalid file path '{domain_request_csv_path}'") + + # Simple check to make sure we don't accidentally pass in the wrong file. Crude but it works. + if "request" not in domain_request_csv_path.lower(): + raise argparse.ArgumentTypeError(f"Invalid file for domain requests: '{domain_request_csv_path}'") + + # Get all ao data. + self.ao_dict = {} + self.ao_dict = self.read_csv_file_and_get_contacts(domain_request_csv_path) + + self.mass_update_records( + DomainRequest, + filter_conditions={ + "senior_official__isnull": True, + }, + fields_to_update=["senior_official"], + ) + + def add_arguments(self, parser): + """Add command line arguments.""" + parser.add_argument( + "--domain_request_csv_path", help="A csv containing the domain request id and the contact id" + ) + + def read_csv_file_and_get_contacts(self, file): + dict_data: dict = {} + with open(file, "r") as requested_file: + reader = csv.DictReader(requested_file) + for row in reader: + domain_request_id = row.get("id") + ao_id = row.get("authorizing_official") + if ao_id: + ao_id = int(ao_id) + if domain_request_id and ao_id: + dict_data[int(domain_request_id)] = ao_id + + return dict_data + + def update_record(self, record: DomainRequest): + """Defines how we update the federal_type field on each record.""" + record.senior_official_id = self.ao_dict.get(record.id) + # record.senior_official = Contact.objects.get(id=contact_id) + logger.info(f"{TerminalColors.OKCYAN}Updating {str(record)} => {record.senior_official}{TerminalColors.ENDC}") + + def should_skip_record(self, record) -> bool: # noqa + """Defines the conditions in which we should skip updating a record.""" + # Don't update this record if there isn't ao data to pull from + if self.ao_dict.get(record.id) is None: + logger.info( + f"{TerminalColors.YELLOW}Skipping update for {str(record)} => " + f"Missing authorizing_official data.{TerminalColors.ENDC}" + ) + return True + else: + return False From 329737023bf7e3e95424010eb36a0f224bdfc655 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 10 Jul 2024 22:08:13 -0400 Subject: [PATCH 29/31] remove trailing whitespace --- ..._official.py => repopulate_domain_request_senior_official.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/registrar/management/commands/{repopulate_domain_request_senior_official.py => repopulate_domain_request_senior_official.py} (100%) diff --git a/src/registrar/management/commands/repopulate_domain_request_senior_official.py b/src/registrar/management/commands/repopulate_domain_request_senior_official.py similarity index 100% rename from src/registrar/management/commands/repopulate_domain_request_senior_official.py rename to src/registrar/management/commands/repopulate_domain_request_senior_official.py From 9b0302e8730f148431a6d2ff86ce0e4a6f685b6b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:02:53 -0600 Subject: [PATCH 30/31] add fix --- src/registrar/models/domain_request.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/registrar/models/domain_request.py b/src/registrar/models/domain_request.py index 078aa9f0d..188adaa52 100644 --- a/src/registrar/models/domain_request.py +++ b/src/registrar/models/domain_request.py @@ -136,6 +136,9 @@ class DomainRequest(TimeStampedModel): @classmethod def get_org_label(cls, org_name: str): """Returns the associated label for a given org name""" + if not org_name: + return None + org_names = org_name.split("_election") if len(org_names) > 0: org_name = org_names[0] From 1d5d7f79f10202a00ba3b85311d79cb510e51dcb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:42:50 -0600 Subject: [PATCH 31/31] Update domain_request.py --- src/registrar/models/domain_request.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/registrar/models/domain_request.py b/src/registrar/models/domain_request.py index 188adaa52..92f5869f7 100644 --- a/src/registrar/models/domain_request.py +++ b/src/registrar/models/domain_request.py @@ -136,6 +136,10 @@ class DomainRequest(TimeStampedModel): @classmethod def get_org_label(cls, org_name: str): """Returns the associated label for a given org name""" + # This is an edgecase on domains with no org. + # This unlikely to happen but + # a break will occur in certain edge cases without this. + # (more specifically, csv exports). if not org_name: return None