mirror of
https://github.com/cisagov/manage.get.gov.git
synced 2025-07-22 10:46:06 +02:00
Use normal request where possible
This commit is contained in:
parent
551e85ed22
commit
3fe1f7bfd1
2 changed files with 87 additions and 100 deletions
|
@ -12,7 +12,7 @@ from registrar.utility.csv_export import (
|
||||||
write_csv_for_domains,
|
write_csv_for_domains,
|
||||||
get_default_start_date,
|
get_default_start_date,
|
||||||
get_default_end_date,
|
get_default_end_date,
|
||||||
write_csv_for_requests,
|
DomainRequestExport,
|
||||||
)
|
)
|
||||||
|
|
||||||
from django.core.management import call_command
|
from django.core.management import call_command
|
||||||
|
@ -668,8 +668,8 @@ class ExportDataTest(MockDb, MockEppLib):
|
||||||
# We'll skip submission date because it's dynamic and therefore
|
# We'll skip submission date because it's dynamic and therefore
|
||||||
# impossible to set in expected_content
|
# impossible to set in expected_content
|
||||||
columns = [
|
columns = [
|
||||||
"Requested domain",
|
"Domain request",
|
||||||
"Organization type",
|
"Domain type",
|
||||||
]
|
]
|
||||||
sort_fields = [
|
sort_fields = [
|
||||||
"requested_domain__name",
|
"requested_domain__name",
|
||||||
|
@ -679,7 +679,15 @@ class ExportDataTest(MockDb, MockEppLib):
|
||||||
"submission_date__lte": self.end_date,
|
"submission_date__lte": self.end_date,
|
||||||
"submission_date__gte": self.start_date,
|
"submission_date__gte": self.start_date,
|
||||||
}
|
}
|
||||||
write_csv_for_requests(writer, columns, sort_fields, filter_condition, should_write_header=True)
|
|
||||||
|
all_requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct()
|
||||||
|
all_requests_dict = all_requests.values(
|
||||||
|
"requested_domain_name",
|
||||||
|
"generic_org_type",
|
||||||
|
"federal_type",
|
||||||
|
"submission_date"
|
||||||
|
)
|
||||||
|
DomainRequestExport.write_csv_for_requests(writer, columns, requests=all_requests_dict, should_write_header=True)
|
||||||
# Reset the CSV file's position to the beginning
|
# Reset the CSV file's position to the beginning
|
||||||
csv_file.seek(0)
|
csv_file.seek(0)
|
||||||
# Read the content into a variable
|
# Read the content into a variable
|
||||||
|
@ -687,7 +695,7 @@ class ExportDataTest(MockDb, MockEppLib):
|
||||||
# We expect READY domains first, created between today-2 and today+2, sorted by created_at then name
|
# We expect READY domains first, created between today-2 and today+2, sorted by created_at then name
|
||||||
# and DELETED domains deleted between today-2 and today+2, sorted by deleted then name
|
# and DELETED domains deleted between today-2 and today+2, sorted by deleted then name
|
||||||
expected_content = (
|
expected_content = (
|
||||||
"Requested domain,Organization type\n"
|
"Domain request,Domain type\n"
|
||||||
"city3.gov,Federal - Executive\n"
|
"city3.gov,Federal - Executive\n"
|
||||||
"city4.gov,Federal - Executive\n"
|
"city4.gov,Federal - Executive\n"
|
||||||
)
|
)
|
||||||
|
|
|
@ -15,6 +15,7 @@ from django.utils import timezone
|
||||||
from django.core.paginator import Paginator
|
from django.core.paginator import Paginator
|
||||||
from django.db.models.functions import Concat, Coalesce
|
from django.db.models.functions import Concat, Coalesce
|
||||||
from django.contrib.postgres.aggregates import StringAgg
|
from django.contrib.postgres.aggregates import StringAgg
|
||||||
|
from registrar.models.utility.generic_helper import Timer
|
||||||
from registrar.templatetags.custom_filters import get_region
|
from registrar.templatetags.custom_filters import get_region
|
||||||
from registrar.utility.enums import DefaultEmail
|
from registrar.utility.enums import DefaultEmail
|
||||||
|
|
||||||
|
@ -719,21 +720,25 @@ class DomainRequestExport:
|
||||||
writer,
|
writer,
|
||||||
columns,
|
columns,
|
||||||
requests,
|
requests,
|
||||||
|
extra_request_fields=None,
|
||||||
should_write_header=True,
|
should_write_header=True,
|
||||||
):
|
):
|
||||||
"""Receives params from the parent methods and outputs a CSV with filtered and sorted requests.
|
"""Receives params from the parent methods and outputs a CSV with filtered and sorted requests.
|
||||||
Works with write_header as long as the same writer object is passed."""
|
Works with write_header as long as the same writer object is passed."""
|
||||||
|
|
||||||
|
if extra_request_fields:
|
||||||
|
extra_request_fields_dict = {request["id"]: request for request in extra_request_fields}
|
||||||
|
|
||||||
# Reduce the memory overhead when performing the write operation
|
# Reduce the memory overhead when performing the write operation
|
||||||
paginator = Paginator(requests, 1000)
|
paginator = Paginator(requests, 1000)
|
||||||
total_body_rows = []
|
total_body_rows = []
|
||||||
|
|
||||||
for page_num in paginator.page_range:
|
for page_num in paginator.page_range:
|
||||||
page = paginator.page(page_num)
|
page = paginator.page(page_num)
|
||||||
rows = []
|
rows = []
|
||||||
for request in page.object_list:
|
for request in page.object_list:
|
||||||
try:
|
try:
|
||||||
row = DomainRequestExport.parse_row_for_requests(columns, request)
|
extra = extra_request_fields_dict.get(request.id) if extra_request_fields else None
|
||||||
|
row = DomainRequestExport.parse_row_for_requests(columns, request, extra)
|
||||||
rows.append(row)
|
rows.append(row)
|
||||||
except ValueError as err:
|
except ValueError as err:
|
||||||
logger.error(f"csv_export -> Error when parsing row: {err}")
|
logger.error(f"csv_export -> Error when parsing row: {err}")
|
||||||
|
@ -745,63 +750,70 @@ class DomainRequestExport:
|
||||||
writer.writerows(total_body_rows)
|
writer.writerows(total_body_rows)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def parse_row_for_requests(columns, request):
|
def parse_row_for_requests(columns, request: DomainRequest, extra_fields: QuerySet):
|
||||||
"""Given a set of columns, generate a new row from cleaned column data"""
|
"""
|
||||||
|
Given a set of columns and a request dictionary,
|
||||||
|
generate a new row from cleaned column data
|
||||||
|
"""
|
||||||
|
|
||||||
# Handle the domain_type field. Defaults to the wrong format.
|
# Handle the domain_type field. Defaults to the wrong format.
|
||||||
org_type = request.get("generic_org_type")
|
org_type = request.organization_type
|
||||||
if org_type and request.get("domain_type") is None:
|
if org_type and extra_fields.get("domain_type") is None:
|
||||||
readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type)
|
readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type)
|
||||||
request["domain_type"] = readable_org_type
|
extra_fields["domain_type"] = readable_org_type
|
||||||
|
|
||||||
# Handle the federal_type field. Defaults to the wrong format.
|
# Handle the federal_type field. Defaults to the wrong format.
|
||||||
federal_type = request.get("federal_type")
|
federal_type = request.federal_type
|
||||||
if federal_type and request.get("human_readable_federal_type") is None:
|
if federal_type and extra_fields.get("human_readable_federal_type") is None:
|
||||||
request["human_readable_federal_type"] = DomainRequest.BranchChoices.get_branch_label(federal_type)
|
extra_fields["human_readable_federal_type"] = DomainRequest.BranchChoices.get_branch_label(federal_type)
|
||||||
|
|
||||||
# Handle the status field. Defaults to the wrong format.
|
# Handle the status field. Defaults to the wrong format.
|
||||||
status = request.get("status")
|
status = request.status
|
||||||
if status and request.get("status_display") is None:
|
if status and extra_fields.get("status_display") is None:
|
||||||
request["status_display"] = DomainRequest.DomainRequestStatus.get_status_label(status)
|
extra_fields["status_display"] = DomainRequest.DomainRequestStatus.get_status_label(status)
|
||||||
|
|
||||||
# Handle the region field.
|
# Handle the region field.
|
||||||
state_territory = request.get("state_territory")
|
state_territory = request.state_territory
|
||||||
if state_territory and request.get("region") is None:
|
if state_territory and extra_fields.get("region") is None:
|
||||||
request["region"] = get_region(state_territory)
|
extra_fields["region"] = get_region(state_territory)
|
||||||
|
|
||||||
|
human_readable_election_board = "N/A"
|
||||||
|
if request.is_election_board is not None:
|
||||||
|
human_readable_election_board = "Yes" if request.is_election_board else "No"
|
||||||
|
|
||||||
# create a dictionary of fields which can be included in output
|
# create a dictionary of fields which can be included in output
|
||||||
FIELDS = {
|
FIELDS = {
|
||||||
"Domain request": request.get("requested_domain_name"),
|
# Precomputed fields (generated in the DB)
|
||||||
"Submitted at": request.get("submission_date"),
|
"Domain request": extra_fields.get("requested_domain_name"),
|
||||||
"Status": request.get("status_display"),
|
"Status": extra_fields.get("status_display"),
|
||||||
"Domain type": request.get("domain_type"),
|
"Domain type": extra_fields.get("domain_type"),
|
||||||
"Federal type": request.get("human_readable_federal_type"),
|
"Federal type": extra_fields.get("human_readable_federal_type"),
|
||||||
"Federal agency": request.get("federal_agency_name"),
|
"Federal agency": extra_fields.get("federal_agency__agency"),
|
||||||
"Organization name": request.get("organization_name"),
|
"Region": extra_fields.get("region"),
|
||||||
"Election office": request.get("human_readable_election_board"),
|
# Creator - performs substantially better when accessed this way
|
||||||
"City": request.get("city"),
|
"Creator first name": extra_fields.get("creator__first_name", ""),
|
||||||
"State/territory": request.get("state_territory"),
|
"Creator last name": extra_fields.get("creator__last_name", ""),
|
||||||
"Region": request.get("region"),
|
"Creator email": extra_fields.get("creator__email", ""),
|
||||||
# Creator
|
"Creator approved domains count": extra_fields.get("creator_approved_domains_count", 0),
|
||||||
"Creator first name": request.get("creator__first_name", ""),
|
"Creator active requests count": extra_fields.get("creator_active_requests_count", 0),
|
||||||
"Creator last name": request.get("creator__last_name", ""),
|
"Alternative domains": extra_fields.get("all_alternative_domains"),
|
||||||
"Creator email": request.get("creator__email", ""),
|
# AO - performs substantially better when accessed this way
|
||||||
"Creator approved domains count": request.get("creator_approved_domains_count", 0),
|
"AO first name": extra_fields.get("authorizing_official__first_name", ""),
|
||||||
"Creator active requests count": request.get("creator_active_requests_count", 0),
|
"AO last name": extra_fields.get("authorizing_official__last_name", ""),
|
||||||
# End of creator
|
"AO email": extra_fields.get("authorizing_official__email", ""),
|
||||||
"Alternative domains": request.get("all_alternative_domains"),
|
"AO title/role": extra_fields.get("authorizing_official__title", ""),
|
||||||
# AO
|
"Request additional details": extra_fields.get("additional_details"),
|
||||||
"AO first name": request.get("authorizing_official__first_name", ""),
|
"Other contacts": extra_fields.get("all_other_contacts"),
|
||||||
"AO last name": request.get("authorizing_official__last_name", ""),
|
"Current websites": extra_fields.get("all_current_websites"),
|
||||||
"AO email": request.get("authorizing_official__email", ""),
|
# Normal fields
|
||||||
"AO title/role": request.get("authorizing_official__title", ""),
|
"Organization name": request.organization_name,
|
||||||
# End of AO
|
"Election office": human_readable_election_board,
|
||||||
"Request purpose": request.get("purpose"),
|
"City": request.city,
|
||||||
"Request additional details": request.get("additional_details"),
|
"State/territory": request.state_territory,
|
||||||
"Other contacts": request.get("all_other_contacts"),
|
"Request purpose": request.purpose,
|
||||||
"CISA regional representative": request.get("cisa_representative_email"), # TODO - same problem as before
|
"CISA regional representative": request.cisa_representative_email,
|
||||||
"Current websites": request.get("all_current_websites"),
|
"Investigator": request.investigator.email if request.investigator else None,
|
||||||
"Investigator": request.get("investigator_email"),
|
"Submitted at": request.submission_date,
|
||||||
}
|
}
|
||||||
|
|
||||||
row = [FIELDS.get(column, "") for column in columns]
|
row = [FIELDS.get(column, "") for column in columns]
|
||||||
|
@ -838,13 +850,12 @@ class DomainRequestExport:
|
||||||
all_requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct()
|
all_requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct()
|
||||||
|
|
||||||
# Convert the request to a querystring for faster processing. Only grab what we need.
|
# Convert the request to a querystring for faster processing. Only grab what we need.
|
||||||
parsed_requests = all_requests.annotate(
|
annotations = all_requests.annotate(
|
||||||
requested_domain_name=DomainRequestExport.get_requested_domain_name_query(),
|
requested_domain_name=DomainRequestExport.get_requested_domain_name_query(),
|
||||||
approved_domain_name=F("approved_domain__name"),
|
|
||||||
).values("requested_domain_name", "generic_org_type", "federal_type", "submission_date")
|
).values("requested_domain_name", "generic_org_type", "federal_type", "submission_date")
|
||||||
|
|
||||||
# Override the default value for domain_type
|
# Override the default value for domain_type
|
||||||
for request in parsed_requests:
|
for request in annotations:
|
||||||
# Handle the domain_type field. Defaults to the wrong variant.
|
# Handle the domain_type field. Defaults to the wrong variant.
|
||||||
org_type = request.get("generic_org_type")
|
org_type = request.get("generic_org_type")
|
||||||
federal_type = request.get("federal_type")
|
federal_type = request.get("federal_type")
|
||||||
|
@ -858,7 +869,7 @@ class DomainRequestExport:
|
||||||
else:
|
else:
|
||||||
request["domain_type"] = readable_org_type
|
request["domain_type"] = readable_org_type
|
||||||
|
|
||||||
DomainRequestExport.write_csv_for_requests(writer, columns, parsed_requests, should_write_header=True)
|
DomainRequestExport.write_csv_for_requests(writer, columns, all_requests, annotations, should_write_header=True)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def export_full_domain_request_report(csv_file):
|
def export_full_domain_request_report(csv_file):
|
||||||
|
@ -898,9 +909,9 @@ class DomainRequestExport:
|
||||||
"requested_domain__name",
|
"requested_domain__name",
|
||||||
]
|
]
|
||||||
requests = DomainRequest.objects.exclude(status__in=excluded_statuses).order_by(*order_by).distinct()
|
requests = DomainRequest.objects.exclude(status__in=excluded_statuses).order_by(*order_by).distinct()
|
||||||
parsed_requests = DomainRequestExport.annotate_and_prepare_domain_request_data(requests)
|
extra_fields = DomainRequestExport.annotate_and_prepare_domain_request_data(requests)
|
||||||
|
|
||||||
DomainRequestExport.write_csv_for_requests(writer, columns, parsed_requests, should_write_header=True)
|
DomainRequestExport.write_csv_for_requests(writer, columns, requests, extra_fields, should_write_header=True)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def annotate_and_prepare_domain_request_data(requests_to_convert: QuerySet[DomainRequest]) -> QuerySet:
|
def annotate_and_prepare_domain_request_data(requests_to_convert: QuerySet[DomainRequest]) -> QuerySet:
|
||||||
|
@ -916,7 +927,6 @@ class DomainRequestExport:
|
||||||
|
|
||||||
Annotations (examples of python-readable equivalents):
|
Annotations (examples of python-readable equivalents):
|
||||||
- requested_domain_name: `requested_domain.name If requested_domain.name is not None else default_message`.
|
- requested_domain_name: `requested_domain.name If requested_domain.name is not None else default_message`.
|
||||||
- request_type: `f"{generic_org_type} | {federal_type}" If request.federal_type is not None else generic_org_type`.
|
|
||||||
- additional_details: `f"{cisa_rep} | {anything_else}" If anything_else or cisa_rep else None`.
|
- additional_details: `f"{cisa_rep} | {anything_else}" If anything_else or cisa_rep else None`.
|
||||||
- all_other_contacts: `[f"{c.first_name} {c.last_name} {c.email}" for c in request.other_contacts.all()].join(" | ")`.
|
- all_other_contacts: `[f"{c.first_name} {c.last_name} {c.email}" for c in request.other_contacts.all()].join(" | ")`.
|
||||||
- all_current_websites: `[w.website for w in request.current_websites.all()].join(" | ")`.
|
- all_current_websites: `[w.website for w in request.current_websites.all()].join(" | ")`.
|
||||||
|
@ -936,53 +946,35 @@ class DomainRequestExport:
|
||||||
additional_details=DomainRequestExport.get_additional_details_query(),
|
additional_details=DomainRequestExport.get_additional_details_query(),
|
||||||
creator_approved_domains_count=DomainRequestExport.get_creator_approved_domains_count_query(),
|
creator_approved_domains_count=DomainRequestExport.get_creator_approved_domains_count_query(),
|
||||||
creator_active_requests_count=DomainRequestExport.get_creator_active_requests_count_query(),
|
creator_active_requests_count=DomainRequestExport.get_creator_active_requests_count_query(),
|
||||||
human_readable_election_board=DomainRequestExport.get_human_readable_election_board_query(),
|
|
||||||
all_other_contacts=DomainRequestExport.get_all_other_contacts_query(),
|
all_other_contacts=DomainRequestExport.get_all_other_contacts_query(),
|
||||||
all_current_websites=StringAgg("current_websites__website", delimiter=" | ", distinct=True),
|
all_current_websites=StringAgg("current_websites__website", delimiter=" | ", distinct=True),
|
||||||
all_alternative_domains=StringAgg("alternative_domains__website", delimiter=" | ", distinct=True),
|
all_alternative_domains=StringAgg("alternative_domains__website", delimiter=" | ", distinct=True),
|
||||||
federal_agency_name=F("federal_agency__agency"),
|
|
||||||
investigator_email=F("investigator__email"),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
requests_dict = parsed_requests.values(
|
requests_queryset = parsed_requests.values(
|
||||||
# Custom fields
|
# Custom fields
|
||||||
"all_alternative_domains",
|
"all_alternative_domains",
|
||||||
"all_other_contacts",
|
"all_other_contacts",
|
||||||
"all_current_websites",
|
"all_current_websites",
|
||||||
"additional_details",
|
"additional_details",
|
||||||
"requested_domain_name",
|
|
||||||
"federal_agency_name",
|
|
||||||
"human_readable_election_board",
|
|
||||||
# Creator
|
|
||||||
"creator__first_name",
|
|
||||||
"creator__last_name",
|
|
||||||
"creator__email",
|
|
||||||
"creator_approved_domains_count",
|
"creator_approved_domains_count",
|
||||||
"creator_active_requests_count",
|
"creator_active_requests_count",
|
||||||
|
# Existing fields
|
||||||
|
"id",
|
||||||
# AO
|
# AO
|
||||||
"authorizing_official__first_name",
|
"authorizing_official__first_name",
|
||||||
"authorizing_official__last_name",
|
"authorizing_official__last_name",
|
||||||
"authorizing_official__email",
|
"authorizing_official__email",
|
||||||
"authorizing_official__title",
|
"authorizing_official__title",
|
||||||
# Investigator
|
# Creator
|
||||||
"investigator_email",
|
"creator__first_name",
|
||||||
# Existing fields
|
"creator__last_name",
|
||||||
"id",
|
"creator__email",
|
||||||
"submission_date",
|
# Federal agency name
|
||||||
"status",
|
"federal_agency__agency",
|
||||||
"federal_type",
|
|
||||||
"organization_name",
|
|
||||||
"is_election_board",
|
|
||||||
"city",
|
|
||||||
"state_territory",
|
|
||||||
"purpose",
|
|
||||||
"cisa_representative_email",
|
|
||||||
"investigator",
|
|
||||||
"generic_org_type",
|
|
||||||
"federal_type",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return requests_dict
|
return requests_queryset
|
||||||
|
|
||||||
# ============================================================= #
|
# ============================================================= #
|
||||||
# Helper functions for django ORM queries. #
|
# Helper functions for django ORM queries. #
|
||||||
|
@ -1086,16 +1078,3 @@ class DomainRequestExport:
|
||||||
distinct=True,
|
distinct=True,
|
||||||
)
|
)
|
||||||
return query
|
return query
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_human_readable_election_board_query():
|
|
||||||
""" """
|
|
||||||
|
|
||||||
query = Case(
|
|
||||||
When(is_election_board=True, then=Value("Yes")),
|
|
||||||
When(is_election_board=False, then=Value("No")),
|
|
||||||
default=Value("N/A"),
|
|
||||||
output_field=CharField(),
|
|
||||||
)
|
|
||||||
|
|
||||||
return query
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue