export updates

This commit is contained in:
CocoByte 2024-11-13 01:16:55 -07:00
parent c481c3c08d
commit 32232c5d56
No known key found for this signature in database
GPG key ID: BBFAA2526384C97F
6 changed files with 236 additions and 55 deletions

View file

@ -1,10 +1,11 @@
import csv
from datetime import date
import logging
import copy
from django import forms
from django.db.models import Value, CharField, Q
from django.db.models.functions import Concat, Coalesce
from django.http import HttpResponseRedirect
from django.http import HttpResponse, HttpResponseRedirect
from registrar.models.federal_agency import FederalAgency
from registrar.utility.admin_helpers import (
get_action_needed_reason_default_email,
@ -42,7 +43,7 @@ from django.utils.html import escape
from django.contrib.auth.forms import UserChangeForm, UsernameField
from django.contrib.admin.views.main import IGNORED_PARAMS
from django_admin_multiple_choice_list_filter.list_filters import MultipleChoiceListFilter
from import_export import resources
from import_export import resources, fields
from import_export.admin import ImportExportModelAdmin
from django.core.exceptions import ObjectDoesNotExist
from django.contrib.admin.widgets import FilteredSelectMultiple
@ -1453,6 +1454,57 @@ class DomainInformationResource(resources.ModelResource):
class Meta:
model = models.DomainInformation
# Override exports for these columns in DomainInformation to use converted values. These values
# come from @Property functions, which are not automatically included in the export and which we
# want to use in place of the native fields.
organization_name = fields.Field(attribute='converted_organization_name', column_name='organization_name')
generic_org_type = fields.Field(attribute='converted_generic_org_type', column_name='generic_org_type')
federal_type = fields.Field(attribute='converted_federal_type', column_name='federal_type')
federal_agency = fields.Field(attribute='converted_federal_agency', column_name='federal_agency')
senior_official = fields.Field(attribute='converted_senior_official', column_name='senior_official')
address_line1 = fields.Field(attribute='converted_address_line1', column_name='address_line1')
address_line2 = fields.Field(attribute='converted_address_line2', column_name='address_line2')
city = fields.Field(attribute='converted_city', column_name='city')
state_territory = fields.Field(attribute='converted_state_territory', column_name='state_territory')
zipcode = fields.Field(attribute='converted_zipcode', column_name='zipcode')
urbanization = fields.Field(attribute='converted_urbanization', column_name='urbanization')
# Custom getters for the above columns that map to @property functions instead of fields
def dehydrate_organization_name(self, obj):
return obj.converted_organization_name
def dehydrate_generic_org_type(self, obj):
return obj.converted_generic_org_type
def dehydrate_federal_type(self, obj):
return obj.converted_federal_type
def dehydrate_federal_agency(self, obj):
return obj.converted_federal_agency
def dehydrate_senior_official(self, obj):
return obj.converted_senior_official
def dehydrate_address_line1(self, obj):
return obj.converted_address_line1
def dehydrate_address_line2(self, obj):
return obj.converted_address_line2
def dehydrate_city(self, obj):
return obj.converted_city
def dehydrate_state_territory(self, obj):
return obj.converted_state_territory
def dehydrate_zipcode(self, obj):
return obj.converted_zipcode
def dehydrate_urbanization(self, obj):
return obj.converted_urbanization
class DomainInformationAdmin(ListHeaderAdmin, ImportExportModelAdmin):
"""Customize domain information admin class."""
@ -1654,6 +1706,56 @@ class DomainRequestResource(FsmModelResource):
class Meta:
model = models.DomainRequest
# Override exports for these columns in DomainInformation to use converted values. These values
# come from @Property functions, which are not automatically included in the export and which we
# want to use in place of the native fields.
organization_name = fields.Field(attribute='converted_organization_name', column_name='GEN organization_name')
generic_org_type = fields.Field(attribute='converted_generic_org_type', column_name='GEN generic_org_type')
federal_type = fields.Field(attribute='converted_federal_type', column_name='GEN federal_type')
federal_agency = fields.Field(attribute='converted_federal_agency', column_name='GEN federal_agency')
senior_official = fields.Field(attribute='converted_senior_official', column_name='GEN senior_official')
address_line1 = fields.Field(attribute='converted_address_line1', column_name='GEN address_line1')
address_line2 = fields.Field(attribute='converted_address_line2', column_name='GEN address_line2')
city = fields.Field(attribute='converted_city', column_name='GEN city')
state_territory = fields.Field(attribute='converted_state_territory', column_name='GEN state_territory')
zipcode = fields.Field(attribute='converted_zipcode', column_name='GEN zipcode')
urbanization = fields.Field(attribute='converted_urbanization', column_name='GEN urbanization')
senior_official = fields.Field(attribute='converted_urbanization', column_name='GEN senior official')
# Custom getters for the above columns that map to @property functions instead of fields
def dehydrate_organization_name(self, obj):
return obj.converted_organization_name
def dehydrate_generic_org_type(self, obj):
return obj.converted_generic_org_type
def dehydrate_federal_type(self, obj):
return obj.converted_federal_type
def dehydrate_federal_agency(self, obj):
return obj.converted_federal_agency
def dehydrate_senior_official(self, obj):
return obj.converted_senior_official
def dehydrate_address_line1(self, obj):
return obj.converted_address_line1
def dehydrate_address_line2(self, obj):
return obj.converted_address_line2
def dehydrate_city(self, obj):
return obj.converted_city
def dehydrate_state_territory(self, obj):
return obj.converted_state_territory
def dehydrate_zipcode(self, obj):
return obj.converted_zipcode
def dehydrate_urbanization(self, obj):
return obj.converted_urbanization
class DomainRequestAdmin(ListHeaderAdmin, ImportExportModelAdmin):
"""Custom domain requests admin class."""
@ -2577,7 +2679,48 @@ class DomainResource(FsmModelResource):
class Meta:
model = models.Domain
#Override the default export so that it matches what is displayed in the admin table for Domains
fields = (
"name",
"converted_generic_org_type",
"federal_type",
"converted_federal_type",
"converted_federal_agency",
"converted_organization_name",
"custom_election_board",
"converted_city",
"converted_state_territory",
"state",
"expiration_date",
"created_at",
"first_ready",
"deleted",
)
# Custom getters to retrieve the values from @Proprerty methods in DomainInfo
converted_generic_org_type = fields.Field(attribute='converted_generic_org_type', column_name='Converted generic org type')
converted_federal_agency = fields.Field(attribute='converted_federal_agency', column_name='Converted federal agency')
converted_organization_name = fields.Field(attribute='converted_organization_name', column_name='Converted organization name')
converted_city = fields.Field(attribute='converted_city', column_name='city')
converted_state_territory = fields.Field(attribute='converted_state_territory', column_name='Converted state territory')
# def dehydrate_generic_org_type(self, obj):
# return obj.domain_info.converted_federal_type
def dehydrate_converted_generic_org_type(self, obj):
return obj.domain_info.converted_generic_org_type
def dehydrate_converted_federal_agency(self, obj):
return obj.domain_info.converted_federal_agency
def dehydrate_converted_organization_name(self, obj):
return obj.domain_info.converted_organization_name
def dehydrate_converted_city(self, obj):
return obj.domain_info.converted_city
def dehydrate_converted_state_territory(self, obj):
return obj.domain_info.converted_state_territory
class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin):
"""Custom domain admin class to add extra buttons."""
@ -3074,8 +3217,6 @@ class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin):
return super().has_change_permission(request, obj)
class DraftDomainResource(resources.ModelResource):
"""defines how each field in the referenced model should be mapped to the corresponding fields in the
import/export file"""

View file

@ -226,11 +226,6 @@ urlpatterns = [
ExportDataTypeRequests.as_view(),
name="export_data_type_requests",
),
path(
"reports/export_data_type_requests/",
ExportDataTypeRequests.as_view(),
name="export_data_type_requests",
),
path(
"domain-request/<int:id>/edit/",
views.DomainRequestWizard.as_view(),

View file

@ -431,8 +431,8 @@ class DomainInformation(TimeStampedModel):
@property
def converted_organization_name(self):
if self.portfolio:
return self.portfolio.organization_name
return self.organization_name
return "portoflio name" #self.portfolio.organization_name
return "self name" #self.organization_name
@property
def converted_generic_org_type(self):
@ -495,7 +495,3 @@ class DomainInformation(TimeStampedModel):
return self.urbanization

View file

@ -1416,8 +1416,8 @@ class DomainRequest(TimeStampedModel):
@property
def converted_organization_name(self):
if self.portfolio:
return self.portfolio.organization_name
return self.organization_name
return "portfolio name" #self.portfolio.organization_name
return "self name" #self.organization_name
@property
def converted_generic_org_type(self):
@ -1449,6 +1449,12 @@ class DomainRequest(TimeStampedModel):
return self.portfolio.state_territory
return self.state_territory
@property
def converted_urbanization(self):
if self.portfolio:
return self.portfolio.urbanization
return self.urbanization
@property
def converted_senior_official(self):
if self.portfolio:

View file

@ -21,6 +21,11 @@ from registrar.utility.constants import BranchChoices
from registrar.utility.enums import DefaultEmail
# ---Logger
import logging
from venv import logger
from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper
logger = logging.getLogger(__name__)
@ -197,6 +202,8 @@ class BaseExport(ABC):
All domain metadata:
Exports domains of all statuses plus domain managers.
"""
TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"Exporting data")
writer = csv.writer(csv_file)
columns = cls.get_columns()
sort_fields = cls.get_sort_fields()
@ -226,6 +233,8 @@ class BaseExport(ABC):
)
models_dict = convert_queryset_to_dict(annotated_queryset, is_model=False)
TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"COLUMNS: {columns}")
# Write to csv file before the write_csv
cls.write_csv_before(writer, **export_kwargs)
@ -374,8 +383,8 @@ class DomainExport(BaseExport):
if first_ready_on is None:
first_ready_on = "(blank)"
# organization_type has generic_org_type AND is_election
domain_org_type = model.get("organization_type")
# organization_type has organization_type AND is_election
domain_org_type = model.get("converted_generic_org_type") or model.get("organization_type")
human_readable_domain_org_type = DomainRequest.OrgChoicesElectionOffice.get_org_label(domain_org_type)
domain_federal_type = model.get("federal_type")
human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type)
@ -392,6 +401,7 @@ class DomainExport(BaseExport):
):
security_contact_email = "(blank)"
# create a dictionary of fields which can be included in output.
# "extra_fields" are precomputed fields (generated in the DB or parsed).
FIELDS = {
@ -400,12 +410,12 @@ class DomainExport(BaseExport):
"First ready on": first_ready_on,
"Expiration date": expiration_date,
"Domain type": domain_type,
"Agency": model.get("federal_agency__agency"),
"Organization name": model.get("organization_name"),
"City": model.get("city"),
"State": model.get("state_territory"),
"Agency": model.get("converted_federal_agency__agency"),
"Organization name": model.get("converted_organization_name"),
"City": model.get("converted_city"),
"State": model.get("converted_state_territory"),
"SO": model.get("so_name"),
"SO email": model.get("senior_official__email"),
"SO email": model.get("converted_senior_official__email"),
"Security contact email": security_contact_email,
"Created at": model.get("domain__created_at"),
"Deleted": model.get("domain__deleted"),
@ -414,8 +424,20 @@ class DomainExport(BaseExport):
}
row = [FIELDS.get(column, "") for column in columns]
TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, f"PARSING ROW: {row}")
return row
def get_filtered_domain_infos_by_org(domain_infos_to_filter, org_to_filter_by):
"""Returns a list of Domain Requests that has been filtered by the given organization value."""
return domain_infos_to_filter.filter(
# Filter based on the generic org value returned by converted_generic_org_type
id__in=[
domainInfos.id for domainInfos in domain_infos_to_filter if domainInfos.converted_generic_org_type and domainInfos.converted_generic_org_type == org_to_filter_by
]
)
@classmethod
def get_sliced_domains(cls, filter_condition):
"""Get filtered domains counts sliced by org type and election office.
@ -423,23 +445,23 @@ class DomainExport(BaseExport):
when a domain has more that one manager.
"""
domains = DomainInformation.objects.all().filter(**filter_condition).distinct()
domains_count = domains.count()
federal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
interstate = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).count()
domain_informations = DomainInformation.objects.all().filter(**filter_condition).distinct()
domains_count = domain_informations.count()
federal = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
interstate = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.INTERSTATE).count()
state_or_territory = (
domains.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count()
cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count()
)
tribal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count()
county = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count()
city = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count()
tribal = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.TRIBAL).distinct().count()
county = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.COUNTY).distinct().count()
city = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.CITY).distinct().count()
special_district = (
domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
)
school_district = (
domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
)
election_board = domains.filter(is_election_board=True).distinct().count()
election_board = domain_informations.filter(is_election_board=True).distinct().count()
return [
domains_count,
@ -461,11 +483,15 @@ class DomainDataType(DomainExport):
Inherits from BaseExport -> DomainExport
"""
TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, f"DomainDataType!!")
@classmethod
def get_columns(cls):
"""
Overrides the columns for CSV export specific to DomainExport.
"""
TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, f"...getting columns")
return [
"Domain name",
"Status",
@ -524,7 +550,7 @@ class DomainDataType(DomainExport):
"""
Get a list of tables to pass to select_related when building queryset.
"""
return ["domain", "senior_official"]
return ["domain", "converted_senior_official"]
@classmethod
def get_prefetch_related(cls):
@ -660,11 +686,11 @@ class DomainRequestsDataType:
cls.safe_get(getattr(request, "all_alternative_domains", None)),
cls.safe_get(getattr(request, "all_other_contacts", None)),
cls.safe_get(getattr(request, "all_current_websites", None)),
cls.safe_get(getattr(request, "converted_federal_agency", None)),
cls.safe_get(getattr(request.converted_senior_official, "first_name", None)),
cls.safe_get(getattr(request.converted_senior_official, "last_name", None)),
cls.safe_get(getattr(request.converted_senior_official, "email", None)),
cls.safe_get(getattr(request.converted_senior_official, "title", None)),
cls.safe_get(getattr(request, "federal_agency", None)),
cls.safe_get(getattr(request.senior_official, "first_name", None)),
cls.safe_get(getattr(request.senior_official, "last_name", None)),
cls.safe_get(getattr(request.senior_official, "email", None)),
cls.safe_get(getattr(request.senior_official, "title", None)),
cls.safe_get(getattr(request.creator, "first_name", None)),
cls.safe_get(getattr(request.creator, "last_name", None)),
cls.safe_get(getattr(request.creator, "email", None)),
@ -1224,24 +1250,34 @@ class DomainRequestExport(BaseExport):
# Return the model class that this export handles
return DomainRequest
def get_filtered_domain_requests_by_org(domain_requests_to_filter, org_to_filter_by):
"""Returns a list of Domain Requests that has been filtered by the given organization value"""
return domain_requests_to_filter.filter(
# Filter based on the generic org value returned by converted_generic_org_type
id__in=[
domainRequest.id for domainRequest in domain_requests_to_filter if domainRequest.converted_generic_org_type and domainRequest.converted_generic_org_type == org_to_filter_by
]
)
@classmethod
def get_sliced_requests(cls, filter_condition):
"""Get filtered requests counts sliced by org type and election office."""
requests = DomainRequest.objects.all().filter(**filter_condition).distinct()
requests_count = requests.count()
federal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
interstate = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count()
federal = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
interstate = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.INTERSTATE).distinct().count()
state_or_territory = (
requests.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count()
cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count()
)
tribal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count()
county = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count()
city = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count()
tribal = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.TRIBAL).distinct().count()
county = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.COUNTY).distinct().count()
city = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.CITY).distinct().count()
special_district = (
requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
)
school_district = (
requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
)
election_board = requests.filter(is_election_board=True).distinct().count()
@ -1269,7 +1305,7 @@ class DomainRequestExport(BaseExport):
human_readable_federal_type = BranchChoices.get_branch_label(federal_type) if federal_type else None
# Handle the org_type field
org_type = model.get("generic_org_type") or model.get("organization_type")
org_type = model.get("converted_generic_org_type") or model.get("organization_type")
human_readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type) if org_type else None
# Handle the status field. Defaults to the wrong format.
@ -1327,9 +1363,9 @@ class DomainRequestExport(BaseExport):
"Creator email": model.get("creator__email"),
"Investigator": model.get("investigator__email"),
# Untouched fields
"Organization name": model.get("organization_name"),
"City": model.get("city"),
"State/territory": model.get("state_territory"),
"Organization name": model.get("converted_organization_name"),
"City": model.get("converted_city"),
"State/territory": model.get("converted_state_territory"),
"Request purpose": model.get("purpose"),
"CISA regional representative": model.get("cisa_representative_email"),
"Last submitted date": model.get("last_submitted_date"),

View file

@ -13,8 +13,12 @@ from registrar.utility import csv_export
import logging
logger = logging.getLogger(__name__)
# ---Logger
import logging
from venv import logger
from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper
logger = logging.getLogger(__name__)
class AnalyticsView(View):
def get(self, request):
@ -161,7 +165,10 @@ class ExportDataType(View):
class ExportDataTypeUser(View):
"""Returns a domain report for a given user on the request"""
TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"ExportDataTypeUser")
def get(self, request, *args, **kwargs):
TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"ExportDataTypeUser -- get")
# match the CSV example with all the fields
response = HttpResponse(content_type="text/csv")
response["Content-Disposition"] = 'attachment; filename="your-domains.csv"'