diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 08461bcdd..957a201aa 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1,10 +1,11 @@ +import csv from datetime import date import logging import copy from django import forms from django.db.models import Value, CharField, Q from django.db.models.functions import Concat, Coalesce -from django.http import HttpResponseRedirect +from django.http import HttpResponse, HttpResponseRedirect from registrar.models.federal_agency import FederalAgency from registrar.utility.admin_helpers import ( get_action_needed_reason_default_email, @@ -42,7 +43,7 @@ from django.utils.html import escape from django.contrib.auth.forms import UserChangeForm, UsernameField from django.contrib.admin.views.main import IGNORED_PARAMS from django_admin_multiple_choice_list_filter.list_filters import MultipleChoiceListFilter -from import_export import resources +from import_export import resources, fields from import_export.admin import ImportExportModelAdmin from django.core.exceptions import ObjectDoesNotExist from django.contrib.admin.widgets import FilteredSelectMultiple @@ -1453,6 +1454,57 @@ class DomainInformationResource(resources.ModelResource): class Meta: model = models.DomainInformation + # Override exports for these columns in DomainInformation to use converted values. These values + # come from @Property functions, which are not automatically included in the export and which we + # want to use in place of the native fields. + organization_name = fields.Field(attribute='converted_organization_name', column_name='organization_name') + generic_org_type = fields.Field(attribute='converted_generic_org_type', column_name='generic_org_type') + federal_type = fields.Field(attribute='converted_federal_type', column_name='federal_type') + federal_agency = fields.Field(attribute='converted_federal_agency', column_name='federal_agency') + senior_official = fields.Field(attribute='converted_senior_official', column_name='senior_official') + address_line1 = fields.Field(attribute='converted_address_line1', column_name='address_line1') + address_line2 = fields.Field(attribute='converted_address_line2', column_name='address_line2') + city = fields.Field(attribute='converted_city', column_name='city') + state_territory = fields.Field(attribute='converted_state_territory', column_name='state_territory') + zipcode = fields.Field(attribute='converted_zipcode', column_name='zipcode') + urbanization = fields.Field(attribute='converted_urbanization', column_name='urbanization') + + # Custom getters for the above columns that map to @property functions instead of fields + def dehydrate_organization_name(self, obj): + return obj.converted_organization_name + + def dehydrate_generic_org_type(self, obj): + return obj.converted_generic_org_type + + def dehydrate_federal_type(self, obj): + return obj.converted_federal_type + + def dehydrate_federal_agency(self, obj): + return obj.converted_federal_agency + + def dehydrate_senior_official(self, obj): + return obj.converted_senior_official + + def dehydrate_address_line1(self, obj): + return obj.converted_address_line1 + + def dehydrate_address_line2(self, obj): + return obj.converted_address_line2 + + def dehydrate_city(self, obj): + return obj.converted_city + + def dehydrate_state_territory(self, obj): + return obj.converted_state_territory + + def dehydrate_zipcode(self, obj): + return obj.converted_zipcode + + def dehydrate_urbanization(self, obj): + return obj.converted_urbanization + + + class DomainInformationAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Customize domain information admin class.""" @@ -1654,6 +1706,56 @@ class DomainRequestResource(FsmModelResource): class Meta: model = models.DomainRequest + # Override exports for these columns in DomainInformation to use converted values. These values + # come from @Property functions, which are not automatically included in the export and which we + # want to use in place of the native fields. + organization_name = fields.Field(attribute='converted_organization_name', column_name='GEN organization_name') + generic_org_type = fields.Field(attribute='converted_generic_org_type', column_name='GEN generic_org_type') + federal_type = fields.Field(attribute='converted_federal_type', column_name='GEN federal_type') + federal_agency = fields.Field(attribute='converted_federal_agency', column_name='GEN federal_agency') + senior_official = fields.Field(attribute='converted_senior_official', column_name='GEN senior_official') + address_line1 = fields.Field(attribute='converted_address_line1', column_name='GEN address_line1') + address_line2 = fields.Field(attribute='converted_address_line2', column_name='GEN address_line2') + city = fields.Field(attribute='converted_city', column_name='GEN city') + state_territory = fields.Field(attribute='converted_state_territory', column_name='GEN state_territory') + zipcode = fields.Field(attribute='converted_zipcode', column_name='GEN zipcode') + urbanization = fields.Field(attribute='converted_urbanization', column_name='GEN urbanization') + senior_official = fields.Field(attribute='converted_urbanization', column_name='GEN senior official') + + # Custom getters for the above columns that map to @property functions instead of fields + def dehydrate_organization_name(self, obj): + return obj.converted_organization_name + + def dehydrate_generic_org_type(self, obj): + return obj.converted_generic_org_type + + def dehydrate_federal_type(self, obj): + return obj.converted_federal_type + + def dehydrate_federal_agency(self, obj): + return obj.converted_federal_agency + + def dehydrate_senior_official(self, obj): + return obj.converted_senior_official + + def dehydrate_address_line1(self, obj): + return obj.converted_address_line1 + + def dehydrate_address_line2(self, obj): + return obj.converted_address_line2 + + def dehydrate_city(self, obj): + return obj.converted_city + + def dehydrate_state_territory(self, obj): + return obj.converted_state_territory + + def dehydrate_zipcode(self, obj): + return obj.converted_zipcode + + def dehydrate_urbanization(self, obj): + return obj.converted_urbanization + class DomainRequestAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom domain requests admin class.""" @@ -2577,7 +2679,48 @@ class DomainResource(FsmModelResource): class Meta: model = models.Domain + #Override the default export so that it matches what is displayed in the admin table for Domains + fields = ( + "name", + "converted_generic_org_type", + "federal_type", + "converted_federal_type", + "converted_federal_agency", + "converted_organization_name", + "custom_election_board", + "converted_city", + "converted_state_territory", + "state", + "expiration_date", + "created_at", + "first_ready", + "deleted", + ) + # Custom getters to retrieve the values from @Proprerty methods in DomainInfo + converted_generic_org_type = fields.Field(attribute='converted_generic_org_type', column_name='Converted generic org type') + converted_federal_agency = fields.Field(attribute='converted_federal_agency', column_name='Converted federal agency') + converted_organization_name = fields.Field(attribute='converted_organization_name', column_name='Converted organization name') + converted_city = fields.Field(attribute='converted_city', column_name='city') + converted_state_territory = fields.Field(attribute='converted_state_territory', column_name='Converted state territory') + + # def dehydrate_generic_org_type(self, obj): + # return obj.domain_info.converted_federal_type + + def dehydrate_converted_generic_org_type(self, obj): + return obj.domain_info.converted_generic_org_type + + def dehydrate_converted_federal_agency(self, obj): + return obj.domain_info.converted_federal_agency + + def dehydrate_converted_organization_name(self, obj): + return obj.domain_info.converted_organization_name + + def dehydrate_converted_city(self, obj): + return obj.domain_info.converted_city + + def dehydrate_converted_state_territory(self, obj): + return obj.domain_info.converted_state_territory class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom domain admin class to add extra buttons.""" @@ -3073,8 +3216,6 @@ class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin): return True return super().has_change_permission(request, obj) - - class DraftDomainResource(resources.ModelResource): """defines how each field in the referenced model should be mapped to the corresponding fields in the diff --git a/src/registrar/config/urls.py b/src/registrar/config/urls.py index d289eaf90..612dcbf77 100644 --- a/src/registrar/config/urls.py +++ b/src/registrar/config/urls.py @@ -226,11 +226,6 @@ urlpatterns = [ ExportDataTypeRequests.as_view(), name="export_data_type_requests", ), - path( - "reports/export_data_type_requests/", - ExportDataTypeRequests.as_view(), - name="export_data_type_requests", - ), path( "domain-request//edit/", views.DomainRequestWizard.as_view(), diff --git a/src/registrar/models/domain_information.py b/src/registrar/models/domain_information.py index 525d7998e..7595eb4f0 100644 --- a/src/registrar/models/domain_information.py +++ b/src/registrar/models/domain_information.py @@ -431,8 +431,8 @@ class DomainInformation(TimeStampedModel): @property def converted_organization_name(self): if self.portfolio: - return self.portfolio.organization_name - return self.organization_name + return "portoflio name" #self.portfolio.organization_name + return "self name" #self.organization_name @property def converted_generic_org_type(self): @@ -495,7 +495,3 @@ class DomainInformation(TimeStampedModel): return self.urbanization - - - - diff --git a/src/registrar/models/domain_request.py b/src/registrar/models/domain_request.py index 0d8bbd5cf..c62a939a3 100644 --- a/src/registrar/models/domain_request.py +++ b/src/registrar/models/domain_request.py @@ -1416,8 +1416,8 @@ class DomainRequest(TimeStampedModel): @property def converted_organization_name(self): if self.portfolio: - return self.portfolio.organization_name - return self.organization_name + return "portfolio name" #self.portfolio.organization_name + return "self name" #self.organization_name @property def converted_generic_org_type(self): @@ -1448,9 +1448,15 @@ class DomainRequest(TimeStampedModel): if self.portfolio: return self.portfolio.state_territory return self.state_territory + + @property + def converted_urbanization(self): + if self.portfolio: + return self.portfolio.urbanization + return self.urbanization @property def converted_senior_official(self): if self.portfolio: return self.portfolio.senior_official - return self.senior_official + return self.senior_official \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 64d960337..0badcc7ea 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -21,6 +21,11 @@ from registrar.utility.constants import BranchChoices from registrar.utility.enums import DefaultEmail + +# ---Logger +import logging +from venv import logger +from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper logger = logging.getLogger(__name__) @@ -197,6 +202,8 @@ class BaseExport(ABC): All domain metadata: Exports domains of all statuses plus domain managers. """ + TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"Exporting data") + writer = csv.writer(csv_file) columns = cls.get_columns() sort_fields = cls.get_sort_fields() @@ -226,6 +233,8 @@ class BaseExport(ABC): ) models_dict = convert_queryset_to_dict(annotated_queryset, is_model=False) + TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"COLUMNS: {columns}") + # Write to csv file before the write_csv cls.write_csv_before(writer, **export_kwargs) @@ -374,8 +383,8 @@ class DomainExport(BaseExport): if first_ready_on is None: first_ready_on = "(blank)" - # organization_type has generic_org_type AND is_election - domain_org_type = model.get("organization_type") + # organization_type has organization_type AND is_election + domain_org_type = model.get("converted_generic_org_type") or model.get("organization_type") human_readable_domain_org_type = DomainRequest.OrgChoicesElectionOffice.get_org_label(domain_org_type) domain_federal_type = model.get("federal_type") human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) @@ -392,6 +401,7 @@ class DomainExport(BaseExport): ): security_contact_email = "(blank)" + # create a dictionary of fields which can be included in output. # "extra_fields" are precomputed fields (generated in the DB or parsed). FIELDS = { @@ -400,12 +410,12 @@ class DomainExport(BaseExport): "First ready on": first_ready_on, "Expiration date": expiration_date, "Domain type": domain_type, - "Agency": model.get("federal_agency__agency"), - "Organization name": model.get("organization_name"), - "City": model.get("city"), - "State": model.get("state_territory"), + "Agency": model.get("converted_federal_agency__agency"), + "Organization name": model.get("converted_organization_name"), + "City": model.get("converted_city"), + "State": model.get("converted_state_territory"), "SO": model.get("so_name"), - "SO email": model.get("senior_official__email"), + "SO email": model.get("converted_senior_official__email"), "Security contact email": security_contact_email, "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), @@ -414,8 +424,20 @@ class DomainExport(BaseExport): } row = [FIELDS.get(column, "") for column in columns] + + TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, f"PARSING ROW: {row}") + return row + def get_filtered_domain_infos_by_org(domain_infos_to_filter, org_to_filter_by): + """Returns a list of Domain Requests that has been filtered by the given organization value.""" + return domain_infos_to_filter.filter( + # Filter based on the generic org value returned by converted_generic_org_type + id__in=[ + domainInfos.id for domainInfos in domain_infos_to_filter if domainInfos.converted_generic_org_type and domainInfos.converted_generic_org_type == org_to_filter_by + ] + ) + @classmethod def get_sliced_domains(cls, filter_condition): """Get filtered domains counts sliced by org type and election office. @@ -423,23 +445,23 @@ class DomainExport(BaseExport): when a domain has more that one manager. """ - domains = DomainInformation.objects.all().filter(**filter_condition).distinct() - domains_count = domains.count() - federal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).count() + domain_informations = DomainInformation.objects.all().filter(**filter_condition).distinct() + domains_count = domain_informations.count() + federal = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.FEDERAL).distinct().count() + interstate = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.INTERSTATE).count() state_or_territory = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() ) - tribal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() + tribal = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.TRIBAL).distinct().count() + county = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.COUNTY).distinct().count() + city = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.CITY).distinct().count() special_district = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() ) school_district = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() ) - election_board = domains.filter(is_election_board=True).distinct().count() + election_board = domain_informations.filter(is_election_board=True).distinct().count() return [ domains_count, @@ -461,11 +483,15 @@ class DomainDataType(DomainExport): Inherits from BaseExport -> DomainExport """ + TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, f"DomainDataType!!") + @classmethod def get_columns(cls): """ Overrides the columns for CSV export specific to DomainExport. """ + + TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, f"...getting columns") return [ "Domain name", "Status", @@ -524,7 +550,7 @@ class DomainDataType(DomainExport): """ Get a list of tables to pass to select_related when building queryset. """ - return ["domain", "senior_official"] + return ["domain", "converted_senior_official"] @classmethod def get_prefetch_related(cls): @@ -660,11 +686,11 @@ class DomainRequestsDataType: cls.safe_get(getattr(request, "all_alternative_domains", None)), cls.safe_get(getattr(request, "all_other_contacts", None)), cls.safe_get(getattr(request, "all_current_websites", None)), - cls.safe_get(getattr(request, "converted_federal_agency", None)), - cls.safe_get(getattr(request.converted_senior_official, "first_name", None)), - cls.safe_get(getattr(request.converted_senior_official, "last_name", None)), - cls.safe_get(getattr(request.converted_senior_official, "email", None)), - cls.safe_get(getattr(request.converted_senior_official, "title", None)), + cls.safe_get(getattr(request, "federal_agency", None)), + cls.safe_get(getattr(request.senior_official, "first_name", None)), + cls.safe_get(getattr(request.senior_official, "last_name", None)), + cls.safe_get(getattr(request.senior_official, "email", None)), + cls.safe_get(getattr(request.senior_official, "title", None)), cls.safe_get(getattr(request.creator, "first_name", None)), cls.safe_get(getattr(request.creator, "last_name", None)), cls.safe_get(getattr(request.creator, "email", None)), @@ -1223,25 +1249,35 @@ class DomainRequestExport(BaseExport): def model(cls): # Return the model class that this export handles return DomainRequest + + def get_filtered_domain_requests_by_org(domain_requests_to_filter, org_to_filter_by): + """Returns a list of Domain Requests that has been filtered by the given organization value""" + return domain_requests_to_filter.filter( + # Filter based on the generic org value returned by converted_generic_org_type + id__in=[ + domainRequest.id for domainRequest in domain_requests_to_filter if domainRequest.converted_generic_org_type and domainRequest.converted_generic_org_type == org_to_filter_by + ] + ) + @classmethod def get_sliced_requests(cls, filter_condition): """Get filtered requests counts sliced by org type and election office.""" requests = DomainRequest.objects.all().filter(**filter_condition).distinct() requests_count = requests.count() - federal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() + federal = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.FEDERAL).distinct().count() + interstate = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() state_or_territory = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() ) - tribal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() + tribal = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.TRIBAL).distinct().count() + county = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.COUNTY).distinct().count() + city = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.CITY).distinct().count() special_district = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() ) school_district = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() ) election_board = requests.filter(is_election_board=True).distinct().count() @@ -1269,7 +1305,7 @@ class DomainRequestExport(BaseExport): human_readable_federal_type = BranchChoices.get_branch_label(federal_type) if federal_type else None # Handle the org_type field - org_type = model.get("generic_org_type") or model.get("organization_type") + org_type = model.get("converted_generic_org_type") or model.get("organization_type") human_readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type) if org_type else None # Handle the status field. Defaults to the wrong format. @@ -1327,9 +1363,9 @@ class DomainRequestExport(BaseExport): "Creator email": model.get("creator__email"), "Investigator": model.get("investigator__email"), # Untouched fields - "Organization name": model.get("organization_name"), - "City": model.get("city"), - "State/territory": model.get("state_territory"), + "Organization name": model.get("converted_organization_name"), + "City": model.get("converted_city"), + "State/territory": model.get("converted_state_territory"), "Request purpose": model.get("purpose"), "CISA regional representative": model.get("cisa_representative_email"), "Last submitted date": model.get("last_submitted_date"), diff --git a/src/registrar/views/report_views.py b/src/registrar/views/report_views.py index d9c4d192c..d34d66daa 100644 --- a/src/registrar/views/report_views.py +++ b/src/registrar/views/report_views.py @@ -13,8 +13,12 @@ from registrar.utility import csv_export import logging -logger = logging.getLogger(__name__) +# ---Logger +import logging +from venv import logger +from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper +logger = logging.getLogger(__name__) class AnalyticsView(View): def get(self, request): @@ -161,7 +165,10 @@ class ExportDataType(View): class ExportDataTypeUser(View): """Returns a domain report for a given user on the request""" + TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"ExportDataTypeUser") + def get(self, request, *args, **kwargs): + TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"ExportDataTypeUser -- get") # match the CSV example with all the fields response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="your-domains.csv"'