export updates

src/registrar/admin.py

@@ -1,10 +1,11 @@
+import csv
 from datetime import date
 import logging
 import copy
 from django import forms
 from django.db.models import Value, CharField, Q
 from django.db.models.functions import Concat, Coalesce
-from django.http import HttpResponseRedirect
+from django.http import HttpResponse, HttpResponseRedirect
 from registrar.models.federal_agency import FederalAgency
 from registrar.utility.admin_helpers import (
     get_action_needed_reason_default_email,
@@ -42,7 +43,7 @@ from django.utils.html import escape
 from django.contrib.auth.forms import UserChangeForm, UsernameField
 from django.contrib.admin.views.main import IGNORED_PARAMS
 from django_admin_multiple_choice_list_filter.list_filters import MultipleChoiceListFilter
-from import_export import resources
+from import_export import resources, fields
 from import_export.admin import ImportExportModelAdmin
 from django.core.exceptions import ObjectDoesNotExist
 from django.contrib.admin.widgets import FilteredSelectMultiple
@@ -1453,6 +1454,57 @@ class DomainInformationResource(resources.ModelResource):
     class Meta:
         model = models.DomainInformation
 
+    # Override exports for these columns in DomainInformation to use converted values.  These values
+    # come from @Property functions, which are not automatically included in the export and which we
+    # want to use in place of the native fields.
+    organization_name = fields.Field(attribute='converted_organization_name', column_name='organization_name')
+    generic_org_type = fields.Field(attribute='converted_generic_org_type', column_name='generic_org_type')
+    federal_type = fields.Field(attribute='converted_federal_type', column_name='federal_type')
+    federal_agency = fields.Field(attribute='converted_federal_agency', column_name='federal_agency')
+    senior_official = fields.Field(attribute='converted_senior_official', column_name='senior_official')
+    address_line1 = fields.Field(attribute='converted_address_line1', column_name='address_line1')
+    address_line2 = fields.Field(attribute='converted_address_line2', column_name='address_line2')
+    city = fields.Field(attribute='converted_city', column_name='city')
+    state_territory = fields.Field(attribute='converted_state_territory', column_name='state_territory')
+    zipcode = fields.Field(attribute='converted_zipcode', column_name='zipcode')
+    urbanization = fields.Field(attribute='converted_urbanization', column_name='urbanization')
+
+    # Custom getters for the above columns that map to @property functions instead of fields
+    def dehydrate_organization_name(self, obj):
+        return obj.converted_organization_name
+    
+    def dehydrate_generic_org_type(self, obj):
+        return obj.converted_generic_org_type
+    
+    def dehydrate_federal_type(self, obj):
+        return obj.converted_federal_type
+    
+    def dehydrate_federal_agency(self, obj):
+        return obj.converted_federal_agency
+    
+    def dehydrate_senior_official(self, obj):
+        return obj.converted_senior_official
+    
+    def dehydrate_address_line1(self, obj):
+        return obj.converted_address_line1
+    
+    def dehydrate_address_line2(self, obj):
+        return obj.converted_address_line2
+    
+    def dehydrate_city(self, obj):
+        return obj.converted_city
+    
+    def dehydrate_state_territory(self, obj):
+        return obj.converted_state_territory
+    
+    def dehydrate_zipcode(self, obj):
+        return obj.converted_zipcode
+    
+    def dehydrate_urbanization(self, obj):
+        return obj.converted_urbanization
+    
+
+
 
 class DomainInformationAdmin(ListHeaderAdmin, ImportExportModelAdmin):
     """Customize domain information admin class."""
@@ -1654,6 +1706,56 @@ class DomainRequestResource(FsmModelResource):
     class Meta:
         model = models.DomainRequest
 
+    # Override exports for these columns in DomainInformation to use converted values.  These values
+    # come from @Property functions, which are not automatically included in the export and which we
+    # want to use in place of the native fields.
+    organization_name = fields.Field(attribute='converted_organization_name', column_name='GEN organization_name')
+    generic_org_type = fields.Field(attribute='converted_generic_org_type', column_name='GEN generic_org_type')
+    federal_type = fields.Field(attribute='converted_federal_type', column_name='GEN federal_type')
+    federal_agency = fields.Field(attribute='converted_federal_agency', column_name='GEN federal_agency')
+    senior_official = fields.Field(attribute='converted_senior_official', column_name='GEN senior_official')
+    address_line1 = fields.Field(attribute='converted_address_line1', column_name='GEN address_line1')
+    address_line2 = fields.Field(attribute='converted_address_line2', column_name='GEN address_line2')
+    city = fields.Field(attribute='converted_city', column_name='GEN city')
+    state_territory = fields.Field(attribute='converted_state_territory', column_name='GEN state_territory')
+    zipcode = fields.Field(attribute='converted_zipcode', column_name='GEN zipcode')
+    urbanization = fields.Field(attribute='converted_urbanization', column_name='GEN urbanization')
+    senior_official = fields.Field(attribute='converted_urbanization', column_name='GEN senior official')
+
+    # Custom getters for the above columns that map to @property functions instead of fields
+    def dehydrate_organization_name(self, obj):
+        return obj.converted_organization_name
+    
+    def dehydrate_generic_org_type(self, obj):
+        return obj.converted_generic_org_type
+    
+    def dehydrate_federal_type(self, obj):
+        return obj.converted_federal_type
+    
+    def dehydrate_federal_agency(self, obj):
+        return obj.converted_federal_agency
+    
+    def dehydrate_senior_official(self, obj):
+        return obj.converted_senior_official
+    
+    def dehydrate_address_line1(self, obj):
+        return obj.converted_address_line1
+    
+    def dehydrate_address_line2(self, obj):
+        return obj.converted_address_line2
+    
+    def dehydrate_city(self, obj):
+        return obj.converted_city
+    
+    def dehydrate_state_territory(self, obj):
+        return obj.converted_state_territory
+    
+    def dehydrate_zipcode(self, obj):
+        return obj.converted_zipcode
+    
+    def dehydrate_urbanization(self, obj):
+        return obj.converted_urbanization
+
 
 class DomainRequestAdmin(ListHeaderAdmin, ImportExportModelAdmin):
     """Custom domain requests admin class."""
@@ -2577,7 +2679,48 @@ class DomainResource(FsmModelResource):
 
     class Meta:
         model = models.Domain
+        #Override the default export so that it matches what is displayed in the admin table for Domains
+        fields = (
+            "name",
+            "converted_generic_org_type",
+            "federal_type",
+            "converted_federal_type",
+            "converted_federal_agency",
+            "converted_organization_name",
+            "custom_election_board",
+            "converted_city",
+            "converted_state_territory",
+            "state",
+            "expiration_date",
+            "created_at",
+            "first_ready",
+            "deleted",
+        )
 
+    # Custom getters to retrieve the values from @Proprerty methods in DomainInfo
+    converted_generic_org_type = fields.Field(attribute='converted_generic_org_type', column_name='Converted generic org type')
+    converted_federal_agency = fields.Field(attribute='converted_federal_agency', column_name='Converted federal agency')
+    converted_organization_name = fields.Field(attribute='converted_organization_name', column_name='Converted organization name')
+    converted_city = fields.Field(attribute='converted_city', column_name='city')
+    converted_state_territory = fields.Field(attribute='converted_state_territory', column_name='Converted state territory')
+    
+    # def dehydrate_generic_org_type(self, obj):
+    #     return obj.domain_info.converted_federal_type
+    
+    def dehydrate_converted_generic_org_type(self, obj):
+        return obj.domain_info.converted_generic_org_type
+    
+    def dehydrate_converted_federal_agency(self, obj):
+        return obj.domain_info.converted_federal_agency
+    
+    def dehydrate_converted_organization_name(self, obj):
+        return obj.domain_info.converted_organization_name
+    
+    def dehydrate_converted_city(self, obj):
+        return obj.domain_info.converted_city
+    
+    def dehydrate_converted_state_territory(self, obj):
+        return obj.domain_info.converted_state_territory
 
 class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin):
     """Custom domain admin class to add extra buttons."""
@@ -3074,8 +3217,6 @@ class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin):
         return super().has_change_permission(request, obj)
 
 
-
-
 class DraftDomainResource(resources.ModelResource):
     """defines how each field in the referenced model should be mapped to the corresponding fields in the
     import/export file"""

src/registrar/config/urls.py

@@ -226,11 +226,6 @@ urlpatterns = [
         ExportDataTypeRequests.as_view(),
         name="export_data_type_requests",
     ),
-    path(
-        "reports/export_data_type_requests/",
-        ExportDataTypeRequests.as_view(),
-        name="export_data_type_requests",
-    ),
     path(
         "domain-request/<int:id>/edit/",
         views.DomainRequestWizard.as_view(),

src/registrar/models/domain_information.py

@@ -431,8 +431,8 @@ class DomainInformation(TimeStampedModel):
     @property
     def converted_organization_name(self):
         if self.portfolio:
-            return self.portfolio.organization_name
+            return "portoflio name" #self.portfolio.organization_name
-        return self.organization_name
+        return "self name" #self.organization_name
 
     @property
     def converted_generic_org_type(self):
@@ -495,7 +495,3 @@ class DomainInformation(TimeStampedModel):
         return self.urbanization
 
 
-
-
-
-

src/registrar/models/domain_request.py

@@ -1416,8 +1416,8 @@ class DomainRequest(TimeStampedModel):
     @property
     def converted_organization_name(self):
         if self.portfolio:
-            return self.portfolio.organization_name
+            return "portfolio name" #self.portfolio.organization_name
-        return self.organization_name
+        return "self name" #self.organization_name
 
     @property
     def converted_generic_org_type(self):
@@ -1449,6 +1449,12 @@ class DomainRequest(TimeStampedModel):
             return self.portfolio.state_territory
         return self.state_territory
     
+    @property
+    def converted_urbanization(self):
+        if self.portfolio:
+            return self.portfolio.urbanization
+        return self.urbanization
+
     @property
     def converted_senior_official(self):
         if self.portfolio:

src/registrar/utility/csv_export.py

@@ -21,6 +21,11 @@ from registrar.utility.constants import BranchChoices
 from registrar.utility.enums import DefaultEmail
 
 
+
+# ---Logger
+import logging
+from venv import logger
+from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper
 logger = logging.getLogger(__name__)
 
 
@@ -197,6 +202,8 @@ class BaseExport(ABC):
         All domain metadata:
         Exports domains of all statuses plus domain managers.
         """
+        TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"Exporting data")
+
         writer = csv.writer(csv_file)
         columns = cls.get_columns()
         sort_fields = cls.get_sort_fields()
@@ -226,6 +233,8 @@ class BaseExport(ABC):
         )
         models_dict = convert_queryset_to_dict(annotated_queryset, is_model=False)
 
+        TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"COLUMNS: {columns}")
+
         # Write to csv file before the write_csv
         cls.write_csv_before(writer, **export_kwargs)
 
@@ -374,8 +383,8 @@ class DomainExport(BaseExport):
         if first_ready_on is None:
             first_ready_on = "(blank)"
 
-        # organization_type has generic_org_type AND is_election
+        # organization_type has organization_type AND is_election
-        domain_org_type = model.get("organization_type")
+        domain_org_type = model.get("converted_generic_org_type") or model.get("organization_type")
         human_readable_domain_org_type = DomainRequest.OrgChoicesElectionOffice.get_org_label(domain_org_type)
         domain_federal_type = model.get("federal_type")
         human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type)
@@ -392,6 +401,7 @@ class DomainExport(BaseExport):
         ):
             security_contact_email = "(blank)"
 
+
         # create a dictionary of fields which can be included in output.
         # "extra_fields" are precomputed fields (generated in the DB or parsed).
         FIELDS = {
@@ -400,12 +410,12 @@ class DomainExport(BaseExport):
             "First ready on": first_ready_on,
             "Expiration date": expiration_date,
             "Domain type": domain_type,
-            "Agency": model.get("federal_agency__agency"),
+            "Agency": model.get("converted_federal_agency__agency"),
-            "Organization name": model.get("organization_name"),
+            "Organization name": model.get("converted_organization_name"),
-            "City": model.get("city"),
+            "City": model.get("converted_city"),
-            "State": model.get("state_territory"),
+            "State": model.get("converted_state_territory"),
             "SO": model.get("so_name"),
-            "SO email": model.get("senior_official__email"),
+            "SO email": model.get("converted_senior_official__email"),
             "Security contact email": security_contact_email,
             "Created at": model.get("domain__created_at"),
             "Deleted": model.get("domain__deleted"),
@@ -414,8 +424,20 @@ class DomainExport(BaseExport):
         }
 
         row = [FIELDS.get(column, "") for column in columns]
+        
+        TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, f"PARSING ROW:  {row}")     
+
         return row
 
+    def get_filtered_domain_infos_by_org(domain_infos_to_filter, org_to_filter_by):
+        """Returns a list of Domain Requests that has been filtered by the given organization value."""
+        return domain_infos_to_filter.filter(
+                    # Filter based on the generic org value returned by converted_generic_org_type
+                    id__in=[
+                        domainInfos.id for domainInfos in domain_infos_to_filter if domainInfos.converted_generic_org_type and domainInfos.converted_generic_org_type == org_to_filter_by
+                    ]
+                )
+    
     @classmethod
     def get_sliced_domains(cls, filter_condition):
         """Get filtered domains counts sliced by org type and election office.
@@ -423,23 +445,23 @@ class DomainExport(BaseExport):
         when a domain has more that one manager.
         """
 
-        domains = DomainInformation.objects.all().filter(**filter_condition).distinct()
+        domain_informations = DomainInformation.objects.all().filter(**filter_condition).distinct()
-        domains_count = domains.count()
+        domains_count = domain_informations.count()
-        federal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
+        federal = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
-        interstate = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).count()
+        interstate = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.INTERSTATE).count()
         state_or_territory = (
-            domains.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count()
+            cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count()
         )
-        tribal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count()
+        tribal = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.TRIBAL).distinct().count()
-        county = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count()
+        county = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.COUNTY).distinct().count()
-        city = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count()
+        city = cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.CITY).distinct().count()
         special_district = (
-            domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
+            cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
         )
         school_district = (
-            domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
+            cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
         )
-        election_board = domains.filter(is_election_board=True).distinct().count()
+        election_board = domain_informations.filter(is_election_board=True).distinct().count()
 
         return [
             domains_count,
@@ -461,11 +483,15 @@ class DomainDataType(DomainExport):
     Inherits from BaseExport -> DomainExport
     """
 
+    TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, f"DomainDataType!!") 
+
     @classmethod
     def get_columns(cls):
         """
         Overrides the columns for CSV export specific to DomainExport.
         """
+
+        TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, f"...getting columns") 
         return [
             "Domain name",
             "Status",
@@ -524,7 +550,7 @@ class DomainDataType(DomainExport):
         """
         Get a list of tables to pass to select_related when building queryset.
         """
-        return ["domain", "senior_official"]
+        return ["domain", "converted_senior_official"]
 
     @classmethod
     def get_prefetch_related(cls):
@@ -660,11 +686,11 @@ class DomainRequestsDataType:
                     cls.safe_get(getattr(request, "all_alternative_domains", None)),
                     cls.safe_get(getattr(request, "all_other_contacts", None)),
                     cls.safe_get(getattr(request, "all_current_websites", None)),
-                    cls.safe_get(getattr(request, "converted_federal_agency", None)),
+                    cls.safe_get(getattr(request, "federal_agency", None)),
-                    cls.safe_get(getattr(request.converted_senior_official, "first_name", None)),
+                    cls.safe_get(getattr(request.senior_official, "first_name", None)),
-                    cls.safe_get(getattr(request.converted_senior_official, "last_name", None)),
+                    cls.safe_get(getattr(request.senior_official, "last_name", None)),
-                    cls.safe_get(getattr(request.converted_senior_official, "email", None)),
+                    cls.safe_get(getattr(request.senior_official, "email", None)),
-                    cls.safe_get(getattr(request.converted_senior_official, "title", None)),
+                    cls.safe_get(getattr(request.senior_official, "title", None)),
                     cls.safe_get(getattr(request.creator, "first_name", None)),
                     cls.safe_get(getattr(request.creator, "last_name", None)),
                     cls.safe_get(getattr(request.creator, "email", None)),
@@ -1224,24 +1250,34 @@ class DomainRequestExport(BaseExport):
         # Return the model class that this export handles
         return DomainRequest
     
+    def get_filtered_domain_requests_by_org(domain_requests_to_filter, org_to_filter_by):
+        """Returns a list of Domain Requests that has been filtered by the given organization value"""
+        return domain_requests_to_filter.filter(
+                    # Filter based on the generic org value returned by converted_generic_org_type
+                    id__in=[
+                        domainRequest.id for domainRequest in domain_requests_to_filter if domainRequest.converted_generic_org_type and domainRequest.converted_generic_org_type == org_to_filter_by
+                    ]
+                )
+
+    
     @classmethod
     def get_sliced_requests(cls, filter_condition):
         """Get filtered requests counts sliced by org type and election office."""
         requests = DomainRequest.objects.all().filter(**filter_condition).distinct()
         requests_count = requests.count()
-        federal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
+        federal = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
-        interstate = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count()
+        interstate = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.INTERSTATE).distinct().count()
         state_or_territory = (
-            requests.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count()
+            cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count()
         )
-        tribal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count()
+        tribal = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.TRIBAL).distinct().count()
-        county = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count()
+        county = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.COUNTY).distinct().count()
-        city = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count()
+        city = cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.CITY).distinct().count()
         special_district = (
-            requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
+            cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
         )
         school_district = (
-            requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
+            cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
         )
         election_board = requests.filter(is_election_board=True).distinct().count()
 
@@ -1269,7 +1305,7 @@ class DomainRequestExport(BaseExport):
         human_readable_federal_type = BranchChoices.get_branch_label(federal_type) if federal_type else None
 
         # Handle the org_type field
-        org_type = model.get("generic_org_type") or model.get("organization_type")
+        org_type = model.get("converted_generic_org_type") or model.get("organization_type")
         human_readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type) if org_type else None
 
         # Handle the status field. Defaults to the wrong format.
@@ -1327,9 +1363,9 @@ class DomainRequestExport(BaseExport):
             "Creator email": model.get("creator__email"),
             "Investigator": model.get("investigator__email"),
             # Untouched fields
-            "Organization name": model.get("organization_name"),
+            "Organization name": model.get("converted_organization_name"),
-            "City": model.get("city"),
+            "City": model.get("converted_city"),
-            "State/territory": model.get("state_territory"),
+            "State/territory": model.get("converted_state_territory"),
             "Request purpose": model.get("purpose"),
             "CISA regional representative": model.get("cisa_representative_email"),
             "Last submitted date": model.get("last_submitted_date"),

src/registrar/views/report_views.py

@@ -13,8 +13,12 @@ from registrar.utility import csv_export
 
 import logging
 
-logger = logging.getLogger(__name__)
 
+# ---Logger
+import logging
+from venv import logger
+from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper
+logger = logging.getLogger(__name__)
 
 class AnalyticsView(View):
     def get(self, request):
@@ -161,7 +165,10 @@ class ExportDataType(View):
 class ExportDataTypeUser(View):
     """Returns a domain report for a given user on the request"""
 
+    TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"ExportDataTypeUser")
+
     def get(self, request, *args, **kwargs):
+        TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, f"ExportDataTypeUser -- get")
         # match the CSV example with all the fields
         response = HttpResponse(content_type="text/csv")
         response["Content-Disposition"] = 'attachment; filename="your-domains.csv"'