From 24a18f1562e50d2210a72ce6916aa41d7e7c8e94 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 May 2024 14:19:25 -0600 Subject: [PATCH 01/12] Remove db_index, add meta index --- src/registrar/models/contact.py | 11 +++++++---- src/registrar/models/domain.py | 7 +++++++ src/registrar/models/domain_information.py | 10 ++++++++-- src/registrar/models/domain_request.py | 11 +++++++++-- src/registrar/models/transition_domain.py | 3 --- src/registrar/models/user.py | 1 - src/registrar/models/verified_by_staff.py | 1 - 7 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/registrar/models/contact.py b/src/registrar/models/contact.py index 3ebd8bc3e..d401102a8 100644 --- a/src/registrar/models/contact.py +++ b/src/registrar/models/contact.py @@ -8,6 +8,13 @@ from phonenumber_field.modelfields import PhoneNumberField # type: ignore class Contact(TimeStampedModel): """Contact information follows a similar pattern for each contact.""" + class Meta: + """Contains meta information about this class""" + indexes = [ + models.Index(fields=["user"]), + models.Index(fields=["email"]), + ] + user = models.OneToOneField( "registrar.User", null=True, @@ -19,7 +26,6 @@ class Contact(TimeStampedModel): null=True, blank=True, verbose_name="first name", - db_index=True, ) middle_name = models.CharField( null=True, @@ -29,7 +35,6 @@ class Contact(TimeStampedModel): null=True, blank=True, verbose_name="last name", - db_index=True, ) title = models.CharField( null=True, @@ -39,13 +44,11 @@ class Contact(TimeStampedModel): email = models.EmailField( null=True, blank=True, - db_index=True, max_length=320, ) phone = PhoneNumberField( null=True, blank=True, - db_index=True, ) def _get_all_relations(self): diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 7f53bb234..537735752 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -65,6 +65,13 @@ class Domain(TimeStampedModel, DomainHelper): domain meets the required checks. """ + class Meta: + """Contains meta information about this class""" + indexes = [ + models.Index(fields=["name"]), + ] + + def __init__(self, *args, **kwargs): self._cache = {} super(Domain, self).__init__(*args, **kwargs) diff --git a/src/registrar/models/domain_information.py b/src/registrar/models/domain_information.py index c724423ce..21fc27a0d 100644 --- a/src/registrar/models/domain_information.py +++ b/src/registrar/models/domain_information.py @@ -22,6 +22,14 @@ class DomainInformation(TimeStampedModel): the domain request once approved, so copying them that way we can make changes after its approved. Most fields here are copied from DomainRequest.""" + class Meta: + """Contains meta information about this class""" + indexes = [ + models.Index(fields=["domain"]), + models.Index(fields=["domain_request"]), + models.Index(fields=["generic_org_type"]), + ] + StateTerritoryChoices = DomainRequest.StateTerritoryChoices # use the short names in Django admin @@ -120,7 +128,6 @@ class DomainInformation(TimeStampedModel): organization_name = models.CharField( null=True, blank=True, - db_index=True, ) address_line1 = models.CharField( null=True, @@ -147,7 +154,6 @@ class DomainInformation(TimeStampedModel): max_length=10, null=True, blank=True, - db_index=True, verbose_name="zip code", ) urbanization = models.CharField( diff --git a/src/registrar/models/domain_request.py b/src/registrar/models/domain_request.py index 75fbadc3e..19f1a66c1 100644 --- a/src/registrar/models/domain_request.py +++ b/src/registrar/models/domain_request.py @@ -24,6 +24,15 @@ logger = logging.getLogger(__name__) class DomainRequest(TimeStampedModel): """A registrant's domain request for a new domain.""" + class Meta: + """Contains meta information about this class""" + indexes = [ + models.Index(fields=["requested_domain"]), + models.Index(fields=["approved_domain"]), + models.Index(fields=["status"]), + models.Index(fields=["generic_org_type"]), + ] + # https://django-auditlog.readthedocs.io/en/latest/usage.html#object-history # If we note any performace degradation due to this addition, # we can query the auditlogs table in admin.py and add the results to @@ -537,7 +546,6 @@ class DomainRequest(TimeStampedModel): organization_name = models.CharField( null=True, blank=True, - db_index=True, ) address_line1 = models.CharField( @@ -566,7 +574,6 @@ class DomainRequest(TimeStampedModel): null=True, blank=True, verbose_name="zip code", - db_index=True, ) urbanization = models.CharField( null=True, diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index 2dafd6da4..0b0cffcec 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -59,7 +59,6 @@ class TransitionDomain(TimeStampedModel): null=True, blank=True, help_text="Organization name", - db_index=True, ) federal_type = models.CharField( max_length=50, @@ -85,7 +84,6 @@ class TransitionDomain(TimeStampedModel): blank=True, help_text="First name / given name", verbose_name="first name", - db_index=True, ) middle_name = models.CharField( null=True, @@ -136,7 +134,6 @@ class TransitionDomain(TimeStampedModel): blank=True, verbose_name="zip code", help_text="Zip code", - db_index=True, ) def __str__(self): diff --git a/src/registrar/models/user.py b/src/registrar/models/user.py index 5e4c88f63..7e82ee1c1 100644 --- a/src/registrar/models/user.py +++ b/src/registrar/models/user.py @@ -70,7 +70,6 @@ class User(AbstractUser): null=True, blank=True, help_text="Phone", - db_index=True, ) verification_type = models.CharField( diff --git a/src/registrar/models/verified_by_staff.py b/src/registrar/models/verified_by_staff.py index c09dce822..1e3e21057 100644 --- a/src/registrar/models/verified_by_staff.py +++ b/src/registrar/models/verified_by_staff.py @@ -9,7 +9,6 @@ class VerifiedByStaff(TimeStampedModel): email = models.EmailField( null=False, blank=False, - db_index=True, ) requestor = models.ForeignKey( From e265008bc86b9a1f094664494ccb4f46776dbf5e Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 May 2024 14:25:52 -0600 Subject: [PATCH 02/12] Add migrations --- ...email_alter_contact_first_name_and_more.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 src/registrar/migrations/0091_alter_contact_email_alter_contact_first_name_and_more.py diff --git a/src/registrar/migrations/0091_alter_contact_email_alter_contact_first_name_and_more.py b/src/registrar/migrations/0091_alter_contact_email_alter_contact_first_name_and_more.py new file mode 100644 index 000000000..a35cc6a22 --- /dev/null +++ b/src/registrar/migrations/0091_alter_contact_email_alter_contact_first_name_and_more.py @@ -0,0 +1,111 @@ +# Generated by Django 4.2.10 on 2024-05-06 20:21 + +from django.db import migrations, models +import phonenumber_field.modelfields + + +class Migration(migrations.Migration): + + dependencies = [ + ("registrar", "0090_waffleflag"), + ] + + operations = [ + migrations.AlterField( + model_name="contact", + name="email", + field=models.EmailField(blank=True, max_length=320, null=True), + ), + migrations.AlterField( + model_name="contact", + name="first_name", + field=models.CharField(blank=True, null=True, verbose_name="first name"), + ), + migrations.AlterField( + model_name="contact", + name="last_name", + field=models.CharField(blank=True, null=True, verbose_name="last name"), + ), + migrations.AlterField( + model_name="contact", + name="phone", + field=phonenumber_field.modelfields.PhoneNumberField(blank=True, max_length=128, null=True, region=None), + ), + migrations.AlterField( + model_name="domaininformation", + name="organization_name", + field=models.CharField(blank=True, null=True), + ), + migrations.AlterField( + model_name="domaininformation", + name="zipcode", + field=models.CharField(blank=True, max_length=10, null=True, verbose_name="zip code"), + ), + migrations.AlterField( + model_name="domainrequest", + name="organization_name", + field=models.CharField(blank=True, null=True), + ), + migrations.AlterField( + model_name="domainrequest", + name="zipcode", + field=models.CharField(blank=True, max_length=10, null=True, verbose_name="zip code"), + ), + migrations.AlterField( + model_name="transitiondomain", + name="first_name", + field=models.CharField( + blank=True, help_text="First name / given name", null=True, verbose_name="first name" + ), + ), + migrations.AlterField( + model_name="transitiondomain", + name="organization_name", + field=models.CharField(blank=True, help_text="Organization name", null=True), + ), + migrations.AlterField( + model_name="transitiondomain", + name="zipcode", + field=models.CharField(blank=True, help_text="Zip code", max_length=10, null=True, verbose_name="zip code"), + ), + migrations.AlterField( + model_name="user", + name="phone", + field=phonenumber_field.modelfields.PhoneNumberField( + blank=True, help_text="Phone", max_length=128, null=True, region=None + ), + ), + migrations.AlterField( + model_name="verifiedbystaff", + name="email", + field=models.EmailField(max_length=254), + ), + migrations.AddIndex( + model_name="contact", + index=models.Index(fields=["user"], name="registrar_c_user_id_4059c4_idx"), + ), + migrations.AddIndex( + model_name="contact", + index=models.Index(fields=["email"], name="registrar_c_email_bde2de_idx"), + ), + migrations.AddIndex( + model_name="domain", + index=models.Index(fields=["name"], name="registrar_d_name_5b1956_idx"), + ), + migrations.AddIndex( + model_name="domainrequest", + index=models.Index(fields=["requested_domain"], name="registrar_d_request_6894eb_idx"), + ), + migrations.AddIndex( + model_name="domainrequest", + index=models.Index(fields=["approved_domain"], name="registrar_d_approve_ac4c46_idx"), + ), + migrations.AddIndex( + model_name="domainrequest", + index=models.Index(fields=["status"], name="registrar_d_status_a32b59_idx"), + ), + migrations.AddIndex( + model_name="domainrequest", + index=models.Index(fields=["generic_org_type"], name="registrar_d_generic_4d1d2a_idx"), + ), + ] From b5784a1a88e6fc3ab870d6783e9798bd92ef8b8c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 May 2024 10:18:21 -0600 Subject: [PATCH 03/12] Add timers --- src/registrar/admin.py | 15 ++- src/registrar/utility/csv_export.py | 190 ++++++++++++++-------------- 2 files changed, 106 insertions(+), 99 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 76d4a6909..c3ba18bbe 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -32,6 +32,7 @@ from django.contrib.auth.forms import UserChangeForm, UsernameField from django_admin_multiple_choice_list_filter.list_filters import MultipleChoiceListFilter from django.utils.translation import gettext_lazy as _ +from registrar.models.utility.generic_helper import Timer logger = logging.getLogger(__name__) @@ -2027,12 +2028,14 @@ class DomainAdmin(ListHeaderAdmin): return HttpResponseRedirect(reverse("domain", args=(obj.id,))) def change_view(self, request, object_id): - # If the analyst was recently editing a domain page, - # delete any associated session values - if "analyst_action" in request.session: - del request.session["analyst_action"] - del request.session["analyst_action_location"] - return super().change_view(request, object_id) + logger.info("Timing change_view on domain") + with Timer(): + # If the analyst was recently editing a domain page, + # delete any associated session values + if "analyst_action" in request.session: + del request.session["analyst_action"] + del request.session["analyst_action_location"] + return super().change_view(request, object_id) def has_change_permission(self, request, obj=None): # Fixes a bug wherein users which are only is_staff diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 8787f9e74..83754f574 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -13,7 +13,7 @@ from django.db.models.functions import Concat, Coalesce from registrar.models.public_contact import PublicContact from registrar.models.user_domain_role import UserDomainRole from registrar.utility.enums import DefaultEmail - +from registrar.models.utility.generic_helper import Timer logger = logging.getLogger(__name__) @@ -379,108 +379,112 @@ def write_csv_for_requests( def export_data_type_to_csv(csv_file): """All domains report with extra columns""" + logger.info("Timing export_data_type_to_csv") + with Timer(): + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Domain name", + "Status", + "Expiration date", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "AO", + "AO email", + "Security contact email", + # For domain manager we are pass it in as a parameter below in write_body + ] - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Status", - "Expiration date", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - # For domain manager we are pass it in as a parameter below in write_body - ] - - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=True, should_write_header=True - ) + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + filter_condition = { + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + } + write_csv_for_domains( + writer, columns, sort_fields, filter_condition, should_get_domain_managers=True, should_write_header=True + ) def export_data_full_to_csv(csv_file): """All domains report""" - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True - ) + logger.info("Timing def export_data_full_to_csv") + with Timer(): + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Security contact email", + ] + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + filter_condition = { + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + } + write_csv_for_domains( + writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True + ) def export_data_federal_to_csv(csv_file): """Federal domains report""" - - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "organization_type__icontains": "federal", - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True - ) + logger.info("Timing def export_data_federal_to_csv") + with Timer(): + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Security contact email", + ] + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + filter_condition = { + "organization_type__icontains": "federal", + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + } + write_csv_for_domains( + writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True + ) def get_default_start_date(): From ca2e2168aa6a8116cc7e1ec818e6978476389ac1 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 May 2024 14:13:55 -0600 Subject: [PATCH 04/12] Remove timer --- src/registrar/admin.py | 13 +- ...email_alter_contact_first_name_and_more.py | 6 +- src/registrar/models/domain_invitation.py | 6 + src/registrar/models/user.py | 7 + src/registrar/utility/csv_export.py | 185 +++++++++--------- 5 files changed, 114 insertions(+), 103 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index c3ba18bbe..efafbcfc4 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -2029,13 +2029,12 @@ class DomainAdmin(ListHeaderAdmin): def change_view(self, request, object_id): logger.info("Timing change_view on domain") - with Timer(): - # If the analyst was recently editing a domain page, - # delete any associated session values - if "analyst_action" in request.session: - del request.session["analyst_action"] - del request.session["analyst_action_location"] - return super().change_view(request, object_id) + # If the analyst was recently editing a domain page, + # delete any associated session values + if "analyst_action" in request.session: + del request.session["analyst_action"] + del request.session["analyst_action_location"] + return super().change_view(request, object_id) def has_change_permission(self, request, obj=None): # Fixes a bug wherein users which are only is_staff diff --git a/src/registrar/migrations/0091_alter_contact_email_alter_contact_first_name_and_more.py b/src/registrar/migrations/0091_alter_contact_email_alter_contact_first_name_and_more.py index a35cc6a22..a3b8fe0e9 100644 --- a/src/registrar/migrations/0091_alter_contact_email_alter_contact_first_name_and_more.py +++ b/src/registrar/migrations/0091_alter_contact_email_alter_contact_first_name_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-05-06 20:21 +# Generated by Django 4.2.10 on 2024-05-07 20:13 from django.db import migrations, models import phonenumber_field.modelfields @@ -92,6 +92,10 @@ class Migration(migrations.Migration): model_name="domain", index=models.Index(fields=["name"], name="registrar_d_name_5b1956_idx"), ), + migrations.AddIndex( + model_name="domaininvitation", + index=models.Index(fields=["status"], name="registrar_d_status_e84571_idx"), + ), migrations.AddIndex( model_name="domainrequest", index=models.Index(fields=["requested_domain"], name="registrar_d_request_6894eb_idx"), diff --git a/src/registrar/models/domain_invitation.py b/src/registrar/models/domain_invitation.py index 12082142d..39473c0be 100644 --- a/src/registrar/models/domain_invitation.py +++ b/src/registrar/models/domain_invitation.py @@ -15,6 +15,12 @@ logger = logging.getLogger(__name__) class DomainInvitation(TimeStampedModel): + class Meta: + """Contains meta information about this class""" + indexes = [ + models.Index(fields=["status"]), + ] + # Constants for status field class DomainInvitationStatus(models.TextChoices): INVITED = "invited", "Invited" diff --git a/src/registrar/models/user.py b/src/registrar/models/user.py index 7e82ee1c1..e630366a4 100644 --- a/src/registrar/models/user.py +++ b/src/registrar/models/user.py @@ -24,6 +24,13 @@ class User(AbstractUser): but can be customized later. """ + class Meta: + """Contains meta information about this class""" + indexes = [ + models.Index(fields=["username"]), + models.Index(fields=["email"]), + ] + class VerificationTypeChoices(models.TextChoices): """ Users achieve access to our system in a few different ways. diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 83754f574..abd91c0ba 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -380,111 +380,106 @@ def write_csv_for_requests( def export_data_type_to_csv(csv_file): """All domains report with extra columns""" logger.info("Timing export_data_type_to_csv") - with Timer(): - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Status", - "Expiration date", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - # For domain manager we are pass it in as a parameter below in write_body - ] + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Domain name", + "Status", + "Expiration date", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "AO", + "AO email", + "Security contact email", + # For domain manager we are pass it in as a parameter below in write_body + ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=True, should_write_header=True - ) + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + filter_condition = { + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + } + write_csv_for_domains( + writer, columns, sort_fields, filter_condition, should_get_domain_managers=True, should_write_header=True + ) def export_data_full_to_csv(csv_file): """All domains report""" - logger.info("Timing def export_data_full_to_csv") - with Timer(): - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True - ) + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Security contact email", + ] + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + filter_condition = { + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + } + write_csv_for_domains( + writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True + ) def export_data_federal_to_csv(csv_file): """Federal domains report""" - logger.info("Timing def export_data_federal_to_csv") - with Timer(): - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "organization_type__icontains": "federal", - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True - ) + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Security contact email", + ] + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + filter_condition = { + "organization_type__icontains": "federal", + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + } + write_csv_for_domains( + writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True + ) def get_default_start_date(): From af272c0cd3a4fb81fe760f3f53c965ca10bc54ae Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 May 2024 14:16:34 -0600 Subject: [PATCH 05/12] Fix migrations after merge --- ..._alter_contact_email_alter_contact_first_name_and_more.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/registrar/migrations/{0091_alter_contact_email_alter_contact_first_name_and_more.py => 0093_alter_contact_email_alter_contact_first_name_and_more.py} (96%) diff --git a/src/registrar/migrations/0091_alter_contact_email_alter_contact_first_name_and_more.py b/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py similarity index 96% rename from src/registrar/migrations/0091_alter_contact_email_alter_contact_first_name_and_more.py rename to src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py index a3b8fe0e9..ca1127f2e 100644 --- a/src/registrar/migrations/0091_alter_contact_email_alter_contact_first_name_and_more.py +++ b/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-05-07 20:13 +# Generated by Django 4.2.10 on 2024-05-07 20:16 from django.db import migrations, models import phonenumber_field.modelfields @@ -7,7 +7,7 @@ import phonenumber_field.modelfields class Migration(migrations.Migration): dependencies = [ - ("registrar", "0090_waffleflag"), + ("registrar", "0092_rename_updated_federal_agency_domaininformation_federal_agency_and_more"), ] operations = [ From 00bcdd16944767d34ee68096f87728de763fb2c6 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 May 2024 14:33:15 -0600 Subject: [PATCH 06/12] Remove timer remnants, cleanup --- src/registrar/admin.py | 3 +-- ..._contact_email_alter_contact_first_name_and_more.py | 10 +++++----- src/registrar/models/contact.py | 1 + src/registrar/models/domain.py | 3 ++- src/registrar/models/domain_information.py | 2 +- src/registrar/models/domain_invitation.py | 1 + src/registrar/models/domain_request.py | 2 +- src/registrar/models/user.py | 1 + src/registrar/utility/csv_export.py | 4 ++-- 9 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index d8b9ff9f9..55cd358c5 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -32,7 +32,7 @@ from django.contrib.auth.forms import UserChangeForm, UsernameField from django_admin_multiple_choice_list_filter.list_filters import MultipleChoiceListFilter from django.utils.translation import gettext_lazy as _ -from registrar.models.utility.generic_helper import Timer + logger = logging.getLogger(__name__) @@ -2024,7 +2024,6 @@ class DomainAdmin(ListHeaderAdmin): return HttpResponseRedirect(reverse("domain", args=(obj.id,))) def change_view(self, request, object_id): - logger.info("Timing change_view on domain") # If the analyst was recently editing a domain page, # delete any associated session values if "analyst_action" in request.session: diff --git a/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py b/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py index ca1127f2e..9696f0c50 100644 --- a/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py +++ b/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-05-07 20:16 +# Generated by Django 4.2.10 on 2024-05-07 20:32 from django.db import migrations, models import phonenumber_field.modelfields @@ -92,6 +92,10 @@ class Migration(migrations.Migration): model_name="domain", index=models.Index(fields=["name"], name="registrar_d_name_5b1956_idx"), ), + migrations.AddIndex( + model_name="domain", + index=models.Index(fields=["state"], name="registrar_d_state_84c134_idx"), + ), migrations.AddIndex( model_name="domaininvitation", index=models.Index(fields=["status"], name="registrar_d_status_e84571_idx"), @@ -108,8 +112,4 @@ class Migration(migrations.Migration): model_name="domainrequest", index=models.Index(fields=["status"], name="registrar_d_status_a32b59_idx"), ), - migrations.AddIndex( - model_name="domainrequest", - index=models.Index(fields=["generic_org_type"], name="registrar_d_generic_4d1d2a_idx"), - ), ] diff --git a/src/registrar/models/contact.py b/src/registrar/models/contact.py index d401102a8..5084ea955 100644 --- a/src/registrar/models/contact.py +++ b/src/registrar/models/contact.py @@ -10,6 +10,7 @@ class Contact(TimeStampedModel): class Meta: """Contains meta information about this class""" + indexes = [ models.Index(fields=["user"]), models.Index(fields=["email"]), diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 537735752..fbf2822a8 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -67,11 +67,12 @@ class Domain(TimeStampedModel, DomainHelper): class Meta: """Contains meta information about this class""" + indexes = [ models.Index(fields=["name"]), + models.Index(fields=["state"]), ] - def __init__(self, *args, **kwargs): self._cache = {} super(Domain, self).__init__(*args, **kwargs) diff --git a/src/registrar/models/domain_information.py b/src/registrar/models/domain_information.py index e0b5ad237..81a8c7296 100644 --- a/src/registrar/models/domain_information.py +++ b/src/registrar/models/domain_information.py @@ -24,10 +24,10 @@ class DomainInformation(TimeStampedModel): class Meta: """Contains meta information about this class""" + indexes = [ models.Index(fields=["domain"]), models.Index(fields=["domain_request"]), - models.Index(fields=["generic_org_type"]), ] StateTerritoryChoices = DomainRequest.StateTerritoryChoices diff --git a/src/registrar/models/domain_invitation.py b/src/registrar/models/domain_invitation.py index 39473c0be..c9cbc8b39 100644 --- a/src/registrar/models/domain_invitation.py +++ b/src/registrar/models/domain_invitation.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) class DomainInvitation(TimeStampedModel): class Meta: """Contains meta information about this class""" + indexes = [ models.Index(fields=["status"]), ] diff --git a/src/registrar/models/domain_request.py b/src/registrar/models/domain_request.py index a947aba2f..02e30b266 100644 --- a/src/registrar/models/domain_request.py +++ b/src/registrar/models/domain_request.py @@ -26,11 +26,11 @@ class DomainRequest(TimeStampedModel): class Meta: """Contains meta information about this class""" + indexes = [ models.Index(fields=["requested_domain"]), models.Index(fields=["approved_domain"]), models.Index(fields=["status"]), - models.Index(fields=["generic_org_type"]), ] # https://django-auditlog.readthedocs.io/en/latest/usage.html#object-history diff --git a/src/registrar/models/user.py b/src/registrar/models/user.py index e630366a4..1ff700239 100644 --- a/src/registrar/models/user.py +++ b/src/registrar/models/user.py @@ -26,6 +26,7 @@ class User(AbstractUser): class Meta: """Contains meta information about this class""" + indexes = [ models.Index(fields=["username"]), models.Index(fields=["email"]), diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index abd91c0ba..19d66e598 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -13,7 +13,8 @@ from django.db.models.functions import Concat, Coalesce from registrar.models.public_contact import PublicContact from registrar.models.user_domain_role import UserDomainRole from registrar.utility.enums import DefaultEmail -from registrar.models.utility.generic_helper import Timer + + logger = logging.getLogger(__name__) @@ -379,7 +380,6 @@ def write_csv_for_requests( def export_data_type_to_csv(csv_file): """All domains report with extra columns""" - logger.info("Timing export_data_type_to_csv") writer = csv.writer(csv_file) # define columns to include in export columns = [ From 87aa44cc7788880df863b0ec4685d560224ba098 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 May 2024 15:17:40 -0600 Subject: [PATCH 07/12] Cleanup --- src/registrar/admin.py | 1 - ..._email_alter_contact_first_name_and_more.py | 18 +++++++++++++++++- src/registrar/models/domain_information.py | 5 ++--- src/registrar/models/user.py | 13 +++++-------- src/registrar/utility/csv_export.py | 1 - 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 55cd358c5..3eea86871 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -33,7 +33,6 @@ from django_admin_multiple_choice_list_filter.list_filters import MultipleChoice from django.utils.translation import gettext_lazy as _ - logger = logging.getLogger(__name__) diff --git a/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py b/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py index 9696f0c50..3c45c4ef2 100644 --- a/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py +++ b/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-05-07 20:32 +# Generated by Django 4.2.10 on 2024-05-07 21:17 from django.db import migrations, models import phonenumber_field.modelfields @@ -96,6 +96,14 @@ class Migration(migrations.Migration): model_name="domain", index=models.Index(fields=["state"], name="registrar_d_state_84c134_idx"), ), + migrations.AddIndex( + model_name="domaininformation", + index=models.Index(fields=["domain"], name="registrar_d_domain__88838a_idx"), + ), + migrations.AddIndex( + model_name="domaininformation", + index=models.Index(fields=["domain_request"], name="registrar_d_domain__d1fba8_idx"), + ), migrations.AddIndex( model_name="domaininvitation", index=models.Index(fields=["status"], name="registrar_d_status_e84571_idx"), @@ -112,4 +120,12 @@ class Migration(migrations.Migration): model_name="domainrequest", index=models.Index(fields=["status"], name="registrar_d_status_a32b59_idx"), ), + migrations.AddIndex( + model_name="user", + index=models.Index(fields=["username"], name="registrar_u_usernam_964b1b_idx"), + ), + migrations.AddIndex( + model_name="user", + index=models.Index(fields=["email"], name="registrar_u_email_c8f2c4_idx"), + ), ] diff --git a/src/registrar/models/domain_information.py b/src/registrar/models/domain_information.py index 81a8c7296..23c9e4f32 100644 --- a/src/registrar/models/domain_information.py +++ b/src/registrar/models/domain_information.py @@ -30,6 +30,8 @@ class DomainInformation(TimeStampedModel): models.Index(fields=["domain_request"]), ] + verbose_name_plural = "Domain information" + StateTerritoryChoices = DomainRequest.StateTerritoryChoices # use the short names in Django admin @@ -342,6 +344,3 @@ class DomainInformation(TimeStampedModel): def _get_many_to_many_fields(): """Returns a set of each field.name that has the many to many relation""" return {field.name for field in DomainInformation._meta.many_to_many} # type: ignore - - class Meta: - verbose_name_plural = "Domain information" diff --git a/src/registrar/models/user.py b/src/registrar/models/user.py index 1ff700239..307a7cbcc 100644 --- a/src/registrar/models/user.py +++ b/src/registrar/models/user.py @@ -25,13 +25,16 @@ class User(AbstractUser): """ class Meta: - """Contains meta information about this class""" - indexes = [ models.Index(fields=["username"]), models.Index(fields=["email"]), ] + permissions = [ + ("analyst_access_permission", "Analyst Access Permission"), + ("full_access_permission", "Full Access Permission"), + ] + class VerificationTypeChoices(models.TextChoices): """ Users achieve access to our system in a few different ways. @@ -252,9 +255,3 @@ class User(AbstractUser): """ self.check_domain_invitations_on_login() - - class Meta: - permissions = [ - ("analyst_access_permission", "Analyst Access Permission"), - ("full_access_permission", "Full Access Permission"), - ] diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 19d66e598..9ac83ed86 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -14,7 +14,6 @@ from registrar.models.public_contact import PublicContact from registrar.models.user_domain_role import UserDomainRole from registrar.utility.enums import DefaultEmail - logger = logging.getLogger(__name__) From 08ea56b9893188af7b26b823d96979aa717243c2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 28 May 2024 08:35:05 -0600 Subject: [PATCH 08/12] Recreate migration --- ...ogether.py => 0094_alter_publiccontact_unique_together.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/registrar/migrations/{0093_alter_publiccontact_unique_together.py => 0094_alter_publiccontact_unique_together.py} (65%) diff --git a/src/registrar/migrations/0093_alter_publiccontact_unique_together.py b/src/registrar/migrations/0094_alter_publiccontact_unique_together.py similarity index 65% rename from src/registrar/migrations/0093_alter_publiccontact_unique_together.py rename to src/registrar/migrations/0094_alter_publiccontact_unique_together.py index 08c71e122..a18c077b9 100644 --- a/src/registrar/migrations/0093_alter_publiccontact_unique_together.py +++ b/src/registrar/migrations/0094_alter_publiccontact_unique_together.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-05-08 17:35 +# Generated by Django 4.2.10 on 2024-05-28 14:34 from django.db import migrations @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ("registrar", "0092_rename_updated_federal_agency_domaininformation_federal_agency_and_more"), + ("registrar", "0093_alter_contact_email_alter_contact_first_name_and_more"), ] operations = [ From 069a2435fa85d2950865b9463ce01f11372ab6df Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 28 May 2024 08:38:46 -0600 Subject: [PATCH 09/12] Merge latest and fix migrations (once again) --- src/registrar/migrations/0094_create_groups_v12.py | 2 +- ...ogether.py => 0096_alter_publiccontact_unique_together.py} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename src/registrar/migrations/{0094_alter_publiccontact_unique_together.py => 0096_alter_publiccontact_unique_together.py} (68%) diff --git a/src/registrar/migrations/0094_create_groups_v12.py b/src/registrar/migrations/0094_create_groups_v12.py index 42106cdb5..0a4e1f51f 100644 --- a/src/registrar/migrations/0094_create_groups_v12.py +++ b/src/registrar/migrations/0094_create_groups_v12.py @@ -25,7 +25,7 @@ def create_groups(apps, schema_editor) -> Any: class Migration(migrations.Migration): dependencies = [ - ("registrar", "0093_alter_publiccontact_unique_together"), + ("registrar", "0093_alter_contact_email_alter_contact_first_name_and_more"), ] operations = [ diff --git a/src/registrar/migrations/0094_alter_publiccontact_unique_together.py b/src/registrar/migrations/0096_alter_publiccontact_unique_together.py similarity index 68% rename from src/registrar/migrations/0094_alter_publiccontact_unique_together.py rename to src/registrar/migrations/0096_alter_publiccontact_unique_together.py index a18c077b9..21e2c8c23 100644 --- a/src/registrar/migrations/0094_alter_publiccontact_unique_together.py +++ b/src/registrar/migrations/0096_alter_publiccontact_unique_together.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-05-28 14:34 +# Generated by Django 4.2.10 on 2024-05-28 14:38 from django.db import migrations @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ("registrar", "0093_alter_contact_email_alter_contact_first_name_and_more"), + ("registrar", "0095_user_middle_name_user_title"), ] operations = [ From 1f2da8f37ccd3287337d0fcedbb7c39985d1a8ee Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 28 May 2024 08:39:32 -0600 Subject: [PATCH 10/12] Revert "Merge latest and fix migrations (once again)" This reverts commit 069a2435fa85d2950865b9463ce01f11372ab6df. --- ...ogether.py => 0094_alter_publiccontact_unique_together.py} | 4 ++-- src/registrar/migrations/0094_create_groups_v12.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename src/registrar/migrations/{0096_alter_publiccontact_unique_together.py => 0094_alter_publiccontact_unique_together.py} (68%) diff --git a/src/registrar/migrations/0096_alter_publiccontact_unique_together.py b/src/registrar/migrations/0094_alter_publiccontact_unique_together.py similarity index 68% rename from src/registrar/migrations/0096_alter_publiccontact_unique_together.py rename to src/registrar/migrations/0094_alter_publiccontact_unique_together.py index 21e2c8c23..a18c077b9 100644 --- a/src/registrar/migrations/0096_alter_publiccontact_unique_together.py +++ b/src/registrar/migrations/0094_alter_publiccontact_unique_together.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-05-28 14:38 +# Generated by Django 4.2.10 on 2024-05-28 14:34 from django.db import migrations @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ("registrar", "0095_user_middle_name_user_title"), + ("registrar", "0093_alter_contact_email_alter_contact_first_name_and_more"), ] operations = [ diff --git a/src/registrar/migrations/0094_create_groups_v12.py b/src/registrar/migrations/0094_create_groups_v12.py index 0a4e1f51f..42106cdb5 100644 --- a/src/registrar/migrations/0094_create_groups_v12.py +++ b/src/registrar/migrations/0094_create_groups_v12.py @@ -25,7 +25,7 @@ def create_groups(apps, schema_editor) -> Any: class Migration(migrations.Migration): dependencies = [ - ("registrar", "0093_alter_contact_email_alter_contact_first_name_and_more"), + ("registrar", "0093_alter_publiccontact_unique_together"), ] operations = [ From 7f126c0c4d90de45f593e207eba392491ae37b74 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 28 May 2024 08:39:57 -0600 Subject: [PATCH 11/12] Revert "Recreate migration" This reverts commit 08ea56b9893188af7b26b823d96979aa717243c2. --- ...ogether.py => 0093_alter_publiccontact_unique_together.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/registrar/migrations/{0094_alter_publiccontact_unique_together.py => 0093_alter_publiccontact_unique_together.py} (65%) diff --git a/src/registrar/migrations/0094_alter_publiccontact_unique_together.py b/src/registrar/migrations/0093_alter_publiccontact_unique_together.py similarity index 65% rename from src/registrar/migrations/0094_alter_publiccontact_unique_together.py rename to src/registrar/migrations/0093_alter_publiccontact_unique_together.py index a18c077b9..08c71e122 100644 --- a/src/registrar/migrations/0094_alter_publiccontact_unique_together.py +++ b/src/registrar/migrations/0093_alter_publiccontact_unique_together.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-05-28 14:34 +# Generated by Django 4.2.10 on 2024-05-08 17:35 from django.db import migrations @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ("registrar", "0093_alter_contact_email_alter_contact_first_name_and_more"), + ("registrar", "0092_rename_updated_federal_agency_domaininformation_federal_agency_and_more"), ] operations = [ From cd5aa6541d4514c154be719c11947d46df8bf844 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 28 May 2024 08:42:16 -0600 Subject: [PATCH 12/12] Migration fix (actual) --- ..._alter_contact_email_alter_contact_first_name_and_more.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/registrar/migrations/{0093_alter_contact_email_alter_contact_first_name_and_more.py => 0096_alter_contact_email_alter_contact_first_name_and_more.py} (96%) diff --git a/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py b/src/registrar/migrations/0096_alter_contact_email_alter_contact_first_name_and_more.py similarity index 96% rename from src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py rename to src/registrar/migrations/0096_alter_contact_email_alter_contact_first_name_and_more.py index 3c45c4ef2..68cbc625b 100644 --- a/src/registrar/migrations/0093_alter_contact_email_alter_contact_first_name_and_more.py +++ b/src/registrar/migrations/0096_alter_contact_email_alter_contact_first_name_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-05-07 21:17 +# Generated by Django 4.2.10 on 2024-05-28 14:40 from django.db import migrations, models import phonenumber_field.modelfields @@ -7,7 +7,7 @@ import phonenumber_field.modelfields class Migration(migrations.Migration): dependencies = [ - ("registrar", "0092_rename_updated_federal_agency_domaininformation_federal_agency_and_more"), + ("registrar", "0095_user_middle_name_user_title"), ] operations = [