Merge pull request #2656 from cisagov/ms/2560-update-first-ready-values

Issue #2560 update first ready values

docs/developer/management_script_helpers.md

@@ -62,4 +62,5 @@ The class provides the following optional configuration variables:
 The class also provides helper methods:
 - `get_class_name`: Returns a display-friendly class name for the terminal prompt
 - `get_failure_message`: Returns the message to display if a record fails to update
-- `should_skip_record`: Defines the condition for skipping a record (by default, no records are skipped)
+- `should_skip_record`: Defines the condition for skipping a record (by default, no records are skipped)
+- `custom_filter`: Allows for additional filters that cannot be expressed using django queryset field lookups

docs/operations/data_migration.md

@@ -817,6 +817,28 @@ Example: `cf ssh getgov-za`
 |:-:|:-------------------------- |:-----------------------------------------------------------------------------------|
 | 1 | **federal_cio_csv_path**   | Specifies where the federal CIO csv is                                             |
 
+## Update First Ready Values
+This section outlines how to run the populate_first_ready script
+
+### Running on sandboxes
+
+#### Step 1: Login to CloudFoundry
+```cf login -a api.fr.cloud.gov --sso```
+
+#### Step 2: SSH into your environment
+```cf ssh getgov-{space}```
+
+Example: `cf ssh getgov-za`
+
+#### Step 3: Create a shell instance
+```/tmp/lifecycle/shell```
+
+#### Step 4: Running the script
+```./manage.py update_first_ready```
+
+### Running locally
+```docker-compose exec app ./manage.py update_first_ready```
+
 ## Populate Domain Request Dates
 This section outlines how to run the populate_domain_request_dates script
 

src/registrar/management/commands/clean_tables.py

@@ -21,7 +21,7 @@ class Command(BaseCommand):
 
         TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect="""
+            prompt_message="""
             This script will delete all rows from the following tables:
              * Contact
              * Domain

src/registrar/management/commands/extend_expiration_dates.py

@@ -130,7 +130,7 @@ class Command(BaseCommand):
         """Asks if the user wants to proceed with this action"""
         TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect=f"""
+            prompt_message=f"""
             ==Extension Amount==
             Period: {extension_amount} year(s)
 

src/registrar/management/commands/load_organization_data.py

@@ -64,7 +64,7 @@ class Command(BaseCommand):
         # Will sys.exit() when prompt is "n"
         TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect=f"""
+            prompt_message=f"""
             ==Master data file==
             domain_additional_filename: {org_args.domain_additional_filename}
 
@@ -84,7 +84,7 @@ class Command(BaseCommand):
         # Will sys.exit() when prompt is "n"
         TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect=f"""
+            prompt_message=f"""
             ==Master data file==
             domain_additional_filename: {org_args.domain_additional_filename}
 

src/registrar/management/commands/load_senior_official_table.py

@@ -27,7 +27,7 @@ class Command(BaseCommand):
 
         TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect=f"""
+            prompt_message=f"""
             ==Proposed Changes==
             CSV: {federal_cio_csv_path}
 

src/registrar/management/commands/load_transition_domain.py

@@ -651,7 +651,7 @@ class Command(BaseCommand):
         title = "Do you wish to load additional data for TransitionDomains?"
         proceed = TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect=f"""
+            prompt_message=f"""
             !!! ENSURE THAT ALL FILENAMES ARE CORRECT BEFORE PROCEEDING
             ==Master data file==
             domain_additional_filename: {domain_additional_filename}

src/registrar/management/commands/patch_federal_agency_info.py

@@ -91,7 +91,7 @@ class Command(BaseCommand):
         # Code execution will stop here if the user prompts "N"
         TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect=f"""
+            prompt_message=f"""
             ==Proposed Changes==
             Number of DomainInformation objects to change: {len(human_readable_domain_names)}
             The following DomainInformation objects will be modified: {human_readable_domain_names}
@@ -148,7 +148,7 @@ class Command(BaseCommand):
         # Code execution will stop here if the user prompts "N"
         TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect=f"""
+            prompt_message=f"""
             ==File location==
             current-full.csv filepath: {file_path}
 

src/registrar/management/commands/populate_first_ready.py

@@ -31,7 +31,7 @@ class Command(BaseCommand):
         # Code execution will stop here if the user prompts "N"
         TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect=f"""
+            prompt_message=f"""
             ==Proposed Changes==
             Number of Domain objects to change: {len(domains)}
             """,

src/registrar/management/commands/populate_organization_type.py

@@ -54,7 +54,7 @@ class Command(BaseCommand):
         # Code execution will stop here if the user prompts "N"
         TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect=f"""
+            prompt_message=f"""
             ==Proposed Changes==
             Number of DomainRequest objects to change: {len(domain_requests)}
 
@@ -72,7 +72,7 @@ class Command(BaseCommand):
         # Code execution will stop here if the user prompts "N"
         TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect=f"""
+            prompt_message=f"""
             ==Proposed Changes==
             Number of DomainInformation objects to change: {len(domain_infos)}
 

src/registrar/management/commands/update_first_ready.py

@@ -0,0 +1,39 @@
+import logging
+from django.core.management import BaseCommand
+from django.db.models.manager import BaseManager
+from registrar.management.commands.utility.terminal_helper import PopulateScriptTemplate, TerminalColors
+from registrar.models import Domain, TransitionDomain
+
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand, PopulateScriptTemplate):
+    help = "Loops through each domain object and populates the last_status_update and first_submitted_date"
+
+    def handle(self, **kwargs):
+        """Loops through each valid Domain object and updates it's first_ready value if it is out of sync"""
+        filter_conditions = {"state__in": [Domain.State.READY, Domain.State.ON_HOLD, Domain.State.DELETED]}
+        self.mass_update_records(Domain, filter_conditions, ["first_ready"], verbose=True)
+
+    def update_record(self, record: Domain):
+        """Defines how we update the first_ready field"""
+        # update the first_ready value based on the creation date.
+        record.first_ready = record.created_at.date()
+
+        logger.info(
+            f"{TerminalColors.OKCYAN}Updating {record} => first_ready: " f"{record.first_ready}{TerminalColors.ENDC}"
+        )
+
+    # check if a transition domain object for this domain name exists,
+    # or if so whether its first_ready value matches its created_at date
+    def custom_filter(self, records: BaseManager[Domain]) -> BaseManager[Domain]:
+        to_include_pks = []
+        for record in records:
+            if (
+                TransitionDomain.objects.filter(domain_name=record.name).exists()
+                and record.first_ready != record.created_at.date()
+            ):  # noqa
+                to_include_pks.append(record.pk)
+
+        return records.filter(pk__in=to_include_pks)

src/registrar/management/commands/utility/terminal_helper.py

@@ -2,9 +2,12 @@ import logging
 import sys
 from abc import ABC, abstractmethod
 from django.core.paginator import Paginator
+from django.db.models import Model
+from django.db.models.manager import BaseManager
 from typing import List
 from registrar.utility.enums import LogCode
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -75,28 +78,61 @@ class PopulateScriptTemplate(ABC):
     run_summary_header = None
 
     @abstractmethod
-    def update_record(self, record):
-        """Defines how we update each field. Must be defined before using mass_update_records."""
+    def update_record(self, record: Model):
+        """Defines how we update each field.
+
+        raises:
+            NotImplementedError: If not defined before calling mass_update_records.
+        """
         raise NotImplementedError
 
-    def mass_update_records(self, object_class, filter_conditions, fields_to_update, debug=True):
+    def mass_update_records(self, object_class: Model, filter_conditions, fields_to_update, debug=True, verbose=False):
         """Loops through each valid "object_class" object - specified by filter_conditions - and
         updates fields defined by fields_to_update using update_record.
 
-        You must define update_record before you can use this function.
+        Parameters:
+            object_class: The Django model class that you want to perform the bulk update on.
+                This should be the actual class, not a string of the class name.
+
+            filter_conditions: dictionary of valid Django Queryset filter conditions
+                (e.g. {'verification_type__isnull'=True}).
+
+            fields_to_update: List of strings specifying which fields to update.
+                (e.g. ["first_ready_date", "last_submitted_date"])
+
+            debug: Whether to log script run summary in debug mode.
+                Default: True.
+
+            verbose: Whether to print a detailed run summary *before* run confirmation.
+                Default: False.
+
+        Raises:
+            NotImplementedError: If you do not define update_record before using this function.
+            TypeError: If custom_filter is not Callable.
         """
 
         records = object_class.objects.filter(**filter_conditions) if filter_conditions else object_class.objects.all()
+
+        # apply custom filter
+        records = self.custom_filter(records)
+
         readable_class_name = self.get_class_name(object_class)
 
+        # for use in the execution prompt.
+        proposed_changes = f"""==Proposed Changes==
+            Number of {readable_class_name} objects to change: {len(records)}
+            These fields will be updated on each record: {fields_to_update}
+            """
+
+        if verbose:
+            proposed_changes = f"""{proposed_changes}
+            These records will be updated: {list(records.all())}
+            """
+
         # Code execution will stop here if the user prompts "N"
         TerminalHelper.prompt_for_execution(
             system_exit_on_terminate=True,
-            info_to_inspect=f"""
-            ==Proposed Changes==
-            Number of {readable_class_name} objects to change: {len(records)}
-            These fields will be updated on each record: {fields_to_update}
-            """,
+            prompt_message=proposed_changes,
             prompt_title=self.prompt_title,
         )
         logger.info("Updating...")
@@ -141,10 +177,17 @@ class PopulateScriptTemplate(ABC):
         return f"{TerminalColors.FAIL}" f"Failed to update {record}" f"{TerminalColors.ENDC}"
 
     def should_skip_record(self, record) -> bool:  # noqa
-        """Defines the condition in which we should skip updating a record. Override as needed."""
+        """Defines the condition in which we should skip updating a record. Override as needed.
+        The difference between this and custom_filter is that records matching these conditions
+        *will* be included in the run but will be skipped (and logged as such)."""
         # By default - don't skip
         return False
 
+    def custom_filter(self, records: BaseManager[Model]) -> BaseManager[Model]:
+        """Override to define filters that can't be represented by django queryset field lookups.
+        Applied to individual records *after* filter_conditions. True means"""
+        return records
+
 
 class TerminalHelper:
     @staticmethod
@@ -220,6 +263,9 @@ class TerminalHelper:
                 an answer is required of the user).
 
         The "answer" return value is True for "yes" or False for "no".
+
+        Raises:
+            ValueError: When "default" is not "yes", "no", or None.
         """
         valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
         if default is None:
@@ -244,6 +290,7 @@ class TerminalHelper:
     @staticmethod
     def query_yes_no_exit(question: str, default="yes"):
         """Ask a yes/no question via raw_input() and return their answer.
+        Allows for answer "e" to exit.
 
         "question" is a string that is presented to the user.
         "default" is the presumed answer if the user just hits <Enter>.
@@ -251,6 +298,9 @@ class TerminalHelper:
                 an answer is required of the user).
 
         The "answer" return value is True for "yes" or False for "no".
+
+        Raises:
+            ValueError: When "default" is not "yes", "no", or None.
         """
         valid = {
             "yes": True,
@@ -317,9 +367,8 @@ class TerminalHelper:
                 case _:
                     logger.info(print_statement)
 
-    # TODO - "info_to_inspect" should be refactored to "prompt_message"
     @staticmethod
-    def prompt_for_execution(system_exit_on_terminate: bool, info_to_inspect: str, prompt_title: str) -> bool:
+    def prompt_for_execution(system_exit_on_terminate: bool, prompt_message: str, prompt_title: str) -> bool:
         """Create to reduce code complexity.
         Prompts the user to inspect the given string
         and asks if they wish to proceed.
@@ -340,7 +389,7 @@ class TerminalHelper:
             =====================================================
             *** IMPORTANT:  VERIFY THE FOLLOWING LOOKS CORRECT ***
 
-            {info_to_inspect}
+            {prompt_message}
             {TerminalColors.FAIL}
             Proceed? (Y = proceed, N = {action_description_for_selecting_no})
             {TerminalColors.ENDC}"""