diff --git a/docs/developer/management_script_helpers.md b/docs/developer/management_script_helpers.md index 104e4dc13..a43bb16aa 100644 --- a/docs/developer/management_script_helpers.md +++ b/docs/developer/management_script_helpers.md @@ -62,4 +62,5 @@ The class provides the following optional configuration variables: The class also provides helper methods: - `get_class_name`: Returns a display-friendly class name for the terminal prompt - `get_failure_message`: Returns the message to display if a record fails to update -- `should_skip_record`: Defines the condition for skipping a record (by default, no records are skipped) \ No newline at end of file +- `should_skip_record`: Defines the condition for skipping a record (by default, no records are skipped) +- `custom_filter`: Allows for additional filters that cannot be expressed using django queryset field lookups \ No newline at end of file diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 5914eb179..4301ca878 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -817,6 +817,28 @@ Example: `cf ssh getgov-za` |:-:|:-------------------------- |:-----------------------------------------------------------------------------------| | 1 | **federal_cio_csv_path** | Specifies where the federal CIO csv is | +## Update First Ready Values +This section outlines how to run the populate_first_ready script + +### Running on sandboxes + +#### Step 1: Login to CloudFoundry +```cf login -a api.fr.cloud.gov --sso``` + +#### Step 2: SSH into your environment +```cf ssh getgov-{space}``` + +Example: `cf ssh getgov-za` + +#### Step 3: Create a shell instance +```/tmp/lifecycle/shell``` + +#### Step 4: Running the script +```./manage.py update_first_ready``` + +### Running locally +```docker-compose exec app ./manage.py update_first_ready``` + ## Populate Domain Request Dates This section outlines how to run the populate_domain_request_dates script diff --git a/src/registrar/management/commands/clean_tables.py b/src/registrar/management/commands/clean_tables.py index 5d4439d95..66b3e772f 100644 --- a/src/registrar/management/commands/clean_tables.py +++ b/src/registrar/management/commands/clean_tables.py @@ -21,7 +21,7 @@ class Command(BaseCommand): TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=""" + prompt_message=""" This script will delete all rows from the following tables: * Contact * Domain diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index cefc38b9e..ac083da1d 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -130,7 +130,7 @@ class Command(BaseCommand): """Asks if the user wants to proceed with this action""" TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=f""" + prompt_message=f""" ==Extension Amount== Period: {extension_amount} year(s) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 122795400..35cc248ee 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -64,7 +64,7 @@ class Command(BaseCommand): # Will sys.exit() when prompt is "n" TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=f""" + prompt_message=f""" ==Master data file== domain_additional_filename: {org_args.domain_additional_filename} @@ -84,7 +84,7 @@ class Command(BaseCommand): # Will sys.exit() when prompt is "n" TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=f""" + prompt_message=f""" ==Master data file== domain_additional_filename: {org_args.domain_additional_filename} diff --git a/src/registrar/management/commands/load_senior_official_table.py b/src/registrar/management/commands/load_senior_official_table.py index 43f61d57a..cdbc607bf 100644 --- a/src/registrar/management/commands/load_senior_official_table.py +++ b/src/registrar/management/commands/load_senior_official_table.py @@ -27,7 +27,7 @@ class Command(BaseCommand): TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=f""" + prompt_message=f""" ==Proposed Changes== CSV: {federal_cio_csv_path} diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 4132096c8..c2dd66f55 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -651,7 +651,7 @@ class Command(BaseCommand): title = "Do you wish to load additional data for TransitionDomains?" proceed = TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=f""" + prompt_message=f""" !!! ENSURE THAT ALL FILENAMES ARE CORRECT BEFORE PROCEEDING ==Master data file== domain_additional_filename: {domain_additional_filename} diff --git a/src/registrar/management/commands/patch_federal_agency_info.py b/src/registrar/management/commands/patch_federal_agency_info.py index b286f1516..51a98ffaa 100644 --- a/src/registrar/management/commands/patch_federal_agency_info.py +++ b/src/registrar/management/commands/patch_federal_agency_info.py @@ -91,7 +91,7 @@ class Command(BaseCommand): # Code execution will stop here if the user prompts "N" TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=f""" + prompt_message=f""" ==Proposed Changes== Number of DomainInformation objects to change: {len(human_readable_domain_names)} The following DomainInformation objects will be modified: {human_readable_domain_names} @@ -148,7 +148,7 @@ class Command(BaseCommand): # Code execution will stop here if the user prompts "N" TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=f""" + prompt_message=f""" ==File location== current-full.csv filepath: {file_path} diff --git a/src/registrar/management/commands/populate_first_ready.py b/src/registrar/management/commands/populate_first_ready.py index 9636476c2..04468029a 100644 --- a/src/registrar/management/commands/populate_first_ready.py +++ b/src/registrar/management/commands/populate_first_ready.py @@ -31,7 +31,7 @@ class Command(BaseCommand): # Code execution will stop here if the user prompts "N" TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=f""" + prompt_message=f""" ==Proposed Changes== Number of Domain objects to change: {len(domains)} """, diff --git a/src/registrar/management/commands/populate_organization_type.py b/src/registrar/management/commands/populate_organization_type.py index a7dd98b24..60d179cb8 100644 --- a/src/registrar/management/commands/populate_organization_type.py +++ b/src/registrar/management/commands/populate_organization_type.py @@ -54,7 +54,7 @@ class Command(BaseCommand): # Code execution will stop here if the user prompts "N" TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=f""" + prompt_message=f""" ==Proposed Changes== Number of DomainRequest objects to change: {len(domain_requests)} @@ -72,7 +72,7 @@ class Command(BaseCommand): # Code execution will stop here if the user prompts "N" TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=f""" + prompt_message=f""" ==Proposed Changes== Number of DomainInformation objects to change: {len(domain_infos)} diff --git a/src/registrar/management/commands/update_first_ready.py b/src/registrar/management/commands/update_first_ready.py new file mode 100644 index 000000000..f1ebdd555 --- /dev/null +++ b/src/registrar/management/commands/update_first_ready.py @@ -0,0 +1,39 @@ +import logging +from django.core.management import BaseCommand +from django.db.models.manager import BaseManager +from registrar.management.commands.utility.terminal_helper import PopulateScriptTemplate, TerminalColors +from registrar.models import Domain, TransitionDomain + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand, PopulateScriptTemplate): + help = "Loops through each domain object and populates the last_status_update and first_submitted_date" + + def handle(self, **kwargs): + """Loops through each valid Domain object and updates it's first_ready value if it is out of sync""" + filter_conditions = {"state__in": [Domain.State.READY, Domain.State.ON_HOLD, Domain.State.DELETED]} + self.mass_update_records(Domain, filter_conditions, ["first_ready"], verbose=True) + + def update_record(self, record: Domain): + """Defines how we update the first_ready field""" + # update the first_ready value based on the creation date. + record.first_ready = record.created_at.date() + + logger.info( + f"{TerminalColors.OKCYAN}Updating {record} => first_ready: " f"{record.first_ready}{TerminalColors.ENDC}" + ) + + # check if a transition domain object for this domain name exists, + # or if so whether its first_ready value matches its created_at date + def custom_filter(self, records: BaseManager[Domain]) -> BaseManager[Domain]: + to_include_pks = [] + for record in records: + if ( + TransitionDomain.objects.filter(domain_name=record.name).exists() + and record.first_ready != record.created_at.date() + ): # noqa + to_include_pks.append(record.pk) + + return records.filter(pk__in=to_include_pks) diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index b9e11be5d..e69d54c07 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -2,9 +2,12 @@ import logging import sys from abc import ABC, abstractmethod from django.core.paginator import Paginator +from django.db.models import Model +from django.db.models.manager import BaseManager from typing import List from registrar.utility.enums import LogCode + logger = logging.getLogger(__name__) @@ -75,28 +78,61 @@ class PopulateScriptTemplate(ABC): run_summary_header = None @abstractmethod - def update_record(self, record): - """Defines how we update each field. Must be defined before using mass_update_records.""" + def update_record(self, record: Model): + """Defines how we update each field. + + raises: + NotImplementedError: If not defined before calling mass_update_records. + """ raise NotImplementedError - def mass_update_records(self, object_class, filter_conditions, fields_to_update, debug=True): + def mass_update_records(self, object_class: Model, filter_conditions, fields_to_update, debug=True, verbose=False): """Loops through each valid "object_class" object - specified by filter_conditions - and updates fields defined by fields_to_update using update_record. - You must define update_record before you can use this function. + Parameters: + object_class: The Django model class that you want to perform the bulk update on. + This should be the actual class, not a string of the class name. + + filter_conditions: dictionary of valid Django Queryset filter conditions + (e.g. {'verification_type__isnull'=True}). + + fields_to_update: List of strings specifying which fields to update. + (e.g. ["first_ready_date", "last_submitted_date"]) + + debug: Whether to log script run summary in debug mode. + Default: True. + + verbose: Whether to print a detailed run summary *before* run confirmation. + Default: False. + + Raises: + NotImplementedError: If you do not define update_record before using this function. + TypeError: If custom_filter is not Callable. """ records = object_class.objects.filter(**filter_conditions) if filter_conditions else object_class.objects.all() + + # apply custom filter + records = self.custom_filter(records) + readable_class_name = self.get_class_name(object_class) + # for use in the execution prompt. + proposed_changes = f"""==Proposed Changes== + Number of {readable_class_name} objects to change: {len(records)} + These fields will be updated on each record: {fields_to_update} + """ + + if verbose: + proposed_changes = f"""{proposed_changes} + These records will be updated: {list(records.all())} + """ + # Code execution will stop here if the user prompts "N" TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, - info_to_inspect=f""" - ==Proposed Changes== - Number of {readable_class_name} objects to change: {len(records)} - These fields will be updated on each record: {fields_to_update} - """, + prompt_message=proposed_changes, prompt_title=self.prompt_title, ) logger.info("Updating...") @@ -141,10 +177,17 @@ class PopulateScriptTemplate(ABC): return f"{TerminalColors.FAIL}" f"Failed to update {record}" f"{TerminalColors.ENDC}" def should_skip_record(self, record) -> bool: # noqa - """Defines the condition in which we should skip updating a record. Override as needed.""" + """Defines the condition in which we should skip updating a record. Override as needed. + The difference between this and custom_filter is that records matching these conditions + *will* be included in the run but will be skipped (and logged as such).""" # By default - don't skip return False + def custom_filter(self, records: BaseManager[Model]) -> BaseManager[Model]: + """Override to define filters that can't be represented by django queryset field lookups. + Applied to individual records *after* filter_conditions. True means""" + return records + class TerminalHelper: @staticmethod @@ -220,6 +263,9 @@ class TerminalHelper: an answer is required of the user). The "answer" return value is True for "yes" or False for "no". + + Raises: + ValueError: When "default" is not "yes", "no", or None. """ valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} if default is None: @@ -244,6 +290,7 @@ class TerminalHelper: @staticmethod def query_yes_no_exit(question: str, default="yes"): """Ask a yes/no question via raw_input() and return their answer. + Allows for answer "e" to exit. "question" is a string that is presented to the user. "default" is the presumed answer if the user just hits . @@ -251,6 +298,9 @@ class TerminalHelper: an answer is required of the user). The "answer" return value is True for "yes" or False for "no". + + Raises: + ValueError: When "default" is not "yes", "no", or None. """ valid = { "yes": True, @@ -317,9 +367,8 @@ class TerminalHelper: case _: logger.info(print_statement) - # TODO - "info_to_inspect" should be refactored to "prompt_message" @staticmethod - def prompt_for_execution(system_exit_on_terminate: bool, info_to_inspect: str, prompt_title: str) -> bool: + def prompt_for_execution(system_exit_on_terminate: bool, prompt_message: str, prompt_title: str) -> bool: """Create to reduce code complexity. Prompts the user to inspect the given string and asks if they wish to proceed. @@ -340,7 +389,7 @@ class TerminalHelper: ===================================================== *** IMPORTANT: VERIFY THE FOLLOWING LOOKS CORRECT *** - {info_to_inspect} + {prompt_message} {TerminalColors.FAIL} Proceed? (Y = proceed, N = {action_description_for_selecting_no}) {TerminalColors.ENDC}"""