diff --git a/src/registrar/management/commands/agency_data_extractor.py b/src/registrar/management/commands/agency_data_extractor.py index 3b7931f8d..d5b304a8a 100644 --- a/src/registrar/management/commands/agency_data_extractor.py +++ b/src/registrar/management/commands/agency_data_extractor.py @@ -78,6 +78,12 @@ class Command(BaseCommand): possibly_unused_agencies.append(agency) TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}") + matched_agencies = [] + for agency in provided_agencies: + if agency in existing_agencies: + matched_agencies.append(agency) + TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Matched agencies: {agency}{TerminalColors.ENDC}") + # Print the summary of findings # 1 - Print the list of agencies in the NEW list, which we do not already have # 2 - Print the list of agencies that we currently have, which are NOT in the new list (these might be eligible for removal?) TODO: would we ever want to remove existing agencies? @@ -87,6 +93,9 @@ class Command(BaseCommand): possibly_unused_agencies_as_string = "{}".format( ",\n ".join(map(str, possibly_unused_agencies)) ) + matched_agencies_as_string = "{}".format( + ",\n ".join(map(str, matched_agencies)) + ) logger.info(f""" {TerminalColors.OKGREEN} @@ -96,6 +105,8 @@ class Command(BaseCommand): {len(provided_agencies)-len(new_agencies)} AGENCIES MATCHED (These are agencies that are in the given agency file AND in our system already) + {TerminalColors.YELLOW}{matched_agencies_as_string} + {TerminalColors.OKGREEN} {len(new_agencies)} AGENCIES TO ADD: These agencies were in the provided agency file, but are not in our system. @@ -139,13 +150,19 @@ class Command(BaseCommand): new_agencies = self.extract_agencies(agency_data_file, sep, debug) hard_coded_agencies = DomainApplication.AGENCIES + transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency', flat=True).distinct() + print(transition_domain_agencies) + + merged_agencies = new_agencies for agency in hard_coded_agencies: if agency not in merged_agencies: merged_agencies.append(agency) - - transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency').distinct() - print(transition_domain_agencies) + + merged_transition_agencies = new_agencies + for agency in transition_domain_agencies: + if agency not in merged_transition_agencies: + merged_transition_agencies.append(agency) prompt_successful = False @@ -208,4 +225,16 @@ class Command(BaseCommand): f"\nThese are all the agencies our dropdown plus all the agencies in the agency file." f"\n\n{len(merged_agencies)} TOTAL\n\n" ) - self.print_agency_list(merged_agencies, "Merged_Dropdown_Agency_List") \ No newline at end of file + self.print_agency_list(merged_agencies, "Merged_Dropdown_Agency_List") + + # OPTION to print out the full list of agencies from the agency file + if prompt: + prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}") + if prompt_successful or not prompt: + logger.info( + f"\n{TerminalColors.OKGREEN}" + f"\n======================== MERGED LISTS (transition domain + agency file) ============================" + f"\nThese are all the agencies our transition domains table plus all the agencies in the agency file." + f"\n\n{len(merged_agencies)} TOTAL\n\n" + ) + self.print_agency_list(merged_transition_agencies, "Merged_Transition_Domain_Agency_List") \ No newline at end of file diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 31ccf1cd6..cb85bd2d8 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -381,8 +381,10 @@ class Command(BaseCommand): # Start parsing the main file and create TransitionDomain objects logger.info("Reading domain-contacts data file %s", domain_contacts_filename) + total_lines = TerminalHelper.get_file_line_count(domain_contacts_filename) with open(domain_contacts_filename, "r") as domain_contacts_file: for row in csv.reader(domain_contacts_file, delimiter=sep): + # TerminalHelper.printProgressBar(total_rows_parsed, total_lines) total_rows_parsed += 1 # fields are just domain, userid, role @@ -394,6 +396,8 @@ class Command(BaseCommand): new_entry_email = "" new_entry_emailSent = False # set to False by default + TerminalHelper.print_conditional(debug_on, f"Processing item {total_rows_parsed}: {new_entry_domain_name}") + # PART 1: Get the status if new_entry_domain_name not in domain_status_dictionary: # This domain has no status...default to "Create" diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index faba8a037..ea0e827fa 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -296,25 +296,27 @@ class Command(BaseCommand): command_string += f"--limitParse {debug_max_entries_to_parse} " # Execute the command string + proceed = False if prompts_enabled: - system_exit_on_terminate = True - TerminalHelper.prompt_for_execution( - system_exit_on_terminate, + proceed = TerminalHelper.prompt_for_execution( + False, command_string, "Running load_transition_domain script", ) # TODO: make this somehow run inside TerminalHelper prompt - call_command( - command_script, - f"{file_location+domain_contacts_filename}", - f"{file_location+contacts_filename}", - f"{file_location+domain_statuses_filename}", - sep=sep, - resetTable=reset_table, - debug=debug_on, - limitParse=debug_max_entries_to_parse, - ) + if proceed: + call_command( + command_script, + f"{file_location+domain_contacts_filename}", + f"{file_location+contacts_filename}", + f"{file_location+domain_statuses_filename}", + sep=sep, + resetTable=reset_table, + debug=debug_on, + limitParse=debug_max_entries_to_parse, + directory=file_location + ) def run_transfer_script(self, debug_on: bool, prompts_enabled: bool): """Runs the transfer_transition_domains_to_domains script""" @@ -324,16 +326,16 @@ class Command(BaseCommand): if debug_on: command_string += "--debug " # Execute the command string + proceed = False if prompts_enabled: - system_exit_on_terminate = True - TerminalHelper.prompt_for_execution( - system_exit_on_terminate, + proceed = TerminalHelper.prompt_for_execution( + False, command_string, "Running transfer_transition_domains_to_domains script", ) - # TODO: make this somehow run inside TerminalHelper prompt - call_command(command_script) + if proceed: + call_command(command_script) def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): """Runs the send_domain_invitations script""" @@ -341,16 +343,17 @@ class Command(BaseCommand): command_script = "send_domain_invitations" command_string = f"./manage.py {command_script} -s" # Execute the command string + proceed = False if prompts_enabled: - system_exit_on_terminate = True - TerminalHelper.prompt_for_execution( - system_exit_on_terminate, + proceed = TerminalHelper.prompt_for_execution( + False, command_string, "Running send_domain_invitations script", ) # TODO: make this somehow run inside TerminalHelper prompt - call_command(command_script, send_emails=True) + if proceed: + call_command(command_script, send_emails=True) def run_migration_scripts( self, diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index f00f72bc2..734beb408 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -197,7 +197,8 @@ class LoadExtraTransitionDomain: logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== - updated {len(updated_transition_domains)} transition domain entries + updated {len(updated_trasition_domains)} transition domain entries: + {TerminalHelper.array_as_string(updated_trasition_domains)} {TerminalColors.ENDC} """ ) diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 78ffb525f..5ec9408cd 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -70,12 +70,45 @@ class TerminalHelper: else: logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") + @staticmethod + def query_yes_no_exit(question: str, default="yes") -> bool: + """Ask a yes/no question via raw_input() and return their answer. + + "question" is a string that is presented to the user. + "default" is the presumed answer if the user just hits . + It must be "yes" (the default), "no" or None (meaning + an answer is required of the user). + + The "answer" return value is True for "yes" or False for "no". + """ + valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False, "e": "exit", "s": "skip"} + if default is None: + prompt = " [y/n] " + elif default == "yes": + prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " + else: + raise ValueError("invalid default answer: '%s'" % default) + + while True: + logger.info(question + prompt) + choice = input().lower() + if default is not None and choice == "": + return valid[default] + elif choice in valid: + if valid[choice] == "exit": + sys.exit() + return valid[choice] + else: + logger.info("Please respond with a valid selection.\n") + # @staticmethod - # def array_as_string(array_to_convert: []) -> str: - # array_as_string = "{}".format( - # ", ".join(map(str, array_to_convert)) - # ) - # return array_as_string + def array_as_string(array_to_convert: []) -> str: + array_as_string = "{}".format( + "\n".join(map(str, array_to_convert)) + ) + return array_as_string @staticmethod def print_conditional( @@ -118,13 +151,13 @@ class TerminalHelper: Returns true if the user responds (y), Returns false if the user responds (n)""" - action_description_for_selecting_no = "skip" + action_description_for_selecting_no = "skip, E = exit" if system_exit_on_terminate: - action_description_for_selecting_no = "exit" + action_description_for_selecting_no = "exit, S = skip" # Allow the user to inspect the command string # and ask if they wish to proceed - proceed_execution = TerminalHelper.query_yes_no( + proceed_execution = TerminalHelper.query_yes_no_exit( f"""{TerminalColors.OKCYAN} ===================================================== {prompt_title} @@ -139,10 +172,52 @@ class TerminalHelper: # If the user decided to proceed return true. # Otherwise, either return false or exit this subroutine. - if not proceed_execution: + if proceed_execution == False: if system_exit_on_terminate: sys.exit() return False - + if proceed_execution == "skip": + return False return True + @staticmethod + def get_file_line_count(filepath: str) -> int: + with open(filepath,'r') as file: + li = file.readlines() + total_line = len(li) + return total_line + + + + @staticmethod + def printProgressBar (iteration, total, prefix = 'Progress:', suffix = 'Complete', decimals = 1, length = 100, fill = '█', printEnd = "\r"): + """ + Call in a loop to create terminal progress bar + @params: + iteration - Required : current iteration (Int) + total - Required : total iterations (Int) + prefix - Optional : prefix string (Str) + suffix - Optional : suffix string (Str) + decimals - Optional : positive number of decimals in percent complete (Int) + length - Optional : character length of bar (Int) + fill - Optional : bar fill character (Str) + printEnd - Optional : end character (e.g. "\r", "\r\n") (Str) + """ + + """ + # Initial call to print 0% progress + printProgressBar(0, l, prefix = 'Progress:', suffix = 'Complete', length = 50) + for i, item in enumerate(items): + # Do stuff... + time.sleep(0.1) + # Update Progress Bar + printProgressBar(i + 1, l, prefix = 'Progress:', suffix = 'Complete', length = 50) + """ + + percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total))) + filledLength = int(length * iteration // total) + bar = fill * filledLength + '-' * (length - filledLength) + print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd) + # Print New Line on Complete + if iteration == total: + print() \ No newline at end of file