Updates to prompts and print statements for master migration to be easier to work with

src/registrar/management/commands/agency_data_extractor.py

@@ -78,6 +78,12 @@ class Command(BaseCommand):
                 possibly_unused_agencies.append(agency)
                 TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}")
 
+        matched_agencies = []
+        for agency in provided_agencies:
+            if agency in existing_agencies:
+                matched_agencies.append(agency)
+                TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Matched agencies: {agency}{TerminalColors.ENDC}")
+
         # Print the summary of findings
         # 1 - Print the list of agencies in the NEW list, which we do not already have
         # 2 - Print the list of agencies that we currently have, which are NOT in the new list (these might be eligible for removal?) TODO: would we ever want to remove existing agencies?
@@ -87,6 +93,9 @@ class Command(BaseCommand):
         possibly_unused_agencies_as_string = "{}".format(
             ",\n        ".join(map(str, possibly_unused_agencies))
         )
+        matched_agencies_as_string = "{}".format(
+            ",\n        ".join(map(str, matched_agencies))
+        )
 
         logger.info(f"""
         {TerminalColors.OKGREEN}
@@ -96,6 +105,8 @@ class Command(BaseCommand):
 
         {len(provided_agencies)-len(new_agencies)} AGENCIES MATCHED
         (These are agencies that are in the given agency file AND in our system already)
+        {TerminalColors.YELLOW}{matched_agencies_as_string}
+        {TerminalColors.OKGREEN}
         
         {len(new_agencies)} AGENCIES TO ADD:
         These agencies were in the provided agency file, but are not in our system.
@@ -139,13 +150,19 @@ class Command(BaseCommand):
 
         new_agencies = self.extract_agencies(agency_data_file, sep, debug)
         hard_coded_agencies = DomainApplication.AGENCIES
+        transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency', flat=True).distinct()
+        print(transition_domain_agencies)
+
+
         merged_agencies = new_agencies
         for agency in hard_coded_agencies:
             if agency not in merged_agencies:
                 merged_agencies.append(agency)
         
-        transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency').distinct()
+        merged_transition_agencies = new_agencies
-        print(transition_domain_agencies)
+        for agency in transition_domain_agencies:
+            if agency not in merged_transition_agencies:
+                merged_transition_agencies.append(agency)
 
         prompt_successful = False
 
@@ -209,3 +226,15 @@ class Command(BaseCommand):
             f"\n\n{len(merged_agencies)} TOTAL\n\n"
             )
             self.print_agency_list(merged_agencies, "Merged_Dropdown_Agency_List")
+        
+         # OPTION to print out the full list of agencies from the agency file
+        if prompt:
+            prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}")
+        if prompt_successful or not prompt:
+            logger.info(
+            f"\n{TerminalColors.OKGREEN}"
+            f"\n======================== MERGED LISTS (transition domain + agency file) ============================"
+            f"\nThese are all the agencies our transition domains table plus all the agencies in the agency file."
+            f"\n\n{len(merged_agencies)} TOTAL\n\n"
+            )
+            self.print_agency_list(merged_transition_agencies, "Merged_Transition_Domain_Agency_List")

src/registrar/management/commands/load_transition_domain.py

@@ -381,8 +381,10 @@ class Command(BaseCommand):
 
         # Start parsing the main file and create TransitionDomain objects
         logger.info("Reading domain-contacts data file %s", domain_contacts_filename)
+        total_lines = TerminalHelper.get_file_line_count(domain_contacts_filename)
         with open(domain_contacts_filename, "r") as domain_contacts_file:
             for row in csv.reader(domain_contacts_file, delimiter=sep):
+                # TerminalHelper.printProgressBar(total_rows_parsed, total_lines)
                 total_rows_parsed += 1
 
                 # fields are just domain, userid, role
@@ -394,6 +396,8 @@ class Command(BaseCommand):
                 new_entry_email = ""
                 new_entry_emailSent = False  # set to False by default
 
+                TerminalHelper.print_conditional(debug_on, f"Processing item {total_rows_parsed}: {new_entry_domain_name}")
+
                 # PART 1: Get the status
                 if new_entry_domain_name not in domain_status_dictionary:
                     # This domain has no status...default to "Create"

src/registrar/management/commands/master_domain_migrations.py

@@ -296,25 +296,27 @@ class Command(BaseCommand):
             command_string += f"--limitParse {debug_max_entries_to_parse} "
 
         # Execute the command string
+        proceed = False
         if prompts_enabled:
-            system_exit_on_terminate = True
+            proceed = TerminalHelper.prompt_for_execution(
-            TerminalHelper.prompt_for_execution(
+                False,
-                system_exit_on_terminate,
                 command_string,
                 "Running load_transition_domain script",
             )
 
         # TODO: make this somehow run inside TerminalHelper prompt
-        call_command(
+        if proceed:
-            command_script,
+            call_command(
-            f"{file_location+domain_contacts_filename}",
+                command_script,
-            f"{file_location+contacts_filename}",
+                f"{file_location+domain_contacts_filename}",
-            f"{file_location+domain_statuses_filename}",
+                f"{file_location+contacts_filename}",
-            sep=sep,
+                f"{file_location+domain_statuses_filename}",
-            resetTable=reset_table,
+                sep=sep,
-            debug=debug_on,
+                resetTable=reset_table,
-            limitParse=debug_max_entries_to_parse,
+                debug=debug_on,
-        )
+                limitParse=debug_max_entries_to_parse,
+                directory=file_location
+            )
 
     def run_transfer_script(self, debug_on: bool, prompts_enabled: bool):
         """Runs the transfer_transition_domains_to_domains script"""
@@ -324,16 +326,16 @@ class Command(BaseCommand):
         if debug_on:
             command_string += "--debug "
         # Execute the command string
+        proceed = False
         if prompts_enabled:
-            system_exit_on_terminate = True
+            proceed = TerminalHelper.prompt_for_execution(
-            TerminalHelper.prompt_for_execution(
+                False,
-                system_exit_on_terminate,
                 command_string,
                 "Running transfer_transition_domains_to_domains script",
             )
-
         # TODO: make this somehow run inside TerminalHelper prompt
-        call_command(command_script)
+        if proceed:
+                call_command(command_script)
 
     def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool):
         """Runs the send_domain_invitations script"""
@@ -341,16 +343,17 @@ class Command(BaseCommand):
         command_script = "send_domain_invitations"
         command_string = f"./manage.py {command_script} -s"
         # Execute the command string
+        proceed = False
         if prompts_enabled:
-            system_exit_on_terminate = True
+            proceed = TerminalHelper.prompt_for_execution(
-            TerminalHelper.prompt_for_execution(
+                False,
-                system_exit_on_terminate,
                 command_string,
                 "Running send_domain_invitations script",
             )
 
         # TODO: make this somehow run inside TerminalHelper prompt
-        call_command(command_script, send_emails=True)
+        if proceed:
+            call_command(command_script, send_emails=True)
 
     def run_migration_scripts(
         self,

src/registrar/management/commands/utility/extra_transition_domain_helper.py

@@ -197,7 +197,8 @@ class LoadExtraTransitionDomain:
         logger.info(
             f"""{TerminalColors.OKGREEN}
             ============= FINISHED ===============
-            updated {len(updated_transition_domains)} transition domain entries
+            updated {len(updated_trasition_domains)} transition domain entries:
+            {TerminalHelper.array_as_string(updated_trasition_domains)}
             {TerminalColors.ENDC}
             """
         )

src/registrar/management/commands/utility/terminal_helper.py

@@ -70,12 +70,45 @@ class TerminalHelper:
             else:
                 logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n")
 
+    @staticmethod
+    def query_yes_no_exit(question: str, default="yes") -> bool:
+        """Ask a yes/no question via raw_input() and return their answer.
+
+        "question" is a string that is presented to the user.
+        "default" is the presumed answer if the user just hits <Enter>.
+                It must be "yes" (the default), "no" or None (meaning
+                an answer is required of the user).
+
+        The "answer" return value is True for "yes" or False for "no".
+        """
+        valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False, "e": "exit", "s": "skip"}
+        if default is None:
+            prompt = " [y/n] "
+        elif default == "yes":
+            prompt = " [Y/n] "
+        elif default == "no":
+            prompt = " [y/N] "
+        else:
+            raise ValueError("invalid default answer: '%s'" % default)
+
+        while True:
+            logger.info(question + prompt)
+            choice = input().lower()
+            if default is not None and choice == "":
+                return valid[default]
+            elif choice in valid:
+                if valid[choice] == "exit":
+                    sys.exit()
+                return valid[choice]
+            else:
+                logger.info("Please respond with a valid selection.\n")
+
     # @staticmethod
-    # def array_as_string(array_to_convert: []) -> str:
+    def array_as_string(array_to_convert: []) -> str:
-    #     array_as_string = "{}".format(
+        array_as_string = "{}".format(
-    #         ", ".join(map(str, array_to_convert))
+            "\n".join(map(str, array_to_convert))
-    #     )
+        )
-    #     return array_as_string
+        return array_as_string
 
     @staticmethod
     def print_conditional(
@@ -118,13 +151,13 @@ class TerminalHelper:
         Returns true if the user responds (y),
         Returns false if the user responds (n)"""
 
-        action_description_for_selecting_no = "skip"
+        action_description_for_selecting_no = "skip, E = exit"
         if system_exit_on_terminate:
-            action_description_for_selecting_no = "exit"
+            action_description_for_selecting_no = "exit, S = skip"
 
         # Allow the user to inspect the command string
         # and ask if they wish to proceed
-        proceed_execution = TerminalHelper.query_yes_no(
+        proceed_execution = TerminalHelper.query_yes_no_exit(
             f"""{TerminalColors.OKCYAN}
             =====================================================
             {prompt_title}
@@ -139,10 +172,52 @@ class TerminalHelper:
 
         # If the user decided to proceed return true.
         # Otherwise, either return false or exit this subroutine.
-        if not proceed_execution:
+        if proceed_execution == False:
             if system_exit_on_terminate:
                 sys.exit()
             return False
-
+        if proceed_execution == "skip":
+            return False
         return True
 
+    @staticmethod
+    def get_file_line_count(filepath: str) -> int:
+        with open(filepath,'r') as file:
+            li = file.readlines()
+        total_line = len(li)
+        return total_line
+    
+
+
+    @staticmethod
+    def printProgressBar (iteration, total, prefix = 'Progress:', suffix = 'Complete', decimals = 1, length = 100, fill = '█', printEnd = "\r"):
+        """
+        Call in a loop to create terminal progress bar
+        @params:
+            iteration   - Required  : current iteration (Int)
+            total       - Required  : total iterations (Int)
+            prefix      - Optional  : prefix string (Str)
+            suffix      - Optional  : suffix string (Str)
+            decimals    - Optional  : positive number of decimals in percent complete (Int)
+            length      - Optional  : character length of bar (Int)
+            fill        - Optional  : bar fill character (Str)
+            printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
+        """
+
+        """
+        # Initial call to print 0% progress
+        printProgressBar(0, l, prefix = 'Progress:', suffix = 'Complete', length = 50)
+        for i, item in enumerate(items):
+            # Do stuff...
+            time.sleep(0.1)
+            # Update Progress Bar
+            printProgressBar(i + 1, l, prefix = 'Progress:', suffix = 'Complete', length = 50)
+        """
+
+        percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
+        filledLength = int(length * iteration // total)
+        bar = fill * filledLength + '-' * (length - filledLength)
+        print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd)
+        # Print New Line on Complete
+        if iteration == total: 
+            print()