diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index b577f062d..1739b5cab 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -97,10 +97,10 @@ Do not use these environments to store data you want to keep around permanently. #### STEP 1: Using cat to transfer data to sandboxes ```bash -cat {LOCAL_PATH_TO_FILE} | cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" +cat {LOCAL_PATH_TO_FILE} | cf ssh {APP_NAME_IN_ENVIRONMENT} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" ``` -* FULL_NAME_OF_YOUR_SANDBOX_HERE - Name of your sandbox, ex: getgov-za +* APP_NAME_IN_ENVIRONMENT - Name of the app running in your environment, e.g. getgov-za or getgov-stable * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt @@ -130,7 +130,7 @@ cf target -o cisa-dotgov -s {SANDBOX_NAME} Use the following command to transfer the desired file: ```shell -scp -P 2222 -o User=cf:$(cf curl /v3/apps/$(cf app {FULL_NAME_OF_YOUR_SANDBOX_HERE} --guid)/processes | jq -r '.resources[] +scp -P 2222 -o User=cf:$(cf curl /v3/apps/$(cf app {APP_NAME_IN_ENVIRONMENT} --guid)/processes | jq -r '.resources[] | select(.type=="web") | .guid')/0 {LOCAL_PATH_TO_FILE} ssh.fr.cloud.gov:tmp/{DESIRED_NAME_OF_FILE} ``` The items in curly braces are the values that you will manually replace. @@ -139,8 +139,6 @@ These are as follows: * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt -NOTE: If you'd wish to change what directory these files are uploaded to, you can change `ssh.fr.cloud.gov:tmp/` to `ssh.fr.cloud.gov:{DIRECTORY_YOU_WANT}/`, but be aware that this makes data migration more tricky than it has to be. - ##### Get a temp auth code The scp command requires a temporary authentication code. Open a new terminal instance (while keeping the current one open), @@ -158,7 +156,7 @@ Due to the nature of how Cloud.gov operates, the getgov directory is dynamically ##### SSH into your sandbox ```shell -cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} +cf ssh {APP_NAME_IN_ENVIRONMENT} ``` ##### Open a shell @@ -196,7 +194,7 @@ cat ../tmp/{filename} > migrationdata/{filename} ``` -*You are now ready to run migration scripts (see "Running the Migration Scripts")* +*You are now ready to run migration scripts (see [Running the Migration Scripts](running-the-migration-scripts))* ### SECTION 2 - LOCAL MIGRATION SETUP (TESTING PURPOSES ONLY) diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index 6c46994ea..c35d2f21a 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -3,6 +3,7 @@ import glob import logging import os +import shutil import string from django.core.management import BaseCommand @@ -36,65 +37,22 @@ class Command(BaseCommand): if not file_extension or not isinstance(file_extension, str): raise ValueError(f"Invalid file extension '{file_extension}'") - matching_extensions = glob.glob(f"../tmp/*.{file_extension}") - if not matching_extensions: + matching_files = glob.glob(f"../tmp/*.{file_extension}") + if not matching_files: logger.error(f"No files with the extension {file_extension} found") + return None - for src_file_path in matching_extensions: + for src_file_path in matching_files: filename = os.path.basename(src_file_path) - exit_status = -1 - do_command = True - - desired_file_path = f"{directory}/{filename}" + + desired_file_path = os.path.join(directory, filename) if os.path.exists(desired_file_path): # For linter prompt = "Do you want to replace it?" replace = f"{desired_file_path} already exists. {prompt}" if not helper.query_yes_no(replace): - do_command = False + continue + + src_file_path = f"../tmp/{filename}" + shutil.copy(src_file_path, desired_file_path) - try: - if do_command: - copy_from = f"../tmp/{filename}" - exit_status = self.cat(copy_from, desired_file_path) - except ValueError as err: - raise err - finally: - if exit_status == 0: - logger.info(f"Successfully copied {filename}") - else: - logger.error(f"Failed to copy {filename}") - - def cat(self, copy_from, copy_to): - """Runs the cat command to - copy_from a location to copy_to a location""" - - # copy_from will be system defined - self.check_file_path(copy_from, check_directory=False) - self.check_file_path(copy_to) - - # This command can only be ran from inside cf ssh getgov-{sandbox} - # It has no utility when running locally, and to exploit this - # you would have to have ssh access anyway, which is a bigger problem. - exit_status = os.system(f"cat {copy_from} > {copy_to}") # nosec - return exit_status - - def check_file_path(self, file_path: str, check_directory=True): - """Does a check on user input to ensure validity""" - if not isinstance(file_path, str): - raise ValueError("Invalid path provided") - - # Remove any initial/final whitespace - file_path = file_path.strip() - - # Check for any attempts to move up in the directory structure - if ".." in file_path and check_directory: - raise ValueError("Moving up in the directory structure is not allowed") - - # Check for any invalid characters - valid_chars = f"/-_.() {string.ascii_letters}{string.digits}" - for char in file_path: - if char not in valid_chars: - raise ValueError(f"Invalid character {char} in file path") - - return file_path