From 6031e2c3c9c6db95282eb390b6637185d7f7af67 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 11:39:33 -0600 Subject: [PATCH 01/10] Update docs/operations/data_migration.md Co-authored-by: Neil MartinsenBurrell --- docs/operations/data_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 32d349278..425065207 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -92,7 +92,7 @@ We can do this both locally and in a sandbox. Load migration data onto a production or sandbox environment **WARNING:** All files uploaded in this manner are temporary, i.e. they will be deleted when the app is restaged. -Do not use this method to store data you want to keep around permanently. +Do not use these environments to store data you want to keep around permanently. We don't want sensitive data to be accidentally present in our application environments. #### STEP 1: Using cat to transfer data to sandboxes From 27465169aa05eb7678e5e2f0f4e974c08f9089b3 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 11:41:33 -0600 Subject: [PATCH 02/10] Update docs/operations/data_migration.md Co-authored-by: Neil MartinsenBurrell --- docs/operations/data_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 425065207..b577f062d 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -135,7 +135,7 @@ scp -P 2222 -o User=cf:$(cf curl /v3/apps/$(cf app {FULL_NAME_OF_YOUR_SANDBOX_HE ``` The items in curly braces are the values that you will manually replace. These are as follows: -* FULL_NAME_OF_YOUR_SANDBOX_HERE - Name of your sandbox, ex: getgov-za +* APP_NAME_IN_ENVIRONMENT - Name of the app running in your environment, e.g. getgov-za or getgov-stable * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt From a0b405f713a9361e7b3fcca083e174204d277e73 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 2 Nov 2023 12:49:25 -0500 Subject: [PATCH 03/10] Scrubbed test files of any possible PII --- src/registrar/tests/data/test_contacts.txt | 14 +++++++------- .../tests/data/test_domain_contacts.txt | 16 ++++++++-------- .../tests/data/test_domain_statuses.txt | 6 +++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/registrar/tests/data/test_contacts.txt b/src/registrar/tests/data/test_contacts.txt index dec8f6816..89f57ccf8 100644 --- a/src/registrar/tests/data/test_contacts.txt +++ b/src/registrar/tests/data/test_contacts.txt @@ -1,7 +1,7 @@ -TESTUSER|52563_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||testuser@gmail.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:18:09Z| -RJD1|52545_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||agustina.wyman7@test.com|GSA|VERISIGN|ctldbatch|2021-06-29T18:53:09Z|VERISIGN|ctldbatch|2021-06-29T18:58:08Z| -JAKING|52555_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||susy.martin4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-06-30T15:38:10Z| -JBOONE|52556_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||stephania.winters4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-06-30T18:28:09Z| -MKELLEY|52557_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||alexandra.bobbitt5@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-08-02T22:13:09Z| -CWILSON|52562_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||jospeh.mcdowell3@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:33:09Z| -LMCCADE|52563_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||reginald.ratcliff4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:18:09Z| \ No newline at end of file +TESTUSER|12363_CONTACT|123-123-1234||918-000-0000||testuser@gmail.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:18:09Z| +USER1|12345_CONTACT|123-123-1234||918-000-0000||agustina.wyman7@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-29T18:53:09Z|SOMECOMPANY|ctldbatch|2021-06-29T18:58:08Z| +USER2|12355_CONTACT|123-123-1234||918-000-0000||susy.martin4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T15:23:10Z|SOMECOMPANY|ctldbatch|2021-06-30T15:38:10Z| +USER3|12356_CONTACT|123-123-1234||918-000-0000||stephania.winters4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T15:23:10Z|SOMECOMPANY|ctldbatch|2021-06-30T18:28:09Z| +USER4|12357_CONTACT|123-123-1234||918-000-0000||alexandra.bobbitt5@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T15:23:10Z|SOMECOMPANY|ctldbatch|2021-08-02T22:13:09Z| +USER5|12362_CONTACT|123-123-1234||918-000-0000||jospeh.mcdowell3@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:33:09Z| +USER6|12363_CONTACT|123-123-1234||918-000-0000||reginald.ratcliff4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:18:09Z| \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_contacts.txt b/src/registrar/tests/data/test_domain_contacts.txt index 069e5231e..3a1ed745f 100644 --- a/src/registrar/tests/data/test_domain_contacts.txt +++ b/src/registrar/tests/data/test_domain_contacts.txt @@ -1,8 +1,8 @@ -Anomaly.gov|ANOMALY|tech -TestDomain.gov|TESTUSER|admin -NEHRP.GOV|RJD1|admin -NEHRP.GOV|JAKING|tech -NEHRP.GOV|JBOONE|billing -NELSONCOUNTY-VA.GOV|MKELLEY|admin -NELSONCOUNTY-VA.GOV|CWILSON|billing -NELSONCOUNTY-VA.GOV|LMCCADE|tech \ No newline at end of file +Anomaly.gov|ANOMALY|tech +TestDomain.gov|TESTUSER|admin +FakeWebsite1|USER1|admin +FakeWebsite1|USER2|tech +FakeWebsite1|USER3|billing +FakeWebsite2.GOV|USER4|admin +FakeWebsite2.GOV|USER5|billing +FakeWebsite2.GOV|USER6|tech \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_statuses.txt b/src/registrar/tests/data/test_domain_statuses.txt index 1f3cc8998..021e52ae7 100644 --- a/src/registrar/tests/data/test_domain_statuses.txt +++ b/src/registrar/tests/data/test_domain_statuses.txt @@ -1,4 +1,4 @@ Anomaly.gov|muahaha| -TestDomain.gov|ok| -NEHRP.GOV|serverHold| -NELSONCOUNTY-VA.GOV|Hold| \ No newline at end of file +TestDomain.gov|ok| +FakeWebsite1.GOV|serverHold| +FakeWebsite2.GOV|Hold| \ No newline at end of file From 183a4bf80156c13735d47eebc18efcbd6dcd413b Mon Sep 17 00:00:00 2001 From: CuriousX Date: Thu, 2 Nov 2023 12:58:50 -0500 Subject: [PATCH 04/10] Update data_migration.md --- docs/operations/data_migration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index b577f062d..68a6419d5 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -97,10 +97,10 @@ Do not use these environments to store data you want to keep around permanently. #### STEP 1: Using cat to transfer data to sandboxes ```bash -cat {LOCAL_PATH_TO_FILE} | cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" +cat {LOCAL_PATH_TO_FILE} | cf ssh {APP_NAME_IN_ENVIRONMENT} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" ``` -* FULL_NAME_OF_YOUR_SANDBOX_HERE - Name of your sandbox, ex: getgov-za +* APP_NAME_IN_ENVIRONMENT - Name of your sandbox, ex: getgov-za * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt From 01d50fe1b32b166627ee96d4ffb56416b1991aa2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:15:51 -0600 Subject: [PATCH 05/10] Documentation updates / fix script --- docs/operations/data_migration.md | 12 ++-- .../commands/cat_files_into_getgov.py | 64 ++++--------------- 2 files changed, 16 insertions(+), 60 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index b577f062d..1739b5cab 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -97,10 +97,10 @@ Do not use these environments to store data you want to keep around permanently. #### STEP 1: Using cat to transfer data to sandboxes ```bash -cat {LOCAL_PATH_TO_FILE} | cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" +cat {LOCAL_PATH_TO_FILE} | cf ssh {APP_NAME_IN_ENVIRONMENT} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" ``` -* FULL_NAME_OF_YOUR_SANDBOX_HERE - Name of your sandbox, ex: getgov-za +* APP_NAME_IN_ENVIRONMENT - Name of the app running in your environment, e.g. getgov-za or getgov-stable * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt @@ -130,7 +130,7 @@ cf target -o cisa-dotgov -s {SANDBOX_NAME} Use the following command to transfer the desired file: ```shell -scp -P 2222 -o User=cf:$(cf curl /v3/apps/$(cf app {FULL_NAME_OF_YOUR_SANDBOX_HERE} --guid)/processes | jq -r '.resources[] +scp -P 2222 -o User=cf:$(cf curl /v3/apps/$(cf app {APP_NAME_IN_ENVIRONMENT} --guid)/processes | jq -r '.resources[] | select(.type=="web") | .guid')/0 {LOCAL_PATH_TO_FILE} ssh.fr.cloud.gov:tmp/{DESIRED_NAME_OF_FILE} ``` The items in curly braces are the values that you will manually replace. @@ -139,8 +139,6 @@ These are as follows: * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt -NOTE: If you'd wish to change what directory these files are uploaded to, you can change `ssh.fr.cloud.gov:tmp/` to `ssh.fr.cloud.gov:{DIRECTORY_YOU_WANT}/`, but be aware that this makes data migration more tricky than it has to be. - ##### Get a temp auth code The scp command requires a temporary authentication code. Open a new terminal instance (while keeping the current one open), @@ -158,7 +156,7 @@ Due to the nature of how Cloud.gov operates, the getgov directory is dynamically ##### SSH into your sandbox ```shell -cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} +cf ssh {APP_NAME_IN_ENVIRONMENT} ``` ##### Open a shell @@ -196,7 +194,7 @@ cat ../tmp/{filename} > migrationdata/{filename} ``` -*You are now ready to run migration scripts (see "Running the Migration Scripts")* +*You are now ready to run migration scripts (see [Running the Migration Scripts](running-the-migration-scripts))* ### SECTION 2 - LOCAL MIGRATION SETUP (TESTING PURPOSES ONLY) diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index 6c46994ea..c35d2f21a 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -3,6 +3,7 @@ import glob import logging import os +import shutil import string from django.core.management import BaseCommand @@ -36,65 +37,22 @@ class Command(BaseCommand): if not file_extension or not isinstance(file_extension, str): raise ValueError(f"Invalid file extension '{file_extension}'") - matching_extensions = glob.glob(f"../tmp/*.{file_extension}") - if not matching_extensions: + matching_files = glob.glob(f"../tmp/*.{file_extension}") + if not matching_files: logger.error(f"No files with the extension {file_extension} found") + return None - for src_file_path in matching_extensions: + for src_file_path in matching_files: filename = os.path.basename(src_file_path) - exit_status = -1 - do_command = True - - desired_file_path = f"{directory}/{filename}" + + desired_file_path = os.path.join(directory, filename) if os.path.exists(desired_file_path): # For linter prompt = "Do you want to replace it?" replace = f"{desired_file_path} already exists. {prompt}" if not helper.query_yes_no(replace): - do_command = False + continue + + src_file_path = f"../tmp/{filename}" + shutil.copy(src_file_path, desired_file_path) - try: - if do_command: - copy_from = f"../tmp/{filename}" - exit_status = self.cat(copy_from, desired_file_path) - except ValueError as err: - raise err - finally: - if exit_status == 0: - logger.info(f"Successfully copied {filename}") - else: - logger.error(f"Failed to copy {filename}") - - def cat(self, copy_from, copy_to): - """Runs the cat command to - copy_from a location to copy_to a location""" - - # copy_from will be system defined - self.check_file_path(copy_from, check_directory=False) - self.check_file_path(copy_to) - - # This command can only be ran from inside cf ssh getgov-{sandbox} - # It has no utility when running locally, and to exploit this - # you would have to have ssh access anyway, which is a bigger problem. - exit_status = os.system(f"cat {copy_from} > {copy_to}") # nosec - return exit_status - - def check_file_path(self, file_path: str, check_directory=True): - """Does a check on user input to ensure validity""" - if not isinstance(file_path, str): - raise ValueError("Invalid path provided") - - # Remove any initial/final whitespace - file_path = file_path.strip() - - # Check for any attempts to move up in the directory structure - if ".." in file_path and check_directory: - raise ValueError("Moving up in the directory structure is not allowed") - - # Check for any invalid characters - valid_chars = f"/-_.() {string.ascii_letters}{string.digits}" - for char in file_path: - if char not in valid_chars: - raise ValueError(f"Invalid character {char} in file path") - - return file_path From 32d5cfac6fb83e07e679a9ed533418d83ba8a6de Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:17:46 -0600 Subject: [PATCH 06/10] Run black --- src/registrar/management/commands/cat_files_into_getgov.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index c35d2f21a..4cd17f6e2 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -44,7 +44,7 @@ class Command(BaseCommand): for src_file_path in matching_files: filename = os.path.basename(src_file_path) - + desired_file_path = os.path.join(directory, filename) if os.path.exists(desired_file_path): # For linter @@ -52,7 +52,6 @@ class Command(BaseCommand): replace = f"{desired_file_path} already exists. {prompt}" if not helper.query_yes_no(replace): continue - + src_file_path = f"../tmp/{filename}" shutil.copy(src_file_path, desired_file_path) - From 4d5f104683de4d87c206537f3ca4f9a95dccc63c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:28:10 -0600 Subject: [PATCH 07/10] Fix md --- docs/operations/data_migration.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 1739b5cab..6a42962e8 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -122,9 +122,9 @@ cf login -a api.fr.cloud.gov --sso ##### Target your workspace ```bash -cf target -o cisa-dotgov -s {SANDBOX_NAME} +cf target -o cisa-dotgov -s {ENVIRONMENT_NAME} ``` -*SANDBOX_NAME* - Name of your sandbox, ex: za or ab +*ENVIRONMENT_NAME* - Name of your sandbox, ex: za or ab ##### Run the scp command @@ -203,8 +203,7 @@ cat ../tmp/{filename} > migrationdata/{filename} In order to run the scripts locally, we need to add the files to a folder under `src/`. This will allow Docker to mount the files to a container (under `/app`) for our use. - - Create a folder called `tmp` underneath `src/` - - Add the above files to this folder + - Add the above files to the `migrationdata/` folder - Open a terminal and navigate to `src/` From 4b7fd9c34b716d33f07e180713715073e17bceb1 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:30:12 -0600 Subject: [PATCH 08/10] Update docs/operations/data_migration.md Co-authored-by: Neil MartinsenBurrell --- docs/operations/data_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 6a42962e8..3821b0029 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -207,7 +207,7 @@ This will allow Docker to mount the files to a container (under `/app`) for our - Open a terminal and navigate to `src/` -*You are now ready to run migration scripts (see "Running the Migration Scripts")* +*You are now ready to run migration scripts.* ## Transition Domains (Part 2) - Running the Migration Scripts From 997f0c88386da0c8ca4fb31f5903c67172d8b228 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:30:35 -0600 Subject: [PATCH 09/10] Update docs/operations/data_migration.md Co-authored-by: Neil MartinsenBurrell --- docs/operations/data_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 3821b0029..2dd00a39f 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -215,7 +215,7 @@ This will allow Docker to mount the files to a container (under `/app`) for our ### STEP 1: Load Transition Domains -Run the following command, making sure the filepaths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. (NOTE: If working in the sandbox, change "/app/tmp" to point to the sandbox directory) +Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. (NOTE: If working in cloud.gov, change "/app/tmp" to point to the `migrationdata/` directory) ```shell docker compose run -T app ./manage.py load_transition_domain /app/tmp/escrow_domain_contacts.daily.gov.GOV.txt /app/tmp/escrow_contacts.daily.gov.GOV.txt /app/tmp/escrow_domain_statuses.daily.gov.GOV.txt --debug ``` From dcbe41f10791a4fa16cee00db11f647a31108bda Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:33:14 -0600 Subject: [PATCH 10/10] Fix linter issue --- src/registrar/management/commands/cat_files_into_getgov.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index 4cd17f6e2..993643ee4 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -4,7 +4,6 @@ import logging import os import shutil -import string from django.core.management import BaseCommand