From a41225be7ebe3500f8af6061887d1d745e5cbf0a Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 19 Jul 2024 18:35:55 -0400 Subject: [PATCH 01/21] updated clean_tables to run in batches of 1000 rows --- .../management/commands/clean_tables.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/registrar/management/commands/clean_tables.py b/src/registrar/management/commands/clean_tables.py index f0c51390b..f7bc37df7 100644 --- a/src/registrar/management/commands/clean_tables.py +++ b/src/registrar/management/commands/clean_tables.py @@ -56,14 +56,24 @@ class Command(BaseCommand): self.clean_table(table_name) def clean_table(self, table_name): - """Delete all rows in the given table""" + """Delete all rows in the given table. + + Delete in batches to be able to handle large tables""" try: # Get the model class dynamically model = apps.get_model("registrar", table_name) - # Use a transaction to ensure database integrity - with transaction.atomic(): - model.objects.all().delete() - logger.info(f"Successfully cleaned table {table_name}") + BATCH_SIZE = 1000 + total_deleted = 0 + while True: + pks = list(model.objects.values_list("pk", flat=True)[:BATCH_SIZE]) + if not pks: + break + # Use a transaction to ensure database integrity + with transaction.atomic(): + deleted, _ = model.objects.filter(pk__in=pks).delete() + total_deleted += deleted + logger.debug(f"Deleted {deleted} objects, total deleted: {total_deleted}") + logger.info(f"Successfully cleaned table {table_name}, deleted {total_deleted} rows") except LookupError: logger.error(f"Model for table {table_name} not found.") except Exception as e: From 2ee6aec639932ba028cdecd3049399f1b97499c8 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 19 Jul 2024 18:45:59 -0400 Subject: [PATCH 02/21] improved logging --- src/registrar/management/commands/clean_tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/clean_tables.py b/src/registrar/management/commands/clean_tables.py index f7bc37df7..dd84b0b62 100644 --- a/src/registrar/management/commands/clean_tables.py +++ b/src/registrar/management/commands/clean_tables.py @@ -72,7 +72,7 @@ class Command(BaseCommand): with transaction.atomic(): deleted, _ = model.objects.filter(pk__in=pks).delete() total_deleted += deleted - logger.debug(f"Deleted {deleted} objects, total deleted: {total_deleted}") + logger.debug(f"Deleted {deleted} {table_name}s, total deleted: {total_deleted}") logger.info(f"Successfully cleaned table {table_name}, deleted {total_deleted} rows") except LookupError: logger.error(f"Model for table {table_name} not found.") From ffbda787ae300d0a0989c3d8f64d81a835c85b1f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 19 Jul 2024 22:10:42 -0400 Subject: [PATCH 03/21] updated tests --- .../tests/test_management_scripts.py | 113 +++++++++++++----- 1 file changed, 84 insertions(+), 29 deletions(-) diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index cfe19b091..8fb754ba0 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -33,6 +33,10 @@ from api.tests.common import less_console_noise_decorator logger = logging.getLogger(__name__) +class CustomDeleteException(Exception): + pass + + class TestPopulateVerificationType(MockEppLib): """Tests for the populate_organization_type script""" @@ -800,36 +804,69 @@ class TestCleanTables(TestCase): @override_settings(IS_PRODUCTION=False) def test_command_cleans_tables(self): """test that the handle method functions properly to clean tables""" - with less_console_noise(): - with patch("django.apps.apps.get_model") as get_model_mock: - model_mock = MagicMock() - get_model_mock.return_value = model_mock - with patch( - "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa - return_value=True, - ): - call_command("clean_tables") + with patch("django.apps.apps.get_model") as get_model_mock: + model_mock = MagicMock() + get_model_mock.return_value = model_mock - table_names = [ - "DomainInformation", - "DomainRequest", - "PublicContact", - "Domain", - "User", - "Contact", - "Website", - "DraftDomain", - "HostIp", - "Host", - ] + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa + return_value=True, + ): + + # List of pks to be returned in batches, one list for each of 11 tables + pk_batch = [1, 2, 3, 4, 5, 6] + # Create a list of batches with alternating non-empty and empty lists + pk_batches = [pk_batch, []] * 11 + + # Set the side effect of values_list to return different pk batches + # First time values_list is called it returns list of 6 objects to delete; + # Next time values_list is called it returns empty list + def values_list_side_effect(*args, **kwargs): + if args == ("pk",) and kwargs.get("flat", False): + return pk_batches.pop(0) + return [] + + model_mock.objects.values_list.side_effect = values_list_side_effect + # Mock the return value of `delete()` to be (6, ...) + model_mock.objects.filter.return_value.delete.return_value = (6, None) + + call_command("clean_tables") + + table_names = [ + "DomainInformation", + "DomainRequest", + "FederalAgency", + "PublicContact", + "HostIp", + "Host", + "Domain", + "User", + "Contact", + "Website", + "DraftDomain", + ] + + expected_filter_calls = [call(pk__in=[1, 2, 3, 4, 5, 6]) for _ in range(11)] + + actual_filter_calls = [c for c in model_mock.objects.filter.call_args_list if "pk__in" in c[1]] + + try: + # Assert that filter(pk__in=...) was called with expected arguments + self.assertEqual(actual_filter_calls, expected_filter_calls) + + # Check that delete() was called for each batch + for batch in [[1, 2, 3, 4, 5, 6]]: + model_mock.objects.filter(pk__in=batch).delete.assert_called() - # Check that each model's delete method was called for table_name in table_names: get_model_mock.assert_any_call("registrar", table_name) - model_mock.objects.all().delete.assert_called() - - self.logger_mock.info.assert_any_call("Successfully cleaned table DomainInformation") + self.logger_mock.info.assert_any_call( + f"Successfully cleaned table {table_name}, deleted 6 rows" + ) + except AssertionError as e: + print(f"AssertionError: {e}") + raise @override_settings(IS_PRODUCTION=False) def test_command_handles_nonexistent_model(self): @@ -860,15 +897,33 @@ class TestCleanTables(TestCase): with patch("django.apps.apps.get_model") as get_model_mock: model_mock = MagicMock() get_model_mock.return_value = model_mock - model_mock.objects.all().delete.side_effect = Exception("Some error") + + # Mock the values_list so that DomainInformation attempts a delete + pk_batches = [[1, 2, 3, 4, 5, 6], []] + + def values_list_side_effect(*args, **kwargs): + if args == ("pk",) and kwargs.get("flat", False): + return pk_batches.pop(0) + return [] + + model_mock.objects.values_list.side_effect = values_list_side_effect + + # Mock delete to raise a generic exception + model_mock.objects.filter.return_value.delete.side_effect = Exception("Mocked delete exception") with patch( - "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", return_value=True, ): - call_command("clean_tables") + with self.assertRaises(Exception) as context: + # Execute the command + call_command("clean_tables") - self.logger_mock.error.assert_any_call("Error cleaning table DomainInformation: Some error") + # Check the exception message + self.assertEqual(str(context.exception), "Custom delete error") + + # Assert that delete was called + model_mock.objects.filter.return_value.delete.assert_called() class TestExportTables(MockEppLib): From 6823d5a2b022b1f93f4215e112f612874fcb6a98 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Mon, 29 Jul 2024 10:52:06 -0500 Subject: [PATCH 04/21] Initial changes --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index d63cf2f94..3c5812235 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -21,6 +21,21 @@ There are several tools we use locally that you will need to have. - [ ] Make sure you have `gpg` >2.1.7. Run `gpg --version` to check. If not, [install gnupg](https://formulae.brew.sh/formula/gnupg) - Alternatively, you can skip this step and [use ssh keys](#setting-up-commit-signing-with-ssh) instead - [ ] Install the [Github CLI](https://cli.github.com/) +Optional +- [ ] Install the Slack Desktop App + +## For Developing on a DHS furnished device + +The following tools much be requested through the DHS IT portal: +- [ ] Docker Community Edition +- [ ] Git +- [ ] VSCode (our preferred editor) +The following tools are optional, but also must be requested through the IT portal: +- [ ] Python 3.10 +- [ ] NodeJS (latest version available) +- [ ] Putty +- [ ] Windows Subsystem for Linux +- [ ] Github Desktop ## Access From bdbc66ec518a7d4a802b77664d6f28fcc52d340e Mon Sep 17 00:00:00 2001 From: Erin Song <121973038+erinysong@users.noreply.github.com> Date: Mon, 29 Jul 2024 09:24:17 -0700 Subject: [PATCH 05/21] Initial update to dev onboarding ticket --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 3c5812235..0c241bc85 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -30,7 +30,8 @@ The following tools much be requested through the DHS IT portal: - [ ] Docker Community Edition - [ ] Git - [ ] VSCode (our preferred editor) -The following tools are optional, but also must be requested through the IT portal: + +The following tools are optional, but also can be requested through the DHS IT portal: - [ ] Python 3.10 - [ ] NodeJS (latest version available) - [ ] Putty From 2b0d5e8e718dae5832aced212ef6121c96d662bd Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Mon, 29 Jul 2024 11:39:30 -0500 Subject: [PATCH 06/21] Add known issues section. --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 3c5812235..117886b65 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -26,7 +26,7 @@ Optional ## For Developing on a DHS furnished device -The following tools much be requested through the DHS IT portal: +The following tools must be requested through the DHS IT portal: - [ ] Docker Community Edition - [ ] Git - [ ] VSCode (our preferred editor) @@ -139,3 +139,11 @@ Additionally, consider a gpg key manager like Kleopatra if you run into issues w We have three types of environments: stable, staging, and sandbox. Stable (production)and staging (pre-prod) get deployed via tagged release, and developer sandboxes are given to get.gov developers to mess around in a production-like environment without disrupting stable or staging. Each sandbox is namespaced and will automatically be deployed too when the appropriate branch syntax is used for that space in an open pull request. There are several things you need to setup to make the sandbox work for a developer. All automation for setting up a developer sandbox is documented in the scripts for [creating a developer sandbox](../../ops/scripts/create_dev_sandbox.sh) and [removing a developer sandbox](../../ops/scripts/destroy_dev_sandbox.sh). A Cloud.gov organization administrator will have to perform the script in order to create the sandbox. + +# Known Issues + +## SSL Verification Failure +Some developers, especially those using Government Furnished Equipment (GFE), have problems installing python packages due to an SSL verification failure. This happens because GFE has a custom certificate chain installed, but python uses its own certificate bundle. As a result, when pip tries to verify the TLS connection to download a package, it cannot and so the download fails. To resolve this, if you are running locally you can use --use-feature=truststore to direct pip to use the local certificate store. If you are running a docker container, you will need to export the root certificate and pull it into the container. Ask another developer how to do this properly. + +## Checksum Error +There is an unresolved issue with python package installation that occurs after the above SSL Verification failure has been resolved. It often manifests as a checksum error, where the hash of a download .whl file (python package) does not match the expected value. This appears to be because pythonhosted.org is cutting off download connections to some devices for some packages (the behavior is somewhat inconsistent). We have outstanding issues with PyPA and DHS IT to fix this. \ No newline at end of file From 30182921ce8539c27c3be256a5b9d2f69c6c76b9 Mon Sep 17 00:00:00 2001 From: Erin Song <121973038+erinysong@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:16:23 -0700 Subject: [PATCH 07/21] Add login and login sandbox tasks --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 84db58ca3..f564da217 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -53,7 +53,12 @@ cf login -a api.fr.cloud.gov --sso **Note:** As mentioned in the [Login documentation](https://developers.login.gov/testing/), the sandbox Login account is different account from your regular, production Login account. If you have not created a Login account for the sandbox before, you will need to create a new account first. -- [ ] Optional- add yourself as a codeowner if desired. See the [Developer readme](https://github.com/cisagov/getgov/blob/main/docs/developer/README.md) for how to do this and what it does. +Follow the [.gov onboarding dev setup instructions](https://docs.google.com/document/d/1ukbpW4LSqkb_CCt8LWfpehP03qqfyYfvK3Fl21NaEq8/edit#heading=h.94jwfwkpkhdx). Mark the below checkboxes as complete to indicate you successfully set up the following accounts. +- [ ] Identity sandbox accounts - 1 superuser access account and 1 analyst access account. +- [ ] Login.gov account to access stable + +**Optional** +- [ ] Add yourself as a codeowner if desired. See the [Developer readme](https://github.com/cisagov/getgov/blob/main/docs/developer/README.md) for how to do this and what it does. ### Steps for the onboarder - [ ] Add the onboardee to cloud.gov org (cisa-dotgov) @@ -147,4 +152,4 @@ All automation for setting up a developer sandbox is documented in the scripts f Some developers, especially those using Government Furnished Equipment (GFE), have problems installing python packages due to an SSL verification failure. This happens because GFE has a custom certificate chain installed, but python uses its own certificate bundle. As a result, when pip tries to verify the TLS connection to download a package, it cannot and so the download fails. To resolve this, if you are running locally you can use --use-feature=truststore to direct pip to use the local certificate store. If you are running a docker container, you will need to export the root certificate and pull it into the container. Ask another developer how to do this properly. ## Checksum Error -There is an unresolved issue with python package installation that occurs after the above SSL Verification failure has been resolved. It often manifests as a checksum error, where the hash of a download .whl file (python package) does not match the expected value. This appears to be because pythonhosted.org is cutting off download connections to some devices for some packages (the behavior is somewhat inconsistent). We have outstanding issues with PyPA and DHS IT to fix this. \ No newline at end of file +There is an unresolved issue with python package installation that occurs after the above SSL Verification failure has been resolved. It often manifests as a checksum error, where the hash of a download .whl file (python package) does not match the expected value. This appears to be because pythonhosted.org is cutting off download connections to some devices for some packages (the behavior is somewhat inconsistent). We have outstanding issues with PyPA and DHS IT to fix this. From ee402120856ddf450249d30f2e0291b94667d4fd Mon Sep 17 00:00:00 2001 From: Erin Song <121973038+erinysong@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:20:56 -0700 Subject: [PATCH 08/21] Update developer-onboarding.md --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index f564da217..fecb10a71 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -53,7 +53,7 @@ cf login -a api.fr.cloud.gov --sso **Note:** As mentioned in the [Login documentation](https://developers.login.gov/testing/), the sandbox Login account is different account from your regular, production Login account. If you have not created a Login account for the sandbox before, you will need to create a new account first. -Follow the [.gov onboarding dev setup instructions](https://docs.google.com/document/d/1ukbpW4LSqkb_CCt8LWfpehP03qqfyYfvK3Fl21NaEq8/edit#heading=h.94jwfwkpkhdx). Mark the below checkboxes as complete to indicate you successfully set up the following accounts. +Follow the [.gov onboarding dev setup instructions](https://docs.google.com/document/d/1ukbpW4LSqkb_CCt8LWfpehP03qqfyYfvK3Fl21NaEq8/edit#heading=h.94jwfwkpkhdx). Mark the below checkboxes as complete to confirm you successfully set up the following accounts. - [ ] Identity sandbox accounts - 1 superuser access account and 1 analyst access account. - [ ] Login.gov account to access stable From 309227151488b3b4f754773b57e915b59e8c0941 Mon Sep 17 00:00:00 2001 From: Erin Song <121973038+erinysong@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:23:23 -0700 Subject: [PATCH 09/21] Update developer-onboarding.md --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index fecb10a71..05a1f1cd3 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -53,7 +53,7 @@ cf login -a api.fr.cloud.gov --sso **Note:** As mentioned in the [Login documentation](https://developers.login.gov/testing/), the sandbox Login account is different account from your regular, production Login account. If you have not created a Login account for the sandbox before, you will need to create a new account first. -Follow the [.gov onboarding dev setup instructions](https://docs.google.com/document/d/1ukbpW4LSqkb_CCt8LWfpehP03qqfyYfvK3Fl21NaEq8/edit#heading=h.94jwfwkpkhdx). Mark the below checkboxes as complete to confirm you successfully set up the following accounts. +Follow the [.gov onboarding dev setup instructions](https://docs.google.com/document/d/1ukbpW4LSqkb_CCt8LWfpehP03qqfyYfvK3Fl21NaEq8/edit#heading=h.94jwfwkpkhdx). Confirm you successfully set up the following accounts. - [ ] Identity sandbox accounts - 1 superuser access account and 1 analyst access account. - [ ] Login.gov account to access stable From 66a5b67b27332482089b112257a3c2e5f2fdf436 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Mon, 29 Jul 2024 13:51:03 -0500 Subject: [PATCH 10/21] new format for software --- .../ISSUE_TEMPLATE/developer-onboarding.md | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 84db58ca3..3e25ca085 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -12,31 +12,26 @@ assignees: abroddrick - Onboardee: _GH handle of person being onboarded_ - Onboarder: _GH handle of onboard buddy_ -## Installation +## Installation -There are several tools we use locally that you will need to have. -- [ ] [Install the cf CLI v7](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html#pkg-mac) for the ability to deploy +There are several tools we use locally that you will need to have. Tools marked with * need to be requested through the DHS IT portal when operating on a DHS device + +- [ ] [Cloudfoundry CLI](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html#pkg-mac) - If you are using Windows, installation information can be found [here](https://github.com/cloudfoundry/cli/wiki/V8-CLI-Installation-Guide#installers-and-compressed-binaries) - Alternatively, for Windows, [consider using chocolately](https://community.chocolatey.org/packages/cloudfoundry-cli/7.2.0) -- [ ] Make sure you have `gpg` >2.1.7. Run `gpg --version` to check. If not, [install gnupg](https://formulae.brew.sh/formula/gnupg) - - Alternatively, you can skip this step and [use ssh keys](#setting-up-commit-signing-with-ssh) instead -- [ ] Install the [Github CLI](https://cli.github.com/) -Optional -- [ ] Install the Slack Desktop App - -## For Developing on a DHS furnished device - -The following tools must be requested through the DHS IT portal: -- [ ] Docker Community Edition -- [ ] Git -- [ ] VSCode (our preferred editor) +- [ ] [GPG](https://gnupg.org/download/) + - This may not work on DHS devices. Instead, you can [use ssh keys](#setting-up-commit-signing-with-ssh) instead +- [ ] *Docker Community Edition +- [ ] *Git +- [ ] *VSCode (our preferred editor) +- [ ] *Github Desktop The following tools are optional, but also can be requested through the DHS IT portal: -- [ ] Python 3.10 -- [ ] NodeJS (latest version available) -- [ ] Putty -- [ ] Windows Subsystem for Linux -- [ ] Github Desktop +- [ ] Slack Desktop App +- [ ] *Python 3.10 +- [ ] *NodeJS (latest version available) +- [ ] *Putty +- [ ] *Windows Subsystem for Linux ## Access From eaaeb56b1df657cf471c6cd6df6fbe1c8f5a3c13 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Mon, 29 Jul 2024 13:56:22 -0500 Subject: [PATCH 11/21] minor clarity edits --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index ba0deaa70..61420f740 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -14,7 +14,7 @@ assignees: abroddrick ## Installation -There are several tools we use locally that you will need to have. Tools marked with * need to be requested through the DHS IT portal when operating on a DHS device +There are several tools we use locally that you will need to have. - [ ] [Cloudfoundry CLI](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html#pkg-mac) - If you are using Windows, installation information can be found [here](https://github.com/cloudfoundry/cli/wiki/V8-CLI-Installation-Guide#installers-and-compressed-binaries) @@ -27,12 +27,15 @@ There are several tools we use locally that you will need to have. Tools marked - [ ] *Github Desktop The following tools are optional, but also can be requested through the DHS IT portal: -- [ ] Slack Desktop App +- [ ] **Slack Desktop App - [ ] *Python 3.10 - [ ] *NodeJS (latest version available) - [ ] *Putty - [ ] *Windows Subsystem for Linux +* -> must be requested through DHS IT portal on DHS devices +** -> can be found in software center on DHS devices + ## Access ### Steps for the onboardee From 45b948ef7137c2f56521450a9a16b0d07fc40a94 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Mon, 29 Jul 2024 14:06:15 -0500 Subject: [PATCH 12/21] Add known issue instructions --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 61420f740..8fa830a01 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -144,10 +144,16 @@ We have three types of environments: stable, staging, and sandbox. Stable (produ All automation for setting up a developer sandbox is documented in the scripts for [creating a developer sandbox](../../ops/scripts/create_dev_sandbox.sh) and [removing a developer sandbox](../../ops/scripts/destroy_dev_sandbox.sh). A Cloud.gov organization administrator will have to perform the script in order to create the sandbox. -# Known Issues +## Known Issues -## SSL Verification Failure +### SSL Verification Failure Some developers, especially those using Government Furnished Equipment (GFE), have problems installing python packages due to an SSL verification failure. This happens because GFE has a custom certificate chain installed, but python uses its own certificate bundle. As a result, when pip tries to verify the TLS connection to download a package, it cannot and so the download fails. To resolve this, if you are running locally you can use --use-feature=truststore to direct pip to use the local certificate store. If you are running a docker container, you will need to export the root certificate and pull it into the container. Ask another developer how to do this properly. -## Checksum Error -There is an unresolved issue with python package installation that occurs after the above SSL Verification failure has been resolved. It often manifests as a checksum error, where the hash of a download .whl file (python package) does not match the expected value. This appears to be because pythonhosted.org is cutting off download connections to some devices for some packages (the behavior is somewhat inconsistent). We have outstanding issues with PyPA and DHS IT to fix this. +### Puppeteer Download Error +When building the node image either individually or with docker compose, there may be an error caused by a node package call puppeteer. This can be resolved by adding `ENV PUPPETEER_SKIP_DOWNLOAD=true` to [node.Dockerfile](../../src/node.Dockerfile) after the COPY command. + +### Checksum Error +There is an unresolved issue with python package installation that occurs after the above SSL Verification failure has been resolved. It often manifests as a checksum error, where the hash of a download .whl file (python package) does not match the expected value. This appears to be because pythonhosted.org is cutting off download connections to some devices for some packages (the behavior is somewhat inconsistent). We have outstanding issues with PyPA and DHS IT to fix this. In the meantime we have a [workaround](#developing-using-docker) + +## Developing Using Docker +While we have unresolved issues with certain devices, you can pull a pre-built docker image from matthewswspence/getgov-base that comes with all the needed packages installed. To do this, you will need to change the very first line in the main [Dockerfile](../../src/Dockerfile) to `FROM matthewswspence/getgov-base:latest`. Note: this change will need to be reverted before any branch can be merged. Additionally, this will only resolve the [checksum error](#checksum-error), you will still need to resolve any other issues through the listed instructions. We are actively working to resolve this inconvenience. From 3ea9649604240f74c1ea896f219edf4e6a9d663e Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Tue, 30 Jul 2024 13:41:05 -0500 Subject: [PATCH 13/21] add line about cf cli --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 8fa830a01..71240cf8f 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -16,7 +16,7 @@ assignees: abroddrick There are several tools we use locally that you will need to have. -- [ ] [Cloudfoundry CLI](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html#pkg-mac) +- [ ] [Cloudfoundry CLI](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html#pkg-mac) Note: If you are on Windows the cli will be under `cf8` or `cf7` depending on which version you install. - If you are using Windows, installation information can be found [here](https://github.com/cloudfoundry/cli/wiki/V8-CLI-Installation-Guide#installers-and-compressed-binaries) - Alternatively, for Windows, [consider using chocolately](https://community.chocolatey.org/packages/cloudfoundry-cli/7.2.0) - [ ] [GPG](https://gnupg.org/download/) From e316989057df49ae27a8a0c2997681fde6706807 Mon Sep 17 00:00:00 2001 From: Erin Song <121973038+erinysong@users.noreply.github.com> Date: Tue, 30 Jul 2024 15:39:24 -0700 Subject: [PATCH 14/21] Update .github/ISSUE_TEMPLATE/developer-onboarding.md Remove whitespace Co-authored-by: zandercymatics <141044360+zandercymatics@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 71240cf8f..a5aba98e8 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -12,7 +12,7 @@ assignees: abroddrick - Onboardee: _GH handle of person being onboarded_ - Onboarder: _GH handle of onboard buddy_ -## Installation +## Installation There are several tools we use locally that you will need to have. From 6dafe9fa2e53f1cf3665ee2b79f394b4830ab423 Mon Sep 17 00:00:00 2001 From: Erin Song <121973038+erinysong@users.noreply.github.com> Date: Tue, 30 Jul 2024 15:48:37 -0700 Subject: [PATCH 15/21] Update developer-onboarding.md Format required software to more readable format --- .../ISSUE_TEMPLATE/developer-onboarding.md | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index a5aba98e8..bac8fafd7 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -21,20 +21,21 @@ There are several tools we use locally that you will need to have. - Alternatively, for Windows, [consider using chocolately](https://community.chocolatey.org/packages/cloudfoundry-cli/7.2.0) - [ ] [GPG](https://gnupg.org/download/) - This may not work on DHS devices. Instead, you can [use ssh keys](#setting-up-commit-signing-with-ssh) instead -- [ ] *Docker Community Edition -- [ ] *Git -- [ ] *VSCode (our preferred editor) -- [ ] *Github Desktop +- [ ] Docker Community Edition* +- [ ] Git* +- [ ] VSCode (our preferred editor)* +- [ ] Github Desktop* The following tools are optional, but also can be requested through the DHS IT portal: -- [ ] **Slack Desktop App -- [ ] *Python 3.10 -- [ ] *NodeJS (latest version available) -- [ ] *Putty -- [ ] *Windows Subsystem for Linux +- [ ] Slack Desktop App** +- [ ] Python 3.10* +- [ ] NodeJS (latest version available)* +- [ ] Putty* +- [ ] Windows Subsystem for Linux* -* -> must be requested through DHS IT portal on DHS devices -** -> can be found in software center on DHS devices +\* Must be requested through DHS IT portal on DHS devices + +** Downloadable via DHS Software Center ## Access From a9c15162728e4fb4ef5ac78b1fb7579d829c0a6c Mon Sep 17 00:00:00 2001 From: Erin Song <121973038+erinysong@users.noreply.github.com> Date: Tue, 30 Jul 2024 15:49:53 -0700 Subject: [PATCH 16/21] Update .github/ISSUE_TEMPLATE/developer-onboarding.md Co-authored-by: zandercymatics <141044360+zandercymatics@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index bac8fafd7..77795ab9e 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -52,7 +52,7 @@ cf login -a api.fr.cloud.gov --sso **Note:** As mentioned in the [Login documentation](https://developers.login.gov/testing/), the sandbox Login account is different account from your regular, production Login account. If you have not created a Login account for the sandbox before, you will need to create a new account first. -Follow the [.gov onboarding dev setup instructions](https://docs.google.com/document/d/1ukbpW4LSqkb_CCt8LWfpehP03qqfyYfvK3Fl21NaEq8/edit#heading=h.94jwfwkpkhdx). Confirm you successfully set up the following accounts. +Follow the [.gov onboarding dev setup instructions](https://docs.google.com/document/d/1ukbpW4LSqkb_CCt8LWfpehP03qqfyYfvK3Fl21NaEq8/edit#heading=h.94jwfwkpkhdx). Confirm you successfully set up the following accounts: - [ ] Identity sandbox accounts - 1 superuser access account and 1 analyst access account. - [ ] Login.gov account to access stable From 5a8842b435ab370ae91b6054ed16a7c8610a4a1c Mon Sep 17 00:00:00 2001 From: Erin Song <121973038+erinysong@users.noreply.github.com> Date: Tue, 30 Jul 2024 15:50:04 -0700 Subject: [PATCH 17/21] Update .github/ISSUE_TEMPLATE/developer-onboarding.md Co-authored-by: zandercymatics <141044360+zandercymatics@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 77795ab9e..3966e6d41 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -154,7 +154,7 @@ Some developers, especially those using Government Furnished Equipment (GFE), ha When building the node image either individually or with docker compose, there may be an error caused by a node package call puppeteer. This can be resolved by adding `ENV PUPPETEER_SKIP_DOWNLOAD=true` to [node.Dockerfile](../../src/node.Dockerfile) after the COPY command. ### Checksum Error -There is an unresolved issue with python package installation that occurs after the above SSL Verification failure has been resolved. It often manifests as a checksum error, where the hash of a download .whl file (python package) does not match the expected value. This appears to be because pythonhosted.org is cutting off download connections to some devices for some packages (the behavior is somewhat inconsistent). We have outstanding issues with PyPA and DHS IT to fix this. In the meantime we have a [workaround](#developing-using-docker) +There is an unresolved issue with python package installation that occurs after the above SSL Verification failure has been resolved. It often manifests as a checksum error, where the hash of a download .whl file (python package) does not match the expected value. This appears to be because pythonhosted.org is cutting off download connections to some devices for some packages (the behavior is somewhat inconsistent). We have outstanding issues with PyPA and DHS IT to fix this. In the meantime we have a [workaround](#developing-using-docker). ## Developing Using Docker While we have unresolved issues with certain devices, you can pull a pre-built docker image from matthewswspence/getgov-base that comes with all the needed packages installed. To do this, you will need to change the very first line in the main [Dockerfile](../../src/Dockerfile) to `FROM matthewswspence/getgov-base:latest`. Note: this change will need to be reverted before any branch can be merged. Additionally, this will only resolve the [checksum error](#checksum-error), you will still need to resolve any other issues through the listed instructions. We are actively working to resolve this inconvenience. From 98d8e4ec93e1715c7ed31a6a206beb2f11ee955e Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 30 Jul 2024 22:10:05 -0400 Subject: [PATCH 18/21] updates --- src/registrar/management/commands/clean_tables.py | 11 +++++++---- src/registrar/tests/test_management_scripts.py | 4 ---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/registrar/management/commands/clean_tables.py b/src/registrar/management/commands/clean_tables.py index dd84b0b62..5d4439d95 100644 --- a/src/registrar/management/commands/clean_tables.py +++ b/src/registrar/management/commands/clean_tables.py @@ -64,15 +64,18 @@ class Command(BaseCommand): model = apps.get_model("registrar", table_name) BATCH_SIZE = 1000 total_deleted = 0 - while True: - pks = list(model.objects.values_list("pk", flat=True)[:BATCH_SIZE]) - if not pks: - break + + # Get initial batch of primary keys + pks = list(model.objects.values_list("pk", flat=True)[:BATCH_SIZE]) + + while pks: # Use a transaction to ensure database integrity with transaction.atomic(): deleted, _ = model.objects.filter(pk__in=pks).delete() total_deleted += deleted logger.debug(f"Deleted {deleted} {table_name}s, total deleted: {total_deleted}") + # Get the next batch of primary keys + pks = list(model.objects.values_list("pk", flat=True)[:BATCH_SIZE]) logger.info(f"Successfully cleaned table {table_name}, deleted {total_deleted} rows") except LookupError: logger.error(f"Model for table {table_name} not found.") diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index 8fb754ba0..bfdeefadd 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -33,10 +33,6 @@ from api.tests.common import less_console_noise_decorator logger = logging.getLogger(__name__) -class CustomDeleteException(Exception): - pass - - class TestPopulateVerificationType(MockEppLib): """Tests for the populate_organization_type script""" From 75f86737075fc8e8a6c04401ad052b33205a411a Mon Sep 17 00:00:00 2001 From: Matt-Spence Date: Wed, 31 Jul 2024 14:53:21 -0500 Subject: [PATCH 19/21] Update developer-onboarding.md --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 3966e6d41..cacc84b89 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -148,7 +148,9 @@ All automation for setting up a developer sandbox is documented in the scripts f ## Known Issues ### SSL Verification Failure -Some developers, especially those using Government Furnished Equipment (GFE), have problems installing python packages due to an SSL verification failure. This happens because GFE has a custom certificate chain installed, but python uses its own certificate bundle. As a result, when pip tries to verify the TLS connection to download a package, it cannot and so the download fails. To resolve this, if you are running locally you can use --use-feature=truststore to direct pip to use the local certificate store. If you are running a docker container, you will need to export the root certificate and pull it into the container. Ask another developer how to do this properly. +Some developers using Government Furnished Equipment (GFE) have problems using tools such as git and pip due to SSL verification failurse. This happens because GFE has a custom certificate chain installed, but these tools use their own certificate bundles. As a result, when they try to verify an ssl connection, they cannot and so the connection fails. To resolve this in pip you can use --use-feature=truststore to direct pip to use the local certificate store. If you are running into this issue when using git on windows, run ```git config --global http.sslbackend schannel```. + +If you are running into these issues in a docker container you will need to export the root certificate and pull it into the container. Ask another developer how to do this properly. ### Puppeteer Download Error When building the node image either individually or with docker compose, there may be an error caused by a node package call puppeteer. This can be resolved by adding `ENV PUPPETEER_SKIP_DOWNLOAD=true` to [node.Dockerfile](../../src/node.Dockerfile) after the COPY command. From 2d8e63d23ed6d403db763aa562ccf7dd8ab49bdc Mon Sep 17 00:00:00 2001 From: Matt-Spence Date: Wed, 31 Jul 2024 15:07:02 -0500 Subject: [PATCH 20/21] Apply suggestions from code review Co-authored-by: zandercymatics <141044360+zandercymatics@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index cacc84b89..ebc1800b6 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -20,13 +20,14 @@ There are several tools we use locally that you will need to have. - If you are using Windows, installation information can be found [here](https://github.com/cloudfoundry/cli/wiki/V8-CLI-Installation-Guide#installers-and-compressed-binaries) - Alternatively, for Windows, [consider using chocolately](https://community.chocolatey.org/packages/cloudfoundry-cli/7.2.0) - [ ] [GPG](https://gnupg.org/download/) - - This may not work on DHS devices. Instead, you can [use ssh keys](#setting-up-commit-signing-with-ssh) instead + - Make sure you have `gpg` >2.1.7. Run `gpg --version` to check. If not, [install gnupg](https://formulae.brew.sh/formula/gnupg) + - This may not work on DHS devices. Alternatively, you can [use ssh keys](#setting-up-commit-signing-with-ssh) instead. - [ ] Docker Community Edition* - [ ] Git* - [ ] VSCode (our preferred editor)* - [ ] Github Desktop* -The following tools are optional, but also can be requested through the DHS IT portal: +The following tools are optional but recommended. For DHS devices, these can be requested through the DHS IT portal: - [ ] Slack Desktop App** - [ ] Python 3.10* - [ ] NodeJS (latest version available)* From 6d25c1fe161c64c12c2aa957d3d4950984760bef Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Wed, 31 Jul 2024 15:49:56 -0500 Subject: [PATCH 21/21] review suggestions --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index ebc1800b6..a0825ab52 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -25,7 +25,7 @@ There are several tools we use locally that you will need to have. - [ ] Docker Community Edition* - [ ] Git* - [ ] VSCode (our preferred editor)* -- [ ] Github Desktop* +- [ ] Github Desktop* or the Github CLI* The following tools are optional but recommended. For DHS devices, these can be requested through the DHS IT portal: - [ ] Slack Desktop App** @@ -34,7 +34,7 @@ The following tools are optional but recommended. For DHS devices, these can be - [ ] Putty* - [ ] Windows Subsystem for Linux* -\* Must be requested through DHS IT portal on DHS devices +* Must be requested through DHS IT portal on DHS devices ** Downloadable via DHS Software Center