Create new stable environment (#367)

update documentation for environments and setup new stable environment

.github/ISSUE_TEMPLATE/developer-onboarding.md

@@ -28,18 +28,15 @@ There are several tools we use locally that you will need to have.
 ```bash
 cf login -a api.fr.cloud.gov  --sso
 ```
-- [ ] Have an admin add you to cloud.gov org and relevant spaces as a SpaceDeveloper
+- [ ] Have an admin add you to cloud.gov org and set up your [sandbox developer space](#setting-up-developer-sandbox). Ensure you can deploy to your sandbox space.
 - [ ] Have an admin add you to our login.gov sandbox team (`.gov registrar poc`) via the [dashboard](https://dashboard.int.identitysandbox.gov/).
 
  **Note:** As mentioned in the [Login documentation](https://developers.login.gov/testing/), the sandbox Login account is different account from your regular, production Login account. If you have not created a Login account for the sandbox before, you will need to create a new account first.
 - [ ] Setup [commit signing in Github](#setting-up-commit-signing) and with git locally.
 
 ### Steps for the onboarder
-- [ ] Add the onboardee to cloud.gov org (cisa-getgov-prototyping) and relevant spaces (unstable) as a SpaceDeveloper
+- [ ] Add the onboardee to cloud.gov org (cisa-getgov-prototyping) 
-
+- [ ] Setup a [developer specific space for the new developer](#setting-up-developer-sandbox)
- ```bash
-cf set-space-role <cloud.account@email.gov> cisa-getgov-prototyping unstable SpaceDeveloper
-```
 - [ ] Add the onboardee to our login.gov sandbox team (`.gov registrar poc`) via the [dashboard](https://dashboard.int.identitysandbox.gov/)
 
 
@@ -81,3 +78,9 @@ You may need to add these two lines to your shell's rc file (e.g. `.bashrc` or `
 GPG_TTY=$(tty)
 export GPG_TTY
 ```
+
+## Setting up developer sandbox
+
+We have two types of environments: stable, and sandbox. Stable gets deployed via tagged release every sprint, and developer sandboxes are given to get.gov developers to mess around in a production-like environment without disrupting stable. Each sandbox is namespaced and will automatically be deployed too when the appropriate branch syntax is used for that space in an open pull request. There are several things you need to setup to make the sandbox work for a developer. 
+
+TKTK: Actual steps for setting up a new developer sandbox automation.

.github/workflows/deploy.yaml

@@ -1,33 +1,22 @@
 name: Build and deploy
 
-# This workflow runs on pushes to main (typically,
+# This workflow runs on pushes of tagged commits.
-# a merged pull request) and on pushes of tagged commits.
 
-# Pushes to main will deploy to Staging
+# "Releases" of tagged commits will deploy `main` to Stable.
-# This will also deploy nightly
 
 on:
-  schedule:
-    # 3am UTC
-    - cron: '0 3 * * *'
-
   push:
     paths-ignore:
       - 'docs/**'
       
-    branches:
-      - main
-
     tags:
       - v*
 
-  workflow_dispatch:
-
 jobs:
-  deploy-staging:
+  deploy-stable:
-    # if this job runs on a branch, we deduce that code
+    # if this job runs on a tag, we deduce that code
-    # has been pushed to main and should be deployed to staging
+    # has been tagged for release and should be deployed to stable
-    if: ${{ github.ref_type == 'branch' }}
+    if: ${{ github.ref_type == 'tag' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -46,8 +35,8 @@ jobs:
         env:
           DEPLOY_NOW: thanks
         with:
-          cf_username: ${{ secrets.CF_STAGING_USERNAME }}
+          cf_username: ${{ secrets.CF_STABLE_USERNAME }}
-          cf_password: ${{ secrets.CF_STAGING_PASSWORD }}
+          cf_password: ${{ secrets.CF_STABLE_PASSWORD }}
           cf_org: cisa-getgov-prototyping
-          cf_space: staging
+          cf_space: stable
-          push_arguments: "-f ops/manifests/manifest-staging.yaml"
+          push_arguments: "-f ops/manifests/manifest-stable.yaml"

.github/workflows/loaddata.yaml

@@ -1,11 +1,11 @@
 name: Reset database
 
 # This workflow can be run from the CLI
-#     gh workflow run loaddata.yaml -f environment=staging
+#     gh workflow run loaddata.yaml -f environment=stable
 # OR
-#     cf run-task getgov-staging --wait \
+#     cf run-task getgov-stable --wait \
 #        --command 'python manage.py flush' --name flush
-#     cf run-task getgov-staging --wait \
+#     cf run-task getgov-stable --wait \
 #        --command 'python manage.py load' --name loaddata
 on:
   workflow_dispatch:
@@ -14,27 +14,27 @@ on:
         type: choice
         description: Where should we load data
         options:
-        - staging
+        - stable
 
 jobs:
-  reset-db-staging:
+  reset-db-stable:
-    if: ${{ github.event.inputs.environment == 'staging' }}
+    if: ${{ github.event.inputs.environment == 'stable' }}
     runs-on: ubuntu-latest
     steps:
-      - name: Delete existing data for staging
+      - name: Delete existing data for stable
         uses: 18f/cg-deploy-action@main
         with:
-          cf_username: ${{ secrets.CF_STAGING_USERNAME }}
+          cf_username: ${{ secrets.CF_STABLE_USERNAME }}
-          cf_password: ${{ secrets.CF_STAGING_PASSWORD }}
+          cf_password: ${{ secrets.CF_STABLE_PASSWORD }}
           cf_org: cisa-getgov-prototyping
-          cf_space: staging
+          cf_space: stable
-          full_command: "cf run-task getgov-staging --wait --command 'python manage.py flush --no-input' --name flush"
+          full_command: "cf run-task getgov-stable --wait --command 'python manage.py flush --no-input' --name flush"
 
-      - name: Load fake data for staging
+      - name: Load fake data for stable
         uses: 18f/cg-deploy-action@main
         with:
-          cf_username: ${{ secrets.CF_STAGING_USERNAME }}
+          cf_username: ${{ secrets.CF_STABLE_USERNAME }}
-          cf_password: ${{ secrets.CF_STAGING_PASSWORD }}
+          cf_password: ${{ secrets.CF_STABLE_PASSWORD }}
           cf_org: cisa-getgov-prototyping
-          cf_space: staging
+          cf_space: stable
-          full_command: "cf run-task getgov-staging --wait --command 'python manage.py load' --name loaddata"
+          full_command: "cf run-task getgov-stable --wait --command 'python manage.py load' --name loaddata"

.github/workflows/migrate.yaml

@@ -3,7 +3,7 @@ name: Run migrations
 # This workflow can be run from the CLI
 #     gh workflow run migrate.yaml -f environment=sandbox
 # OR
-#     cf run-task getgov-staging --wait \
+#     cf run-task getgov-stable --wait \
 #        --command 'python manage.py migrate' --name migrate
 
 on:
@@ -13,19 +13,19 @@ on:
         type: choice
         description: Where should we run migrations
         options:
-        - staging
+        - stable
 
 jobs:
-  migrate-staging:
+  migrate-stable:
-    if: ${{ github.event.inputs.environment == 'staging' }}
+    if: ${{ github.event.inputs.environment == 'stable' }}
     runs-on: ubuntu-latest
     steps:
-      - name: Run Django migrations for staging
+      - name: Run Django migrations for stable
         uses: 18f/cg-deploy-action@main
         with:
-          cf_username: ${{ secrets.CF_STAGING_USERNAME }}
+          cf_username: ${{ secrets.CF_STABLE_USERNAME }}
-          cf_password: ${{ secrets.CF_STAGING_PASSWORD }}
+          cf_password: ${{ secrets.CF_STABLE_PASSWORD }}
           cf_org: cisa-getgov-prototyping
-          cf_space: staging
+          cf_space: stable
-          full_command: "cf run-task getgov-staging --wait --command 'python manage.py migrate' --name migrate"
+          full_command: "cf run-task getgov-stable --wait --command 'python manage.py migrate' --name migrate"
 

docs/architecture/diagrams/get.gov registrar deployment.puml

@@ -16,16 +16,16 @@ Deployment_Node(aws, "AWS GovCloud", "Amazon Web Services Region") {
         System_Ext(cloudgov_logdrain, "logs.fr.cloud.gov", "ELK")
         Boundary(atob, "ATO boundary") {
             Deployment_Node(organization, "get.gov organization") {
-                Deployment_Node(unstable, "unstable space") {
+                Deployment_Node(sandbox, "sandbox space") {
-                    System_Boundary(dashboard_unstable, "get.gov registrar") {
+                    System_Boundary(dashboard_sandbox, "get.gov registrar") {
-                        Container(getgov_app_unstable, "Registrar Application", "Python, Django", "Delivers static HTML/CSS and forms")
+                        Container(getgov_app_sandbox, "Registrar Application", "Python, Django", "Delivers static HTML/CSS and forms")
-                        ContainerDb(dashboard_db_unstable, "Unstable PostgreSQL Database", "AWS RDS", "Stores agency information and reports")
+                        ContainerDb(dashboard_db_sandbox, "sandbox PostgreSQL Database", "AWS RDS", "Stores agency information and reports")
                     }
                 }
-                Deployment_Node(staging, "staging space") {
+                Deployment_Node(stable, "stable space") {
-                    System_Boundary(dashboard_staging, "get.gov registrar") {
+                    System_Boundary(dashboard_stable, "get.gov registrar") {
-                        Container(getgov_app_staging, "Registrar Application", "Python, Django", "Delivers static HTML/CSS and forms")
+                        Container(getgov_app_stable, "Registrar Application", "Python, Django", "Delivers static HTML/CSS and forms")
-                        ContainerDb(dashboard_db_staging, "Staging PostgreSQL Database", "AWS RDS", "Stores agency information and reports")
+                        ContainerDb(dashboard_db_stable, "stable PostgreSQL Database", "AWS RDS", "Stores agency information and reports")
                     }
                 }
             }
@@ -34,7 +34,7 @@ Deployment_Node(aws, "AWS GovCloud", "Amazon Web Services Region") {
 }
 
 ' Logs flow
-Rel(staging, cloudgov_logdrain, "logs to", "stdout/stderr")
+Rel(stable, cloudgov_logdrain, "logs to", "stdout/stderr")
 Rel(team, cloudgov_logdrain, "reviews logs", "https (443)")
 
 Rel(team, cloudgov_uaa, "authenticates with", "https (443)")
@@ -54,12 +54,12 @@ Rel(github_repo, github_actions_deploy, "pushes to main trigger deployment")
 Rel(team, github_repo, "commits code, makes pull-request, approves PRs", "https (443)")
 Rel(github_actions_deploy, cloudgov_controller, "pushes code, invokes tasks", "https (443)")
 Rel(github_actions_deploy, cloudgov_router, "runs smoke tests on URLs", "https (443)")
-Rel(cloudgov_controller, staging, "provisions/operates apps and services", "admin access limited")
+Rel(cloudgov_controller, stable, "provisions/operates apps and services", "admin access limited")
-Rel(cloudgov_controller, unstable, "provisions/operates apps and services")
+Rel(cloudgov_controller, sandbox, "provisions/operates apps and services")
 
-Rel(getgov_app_staging, dashboard_db_staging, "reads agency info, reads/writes reports, ", "postgres (5432)")
+Rel(getgov_app_stable, dashboard_db_stable, "reads agency info, reads/writes reports, ", "postgres (5432)")
-Rel(getgov_app_unstable, dashboard_db_unstable, "reads agency info, reads/writes reports, ", "postgres (5432)")
+Rel(getgov_app_sandbox, dashboard_db_sandbox, "reads agency info, reads/writes reports, ", "postgres (5432)")
 
-Rel(cloudgov_router, getgov_app_staging, "proxies to", "https GET/POST (443)")
+Rel(cloudgov_router, getgov_app_stable, "proxies to", "https GET/POST (443)")
 
 @enduml

docs/developer/README.md

@@ -28,7 +28,7 @@ Secrets (for example, if you'd like to have a working Login.gov authentication)
 DJANGO_SECRET_LOGIN_KEY="<...>"
 ```
 
-You'll need to create the `.env` file yourself. Get the secrets from Cloud.gov by running `cf env getgov-unstable`. More information is available in [rotate_application_secrets.md](../operations/runbooks/rotate_application_secrets.md).
+You'll need to create the `.env` file yourself. Get the secrets from Cloud.gov by running `cf env getgov-YOURSANDBOX`. More information is available in [rotate_application_secrets.md](../operations/runbooks/rotate_application_secrets.md).
 
 ## Viewing Logs
 
@@ -134,10 +134,6 @@ Within the `registrar/assets` folder, the `_theme` folder contains three files i
 
 You can also compile the sass at any time using `npx gulp compile`. Similarly, you can copy over other static assets (images and javascript files), using `npx gulp copyAssets`.
 
-## Nightly Builds
-
-We run a deploy to staging from `main` nightly. You can see the deploys [here](https://github.com/cisagov/getgov/actions/workflows/deploy.yaml).
-
 ## Finite State Machines
 
 In an effort to keep our domain logic centralized, we are representing the state of 

docs/developer/database-access.md

@@ -5,46 +5,46 @@ You can connect to a Cloud.gov database using the
 After installing it, use the command
 
 ```shell
-cf connect-to-service getgov-unstable getgov-unstable-databse
+cf connect-to-service getgov-SANDBOXNAME getgov-SANDBOXNAME-databse
 ```
 
-to get a `psql` shell on the `unstable` environment's database.
+to get a `psql` shell on the sandbox environment's database.
 
 ## Running Migrations
 
 When new code changes the database schema, we need to apply Django's migrations.
 We can run these using CloudFoundry's tasks to run the `manage.py migrate`
-command in the correct environment. For the `unstable` environment, developers
+command in the correct environment. For any developer environment, developers
 can manually run the task with
 
 ```shell
-cf run-task getgov-unstable --command 'python manage.py migrate' --name migrate
+cf run-task getgov-SANDBOXNAME --command 'python manage.py migrate' --name migrate
 ```
 
 Optionally, load data from fixtures as well
 
 ```shell
-cf run-task getgov-unstable --wait --command 'python manage.py load' --name loaddata
+cf run-task getgov-SANDBOXNAME --wait --command 'python manage.py load' --name loaddata
 ```
 
-For the `staging` environment, developers don't have credentials so we need to
+For the `stable` environment, developers don't have credentials so we need to
 run that command using Github Actions. Go to
 <https://github.com/cisagov/getgov/actions/workflows/migrate.yaml> and select
-the "Run workflow" button, making sure that `staging` is selected.
+the "Run workflow" button, making sure that `stable` is selected.
 
 ## Getting data for fixtures
 
 To run the `dumpdata` command, you'll need to ssh to a running container. `cf run-task` is useless for this, as you will not be able to see the output.
 
 ```shell
-cf ssh getgov-unstable
+cf ssh getgov-SANDBOXNAME
 /tmp/lifecycle/shell  # this configures your environment
 ./manage.py dumpdata
 ```
 
 ## Dropping and re-creating the database
 
-For `unstable`, it might be necessary to start the database over from scratch.
+For your sandbox environment, it might be necessary to start the database over from scratch.
 The easiest way to do that is `DROP DATABASE ...` followed by `CREATE DATABASE
 ...`. In the `psql` shell, first run the `\l` command to see all of the
 databases that are present:

docs/operations/README.md

@@ -36,23 +36,13 @@ Binding the database in `manifest-<ENVIRONMENT>.json` automatically inserts the
 
 # Deploy
 
-We have two environments: `unstable` and `staging`. Developers can deploy locally to unstable whenever they want. However, only our CD service can deploy to `staging`, and it does so on every commit to `main`. This is to ensure that we have a "golden" environment to point to, and can still test things out in an unstable space. You should make sure all of the USWDS assets are compiled and collected before deploying to unstable. To deploy locally to `unstable`:
+We have two types of environments: developer "sandboxes" and `stable`. Developers can deploy locally to their sandbox whenever they want. However, only our CD service can deploy to `stable`, and it does so when we make tagged releases of `main`. This is to ensure that we have a "golden" environment to point to, and can still test things out in a sandbox space. You should make sure all of the USWDS assets are compiled and collected before deploying to your sandbox. To deploy locally to `sandbox`:
 
-```bash
+For ease of use, you can run the `deploy.sh <sandbox name>` script in the `/src` directory to build the assets and deploy to your sandbox. Similarly, you could run `build.sh <sandbox name>` script to just compile and collect the assets without deploying.
-# Compile and collect the assets
-docker-compose run node npx gulp compile
-docker-compose run node npx gulp copyAssets
-docker-compose run app ./manage.py collectstatic
-
-# Deploy to unstable
-cf target -o cisa-getgov-prototyping -s unstable
-cf push getgov-unstable -f ops/manifests/manifest-unstable.yaml
-cf run-task getgov-unstable --command 'python manage.py migrate' --name migrate
-```
-Alternatively, you could run the `deploy.sh` script in the `/src` directory to build the assets and deploy to `unstable`. Similarly, you could run `build.sh` script to just compile and collect the assets without deploying.
 
+Your sandbox space should've been setup as part of the onboarding process. If this was not the case, please have an admin follow the instructions [here](../../.github/ISSUE_TEMPLATE/developer-onboarding.md#setting-up-developer-sandbox).
 
 ## Serving static assets
 We are using [WhiteNoise](http://whitenoise.evans.io/en/stable/index.html) plugin to serve our static assets on cloud.gov. This plugin is added to the `MIDDLEWARE` list in our apps `settings.py`.
 
-Note that it’s a good idea to run `collectstatic` locally or in the docker container before pushing files up to `unstable`. This is because `collectstatic` relies on timestamps when deciding to whether to overwrite the existing assets in `/public`. Due the way files are uploaded, the compiled css in the `/assets/css` folder on `unstable` will have a slightly earlier timestamp than the files in `/public/css`, and consequently running `collectstatic` on`unstable` will not update `public/css` as you may expect. For convenience, both the `deploy.sh` and `build.sh` scripts will take care of that. 
+Note that it’s a good idea to run `collectstatic` locally or in the docker container before pushing files up to your sandbox. This is because `collectstatic` relies on timestamps when deciding to whether to overwrite the existing assets in `/public`. Due the way files are uploaded, the compiled css in the `/assets/css` folder on your sandbox will have a slightly earlier timestamp than the files in `/public/css`, and consequently running `collectstatic` on your sandbox will not update `public/css` as you may expect. For convenience, both the `deploy.sh` and `build.sh` scripts will take care of that. 

docs/operations/runbooks/rotate_application_secrets.md

@@ -21,7 +21,7 @@ Where `credentials-<ENVIRONMENT>.json` looks like:
 
 (Specific credentials are mentioned below.)
 
-You can see the current environment with `cf env <APP>`, for example `cf env getgov-unstable`.
+You can see the current environment with `cf env <APP>`, for example `cf env getgov-stable`.
 
 The commands `cups` and `uups` stand for [`create user provided service`](https://docs.cloudfoundry.org/devguide/services/user-provided.html) and `update user provided service`. User provided services are the way currently recommended by Cloud.gov for deploying secrets. The user provided service is bound to the application in `manifest-<ENVIRONMENT>.json`.
 
@@ -30,8 +30,8 @@ To rotate secrets, create a new `credentials-<ENVIRONMENT>.json` file, upload it
 Example:
 
 ```bash
-cf update-user-provided-service getgov-credentials -p credentials-unstable.json
+cf update-user-provided-service getgov-credentials -p credentials-stable.json
-cf restage getgov-unstable --strategy rolling
+cf restage getgov-stable --strategy rolling
 ```
 
 Non-secret environment variables can be declared in `manifest-<ENVIRONMENT>.json` directly.

ops/manifests/manifest-stable.yaml

@@ -1,6 +1,6 @@
 ---
 applications:
-- name: getgov-staging
+- name: getgov-stable
   buildpacks:
     - python_buildpack
   path: ../../src
@@ -17,11 +17,11 @@ applications:
     # Tell Django where to find its configuration
     DJANGO_SETTINGS_MODULE: registrar.config.settings
     # Tell Django where it is being hosted
-    DJANGO_BASE_URL: https://getgov-staging.app.cloud.gov
+    DJANGO_BASE_URL: https://getgov-stable.app.cloud.gov
     # Tell Django how much stuff to log
     DJANGO_LOG_LEVEL: INFO
   routes:
-    - route: getgov-staging.app.cloud.gov
+    - route: getgov-stable.app.cloud.gov
   services:
   - getgov-credentials
-  - getgov-staging-database
+  - getgov-stable-database

ops/manifests/manifest-unstable.yaml

@@ -1,27 +0,0 @@
----
-applications:
-- name: getgov-unstable
-  buildpacks:
-    - python_buildpack
-  path: ../../src
-  instances: 1
-  memory: 512M
-  stack: cflinuxfs3
-  timeout: 180
-  command: ./run.sh
-  health-check-type: http
-  health-check-http-endpoint: /health
-  env:
-    # Send stdout and stderr straight to the terminal without buffering
-    PYTHONUNBUFFERED: yup
-    # Tell Django where to find its configuration
-    DJANGO_SETTINGS_MODULE: registrar.config.settings
-    # Tell Django where it is being hosted
-    DJANGO_BASE_URL: https://getgov-unstable.app.cloud.gov
-    # Tell Django how much stuff to log
-    DJANGO_LOG_LEVEL: INFO
-  routes:
-    - route: getgov-unstable.app.cloud.gov
-  services:
-  - getgov-credentials
-  - getgov-unstable-database

ops/scripts/build.sh

@@ -5,4 +5,5 @@ docker compose run node npx gulp compile;
 docker compose run node npx gulp copyAssets;
 
 # Collect assets
+docker compose build
 docker compose run app python manage.py collectstatic --noinput

ops/scripts/deploy.sh

@@ -3,9 +3,9 @@
 # Compile and collect static assets
 ../ops/scripts/build.sh
 
-# Deploy to unstable 
+# Deploy to sandbox 
-cf target -o cisa-getgov-prototyping -s unstable
+cf target -o cisa-getgov-prototyping -s $1
-cf push getgov-unstable -f ../ops/manifests/manifest-unstable.yaml
+cf push getgov-$1 -f ../ops/manifests/manifest-$1.yaml
 
 # migrations need to be run manually. Developers can use this command
-#cf run-task getgov-unstable --command 'python manage.py migrate' --name migrate
+#cf run-task getgov-SANDBOXNAME --command 'python manage.py migrate' --name migrate

ops/scripts/rotate_cloud_secrets.sh

@@ -5,7 +5,7 @@ if [ ! $(command -v gh) ] || [ ! $(command -v jq) ] || [ ! $(command -v cf) ]; t
 fi
 
 if [ -z "$1" ]; then
-    echo 'Please specify a space to target (i.e. unstable, staging)' >&2
+    echo 'Please specify a space to target (i.e. lmm, stable)' >&2
     exit 1
 fi
 

src/registrar/config/settings.py

@@ -511,8 +511,7 @@ SECURE_SSL_REDIRECT = True
 # which are possible even under many seemingly-safe
 # web server configurations.
 ALLOWED_HOSTS = [
-    "getgov-unstable.app.cloud.gov",
+    "getgov-stable.app.cloud.gov",
-    "getgov-staging.app.cloud.gov",
     "get.gov",
 ]
 

src/run.sh

@@ -3,7 +3,7 @@
 set -o errexit
 set -o pipefail
 
-# Make sure that django's `collectstatic` has been run locally before pushing up to unstable,
+# Make sure that django's `collectstatic` has been run locally before pushing up to any environment,
-# so that the styles and static assets to show up correctly on unstable.
+# so that the styles and static assets to show up correctly on any environment.
 
 gunicorn registrar.config.wsgi -t 60