Add GCB workflows to promote the nomulus tool command after deployment

With https://github.com/spinnaker/spinnaker/issues/4048 Spinnaker now natively supports GCB. We are able to start a GCB job from Spinnaker, and also there is better support to consume GCB pub/sub messages. Some changes are made to remove the workaround no longer needed.

Two new workflows are added, one to rsync a GCS folder to live/ after the deployment is done (so that the nomulus.jar file can then be fetched to x20 by a []cron job), and the other to tag the proxy image as live once it is deployed.

Lastly, the docs/ folders are needed when running tests. Remove it from .gcloudignore so that when a test run is kicked off by running "gcloud builds submit" the folder is sent to GCB. Ideally .gcloudignore should be identical to .gitignore but since they both are version controlled it is hard it make one a symlink of another.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=252625901

.gcloudignore

@@ -1,4 +1,3 @@
-docs/**
 python/**
 bazel-*/**
 gradle/**/build/**

release/build_nomulus_for_env.sh

@@ -24,7 +24,7 @@ then
 fi
 
 environment="$1"
-dest="$2/$1"
+dest="$2"
 gcs_prefix="storage.googleapis.com/domain-registry-maven-repository"
 
 if [ "${environment}" == tool ]
@@ -37,8 +37,9 @@ then
     -PpluginsUrl=https://"${gcs_prefix}"/plugins
   cd -
 
-  mv gradle/core/build/libs/nomulus.jar .
+  mv gradle/core/build/libs/nomulus.jar "${dest}"
 else
+  dest="${dest}/$1"
   mkdir -p "${dest}"
 
   cd gradle
@@ -56,6 +57,7 @@ else
     "${dest}/META-INF"
 
   cd "${dest}"
-  tar cvf ../../"${environment}.tar" .
+  tar cvf ../"${environment}.tar" .
   cd -
+  rm -rf "${dest}"
 fi

release/cloudbuild-nomulus.yaml

@@ -4,14 +4,14 @@
 # credential helper.
 # Then, in the root of a nomulus source tree, run:
 # cloud-build-local --config=cloudbuild-nomulus.yaml --dryrun=false \
-#   --substitutions TAG_NAME=[TAG] ..
+#   --substitutions TAG_NAME=[TAG] .
 # This will build the contents of the current directory and generate the
 # nomulus war-files locally.
 # The PROJECT_ID is the current project name that gcloud uses.
 # You can add "--push true" to have the image pushed to GCR.
 #
 # To manually trigger a build on GCB, run:
-# gcloud builds submit --config cloudbuild-nomulus.yaml --substitutions TAG_NAME=[TAG] ..
+# gcloud builds submit --config cloudbuild-nomulus.yaml --substitutions TAG_NAME=[TAG] .
 #
 # To trigger a build automatically, follow the instructions below and add a trigger:
 # https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
@@ -30,29 +30,27 @@ steps:
   dir: 'gradle'
 # Build the tool binary.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
-  args: ['release/build_nomulus_for_env.sh', 'tool', 'nomulus']
+  args: ['release/build_nomulus_for_env.sh', 'tool', 'output']
 # Build and package the deployment files for alpha.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
-  args: ['release/build_nomulus_for_env.sh', 'alpha', 'nomulus']
+  args: ['release/build_nomulus_for_env.sh', 'alpha', 'output']
 # Build and package the deployment files for crash.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
-  args: ['release/build_nomulus_for_env.sh', 'crash', 'nomulus']
+  args: ['release/build_nomulus_for_env.sh', 'crash', 'output']
 # Build and package the deployment files for sandbox.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
-  args: ['release/build_nomulus_for_env.sh', 'sandbox', 'nomulus']
+  args: ['release/build_nomulus_for_env.sh', 'sandbox', 'output']
 # Build and package the deployment files for production.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
-  args: ['release/build_nomulus_for_env.sh', 'production', 'nomulus']
+  args: ['release/build_nomulus_for_env.sh', 'production', 'output']
 # The tarballs to upload to GCS.
 artifacts:
   objects:
     location: 'gs://${PROJECT_ID}-deploy/${TAG_NAME}'
     paths:
-    - 'nomulus.jar'
-    - 'alpha.tar'
-    - 'crash.tar'
-    - 'sandbox.tar'
-    - 'production.tar'
+    - 'output/*.tar'
+    - 'output/nomulus.jar'
+    - 'release/cloudbuild-sync.yaml'
 timeout: 3600s
 options:
   machineType: 'N1_HIGHCPU_8'

release/cloudbuild-proxy.yaml

@@ -3,12 +3,12 @@
 # credential helper.
 # See: https://cloud.google.com/cloud-build/docs/build-debug-locally
 # Then run:
-# cloud-build-local --config=cloudbuild-proxy.yaml --dryrun=false --substitutions TAG_NAME=[TAG] ..
+# cloud-build-local --config=cloudbuild-proxy.yaml --dryrun=false --substitutions TAG_NAME=[TAG] .
 # This will create a docker image named gcr.io/[PROJECT_ID]/proxy:[TAG] locally.
 # The PROJECT_ID is the current project name that gcloud uses.
 #
 # To manually trigger a build on GCB, run:
-# gcloud builds submit --config cloudbuild-proxy.yaml --substitutions TAG_NAME=[TAG] ..
+# gcloud builds submit --config cloudbuild-proxy.yaml --substitutions TAG_NAME=[TAG] .
 #
 # To trigger a build automatically, follow the instructions below and add a trigger:
 # https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
@@ -31,32 +31,26 @@ steps:
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   args: ['docker', 'build', '--tag', 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}', '.']
   dir: 'gradle/proxy'
-# Move config files to the working directory. This is necessary because of Spinnaker limitations.
-# It will concantinate `location' and `path' in the artifact field to construct the artifact
-# path, even though the artifact is always uploaded to the `location', and `path' can be a regular
-# expression.
-- name: 'gcr.io/${PROJECT_ID}/builder:latest'
-  entrypoint: /bin/bash
-  args: ['-c', 'mv java/google/registry/proxy/kubernetes/* .']
 # Push the image. We can't let Cloud Build's default processing do that for us
 # because we need to push the image before we can sign it in the following
 # step.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   args: ['docker', 'push', 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}']
-# Get the image hash and sign it.
+# Get the image digest, sign it and substitute in the digest in the tagging yaml file.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
   args:
   - -c
   - >
-    hash=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/proxy \
+    digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/proxy \
       --format="get(digest)" --filter="tags = ${TAG_NAME}") && \
     gcloud --project=${PROJECT_ID} alpha container binauthz attestations \
-      sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/proxy@$hash \
+      sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/proxy@$digest \
       --attestor=build-attestor --attestor-project=${PROJECT_ID} \
       --keyversion-project=${PROJECT_ID} --keyversion-location=global \
       --keyversion-keyring=attestor-keys --keyversion-key=signing \
       --keyversion=1
+    sed -i s/proxy:'$${TAG_NAME}'/proxy@$digest/g release/cloudbuild-tag.yaml
 # Images to upload to GCR. Even though the image has already been uploaded, we still include it
 # here so that the GCB pubsub message contains it (for Spinnaker to consume).
 images: ['gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}']
@@ -66,15 +60,8 @@ artifacts:
     location: 'gs://${PROJECT_ID}-deploy/${TAG_NAME}'
     # This cannot be regexs because of how Spinnaker constructs artifact paths.
     paths:
-    - 'proxy-deployment-alpha.yaml'
-    - 'proxy-deployment-crash.yaml'
-    - 'proxy-deployment-sandbox.yaml'
-    - 'proxy-deployment-production.yaml'
-    - 'proxy-deployment-crash-canary.yaml'
-    - 'proxy-deployment-sandbox-canary.yaml'
-    - 'proxy-deployment-production-canary.yaml'
-    - 'proxy-service.yaml'
-    - 'proxy-service-canary.yaml'
+    - 'java/google/registry/proxy/kubernetes/proxy-*.yaml'
+    - 'release/cloudbuild-tag.yaml'
 timeout: 3600s
 options:
   machineType: 'N1_HIGHCPU_8'

release/cloudbuild-release.yaml

@@ -4,10 +4,10 @@
 # See: https://cloud.google.com/cloud-build/docs/build-debug-locally
 # Then run:
 # cloud-build-local --config=cloudbuild-release.yaml --dryrun=false \
-#   --substitutions TAG_NAME=[TAG] ..
+#   --substitutions TAG_NAME=[TAG] .
 #
 # To manually trigger a build on GCB, run:
-# gcloud builds submit --config cloudbuild-proxy.yaml --substitutions TAG_NAME=[TAG] ..
+# gcloud builds submit --config cloudbuild-proxy.yaml --substitutions TAG_NAME=[TAG] .
 #
 # To trigger a build automatically, follow the instructions below and add a trigger:
 # https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
@@ -54,7 +54,7 @@ steps:
     docker push gcr.io/${PROJECT_ID}/builder:latest
     docker push gcr.io/${PROJECT_ID}/builder:${TAG_NAME}
   dir: 'release/builder/'
-# Do text replacement in the merged repo, hardcoding image hashes.
+# Do text replacement in the merged repo, hardcoding image digests.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: /bin/bash
   args:
@@ -68,6 +68,7 @@ steps:
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-proxy.yaml
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-nomulus.yaml
     sed -i s/GCP_PROJECT/${PROJECT_ID}/ java/google/registry/proxy/kubernetes/proxy-*.yaml
+    sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-sync.yaml
 # Upload the gradle binary to GCS if it does not exist and point URL in gradle wrapper to it.
 - name: 'gcr.io/cloud-builders/gsutil'
   entrypoint: /bin/bash

release/cloudbuild-sync.yaml

@@ -0,0 +1,24 @@
+# To run the build locally, install cloud-build-local first.
+# Then run:
+# cloud-build-local --config=cloudbuild-sync.yaml --dryrun=false --substitutions TAG_NAME=[TAG] .
+# This will sync the folder gs://[PROJECT_ID]-deploy/[TAG] to gs://[PROJECT_ID]-deploy/live.
+# The PROJECT_ID is the current project name that gcloud uses.
+#
+# To manually trigger a build on GCB, run:
+# gcloud builds submit --config cloudbuild-sync.yaml --substitutions TAG_NAME=[TAG] .
+#
+# To trigger a build automatically, follow the instructions below and add a trigger:
+# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
+steps:
+# Rsync the folder.
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  args:
+  - 'gsutil'
+  - '-m'
+  - 'rsync'
+  - '-d'
+  - 'gs://${PROJECT_ID}-deploy/${TAG_NAME}'
+  - 'gs://${PROJECT_ID}-deploy/live'
+timeout: 3600s
+options:
+  machineType: 'N1_HIGHCPU_8'

release/cloudbuild-tag.yaml

@@ -0,0 +1,23 @@
+# To run the build locally, install cloud-build-local first.
+# Then run:
+# cloud-build-local --config=cloudbuild-tag.yaml --dryrun=false --substitutions TAG_NAME=[TAG] .
+# This will add a "live" tag to the proxy image in gcr.io/[PROJECT_ID]/proxy:[TAG].
+# The PROJECT_ID is the current project name that gcloud uses.
+#
+# To manually trigger a build on GCB, run:
+# gcloud builds submit --config cloudbuild-tag.yaml --substitutions TAG_NAME=[TAG] .
+#
+# To trigger a build automatically, follow the instructions below and add a trigger:
+# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
+steps:
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  args:
+  - gcloud
+  - container
+  - images
+  - add-tag
+  - gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}
+  - gcr.io/${PROJECT_ID}/proxy:live
+timeout: 3600s
+options:
+  machineType: 'N1_HIGHCPU_8'