Update proxy deployment pipeline

The pipeline is broken into two. The first one is to be triggered when the public repo is tagged. It then tags the private repo, builds and upload the builder and base images, and push a new commit to the release (merged repo). This pipeline also does text manipulation on several files in the release repo to ensure that the images uploaded in this pipeline is always used to reproducibly build the release repo at the same commit.

The second pipeline is then triggered by commit into the release repo, which builds, signs and uploads the proxy image.

Also updated the dependency lock files to use the latest plugins dependencies, which are uploaded to the GCS repo.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=244666211

gradle/gradle/dependency-locks/buildscript-classpath.lockfile

@@ -13,8 +13,8 @@ com.jcraft:jzlib:1.1.1
 com.moowork.gradle:gradle-node-plugin:1.2.0
 com.moowork.node:com.moowork.node.gradle.plugin:1.2.0
 com.netflix.nebula:gradle-lint-plugin:10.4.2
-com.netflix.nebula:nebula-gradle-interop:1.0.6
-com.netflix.nebula:nebula-test:7.2.3
+com.netflix.nebula:nebula-gradle-interop:1.0.7
+com.netflix.nebula:nebula-test:7.2.4
 commons-codec:commons-codec:1.9
 commons-io:commons-io:2.5
 commons-lang:commons-lang:2.6

gradle/proxy/Dockerfile

@@ -1,4 +1,3 @@
-# TODO(jianglai): Peg to a specific sha256 hash to enable reproducible build.
 FROM gcr.io/distroless/java
 ADD build/libs/proxy_server.jar .
 ENTRYPOINT ["java", "-jar", "proxy_server.jar"]

release/builder/Dockerfile

@@ -12,11 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This Dockerfile builds an image that can be used to build the nomulus app.
-# We need the following programs during Gradle build:
+# This Dockerfile builds an image that can be used in Google Cloud Build.
+# We need the following programs to build the Nomulus app and the proxy:
 # 1. Java 8 for compilation.
 # 2. Node.js/NPM for JavaScript compilation.
 # 3. Google Cloud SDK for generating the WARs.
+# 4. Git to manipulate the private and the merged repos.
+# 5. Docker to build and push images.
 FROM marketplace.gcr.io/google/ubuntu1804
 ENV DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF-8
 ADD ./build.sh .

release/builder/build.sh

@@ -33,6 +33,10 @@ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \
   | apt-key add -
 apt-get update -y
 apt-get install google-cloud-sdk-app-engine-java -y
+# Install git
+apt-get install git -y
+# Install docker
+apt-get install docker.io -y
 apt-get remove apt-utils locales lsb-release -y
 apt-get autoclean -y
 apt-get autoremove -y

release/cloudbuild-nomulus.yaml

@@ -16,46 +16,51 @@
 # https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
 steps:
 # Set permissions correctly. Not sure why it is necessary, but it is.
-- name: 'alpine'
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  entrypoint: /bin/bash
   args: ['chown', '-R', 'root:root', '.']
-- name: 'alpine'
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  entrypoint: /bin/bash
   args: ['chmod', '-R', '777', '.']
-# Clone the private repo and merge its contents.
-- name: 'gcr.io/cloud-builders/gcloud'
-  args: ['source', 'repos', 'clone', 'nomulus-internal']
-- name: 'alpine'
-  args: ['sh', '-c', 'cp -r nomulus-internal/* .']
 # Create a directory to store the artifacts
-- name: 'alpine'
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  entrypoint: /bin/bash
   args: ['mkdir', 'nomulus']
+# Run tests
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  args: ['./gradlew', 'test']
+  dir: 'gradle'
 # Build the deployment files for sandbox.
-- name: 'gcr.io/${PROJECT_ID}/builder'
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
   args:
-    - './gradlew'
-    - 'stage'
-    - '-x'
-    - 'autoLintGradle'
-    - '-Penvironment=sandbox'
-    - '-PmavenUrl=gcs://domain-registry-maven-repository/maven'
-    - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins'
+  - './gradlew'
+  - 'stage'
+  - '-x'
+  - 'autoLintGradle'
+  - '-Penvironment=sandbox'
+  - '-PmavenUrl=gcs://domain-registry-maven-repository/maven'
+  - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins'
   dir: 'gradle'
-- name: 'alpine'
-  args: ['sh', './move_artifacts.sh', 'sandbox', 'nomulus']
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  entrypoint: bin/bash
+  args: ['./move_artifacts.sh', 'sandbox', 'nomulus']
 # Build the deployment files for crash.
-- name: 'gcr.io/${PROJECT_ID}/builder'
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
   args:
-    - './gradlew'
-    - 'stage'
-    - '-x'
-    - 'autoLintGradle'
-    - '-Penvironment=crash'
-    - '-PmavenUrl=gcs://domain-registry-maven-repository/maven'
-    - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins'
+  - './gradlew'
+  - 'stage'
+  - '-x'
+  - 'autoLintGradle'
+  - '-Penvironment=crash'
+  - '-PmavenUrl=gcs://domain-registry-maven-repository/maven'
+  - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins'
   dir: 'gradle'
-- name: 'alpine'
-  args: ['sh', './move_artifacts.sh', 'crash', 'nomulus']
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  entrypoint: bin/bash
+  args: ['./move_artifacts.sh', 'crash', 'nomulus']
 # Create the uber tarball including all environments.
-- name: 'alpine'
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  entrypoint: bin/bash
   args: ['tar', 'cvf', '../nomulus.tar', '.']
   dir: 'nomulus'
 # The tarball to upload to GCS.

release/cloudbuild-proxy.yaml

@@ -3,89 +3,87 @@
 # credential helper.
 # See: https://cloud.google.com/cloud-build/docs/build-debug-locally
 # Then run:
-# cloud-build-local --config=cloudbuild-proxy.yaml --dryrun=false --substitutions TAG_NAME=[TAG] .
+# cloud-build-local --config=cloudbuild-proxy.yaml --dryrun=false --substitutions TAG_NAME=[TAG] ..
 # This will create a docker image named gcr.io/[PROJECT_ID]/proxy:[TAG] locally.
 # The PROJECT_ID is the current project name that gcloud uses.
 #
 # To manually trigger a build on GCB, run:
-# gcloud builds submit --config cloudbuild-proxy.yaml --substitutions TAG_NAME=[TAG] .
+# gcloud builds submit --config cloudbuild-proxy.yaml --substitutions TAG_NAME=[TAG] ..
 #
 # To trigger a build automatically, follow the instructions below and add a trigger:
 # https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
 steps:
 # Set permissions correctly. Not sure why it is necessary, but it is.
-- name: 'alpine'
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
   args: ['chown', '-R', 'root:root', '.']
-- name: 'alpine'
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
   args: ['chmod', '-R', '777', '.']
-# Clone the private repo merge its contents.
-- name: 'gcr.io/cloud-builders/gcloud'
-  args: ['source', 'repos', 'clone', 'nomulus-internal']
-- name: 'alpine'
-  args: ['sh', '-c', 'cp -r nomulus-internal/* .']
 # Build the deploy jar.
-- name: 'openjdk:8-slim'
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
   args:
-    - './gradlew'
-    - ':proxy:deployJar'
-    - '-x'
-    - 'autoLintGradle'
-    - '-PmavenUrl=gcs://domain-registry-maven-repository/maven'
-    - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins'
+  - './gradlew'
+  - ':proxy:test'
+  - ':proxy:deployJar'
+  - '-x'
+  - 'autoLintGradle'
+  - '-PmavenUrl=gcs://domain-registry-maven-repository/maven'
+  - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins'
   dir: 'gradle'
 # Build the docker image.
-- name: 'gcr.io/cloud-builders/docker'
-  args: ['build', '--tag', 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}', '.']
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  args: ['docker', 'build', '--tag', 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}', '.']
   dir: 'gradle/proxy'
 # Move config files to the working directory. This is necessary because of Spinnaker limitations.
 # It will concantinate `location' and `path' in the artifact field to construct the artifact
 # path, even though the artifact is always uploaded to the `location', and `path' can be a regular
 # expression.
-- name: 'alpine'
-  args: ['sh', '-c', 'mv java/google/registry/proxy/kubernetes/* .']
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  entrypoint: /bin/bash
+  args: ['-c', 'mv java/google/registry/proxy/kubernetes/* .']
 # Replace the tag "latest" with the git tag that triggered this build. This is due to a bug in
 # Spinnaker where the tag is appended to the image name when the deployment pipeline is triggered
 # by GCB pubsub messages. The bug is fixed in https://github.com/spinnaker/echo/pull/498 and we can
 # remove this step and the "latest" tag in the manifests when Spinnaker 1.13 is deployed.
-- name: 'alpine'
-  args: ['sh', '-c', 'sed -i s/:latest/:${TAG_NAME}/ proxy-*.yaml']
-# Replace project name.
-- name: 'alpine'
-  args: ['sh', '-c', 'sed -i s/GCP_PROJECT/${PROJECT_ID}/ proxy-*.yaml']
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  entrypoint: /bin/bash
+  args: ['-c', 'sed -i s/:latest/:${TAG_NAME}/ proxy-*.yaml']
 # Push the image. We can't let Cloud Build's default processing do that for us
 # because we need to push the image before we can sign it in the following
 # step.
-- name: 'gcr.io/cloud-builders/docker'
-  args: ['push', 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}']
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  args: ['docker', 'push', 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}']
 # Get the image hash and sign it.
-- name: 'gcr.io/domain-registry-dev/builder'
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
   args:
-    - -c
-    - >
-      hash=$(gcloud container images list-tags \
-        gcr.io/${PROJECT_ID}/proxy \
-        --format="get(digest)" --filter="tags = ${TAG_NAME}") && \
-      gcloud --project=${PROJECT_ID} alpha container binauthz attestations \
-        sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/proxy@$hash \
-        --attestor=build-attestor --attestor-project=${PROJECT_ID} \
-        --keyversion-project=${PROJECT_ID} --keyversion-location=global \
-        --keyversion-keyring=attestor-keys --keyversion-key=signing \
-        --keyversion=1
+  - -c
+  - >
+    hash=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/proxy \
+      --format="get(digest)" --filter="tags = ${TAG_NAME}") && \
+    gcloud --project=${PROJECT_ID} alpha container binauthz attestations \
+      sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/proxy@$hash \
+      --attestor=build-attestor --attestor-project=${PROJECT_ID} \
+      --keyversion-project=${PROJECT_ID} --keyversion-location=global \
+      --keyversion-keyring=attestor-keys --keyversion-key=signing \
+      --keyversion=1
+# Images to upload to GCR. Even though the image has already been uploaded, we still include it
+# here so that the GCB pubsub message contains it (for Spinnaker to consume).
+images: ['gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}']
 # Config files to upload to GCS.
 artifacts:
   objects:
     location: 'gs://${PROJECT_ID}-deploy/${TAG_NAME}'
     # This cannot be regexs because of how Spinnaker constructs artifact paths.
     paths:
-      - 'proxy-deployment-crash.yaml'
-      - 'proxy-deployment-sandbox.yaml'
-      - 'proxy-deployment-production.yaml'
-      - 'proxy-deployment-crash-canary.yaml'
-      - 'proxy-deployment-sandbox-canary.yaml'
-      - 'proxy-deployment-production-canary.yaml'
-      - 'proxy-service.yaml'
-      - 'proxy-service-canary.yaml'
+    - 'proxy-deployment-alpha.yaml'
+    - 'proxy-deployment-crash.yaml'
+    - 'proxy-deployment-sandbox.yaml'
+    - 'proxy-deployment-production.yaml'
+    - 'proxy-deployment-crash-canary.yaml'
+    - 'proxy-deployment-sandbox-canary.yaml'
+    - 'proxy-deployment-production-canary.yaml'
+    - 'proxy-service.yaml'
+    - 'proxy-service-canary.yaml'
 timeout: 3600s
 options:
   machineType: 'N1_HIGHCPU_8'

release/cloudbuild-release.yaml

@@ -0,0 +1,92 @@
+# To run the build locally, install cloud-build-local first.
+# You will need access to a private registry, so be sure to install the docker
+# credential helper.
+# See: https://cloud.google.com/cloud-build/docs/build-debug-locally
+# Then run:
+# cloud-build-local --config=cloudbuild-release.yaml --dryrun=false \
+#   --substitutions TAG_NAME=[TAG] ..
+#
+# To manually trigger a build on GCB, run:
+# gcloud builds submit --config cloudbuild-proxy.yaml --substitutions TAG_NAME=[TAG] ..
+#
+# To trigger a build automatically, follow the instructions below and add a trigger:
+# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
+#
+# This pipeline prepares a release. The pipeline should be run against the Nomulus public repo on
+# GitHub. It builds the builder and base images, and hard codes the sha256 hashes of the resulting
+# images in the merged code base (internal + public) , which is tagged and pushed into the release
+# repo. Actual release artifacts are built from the release repo, ensuring reproducibility.
+steps:
+# Check the out internal repo.
+- name: 'gcr.io/cloud-builders/gcloud'
+  args: ['source', 'repos', 'clone', 'nomulus-internal']
+# Tag and push the internal repo.
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    git tag ${TAG_NAME} && git push origin ${TAG_NAME}
+  dir: 'nomulus-internal'
+# Merge the repos.
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    shopt -s dotglob
+    rm -rf .git && rm -rf nomulus-internal/.git
+    cp -rf nomulus-internal/* .
+    rm -rf nomulus-internal
+# Build the builder image and tag the proxy base image, to be uploaded later.
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    docker build -t gcr.io/${PROJECT_ID}/builder:${TAG_NAME} .
+    docker tag gcr.io/${PROJECT_ID}/builder:${TAG_NAME} gcr.io/${PROJECT_ID}/builder:latest
+    docker pull gcr.io/distroless/java
+    docker tag gcr.io/distroless/java gcr.io/${PROJECT_ID}/base:${TAG_NAME}
+    docker tag gcr.io/distroless/java gcr.io/${PROJECT_ID}/base:latest
+    docker push gcr.io/${PROJECT_ID}/base:latest
+    docker push gcr.io/${PROJECT_ID}/base:${TAG_NAME}
+    docker push gcr.io/${PROJECT_ID}/builder:latest
+    docker push gcr.io/${PROJECT_ID}/builder:${TAG_NAME}
+  dir: 'release/builder/'
+# Do text replacement in the merged repo, hardcoding image hashes.
+- name: 'gcr.io/cloud-builders/gcloud'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    builder_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/builder \
+      --format='get(digest)' --filter='tags = ${TAG_NAME}')
+    base_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/base \
+      --format='get(digest)' --filter='tags = ${TAG_NAME}')
+    sed -i s%distroless/java%${PROJECT_ID}/base@$base_digest% gradle/proxy/Dockerfile
+    sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-proxy.yaml
+    sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-nomulus.yaml
+    sed -i s/GCP_PROJECT/${PROJECT_ID}/ java/google/registry/proxy/kubernetes/proxy-*.yaml
+# Check out the release repo.
+- name: 'gcr.io/cloud-builders/gcloud'
+  args: ['source', 'repos', 'clone', 'nomulus-release']
+# Tag and check in the release repo.
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    cp -rf nomulus-release/.git .
+    rm -rf nomulus-release
+    git config --global user.name "Cloud Build"
+    git config --global user.email \
+      $(gcloud auth list --format='get(account)' --filter=active)
+    git add .
+    git commit -m "Release commit for tag ${TAG_NAME}"
+    git push origin master
+    git tag ${TAG_NAME}
+    git push origin ${TAG_NAME}
+timeout: 3600s
+options:
+  machineType: 'N1_HIGHCPU_8'