Deploy SQL schema from Cloud Build (#350)

Defined Docker image for schema deployment.

Included schema deploymer docker in the Cloud Build release process.

Defined Cloud Build config for schema deployment.

TESTED=Used cloud-build-local to test deployment flow
TESTED=Used docker to test schema deployer image in more ways

release/cloudbuild-deploy.yaml

@@ -48,7 +48,7 @@ steps:
   - deploy_invoicing_pipeline
 # Save the deployed tag for the current environment on GCS. Because of b/137891685
 # which causes the for-loop in the next step to fail, this may not be the last step.
-# TODO(weiminyu): do this in last step, and use 'artifacts' section to upload the tag.
+# TODO(weiminyu): do this in last step.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
   args:
@@ -56,7 +56,7 @@ steps:
   - |
     set -e
     echo ${TAG_NAME} | \
-      gsutil cp - gs://${PROJECT_ID}-deploy/deployed_tags/nomulus.${_ENV}.tag
+      gsutil cp - gs://${PROJECT_ID}-deployed-tags/nomulus.${_ENV}.tag
 # Deploy the GAE config files.
 # First authorize the gcloud tool to use the credential json file, then
 # download and unzip the tarball that contains the relevant config files

release/cloudbuild-release.yaml

@@ -98,6 +98,37 @@ steps:
       sed s/'$${_ENV}'/${environment}/g release/cloudbuild-deploy.yaml \
         > release/cloudbuild-deploy-${environment}.yaml
     done
+# Build the schema_deployer image and upload it to GCR.
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    docker build -t gcr.io/${PROJECT_ID}/schema_deployer:${TAG_NAME} .
+    docker tag gcr.io/${PROJECT_ID}/schema_deployer:${TAG_NAME} \
+      gcr.io/${PROJECT_ID}/schema_deployer:latest
+    docker push gcr.io/${PROJECT_ID}/schema_deployer:latest
+    docker push gcr.io/${PROJECT_ID}/schema_deployer:${TAG_NAME}
+  dir: 'release/schema-deployer/'
+# Do text replacement in the schema-deploy config, hardcoding image digests.
+- name: 'gcr.io/cloud-builders/gcloud'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    builder_digest=$( \
+      gcloud container images list-tags gcr.io/${PROJECT_ID}/builder \
+      --format='get(digest)' --filter='tags = ${TAG_NAME}')
+    schema_deployer_digest=$( \
+      gcloud container images list-tags gcr.io/${PROJECT_ID}/schema_deployer \
+      --format='get(digest)' --filter='tags = ${TAG_NAME}')
+    sed -i s/builder:latest/builder@$builder_digest/g \
+      release/cloudbuild-schema-deploy.yaml
+    sed -i s/schema_deployer:latest/schema_deployer@$schema_deployer_digest/g \
+      release/cloudbuild-schema-deploy.yaml
+    sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-schema-deploy.yaml
 # Upload the gradle binary to GCS if it does not exist and point URL in gradle wrapper to it.
 - name: 'gcr.io/cloud-builders/gsutil'
   entrypoint: /bin/bash

release/cloudbuild-schema-deploy.yaml

@@ -0,0 +1,81 @@
+# To run the build locally, install cloud-build-local first.
+# Then run:
+# cloud-build-local --config=cloudbuild-schema-deploy.yaml --dryrun=false \
+# --substitutions=TAG_NAME=[TAG],_ENV=[ENV] ..
+#
+# This will deploy Cloud SQL schema release with tag value ${TAG_NAME} to
+# the environment specified by ${_ENV}.
+#
+# To manually trigger a build on GCB, run:
+# gcloud builds submit --config=cloudbuild-schema-deploy.yaml \
+# --substitutions=TAG_NAME=[TAG],_ENV=[ENV] ..
+#
+# To trigger a build automatically, follow the instructions below and add a trigger:
+# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
+#
+# Note that the release process hardens the tags and variables in this file:
+# - The 'latest' tag on docker images will be replaced by their image digests.
+# - The ${TAG_NAME} pattern will be replaced by the acutal release tag.
+# Please refer to ./cloudbuild-release.yaml for more details.
+steps:
+# Download and decrypt the nomulus tool credential, which has the privilege to
+# start Cloud SQL proxy to all environments.
+# Also download and decrypt the admin_credential file, which has the cloud
+# instance name and database login name and password.
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  volumes:
+  - name: 'secrets'
+    path: '/secrets'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    gsutil cp gs://${PROJECT_ID}-deploy/secrets/tool-credential.json.enc - \
+      | base64 -d \
+      | gcloud kms decrypt \
+        --ciphertext-file=- \
+        --plaintext-file=/secrets/cloud_sql_credential.json \
+        --location=global --keyring=nomulus-tool-keyring --key=nomulus-tool-key
+    gsutil cp gs://${PROJECT_ID}-deploy/cloudsql-credentials/${_ENV}/admin_credential.enc - \
+      | base64 -d \
+      | gcloud kms decrypt \
+        --ciphertext-file=- \
+        --plaintext-file=/secrets/admin_credential.dec \
+        --location global --keyring=nomulus-tool-keyring \
+        --key=nomulus-tool-key
+# Download the schema jar to be deployed.
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  volumes:
+  - name: 'flyway'
+    path: '/flyway/jars'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    gsutil cp gs://domain-registry-dev-deploy/${TAG_NAME}/schema.jar \
+      /flyway/jars
+# Deploy SQL schema
+- name: 'gcr.io/${PROJECT_ID}/schema_deployer:latest'
+  volumes:
+  - name: 'secrets'
+    path: '/secrets'
+  - name: 'flyway'
+    path: '/flyway/jars'
+  args: ['migrate']
+# Save the deployed tag for the current environment on GCS to a well-known.
+# location. Do not use the 'artifacts' section for this since it will
+# upload an extra metadata file every time and pollute the folder.
+# TODO(weiminyu): modify this step so that TAG_NAME may be 'live'.
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    echo ${TAG_NAME} | \
+      gsutil cp - gs://${PROJECT_ID}-deployed-tags/sql.${_ENV}.tag\
+timeout: 3600s
+options:
+  machineType: 'N1_HIGHCPU_8'

release/schema-deployer/Dockerfile

@@ -0,0 +1,56 @@
+# Copyright 2019 The Nomulus Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This Dockerfile builds an image that can be used in Google Cloud Build.
+# We need the following programs to build the schema deployer:
+# 1. Bash to execute a shell script.
+# 2. Java 8 for running the Flywaydb commandline tool.
+# 2. Cloud SQL proxy for connection to the SQL instance.
+# 3. The Flywaydb commandline tool.
+#
+# Please refer to deploy_sql_schema.sh for expected volumes and arguments.
+
+# Although any Linux-based Java image with bash would work (e.g., openjdk:8),
+# as a GCP application we prefer to start with a GCP-approved base image.
+FROM marketplace.gcr.io/google/ubuntu1804
+ENV DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF-8
+# Install openjdk-8
+RUN apt-get update -y \
+    && apt-get install locales -y \
+    && locale-gen en_US.UTF-8 \
+    && apt-get install apt-utils -y \
+    && apt-get upgrade -y \
+    && apt-get install openjdk-8-jdk-headless -y
+
+# Get netstat, used for checking Cloud SQL proxy readiness.
+RUN apt-get install net-tools
+
+COPY deploy_sql_schema.sh /usr/local/bin/
+ADD https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \
+    /usr/local/bin/cloud_sql_proxy
+RUN chmod +x /usr/local/bin/cloud_sql_proxy
+# Adapted from https://github.com/flyway/flyway-docker/blob/master/Dockerfile
+RUN \
+  FLYWAY_MAVEN=https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline \
+  && FLYWAY_VERSION=$(curl ${FLYWAY_MAVEN}/maven-metadata.xml \
+                       | grep -oP "<release>\K.*(?=</release>)") \
+  && echo "Downloading Flyway-commandline-${FLYWAY_VERSION}" \
+  && mkdir -p /flyway \
+  && curl -L ${FLYWAY_MAVEN}/${FLYWAY_VERSION}/flyway-commandline-${FLYWAY_VERSION}.tar.gz \
+     -o flyway-commandline-${FLYWAY_VERSION}.tar.gz \
+  && tar -xzf flyway-commandline-${FLYWAY_VERSION}.tar.gz --strip-components=1 \
+       -C /flyway \
+  && rm flyway-commandline-${FLYWAY_VERSION}.tar.gz
+
+ENTRYPOINT [ "deploy_sql_schema.sh" ]

release/schema-deployer/deploy_sql_schema.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+# Copyright 2019 The Nomulus Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Mounted volumes and required files in them:
+# - /secrets: cloud_sql_credential.json: Cloud SQL proxy credential
+# - /flyway/jars: the schema jar to be deployed.
+#
+# Database login info may be passed in two ways:
+# - Decrypt the admin_credential.enc file on GCS and map it as
+#   /secrets/admin_credential.dec
+# - Provide the content of the admin_credential as command line arguments
+
+set -e
+if [ "$#" -le 1 ]; then
+  if [ ! -f /secrets/admin_credential.dec ]; then
+    echo "Missing /secrets/admin_credential.dec"
+    exit 1
+  fi
+  cloud_sql_instance=$(cut -d' ' -f1 /secrets/admin_credential.dec)
+  db_user=$(cut -d' ' -f2 /secrets/admin_credential.dec)
+  db_password=$(cut -d' ' -f3 /secrets/admin_credential.dec)
+  flyway_action=${1:-validate}
+elif [ "$#" -ge 3 ]; then
+  cloud_sql_instance=$1
+  db_user=$2
+  db_password=$3
+  flyway_action=${4:-validate}
+else
+  echo "Wrong number of arguments."
+  exit 1
+fi
+
+# Disallow the 'clean' command, which drops the entire database.
+# See https://flywaydb.org/documentation/commandline/ for command listing.
+if [ "${flyway_action}" == "clean" ]; then
+  echo "The clean action is not allowed."
+  exit 1;
+fi
+echo "$(date): Starting ${flyway_action} action on ${cloud_sql_instance}."
+
+# Set up connection to the Cloud SQL instance.
+# For now we use Cloud SQL Proxy to set up a SSL tunnel to the Cloud SQL
+# instance. This has two drawbacks:
+# - It starts a background process, which is an anti-pattern in Docker.
+# - The main job needs to wait for a while for the proxy to come up.
+# We will research for a better long-term solution.
+#
+# Other options considered:
+# - Connect using Socket Factory in this script.
+#   * Drawback: need to manage version and transitive dependencies
+#     of the postgres-socket-factory jar.
+# - Create a self-contained Java application that connects using socket factory
+#   * Drawback: Seems an overkill
+cloud_sql_proxy -instances=${cloud_sql_instance}=tcp:5432 \
+  --credential_file=/secrets/cloud_sql_credential.json &
+
+set +e
+# Wait for cloud_sql_proxy to start:
+# first sleep 1 second for the process to launch, then loop until port is ready
+# or the proxy process dies.
+sleep 1
+while ! netstat -an | grep ':5432 ' && pgrep cloud_sql_proxy; do sleep 1; done
+
+if ! pgrep cloud_sql_proxy; then
+  echo "Cloud SQL Proxy failed to set up connection."
+  exit 1
+fi
+
+/flyway/flyway -community -user=${db_user} -password=${db_password} \
+  -url=jdbc:postgresql://localhost:5432/postgres \
+  -locations=classpath:sql/flyway \
+  ${flyway_action}
+migration_result=$?
+
+if [ ${flyway_action} == "migrate" ]; then
+  # After deployment, log the current schema.
+  /flyway/flyway -community -user=${db_user} -password=${db_password} \
+    -url=jdbc:postgresql://localhost:5432/postgres \
+    -locations=classpath:sql/flyway \
+    info
+fi
+# Stop Cloud SQL Proxy
+pkill cloud_sql_proxy
+exit ${migration_result}