Prober ssl cert update automation (#2019)

Defined CloudBuild script and docker image that automatically
updates probers' SSL certs

release/cloudbuild-nomulus.yaml

@@ -74,6 +74,35 @@ steps:
     sed -i s/'nomulus-tool:latest'/nomulus-tool@$digest/g release/cloudbuild-deploy-*.yaml
     # schema-deploy and schema-verify scripts
     sed -i s/'nomulus-tool:latest'/nomulus-tool@$digest/g release/cloudbuild-schema-*.yaml
+# Build the prober_cert_updater image and upload it to GCR. This image extends
+# from the `builder` and the nomulus.jar built earlier.
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    # The nomulus jar is not under the working dir. Must be copied over. 
+    cp ../../output/nomulus.jar .
+    docker build -t gcr.io/${PROJECT_ID}/prober_cert_updater:${TAG_NAME} \
+      --build-arg TAG_NAME=${TAG_NAME} --build-arg PROJECT_ID=${PROJECT_ID} .
+    docker tag gcr.io/${PROJECT_ID}/prober_cert_updater:${TAG_NAME} \
+      gcr.io/${PROJECT_ID}/prober_cert_updater:latest
+    docker push gcr.io/${PROJECT_ID}/prober_cert_updater:latest
+    docker push gcr.io/${PROJECT_ID}/prober_cert_updater:${TAG_NAME}
+  dir: 'release/prober-cert-updater/'
+# Update the prober_updater image digest in relevant GCB files.
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    digest=$(gcloud container images list-tags \
+      gcr.io/${PROJECT_ID}/prober_cert_updater \
+      --format="get(digest)" --filter="tags = ${TAG_NAME}")
+    sed -i s/'prober_cert_updater:latest'/prober_cert_updater@$digest/g \
+      release/cloudbuild-renew-prober-certs-*.yaml
 # Build and stage Dataflow Flex templates.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
@@ -141,6 +170,7 @@ artifacts:
     - 'release/cloudbuild-sync.yaml'
     - 'release/cloudbuild-deploy-*.yaml'
     - 'release/cloudbuild-delete-*.yaml'
+    - 'release/cloudbuild-renew-prober-certs-*.yaml'
     - 'release/cloudbuild-schema-deploy-*.yaml'
     - 'release/cloudbuild-schema-verify-*.yaml'
 

release/cloudbuild-release.yaml

@@ -126,7 +126,8 @@ steps:
     docker push gcr.io/${PROJECT_ID}/schema_verifier:latest
     docker push gcr.io/${PROJECT_ID}/schema_verifier:${TAG_NAME}
   dir: 'release/schema-verifier/'
-# Do text replacement in the schema-deploy and schema-verify configs.
+# Do text replacement in the schema-deploy, schema-verify and
+# prober_cert_updater configs.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: /bin/bash
   args:
@@ -142,10 +143,16 @@ steps:
     schema_verifier_digest=$( \
       gcloud container images list-tags gcr.io/${PROJECT_ID}/schema_verifier \
       --format='get(digest)' --filter='tags = ${TAG_NAME}')
+    prober_cert_updater_digest=$( \
+      gcloud container images list-tags \
+      gcr.io/${PROJECT_ID}/prober_cert_updater \
+      --format='get(digest)' --filter='tags = ${TAG_NAME}')
     sed -i s/builder:latest/builder@$builder_digest/g \
       release/cloudbuild-schema-deploy.yaml
     sed -i s/builder:latest/builder@$builder_digest/g \
       release/cloudbuild-schema-verify.yaml
+    sed -i s/builder:latest/builder@$builder_digest/g \
+      release/cloudbuild-renew-prober-certs.yaml
     sed -i s/schema_deployer:latest/schema_deployer@$schema_deployer_digest/g \
       release/cloudbuild-schema-deploy.yaml
     sed -i s/schema_verifier:latest/schema_verifier@$schema_verifier_digest/g \
@@ -156,6 +163,8 @@ steps:
         > release/cloudbuild-schema-deploy-${environment}.yaml
       sed s/'$${_ENV}'/${environment}/g release/cloudbuild-schema-verify.yaml \
         > release/cloudbuild-schema-verify-${environment}.yaml
+      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-renew-prober-certs.yaml \
+        > release/cloudbuild-renew-prober-certs-${environment}.yaml
     done
 # Upload the gradle binary to GCS if it does not exist and point URL in gradle wrapper to it.
 - name: 'gcr.io/cloud-builders/gsutil'

release/cloudbuild-renew-prober-certs.yaml

@@ -0,0 +1,91 @@
+# To run the build locally, install cloud-build-local first.
+# Then run:
+# cloud-build-local --config=cloudbuild-renew-prober-certs.yaml --dryrun=false \
+# --substitutions=_ENV=[ENV] ..
+#
+# This will generate a new SSL certificate and apply it to the probers in the
+# environment specified by ${_ENV}.
+#
+# To manually trigger a build on GCB, run:
+# gcloud builds submit --config=cloudbuild-renew-prober-certs.yaml \
+# --substitutions=_ENV=[ENV] ..
+#
+# To manually trigger a build on GCB using a released version, with TAG being
+# the release tag and _ENV being the environment, run:
+# cd $(mktemp -d);
+# gcloud storage cp \
+#   gs://domain-registry-dev-deploy/{TAG}/cloudbuild-renew-prober-certs-{_ENV}.yaml .
+# gcloud builds submit --config="./cloudbuild-renew-prober-certs-{_ENV}.yaml"
+#
+# To trigger a build automatically, follow the instructions below and add a trigger:
+# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
+#
+# Note that the release process hardens the tags and variables in this file:
+# - The 'latest' tag on docker images will be replaced by their image digests.
+# - The ${_ENV} pattern will be replaced by the actual environment name.
+# Please refer to ./cloudbuild-release.yaml for more details.
+steps:
+# Generate new SSL certs
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    # Download the CA certificate files for signing the new certificate.
+    gcloud storage cp gs://$PROJECT_ID-prober-certs/ca.* .
+    # Get the passphrase
+    gcloud secrets versions access latest \
+      --secret=epp-prober-passphrase-${_ENV} \
+      --out-file=./passphrase.txt
+    openssl req -new -newkey rsa:4096 -nodes \
+      -out prober-client-tls.req \
+      -keyout prober-client-tls.key \
+      -subj "/C=US/ST=New York/L=New York/O=Google/OU=gTLD/CN=Google Registry/emailAddress=example@example.com" \
+      -passout file:./passphrase.txt
+    openssl x509 -CA ca.pem  -CAkey ca.key -CAserial ca.srl -req -days 398  \
+      -in prober-client-tls.req \
+      -out prober-client-tls.pem
+    openssl pkcs12 -export -in prober-client-tls.pem \
+      -inkey prober-client-tls.key -out prober-client-tls.p12 \
+      -password file:./passphrase.txt
+# Download the nomulus-tools credential, which has the privilege to invoke tools
+# commands. Also download the list of probers.
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    # Download the list of prober registrars in a file.
+    prober_list_name=prober_registrars.${_ENV}
+    gcloud storage cp \
+      gs://$PROJECT_ID-prober-certs/$prober_list_name ./prober-list
+    # Get the nomulus-tools credential, which is needed by the next step for
+    # invoking nomulus-tools commands.
+    gcloud secrets versions access latest \
+      --secret nomulus-tool-cloudbuild-credential \
+      > nomulus_tool_credential.json
+# Install the new pem cert in the Nomulus server. After this step, both the
+# current cert and the new cert are accepted for login by the server.
+- name: 'gcr.io/$PROJECT_ID/prober_cert_updater:latest'
+  args:
+  - ${_ENV}
+  - ./prober-client-tls.pem
+  - ./prober-list
+  - ./nomulus_tool_credential.json
+# Add the p12 cert to SecretManager. Prober instances will start using this
+# cert when they restart.
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    if [ ${_ENV} == production ]; then
+      secret_id="prober-keys"
+    else
+      secret_id="prober-keys-${_ENV}"
+    fi
+    gcloud secrets versions add $secret_id \
+      --data-file="./prober-client-tls.p12"

release/prober-cert-updater/Dockerfile

@@ -0,0 +1,27 @@
+# Copyright 2023 The Nomulus Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This Dockerfile updates the SSL certs used by the probers. It uses the builder
+# image, and should be built after the builder in the release process. It also
+# needs the :core:nomulus jar in this directory, which is the caller's
+# responsibility to copy it here.
+
+ARG PROJECT_ID
+ARG TAG_NAME
+FROM gcr.io/${PROJECT_ID}/builder:${TAG_NAME}
+
+COPY nomulus.jar /
+COPY rotate_prober_certs.sh /usr/local/bin
+
+ENTRYPOINT [ "rotate_prober_certs.sh" ]

release/prober-cert-updater/rotate_prober_certs.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Copyright 2023 The Nomulus Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Rotates SSL certs for prober registars. This script expects the following
+# parameters in order:
+# - env: The Nomulus environment, production, sandbox, etc.
+# - prober_cert: The SSL cert file (.pem) to be installed.
+# - prober_list: The probers' registrar-ids in a file, one per line.
+# - tools_credential: The credential (.json) needed to run the nomulus command.
+
+set -e
+if [ "$#" -ne 4 ]; then
+  echo "Expecting four parameters in order: env prober_cert_file prober_list" \
+      "tools_credential"
+  exit 1
+fi
+
+nomulus_env="${1}"
+cert_file="${2}"
+prober_list="${3}"
+tools_credential="${4}"
+
+echo ${nomulus_env} ${cert_file} ${prober_list}
+
+cat "${prober_list}" | while IFS= read -r prober; do
+  echo "Updating client certificate for ${prober}."
+  java -jar /nomulus.jar -e "${nomulus_env}" \
+      --credential "${tools_credential}" \
+      update_registrar "${prober}" -f \
+      --rotate_primary_cert \
+      --cert_file  "${cert_file}"
+done