Add a Cloud Build job for syncing Tld configuration files from the internal repo with the database (#2174)

* Add a cloudbuild-tld-sync job

This job checks the Tld config files in the internal repo and syncs them with the actual Tld objects in the database using the configure_tld numulus command.

* Add the dockerfile and shell script

* Force the command

* Add comments

* add newline

* Create a separate copy of the job for each environment

* fix file name

* Fix indentation

release/cloudbuild-nomulus.yaml

@@ -91,6 +91,35 @@ steps:
       --format="get(digest)" --filter="tags = ${TAG_NAME}")
     sed -i s/'prober_cert_updater:latest'/prober_cert_updater@$digest/g \
       release/cloudbuild-renew-prober-certs-*.yaml
+# Build the tld_updater image and upload it to GCR. This image extends
+# from the `builder` and the nomulus.jar built earlier.
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    # The nomulus jar is not under the working dir. Must be copied over. 
+    cp ../../output/nomulus.jar .
+    docker build -t gcr.io/${PROJECT_ID}/tld_updater:${TAG_NAME} \
+      --build-arg TAG_NAME=${TAG_NAME} --build-arg PROJECT_ID=${PROJECT_ID} .
+    docker tag gcr.io/${PROJECT_ID}/tld_updater:${TAG_NAME} \
+      gcr.io/${PROJECT_ID}/tld_updater:latest
+    docker push gcr.io/${PROJECT_ID}/tld_updater:latest
+    docker push gcr.io/${PROJECT_ID}/tld_updater:${TAG_NAME}
+  dir: 'release/tld-updater/'
+# Update the tld_updater image digest in relevant GCB files.
+- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    digest=$(gcloud container images list-tags \
+      gcr.io/${PROJECT_ID}/tld_updater \
+      --format="get(digest)" --filter="tags = ${TAG_NAME}")
+    sed -i s/'tld_updater:latest'/tld_updater@$digest/g \
+      release/cloudbuild-tld-sync-*.yaml
 # Build and stage Dataflow Flex templates.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
@@ -161,6 +190,7 @@ artifacts:
     - 'release/cloudbuild-renew-prober-certs-*.yaml'
     - 'release/cloudbuild-schema-deploy-*.yaml'
     - 'release/cloudbuild-schema-verify-*.yaml'
+    - 'release/cloudbuild-tld-sync-*.yaml'
 
 timeout: 7200s
 options:

release/cloudbuild-release.yaml

@@ -139,12 +139,18 @@ steps:
       gcloud container images list-tags \
       gcr.io/${PROJECT_ID}/prober_cert_updater \
       --format='get(digest)' --filter='tags = ${TAG_NAME}')
+     tld_updater_digest=$( \
+      gcloud container images list-tags \
+      gcr.io/${PROJECT_ID}/tld_updater \
+      --format='get(digest)' --filter='tags = ${TAG_NAME}')
     sed -i s/builder:latest/builder@$builder_digest/g \
       release/cloudbuild-schema-deploy.yaml
     sed -i s/builder:latest/builder@$builder_digest/g \
       release/cloudbuild-schema-verify.yaml
     sed -i s/builder:latest/builder@$builder_digest/g \
       release/cloudbuild-renew-prober-certs.yaml
+    sed -i s/builder:latest/builder@$builder_digest/g \
+      release/cloudbuild-tld-sync.yaml
     sed -i s/schema_deployer:latest/schema_deployer@$schema_deployer_digest/g \
       release/cloudbuild-schema-deploy.yaml
     sed -i s/schema_verifier:latest/schema_verifier@$schema_verifier_digest/g \
@@ -157,6 +163,8 @@ steps:
         > release/cloudbuild-schema-verify-${environment}.yaml
       sed s/'$${_ENV}'/${environment}/g release/cloudbuild-renew-prober-certs.yaml \
         > release/cloudbuild-renew-prober-certs-${environment}.yaml
+      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-tld-sync.yaml \
+        > release/cloudbuild-tld-sync-${environment}.yaml
     done
 # Upload the gradle binary to GCS if it does not exist and point URL in gradle wrapper to it.
 - name: 'gcr.io/cloud-builders/gsutil'

release/cloudbuild-tld-sync.yaml

@@ -0,0 +1,45 @@
+# This will sync the Tld configurations in the internal repo with the Tld objects in the database.
+#
+# To manually trigger a build on GCB, run:
+# gcloud builds submit --config cloudbuild-tld-sync.yaml --substitutions \
+#   _INTERNAL_REPO_URL=[URL] ..
+#
+# To trigger a build automatically, follow the instructions below and add a trigger:
+# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
+#
+# Note that the release process hardens the tags and variables in this file:
+# - The 'latest' tag on docker images will be replaced by their image digests.
+# - The ${_ENV} pattern will be replaced by the actual environment name.
+# Please refer to ./cloudbuild-release.yaml for more details.
+#
+steps:
+# Check out the internal repo.
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    git clone https://gerrit.googlesource.com/gcompute-tools
+    ./gcompute-tools/git-cookie-authdaemon
+    git clone ${_INTERNAL_REPO_URL} nomulus-internal
+# Download and decrypt the nomulus tool credential
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    gcloud secrets versions access latest \
+      --secret nomulus-tool-cloudbuild-credential \
+      > nomulus_tool_credential.json
+# Configure the TLDs using the stored configuration files in the internal repo
+- name: 'gcr.io/$PROJECT_ID/tld_updater:latest'
+  args:
+  - ${_ENV}
+  - ./nomulus_tool_credential.json
+  - nomulus-internal/core/src/main/java/google/registry/config/files/tld/
+
+timeout: 7200s
+options:
+  machineType: 'E2_HIGHCPU_32'

release/tld-updater/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright 2023 The Nomulus Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG PROJECT_ID
+ARG TAG_NAME
+FROM gcr.io/${PROJECT_ID}/builder:${TAG_NAME}
+
+COPY nomulus.jar /
+COPY sync_tlds.sh /usr/local/bin
+
+ENTRYPOINT [ "bash", "sync_tlds.sh" ]

release/tld-updater/sync_tlds.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Copyright 2023 The Nomulus Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sync the TLD configuration files from the internal repo with the Tld object
+# in the database. Loops through the Tld configuration files and runs the configure_tld command
+# with the file.
+
+# - env: The Nomulus environment, production, sandbox, etc.
+# - tools_credential: The credential (.json) needed to run the nomulus command.
+# - config_file_directory: The internal directory storing the TLD config files.
+
+set -e
+if [ "$#" -ne 3 ]; then
+  echo "Expecting three parameters in order: env tools_credential config_file_directory"
+  exit 1
+fi
+
+nomulus_env="${1}"
+tools_credential="${2}"
+config_file_directory="${3}"
+
+echo ${config_file_directory}
+
+for FILE in ${config_file_directory}/${nomulus_env}/*; do
+  echo $FILE
+  java -jar /nomulus.jar -e "${nomulus_env}" \
+  --credential "${tools_credential}" \
+  configure_tld -i $FILE --force
+done