Add Cloud Build sync job for reserved and premium lists (#2302)

* Change tld-update to db-object-updater

* rename sync_tlds.sh to sync_db_objects.sh

* Change to configured command name

* Change environment to sandbox explicitly for testing on alpha

* Add remaining object steps and change cloudbuild-tld-sync to cloudbuild-sync-db-objects

* Add build_environment flag

* Change order of command and directory

* Uncomment out reserved list part

release/cloudbuild-nomulus.yaml

@@ -91,7 +91,7 @@ steps:
       --format="get(digest)" --filter="tags = ${TAG_NAME}")
     sed -i s/'prober_cert_updater:latest'/prober_cert_updater@$digest/g \
       release/cloudbuild-renew-prober-certs-*.yaml
-# Build the tld_updater image and upload it to GCR. This image extends
+# Build the db_object_updater image and upload it to GCR. This image extends
 # from the `builder` and the nomulus.jar built earlier.
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: /bin/bash
@@ -101,14 +101,14 @@ steps:
     set -e
     # The nomulus jar is not under the working dir. Must be copied over. 
     cp ../../output/nomulus.jar .
-    docker build -t gcr.io/${PROJECT_ID}/tld_updater:${TAG_NAME} \
+    docker build -t gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME} \
       --build-arg TAG_NAME=${TAG_NAME} --build-arg PROJECT_ID=${PROJECT_ID} .
-    docker tag gcr.io/${PROJECT_ID}/tld_updater:${TAG_NAME} \
+    docker tag gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME} \
-      gcr.io/${PROJECT_ID}/tld_updater:latest
+      gcr.io/${PROJECT_ID}/db_object_updater:latest
-    docker push gcr.io/${PROJECT_ID}/tld_updater:latest
+    docker push gcr.io/${PROJECT_ID}/db_object_updater:latest
-    docker push gcr.io/${PROJECT_ID}/tld_updater:${TAG_NAME}
+    docker push gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME}
-  dir: 'release/tld-updater/'
+  dir: 'release/db-object-updater/'
-# Update the tld_updater image digest in relevant GCB files.
+# Update the db_object_updater image digest in relevant GCB files.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
   args:
@@ -116,10 +116,10 @@ steps:
   - |
     set -e
     digest=$(gcloud container images list-tags \
-      gcr.io/${PROJECT_ID}/tld_updater \
+      gcr.io/${PROJECT_ID}/db_object_updater \
       --format="get(digest)" --filter="tags = ${TAG_NAME}")
-    sed -i s/'tld_updater:latest'/tld_updater@$digest/g \
+    sed -i s/'db_object_updater:latest'/db_object_updater@$digest/g \
-      release/cloudbuild-tld-sync-*.yaml
+      release/cloudbuild-sync-db-objects-*.yaml
 # Build and stage Dataflow Flex templates.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
@@ -190,7 +190,7 @@ artifacts:
     - 'release/cloudbuild-renew-prober-certs-*.yaml'
     - 'release/cloudbuild-schema-deploy-*.yaml'
     - 'release/cloudbuild-schema-verify-*.yaml'
-    - 'release/cloudbuild-tld-sync-*.yaml'
+    - 'release/cloudbuild-sync-db-objects-*.yaml'
 
 timeout: 7200s
 options:

release/cloudbuild-release.yaml

@@ -139,9 +139,9 @@ steps:
       gcloud container images list-tags \
       gcr.io/${PROJECT_ID}/prober_cert_updater \
       --format='get(digest)' --filter='tags = ${TAG_NAME}')
-     tld_updater_digest=$( \
+     db_object_updater_digest=$( \
       gcloud container images list-tags \
-      gcr.io/${PROJECT_ID}/tld_updater \
+      gcr.io/${PROJECT_ID}/db_object_updater \
       --format='get(digest)' --filter='tags = ${TAG_NAME}')
     sed -i s/builder:latest/builder@$builder_digest/g \
       release/cloudbuild-schema-deploy.yaml
@@ -150,7 +150,7 @@ steps:
     sed -i s/builder:latest/builder@$builder_digest/g \
       release/cloudbuild-renew-prober-certs.yaml
     sed -i s/builder:latest/builder@$builder_digest/g \
-      release/cloudbuild-tld-sync.yaml
+      release/cloudbuild-sync-db-objects.yaml
     sed -i s/schema_deployer:latest/schema_deployer@$schema_deployer_digest/g \
       release/cloudbuild-schema-deploy.yaml
     sed -i s/schema_verifier:latest/schema_verifier@$schema_verifier_digest/g \
@@ -163,8 +163,8 @@ steps:
         > release/cloudbuild-schema-verify-${environment}.yaml
       sed s/'$${_ENV}'/${environment}/g release/cloudbuild-renew-prober-certs.yaml \
         > release/cloudbuild-renew-prober-certs-${environment}.yaml
-      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-tld-sync.yaml \
+      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-sync-db-objects.yaml \
-        > release/cloudbuild-tld-sync-${environment}.yaml
+        > release/cloudbuild-sync-db-objects-${environment}.yaml
     done
 # Upload the gradle binary to GCS if it does not exist and point URL in gradle wrapper to it.
 - name: 'gcr.io/cloud-builders/gsutil'

release/cloudbuild-sync-db-objects.yaml

@@ -1,7 +1,8 @@
-# This will sync the Tld configurations in the internal repo with the Tld objects in the database.
+# This will sync the configuration files in the internal repo with their
+# corresponding objects in the database.
 #
 # To manually trigger a build on GCB, run:
-# gcloud builds submit --config cloudbuild-tld-sync.yaml --substitutions \
+# gcloud builds submit --config cloudbuild-sync-db-objects.yaml --substitutions \
 #   _INTERNAL_REPO_URL=[URL] ..
 #
 # To trigger a build automatically, follow the instructions below and add a trigger:
@@ -34,11 +35,26 @@ steps:
       --secret nomulus-tool-cloudbuild-credential \
       > nomulus_tool_credential.json
 # Configure the TLDs using the stored configuration files in the internal repo
-- name: 'gcr.io/$PROJECT_ID/tld_updater:latest'
+- name: 'gcr.io/$PROJECT_ID/db_object_updater:latest'
   args:
   - ${_ENV}
   - ./nomulus_tool_credential.json
+  - configure_tld
   - nomulus-internal/core/src/main/java/google/registry/config/files/tld/
+# Configure the premium lists using the stored configuration files in the internal repo
+- name: 'gcr.io/$PROJECT_ID/db_object_updater:latest'
+  args:
+  - ${_ENV}
+  - ./nomulus_tool_credential.json
+  - update_premium_list
+  - nomulus-internal/core/src/main/java/google/registry/config/files/premium/
+# Configure the reserved lists using the stored configuration files in the internal repo
+- name: 'gcr.io/$PROJECT_ID/db_object_updater:latest'
+  args:
+  - ${_ENV}
+  - ./nomulus_tool_credential.json
+  - update_reserved_list
+  - nomulus-internal/core/src/main/java/google/registry/config/files/reserved/
 
 timeout: 7200s
 options:

release/db-object-updater/Dockerfile

@@ -17,6 +17,6 @@ ARG TAG_NAME
 FROM gcr.io/${PROJECT_ID}/builder:${TAG_NAME}
 
 COPY nomulus.jar /
-COPY sync_tlds.sh /usr/local/bin
+COPY sync_db_objects.sh /usr/local/bin
 
-ENTRYPOINT [ "bash", "sync_tlds.sh" ]
+ENTRYPOINT [ "bash", "sync_db_objects.sh" ]

release/db-object-updater/sync_db_objects.sh

@@ -13,23 +13,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Sync the TLD configuration files from the internal repo with the Tld object
+# Sync the configuration files in the internal repo with the objects in the
-# in the database. Loops through the Tld configuration files and runs the configure_tld command
+# database. Loops through the configuration files in the inputted directory and
-# with the file.
+# runs the passed in nomulus update command with the file.
 
 # - env: The Nomulus environment, production, sandbox, etc.
 # - tools_credential: The credential (.json) needed to run the nomulus command.
+# - nomulus_command: The nomulus command to run.
 # - config_file_directory: The internal directory storing the TLD config files.
 
 set -e
-if [ "$#" -ne 3 ]; then
+if [ "$#" -ne 4 ]; then
-  echo "Expecting three parameters in order: env tools_credential config_file_directory"
+  echo "Expecting four parameters in order: env tools_credential nomulus_command config_file_directory"
   exit 1
 fi
 
 nomulus_env="${1}"
 tools_credential="${2}"
-config_file_directory="${3}"
+nomulus_command="${3}"
+config_file_directory="${4}"
 
 echo ${config_file_directory}
 
@@ -37,5 +39,5 @@ for FILE in ${config_file_directory}/${nomulus_env}/*; do
   echo $FILE
   java -jar /nomulus.jar -e "${nomulus_env}" \
   --credential "${tools_credential}" \
-  configure_tld -i $FILE --force --build_environment
+  "${nomulus_command}" -i $FILE --force --build_environment
 done