From f173b4fb4e8b0f3fccbbb26a932dec87b4f4c56c Mon Sep 17 00:00:00 2001
From: Pavlo Tkach <3469726+ptkach@users.noreply.github.com>
Date: Thu, 4 May 2023 15:57:32 -0400
Subject: [PATCH] Adds cloud scheduler and tasks deployer (#1999)

---
 appengine_war.gradle                          |   2 +-
 build.gradle                                  |  12 +-
 .../default/WEB-INF/cloud-scheduler-tasks.xml |   4 +-
 .../default/WEB-INF/cloud-tasks-queue.xml     | 113 ++++++
 .../{queue.xml => to_be_deleted_queue.xml}    |   1 +
 .../default/WEB-INF/cloud-scheduler-tasks.xml |   4 +-
 .../default/WEB-INF/cloud-scheduler-tasks.xml |   4 +-
 .../default/WEB-INF/cloud-scheduler-tasks.xml |   4 +-
 .../default/WEB-INF/cloud-scheduler-tasks.xml |   4 +-
 docs/architecture.md                          |  12 +-
 docs/configuration.md                         |  11 +-
 release/builder/Dockerfile                    |  14 +-
 release/builder/cloudSchedulerDeployer.go     | 176 ---------
 .../builder/deployCloudSchedulerAndQueue.go   | 339 ++++++++++++++++++
 release/cloudbuild-deploy.yaml                |  10 +-
 15 files changed, 496 insertions(+), 214 deletions(-)
 create mode 100644 core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml
 rename core/src/main/java/google/registry/env/common/default/WEB-INF/{queue.xml => to_be_deleted_queue.xml} (97%)
 delete mode 100644 release/builder/cloudSchedulerDeployer.go
 create mode 100644 release/builder/deployCloudSchedulerAndQueue.go

diff --git a/appengine_war.gradle b/appengine_war.gradle
index 8c745aca2..7a233703b 100644
--- a/appengine_war.gradle
+++ b/appengine_war.gradle
@@ -103,7 +103,7 @@ explodeWar.doLast {
     file("${it.explodedAppDirectory}/WEB-INF/lib/tools.jar").setWritable(true)
 }
 
-appengineDeployAll.finalizedBy ':cloudSchedulerDeployer'
+appengineDeployAll.finalizedBy ':deployCloudSchedulerAndQueue'
 rootProject.deploy.dependsOn appengineDeployAll
 rootProject.stage.dependsOn appengineStage
 tasks['war'].dependsOn ':console-webapp:buildConsoleWebappProd'
diff --git a/build.gradle b/build.gradle
index c7773f2ed..d6416b31b 100644
--- a/build.gradle
+++ b/build.gradle
@@ -558,17 +558,23 @@ task coreDev {
 
 javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }
 
-// Runs the script, which deploys cloud scheduler tasks based on the config
-task cloudSchedulerDeployer {
+// Runs the script, which deploys cloud scheduler and tasks based on the config
+task deployCloudSchedulerAndQueue {
   doLast {
     def env = environment
     if (!prodOrSandboxEnv) {
       exec {
         commandLine 'go', 'run',
-                "${rootDir}/release/builder/cloudSchedulerDeployer.go",
+                "${rootDir}/release/builder/deployCloudSchedulerAndQueue.go",
                 "${rootDir}/core/src/main/java/google/registry/env/${env}/default/WEB-INF/cloud-scheduler-tasks.xml",
                 "domain-registry-${env}"
       }
+      exec {
+        commandLine 'go', 'run',
+                "${rootDir}/release/builder/deployCloudSchedulerAndQueue.go",
+                "${rootDir}/core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml",
+                "domain-registry-${env}"
+      }
     }
   }
 }
diff --git a/core/src/main/java/google/registry/env/alpha/default/WEB-INF/cloud-scheduler-tasks.xml b/core/src/main/java/google/registry/env/alpha/default/WEB-INF/cloud-scheduler-tasks.xml
index 50b543b37..068d7d41a 100644
--- a/core/src/main/java/google/registry/env/alpha/default/WEB-INF/cloud-scheduler-tasks.xml
+++ b/core/src/main/java/google/registry/env/alpha/default/WEB-INF/cloud-scheduler-tasks.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<taskentries>
+<entries>
   <task>
     <url>/_dr/task/rdeStaging</url>
     <name>rdeStaging</name>
@@ -138,4 +138,4 @@
     </description>
     <schedule>*/1 * * * *</schedule>
   </task>
-</taskentries>
+</entries>
diff --git a/core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml b/core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml
new file mode 100644
index 000000000..f285121a5
--- /dev/null
+++ b/core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<entries>
+  <!-- Queue template with all supported params -->
+  <!-- More information - https://cloud.google.com/sdk/gcloud/reference/tasks/queues/create -->
+  <!--
+    <queue>
+      <name></name>
+      <max-attempts></max-attempts>
+      <max-backoff></max-backoff>
+      <max-concurrent-dispatches></max-concurrent-dispatches>
+      <max-dispatches-per-second></max-dispatches-per-second>
+      <max-doublings></max-doublings>
+      <max-retry-duration></max-retry-duration>
+      <min-backoff></min-backoff>
+    </queue>
+  -->
+
+  <!-- Queue for reading DNS update requests and batching them off to the dns-publish queue. -->
+  <queue>
+    <name>dns-refresh</name>
+    <max-dispatches-per-second>100</max-dispatches-per-second>
+  </queue>
+
+  <!-- Queue for publishing DNS updates in batches. -->
+  <queue>
+    <name>dns-publish</name>
+    <max-dispatches-per-second>100</max-dispatches-per-second>
+    <!-- 30 sec backoff increasing linearly up to 30 minutes. -->
+    <min-backoff>30s</min-backoff>
+    <max-backoff>1800s</max-backoff>
+    <max-doublings>0</max-doublings>
+  </queue>
+
+  <!-- Queue for uploading RDE deposits to the escrow provider. -->
+  <queue>
+    <name>rde-upload</name>
+    <max-dispatches-per-second>0.166666667</max-dispatches-per-second>
+    <max-concurrent-dispatches>5</max-concurrent-dispatches>
+    <max-retry-duration>14400s</max-retry-duration>
+  </queue>
+
+  <!-- Queue for uploading RDE reports to ICANN. -->
+  <queue>
+    <name>rde-report</name>
+    <max-dispatches-per-second>1</max-dispatches-per-second>
+    <max-concurrent-dispatches>1</max-concurrent-dispatches>
+    <max-retry-duration>14400s</max-retry-duration>
+  </queue>
+
+  <!-- Queue for copying BRDA deposits to GCS. -->
+  <queue>
+    <name>brda</name>
+    <max-dispatches-per-second>0.016666667</max-dispatches-per-second>
+    <max-concurrent-dispatches>10</max-concurrent-dispatches>
+    <max-retry-duration>82800s</max-retry-duration>
+  </queue>
+
+  <!-- Queue for tasks that trigger domain DNS update upon host rename. -->
+  <queue>
+    <name>async-host-rename</name>
+    <max-dispatches-per-second>1</max-dispatches-per-second>
+  </queue>
+
+  <!-- Queue for tasks that wait for a Beam pipeline to complete (i.e. Spec11 and invoicing). -->
+  <queue>
+    <name>beam-reporting</name>
+    <max-dispatches-per-second>0.016666667</max-dispatches-per-second>
+    <max-concurrent-dispatches>1</max-concurrent-dispatches>
+    <max-attempts>5</max-attempts>
+    <min-backoff>180s</min-backoff>
+    <max-backoff>180s</max-backoff>
+  </queue>
+
+
+  <!-- Queue for tasks that communicate with TMCH MarksDB webserver. -->
+  <queue>
+    <name>marksdb</name>
+    <max-dispatches-per-second>0.016666667</max-dispatches-per-second>
+    <max-concurrent-dispatches>1</max-concurrent-dispatches>
+    <max-retry-duration>39600s</max-retry-duration>  <!-- cron interval minus hour -->
+  </queue>
+
+  <!-- Queue for tasks to produce LORDN CSV reports, populated by a Cloud Scheduler fanout job. -->
+  <queue>
+    <name>nordn</name>
+    <max-dispatches-per-second>1</max-dispatches-per-second>
+    <max-concurrent-dispatches>10</max-concurrent-dispatches>
+    <max-retry-duration>39600s</max-retry-duration>  <!-- cron interval minus hour -->
+  </queue>
+
+  <!-- Queue for tasks that sync data to Google Spreadsheets. -->
+  <queue>
+    <name>sheet</name>
+    <max-dispatches-per-second>1</max-dispatches-per-second>
+    <!-- max-concurrent-dispatches is intentionally omitted. -->
+    <max-retry-duration>3600s</max-retry-duration>
+  </queue>
+
+  <!-- Queue for infrequent cron tasks (i.e. hourly or less often) that should retry three times on failure. -->
+  <queue>
+    <name>retryable-cron-tasks</name>
+    <max-dispatches-per-second>1</max-dispatches-per-second>
+    <max-attempts>3</max-attempts>
+  </queue>
+
+  <!--  &lt;!&ndash; Queue for async actions that should be run at some point in the future. &ndash;&gt;-->
+  <queue>
+    <name>async-actions</name>
+    <max-dispatches-per-second>1</max-dispatches-per-second>
+    <max-concurrent-dispatches>5</max-concurrent-dispatches>
+  </queue>
+
+</entries>
diff --git a/core/src/main/java/google/registry/env/common/default/WEB-INF/queue.xml b/core/src/main/java/google/registry/env/common/default/WEB-INF/to_be_deleted_queue.xml
similarity index 97%
rename from core/src/main/java/google/registry/env/common/default/WEB-INF/queue.xml
rename to core/src/main/java/google/registry/env/common/default/WEB-INF/to_be_deleted_queue.xml
index 5382b3fd0..9d9fd452d 100644
--- a/core/src/main/java/google/registry/env/common/default/WEB-INF/queue.xml
+++ b/core/src/main/java/google/registry/env/common/default/WEB-INF/to_be_deleted_queue.xml
@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- TODO: @ptkach - Delete once Cloud Api deployer is up and running -->
 <queue-entries>
 
   <!-- Queue for reading DNS update requests and batching them off to the dns-publish queue. -->
diff --git a/core/src/main/java/google/registry/env/crash/default/WEB-INF/cloud-scheduler-tasks.xml b/core/src/main/java/google/registry/env/crash/default/WEB-INF/cloud-scheduler-tasks.xml
index d282db1c3..dfb7f3f7c 100644
--- a/core/src/main/java/google/registry/env/crash/default/WEB-INF/cloud-scheduler-tasks.xml
+++ b/core/src/main/java/google/registry/env/crash/default/WEB-INF/cloud-scheduler-tasks.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<taskentries>
+<entries>
 
   <!--
     /cron/fanout params:
@@ -148,4 +148,4 @@
     </description>
     <schedule>7 3 * * *</schedule>
   </task>
-</taskentries>
+</entries>
diff --git a/core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml b/core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml
index 622f9bc17..96ac00ecd 100644
--- a/core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml
+++ b/core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<taskentries>
+<entries>
   <task>
     <url>/_dr/task/rdeStaging</url>
     <name>rdeStaging</name>
@@ -273,4 +273,4 @@
     </description>
     <schedule>0 15 * * 1</schedule>
   </task>
-</taskentries>
+</entries>
diff --git a/core/src/main/java/google/registry/env/qa/default/WEB-INF/cloud-scheduler-tasks.xml b/core/src/main/java/google/registry/env/qa/default/WEB-INF/cloud-scheduler-tasks.xml
index 0ce2ad22e..519817960 100644
--- a/core/src/main/java/google/registry/env/qa/default/WEB-INF/cloud-scheduler-tasks.xml
+++ b/core/src/main/java/google/registry/env/qa/default/WEB-INF/cloud-scheduler-tasks.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<taskentries>
+<entries>
   <task>
     <url>/_dr/task/rdeStaging</url>
     <name>rdeStaging</name>
@@ -68,4 +68,4 @@
     </description>
     <schedule>7 3 * * 6</schedule>
   </task>
-</taskentries>
+</entries>
diff --git a/core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cloud-scheduler-tasks.xml b/core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cloud-scheduler-tasks.xml
index 20eeba06b..4760df67f 100644
--- a/core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cloud-scheduler-tasks.xml
+++ b/core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cloud-scheduler-tasks.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<taskentries>
+<entries>
   <task>
     <url>/_dr/task/rdeStaging</url>
     <name>rdeStaging</name>
@@ -162,4 +162,4 @@
     </description>
     <schedule>0 15 * * 1</schedule>
   </task>
-</taskentries>
+</entries>
diff --git a/docs/architecture.md b/docs/architecture.md
index 21a622b9b..737d92997 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -110,12 +110,12 @@ the push queue will, upon execution, poll the corresponding pull queue for a
 specified number of tasks and execute them in a batch. This allows the code to
 execute in the background while taking advantage of batch processing.
 
-The task queues used by Nomulus are configured in the `queue.xml` file. Note
-that many push queues have a direct one-to-one correspondence with entries in
-`cron.xml` because they need to be fanned-out on a per-TLD or other basis (see
-the Cron section below for more explanation). The exact queue that a given cron
-task will use is passed as the query string parameter "queue" in the url
-specification for the cron task.
+The task queues used by Nomulus are configured in the `cloud-tasks-queue.xml` 
+file. Note that many push queues have a direct one-to-one correspondence with 
+entries in `cloud-scheduler-tasks.xml` because they need to be fanned-out on a 
+per-TLD or other basis (see the Cron section below for more explanation). 
+The exact queue that a given cron task will use is passed as the query string 
+parameter "queue" in the url specification for the cron task.
 
 Here are the task queues in use by the system. All are push queues unless
 explicitly marked as otherwise.
diff --git a/docs/configuration.md b/docs/configuration.md
index 3f1906643..1c59f8fc9 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -35,12 +35,13 @@ App Engine configuration isn't covered in depth in this document as it is
 thoroughly documented in the [App Engine configuration docs][app-engine-config].
 The main files of note that come pre-configured in Nomulus are:
 
-*   `cron.xml` -- Configuration of cronjobs
-*   `web.xml` -- Configuration of URL paths on the webserver
-*   `appengine-web.xml` -- Overall App Engine settings including number and type
+* `cron.xml` -- Configuration of cronjobs
+* `web.xml` -- Configuration of URL paths on the webserver
+* `appengine-web.xml` -- Overall App Engine settings including number and type
     of instances
-*   `queue.xml` -- Configuration of App Engine task queues
-*   `application.xml` -- Configuration of the application name and its services
+* `cloud-scheduler-tasks.xml` -- Configuration of Cloud Scheduler Tasks
+* * `cloud-tasks-queue.xml` -- Configuration of Cloud Tasks Queue
+* `application.xml` -- Configuration of the application name and its services
 
 Cron, web, and queue are covered in more detail in the "App Engine architecture"
 doc, and the rest are covered in the general App Engine documentation.
diff --git a/release/builder/Dockerfile b/release/builder/Dockerfile
index e3f3e4aca..1f6ef503a 100644
--- a/release/builder/Dockerfile
+++ b/release/builder/Dockerfile
@@ -19,18 +19,18 @@
 # 3. Google Cloud SDK for generating the WARs.
 # 4. Git to manipulate the private and the merged repos.
 # 5. Docker to build and push images.
-# 6. cloudSchedulerDeployer for deploying cloud scheduler tasks based on config
+# 6. deployerForSchedulerAndTasks for deploying cloud scheduler and cloud tasks
 
-FROM golang:1.19 as cloudSchedulerBuilder
-WORKDIR /usr/src/cloudSchedulerDeployer
-RUN go mod init cloudSchedulerDeployer
+FROM golang:1.19 as deployCloudSchedulerAndQueueBuilder
+WORKDIR /usr/src/deployCloudSchedulerAndQueue
+RUN go mod init deployCloudSchedulerAndQueue
 COPY *.go ./
-RUN go build -o /cloudSchedulerDeployer
+RUN go build -o /deployCloudSchedulerAndQueue
 
 FROM marketplace.gcr.io/google/debian10
 ENV DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF-8
-# Add script for cloud scheduler deployer
-COPY --from=cloudSchedulerBuilder /cloudSchedulerDeployer /usr/local/bin/cloudSchedulerDeployer
+# Add script for cloud scheduler and cloud tasks deployment
+COPY --from=deployCloudSchedulerAndQueueBuilder /deployCloudSchedulerAndQueue /usr/local/bin/deployCloudSchedulerAndQueue
 # Add Cloud sql connector
 ADD https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \
     /usr/local/bin/cloud_sql_proxy
diff --git a/release/builder/cloudSchedulerDeployer.go b/release/builder/cloudSchedulerDeployer.go
deleted file mode 100644
index c414e2bfa..000000000
--- a/release/builder/cloudSchedulerDeployer.go
+++ /dev/null
@@ -1,176 +0,0 @@
-// Copyright 2023 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// The cloudScheduler tool allows creating, updating and deleting cloud
-// scheduler jobs from xml config file
-
-package main
-
-import (
-	"encoding/json"
-	"encoding/xml"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"os/exec"
-	"strings"
-)
-
-func main() {
-	if len(os.Args) < 3 || os.Args[1] == "" || os.Args[2] == "" {
-		panic("Error - invalid parameters:\nFirst parameter required - config file path;\nSecond parameter required - project name")
-	}
-
-	// Config file path
-	configFileLocation := os.Args[1]
-	// Project name where to submit the tasks
-	projectName := os.Args[2]
-
-	log.Default().Println("Filepath " + configFileLocation)
-
-	xmlFile, err := os.Open(configFileLocation)
-	if err != nil {
-		panic(err)
-	}
-	defer xmlFile.Close()
-
-	type Task struct {
-		XMLName     xml.Name `xml:"task"`
-		URL         string   `xml:"url"`
-		Description string   `xml:"description"`
-		Schedule    string   `xml:"schedule"`
-		Name        string   `xml:"name"`
-	}
-
-	type Taskentries struct {
-		XMLName xml.Name `xml:"taskentries"`
-		Task    []Task   `xml:"task"`
-	}
-
-	type ServiceAccount struct {
-		DisplayName string `json:"displayName"`
-		Email       string `json:"email"`
-	}
-
-	type ExistingJob struct {
-		Name  string `json:"name"`
-		State string `json:"state"`
-	}
-
-	byteValue, _ := io.ReadAll(xmlFile)
-
-	var taskEntries Taskentries
-
-	if err := xml.Unmarshal(byteValue, &taskEntries); err != nil {
-		panic("Failed to unmarshal taskentries: " + err.Error())
-	}
-
-	getArgs := func(taskRecord Task, operationType string, serviceAccountEmail string) []string {
-		// Cloud Schedule doesn't allow description of more than 499 chars and \n
-		var description string
-		if len(taskRecord.Description) > 499 {
-			description = taskRecord.Description[:499]
-		} else {
-			description = taskRecord.Description
-		}
-		description = strings.ReplaceAll(description, "\n", " ")
-
-		return []string{
-			"--project", projectName,
-			"scheduler", "jobs", operationType,
-			"http", taskRecord.Name,
-			"--location", "us-central1",
-			"--schedule", taskRecord.Schedule,
-			"--uri", fmt.Sprintf("https://backend-dot-%s.appspot.com%s", projectName, strings.TrimSpace(taskRecord.URL)),
-			"--description", description,
-			"--http-method", "get",
-			"--oidc-service-account-email", serviceAccountEmail,
-			"--oidc-token-audience", projectName,
-		}
-	}
-
-	// Get existing jobs from Cloud Scheduler
-	var allExistingJobs []ExistingJob
-	cmdGetExistingList := exec.Command("gcloud", "scheduler", "jobs", "list", "--project="+projectName, "--location=us-central1", "--format=json")
-	cmdGetExistingListOutput, cmdGetExistingListError := cmdGetExistingList.CombinedOutput()
-	if cmdGetExistingListError != nil {
-		panic("Can't obtain existing cloud scheduler jobs for " + projectName)
-	}
-	err = json.Unmarshal(cmdGetExistingListOutput, &allExistingJobs)
-	if err != nil {
-		panic("Failed to parse existing jobs from cloud schedule: " + err.Error())
-	}
-
-	// Sync deleted jobs
-	enabledExistingJobs := map[string]bool{}
-	for i := 0; i < len(allExistingJobs); i++ {
-		jobName := strings.Split(allExistingJobs[i].Name, "jobs/")[1]
-		toBeDeleted := true
-		if allExistingJobs[i].State == "ENABLED" {
-			enabledExistingJobs[jobName] = true
-		}
-		for i := 0; i < len(taskEntries.Task); i++ {
-			if taskEntries.Task[i].Name == jobName {
-				toBeDeleted = false
-				break
-			}
-		}
-		if toBeDeleted {
-			cmdDelete := exec.Command("gcloud", "scheduler", "jobs", "delete", jobName, "--project="+projectName, "--quiet")
-			cmdDeleteOutput, cmdDeleteError := cmdDelete.CombinedOutput()
-			log.Default().Println("Deleting cloud scheduler job " + jobName)
-			if cmdDeleteError != nil {
-				panic(string(cmdDeleteOutput))
-			}
-		}
-	}
-
-	// Find service account email
-	var serviceAccounts []ServiceAccount
-	var serviceAccountEmail string
-	cmdGetServiceAccounts := exec.Command("gcloud", "iam", "service-accounts", "list", "--project="+projectName, "--format=json")
-	cmdGetServiceAccountsOutput, cmdGetServiceAccountsError := cmdGetServiceAccounts.CombinedOutput()
-	if cmdGetServiceAccountsError != nil {
-		panic(cmdGetServiceAccountsError)
-	}
-	err = json.Unmarshal(cmdGetServiceAccountsOutput, &serviceAccounts)
-	if err != nil {
-		panic(err.Error())
-	}
-	for i := 0; i < len(serviceAccounts); i++ {
-		if serviceAccounts[i].DisplayName == "cloud-scheduler" {
-			serviceAccountEmail = serviceAccounts[i].Email
-			break
-		}
-	}
-	if serviceAccountEmail == "" {
-		panic("Service account for cloud scheduler is not created for " + projectName)
-	}
-
-	// Sync created and updated jobs
-	for i := 0; i < len(taskEntries.Task); i++ {
-		cmdType := "update"
-		if enabledExistingJobs[taskEntries.Task[i].Name] != true {
-			cmdType = "create"
-		}
-
-		syncCommand := exec.Command("gcloud", getArgs(taskEntries.Task[i], cmdType, serviceAccountEmail)...)
-		syncCommandOutput, syncCommandError := syncCommand.CombinedOutput()
-		log.Default().Println(cmdType + " cloud scheduler job " + taskEntries.Task[i].Name)
-		if syncCommandError != nil {
-			panic(string(syncCommandOutput))
-		}
-	}
-}
diff --git a/release/builder/deployCloudSchedulerAndQueue.go b/release/builder/deployCloudSchedulerAndQueue.go
new file mode 100644
index 000000000..9e4302536
--- /dev/null
+++ b/release/builder/deployCloudSchedulerAndQueue.go
@@ -0,0 +1,339 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The deployerForSchedulerAndTasks tool allows creating, updating and deleting
+// cloud scheduler jobs and tasks from xml config file
+
+package main
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+var projectName string
+
+const GcpLocation = "us-central1"
+
+type SyncManager[T Task | Queue] interface {
+	syncCreated(e T) ([]byte, error)
+	syncDeleted(e string) ([]byte, error)
+	syncUpdated(e T, v ExistingEntry) ([]byte, error)
+	fetchExistingRecords() ExistingEntries
+	getXmlEntries() []T
+	getEntryName(e T) string
+}
+
+type ServiceAccount struct {
+	DisplayName string `json:"displayName"`
+	Email       string `json:"email"`
+}
+
+type Queue struct {
+	Name                    string `xml:"name"`
+	MaxAttempts             string `xml:"max-attempts"`
+	MaxBackoff              string `xml:"max-backoff"`
+	MaxConcurrentDispatches string `xml:"max-concurrent-dispatches"`
+	MaxDispatchesPerSecond  string `xml:"max-dispatches-per-second"`
+	MaxDoublings            string `xml:"max-doublings"`
+	MaxRetryDuration        string `xml:"max-retry-duration"`
+	MinBackoff              string `xml:"min-backoff"`
+	MaxBurstSize            string `xml:"max-burst-size"`
+}
+
+type Task struct {
+	URL         string `xml:"url"`
+	Description string `xml:"description"`
+	Schedule    string `xml:"schedule"`
+	Name        string `xml:"name"`
+}
+
+type QueuesSyncManager struct {
+	Queues []Queue
+}
+
+type TasksSyncManager struct {
+	Tasks               []Task
+	ServiceAccountEmail string
+}
+
+type XmlEntries struct {
+	XMLName xml.Name `xml:"entries"`
+	Tasks   []Task   `xml:"task"`
+	Queues  []Queue  `xml:"queue"`
+}
+
+type ExistingEntry struct {
+	Name  string `json:"name"`
+	State string `json:"state"`
+}
+
+type ExistingEntries struct {
+	asAMap map[string]ExistingEntry
+}
+
+func handleSyncErrors(response []byte, err error) {
+	if err != nil {
+		panic(string(response) + " " + err.Error())
+	}
+}
+
+func sync[T Queue | Task](manager SyncManager[T]) {
+	// State from xml file
+	xmlEntries := manager.getXmlEntries()
+	// Fetch tasks currently in GCP
+	existingTasks := manager.fetchExistingRecords()
+	// Find and sync deleted tasks
+	for taskName := range existingTasks.asAMap {
+		toBeDeleted := true
+		for j := 0; j < len(xmlEntries); j++ {
+			if manager.getEntryName(xmlEntries[j]) == taskName {
+				toBeDeleted = false
+				break
+			}
+		}
+		if toBeDeleted {
+			handleSyncErrors(manager.syncDeleted(taskName))
+		}
+	}
+	// Sync created and updated tasks
+	for j := 0; j < len(xmlEntries); j++ {
+		if val, ok := existingTasks.asAMap[manager.getEntryName(xmlEntries[j])]; ok {
+			handleSyncErrors(manager.syncUpdated(xmlEntries[j], val))
+		} else {
+			handleSyncErrors(manager.syncCreated(xmlEntries[j]))
+		}
+	}
+}
+
+var cloudSchedulerServiceAccountEmail string
+
+func getCloudSchedulerServiceAccountEmail() string {
+	if cloudSchedulerServiceAccountEmail != "" {
+		return cloudSchedulerServiceAccountEmail
+	}
+
+	var serviceAccounts []ServiceAccount
+	cmdGetServiceAccounts := exec.Command("gcloud", "iam", "service-accounts", "list", "--project="+projectName, "--format=json")
+	cmdGetServiceAccountsOutput, cmdGetServiceAccountsError := cmdGetServiceAccounts.CombinedOutput()
+	if cmdGetServiceAccountsError != nil {
+		panic("Failed to fetch service accounts " + string(cmdGetServiceAccountsOutput))
+	}
+	err := json.Unmarshal(cmdGetServiceAccountsOutput, &serviceAccounts)
+	if err != nil {
+		panic("Failed to parse service account response: " + err.Error())
+	}
+	for i := 0; i < len(serviceAccounts); i++ {
+		if serviceAccounts[i].DisplayName == "cloud-scheduler" {
+			cloudSchedulerServiceAccountEmail = serviceAccounts[i].Email
+			return serviceAccounts[i].Email
+		}
+	}
+	panic("Service account for cloud scheduler is not created for " + projectName)
+}
+
+func (manager TasksSyncManager) getEntryName(task Task) string {
+	return task.Name
+}
+
+func (manager TasksSyncManager) getXmlEntries() []Task {
+	return manager.Tasks
+}
+
+func (manager TasksSyncManager) getArgs(task Task, operationType string) []string {
+	// Cloud Schedule doesn't allow description of more than 499 chars and \n
+	var description string
+	if len(task.Description) > 499 {
+		log.Default().Println("Task description exceeds the allowed length of " +
+			"500 characters, clipping the description before submitting the task " +
+			task.Name)
+		description = task.Description[:499]
+	} else {
+		description = task.Description
+	}
+	description = strings.ReplaceAll(description, "\n", " ")
+
+	return []string{
+		"--project", projectName,
+		"scheduler", "jobs", operationType,
+		"http", task.Name,
+		"--location", GcpLocation,
+		"--schedule", task.Schedule,
+		"--uri", fmt.Sprintf("https://backend-dot-%s.appspot.com%s", projectName, strings.TrimSpace(task.URL)),
+		"--description", description,
+		"--http-method", "get",
+		"--oidc-service-account-email", getCloudSchedulerServiceAccountEmail(),
+		"--oidc-token-audience", projectName,
+	}
+}
+
+func (manager TasksSyncManager) fetchExistingRecords() ExistingEntries {
+	return getExistingEntries(exec.Command("gcloud", "scheduler", "jobs", "list", "--project="+projectName, "--location="+GcpLocation, "--format=json"))
+}
+
+func (manager TasksSyncManager) syncDeleted(taskName string) ([]byte, error) {
+	log.Default().Println("Deleting cloud scheduler task " + taskName)
+	cmdDelete := exec.Command("gcloud", "scheduler", "jobs", "delete", taskName, "--project="+projectName, "--quiet")
+	return cmdDelete.CombinedOutput()
+}
+
+func (manager TasksSyncManager) syncCreated(task Task) ([]byte, error) {
+	log.Default().Println("Creating cloud scheduler task " + task.Name)
+	syncCommand := exec.Command("gcloud", manager.getArgs(task, "create")...)
+	return syncCommand.CombinedOutput()
+}
+
+func (manager TasksSyncManager) syncUpdated(task Task, _ ExistingEntry) ([]byte, error) {
+	log.Default().Println("Updating cloud scheduler task " + task.Name)
+	syncCommand := exec.Command("gcloud", manager.getArgs(task, "update")...)
+	return syncCommand.CombinedOutput()
+}
+
+func (manager QueuesSyncManager) getEntryName(queue Queue) string {
+	return queue.Name
+}
+
+func (manager QueuesSyncManager) getXmlEntries() []Queue {
+	return manager.Queues
+}
+
+func (manager QueuesSyncManager) getArgs(queue Queue, operationType string) []string {
+	args := []string{
+		"tasks", "queues", operationType, queue.Name, "--project", projectName,
+	}
+	if queue.MaxAttempts != "" {
+		args = append(args, "--max-attempts", queue.MaxAttempts)
+	}
+	if queue.MaxBackoff != "" {
+		args = append(args, "--max-backoff", queue.MaxBackoff)
+	}
+	if queue.MaxConcurrentDispatches != "" {
+		args = append(args, "--max-concurrent-dispatches", queue.MaxConcurrentDispatches)
+	}
+	if queue.MaxDispatchesPerSecond != "" {
+		args = append(args, "--max-dispatches-per-second", queue.MaxDispatchesPerSecond)
+	}
+	if queue.MaxDoublings != "" {
+		args = append(args, "--max-doublings", queue.MaxDoublings)
+	}
+	if queue.MaxRetryDuration != "" {
+		args = append(args, "--max-retry-duration", queue.MaxRetryDuration)
+	}
+	if queue.MinBackoff != "" {
+		args = append(args, "--min-backoff", queue.MinBackoff)
+	}
+	if queue.MaxBurstSize != "" {
+		args = append(args, "--max-burst-size", queue.MaxBurstSize)
+	}
+	args = append(args, "--format=json")
+	return args
+}
+
+func (manager QueuesSyncManager) fetchExistingRecords() ExistingEntries {
+	return getExistingEntries(exec.Command("gcloud", "tasks", "queues", "list", "--project="+projectName, "--location="+GcpLocation, "--format=json"))
+}
+
+func (manager QueuesSyncManager) syncDeleted(taskName string) ([]byte, error) {
+	// Default queue can't be deleted
+	if taskName == "default" {
+		return []byte{}, nil
+	}
+	log.Default().Println("Pausing cloud tasks queue " + taskName)
+	cmd := exec.Command("gcloud", "tasks", "queues", "pause", taskName, "--project="+projectName, "--quiet")
+	return cmd.CombinedOutput()
+}
+
+func (manager QueuesSyncManager) syncCreated(queue Queue) ([]byte, error) {
+	log.Default().Println("Creating cloud tasks queue " + queue.Name)
+	cmd := exec.Command("gcloud", manager.getArgs(queue, "create")...)
+	return cmd.CombinedOutput()
+}
+
+func (manager QueuesSyncManager) syncUpdated(queue Queue, existingQueue ExistingEntry) ([]byte, error) {
+	if existingQueue.State == "PAUSED" {
+		log.Default().Println("Resuming cloud tasks queue " + queue.Name)
+		cmdResume := exec.Command("gcloud", "tasks", "queues", "resume", queue.Name, "--project="+projectName, "--quiet")
+		r, err := cmdResume.CombinedOutput()
+		if err != nil {
+			return r, err
+		}
+	}
+	log.Default().Println("Updating cloud tasks queue " + queue.Name)
+	cmd := exec.Command("gcloud", manager.getArgs(queue, "update")...)
+	return cmd.CombinedOutput()
+}
+
+func getExistingEntries(cmd *exec.Cmd) ExistingEntries {
+	var allExistingEntries []ExistingEntry
+	cmdGetExistingListOutput, cmdGetExistingListError := cmd.CombinedOutput()
+	if cmdGetExistingListError != nil {
+		panic("Failed to load existing entries: " + cmdGetExistingListError.Error())
+	}
+	err := json.Unmarshal(cmdGetExistingListOutput, &allExistingEntries)
+	if err != nil {
+		panic("Failed to parse existing entries: " + err.Error())
+	}
+	e := ExistingEntries{
+		asAMap: map[string]ExistingEntry{},
+	}
+	for i := 0; i < len(allExistingEntries); i++ {
+		existingEntry := allExistingEntries[i]
+		e.asAMap[existingEntry.Name[strings.LastIndex(existingEntry.Name, "/")+1:]] = existingEntry
+	}
+	return e
+}
+
+func main() {
+	if len(os.Args) < 3 || os.Args[1] == "" || os.Args[2] == "" {
+		panic("Error - Invalid Parameters.\nRequired params: 1 - config file path; 2 - project name;")
+	}
+	// Config file path
+	configFileLocation := os.Args[1]
+	// Project name where to submit the tasks
+	projectName = os.Args[2]
+
+	log.Default().Println("Filepath " + configFileLocation)
+	xmlFile, err := os.Open(configFileLocation)
+	if err != nil {
+		panic(err)
+	}
+	defer xmlFile.Close()
+	byteValue, _ := io.ReadAll(xmlFile)
+	var xmlEntries XmlEntries
+	if err := xml.Unmarshal(byteValue, &xmlEntries); err != nil {
+		panic("Failed to parse xml file entries: " + err.Error())
+	}
+
+	if len(xmlEntries.Tasks) > 0 {
+		log.Default().Println("Syncing cloud scheduler tasks from xml...")
+		tasksSyncManager := TasksSyncManager{
+			Tasks: xmlEntries.Tasks,
+		}
+		sync[Task](tasksSyncManager)
+	}
+
+	if len(xmlEntries.Queues) > 0 {
+		log.Default().Println("Syncing cloud task queues from xml...")
+		queuesSyncManager := QueuesSyncManager{
+			Queues: xmlEntries.Queues,
+		}
+		sync[Queue](queuesSyncManager)
+	}
+}
diff --git a/release/cloudbuild-deploy.yaml b/release/cloudbuild-deploy.yaml
index 8708dea2a..165aecf15 100644
--- a/release/cloudbuild-deploy.yaml
+++ b/release/cloudbuild-deploy.yaml
@@ -28,7 +28,7 @@ steps:
     set -e
     gcloud secrets versions access latest \
       --secret nomulus-tool-cloudbuild-credential > tool-credential.json
-# Create/Update cloud scheduler jobs based on a cloud-scheduler-tasks.xml
+# Create/Update cloud scheduler and cloud tasks based on a cloud-scheduler-tasks.xml
 - name: 'gcr.io/$PROJECT_ID/builder:latest'
   entrypoint: /bin/bash
   args:
@@ -43,7 +43,8 @@ steps:
     fi
     gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/${_ENV}.tar .
     tar -xvf ${_ENV}.tar
-    cloudSchedulerDeployer default/WEB-INF/cloud-scheduler-tasks.xml $project_id
+    deployCloudSchedulerAndQueue default/WEB-INF/cloud-scheduler-tasks.xml $project_id
+    deployCloudSchedulerAndQueue default/WEB-INF/cloud-tasks-queue.xml $project_id
 # Deploy the GAE config files.
 # First authorize the gcloud tool to use the credential json file, then
 # download and unzip the tarball that contains the relevant config files
@@ -59,10 +60,7 @@ steps:
     else
       project_id="domain-registry-${_ENV}"
     fi
-    for filename in dispatch queue; do
-      gcloud -q --project $project_id app deploy \
-        default/WEB-INF/appengine-generated/$filename.yaml
-    done
+    gcloud -q --project $project_id app deploy default/WEB-INF/appengine-generated/dispatch.yaml
 # Save the deployed tag for the current environment on GCS, and update the
 # mappings from Nomulus releases to Appengine versions.
 - name: 'gcr.io/$PROJECT_ID/builder:latest'