zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-04-29 19:47:51 +02:00
Code Issues Projects Releases Packages Wiki Activity

Adds cloud scheduler and tasks deployer (#1999)

Browse source
This commit is contained in:
Pavlo Tkach 2023-05-04 15:57:32 -04:00 • committed by GitHub
parent 26efe67211
commit f173b4fb4e
15 changed files with 496 additions and 214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
appengine_war.gradle
View file

@ -103,7 +103,7 @@ explodeWar.doLast {
file("${it.explodedAppDirectory}/WEB-INF/lib/tools.jar").setWritable(true) file("${it.explodedAppDirectory}/WEB-INF/lib/tools.jar").setWritable(true)
} }
appengineDeployAll.finalizedBy ':cloudSchedulerDeployer' appengineDeployAll.finalizedBy ':deployCloudSchedulerAndQueue'
rootProject.deploy.dependsOn appengineDeployAll rootProject.deploy.dependsOn appengineDeployAll
rootProject.stage.dependsOn appengineStage rootProject.stage.dependsOn appengineStage
tasks['war'].dependsOn ':console-webapp:buildConsoleWebappProd' tasks['war'].dependsOn ':console-webapp:buildConsoleWebappProd'

12
build.gradle
View file

@ -558,17 +558,23 @@ task coreDev {
javadocDependentTasks.each { tasks.javadoc.dependsOn(it) } javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }
// Runs the script, which deploys cloud scheduler tasks based on the config // Runs the script, which deploys cloud scheduler and tasks based on the config
task cloudSchedulerDeployer { task deployCloudSchedulerAndQueue {
doLast { doLast {
def env = environment def env = environment
if (!prodOrSandboxEnv) { if (!prodOrSandboxEnv) {
exec { exec {
commandLine 'go', 'run', commandLine 'go', 'run',
"${rootDir}/release/builder/cloudSchedulerDeployer.go", "${rootDir}/release/builder/deployCloudSchedulerAndQueue.go",
"${rootDir}/core/src/main/java/google/registry/env/${env}/default/WEB-INF/cloud-scheduler-tasks.xml", "${rootDir}/core/src/main/java/google/registry/env/${env}/default/WEB-INF/cloud-scheduler-tasks.xml",
"domain-registry-${env}" "domain-registry-${env}"
} }
exec {
commandLine 'go', 'run',
"${rootDir}/release/builder/deployCloudSchedulerAndQueue.go",
"${rootDir}/core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml",
"domain-registry-${env}"
}
} }
} }
} }

4
core/src/main/java/google/registry/env/alpha/default/WEB-INF/cloud-scheduler-tasks.xml vendored
View file

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<taskentries> <entries>
<task> <task>
<url>/_dr/task/rdeStaging</url> <url>/_dr/task/rdeStaging</url>
<name>rdeStaging</name> <name>rdeStaging</name>
@ -138,4 +138,4 @@
</description> </description>
<schedule>*/1 * * * *</schedule> <schedule>*/1 * * * *</schedule>
</task> </task>
</taskentries> </entries>

113
core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml vendored Normal file
View file

@ -0,0 +1,113 @@
<?xml version="1.0" encoding="UTF-8"?>
<entries>
<!-- Queue template with all supported params -->
<!-- More information - https://cloud.google.com/sdk/gcloud/reference/tasks/queues/create -->
<!--
<queue>
<name></name>
<max-attempts></max-attempts>
<max-backoff></max-backoff>
<max-concurrent-dispatches></max-concurrent-dispatches>
<max-dispatches-per-second></max-dispatches-per-second>
<max-doublings></max-doublings>
<max-retry-duration></max-retry-duration>
<min-backoff></min-backoff>
</queue>
-->
<!-- Queue for reading DNS update requests and batching them off to the dns-publish queue. -->
<queue>
<name>dns-refresh</name>
<max-dispatches-per-second>100</max-dispatches-per-second>
</queue>
<!-- Queue for publishing DNS updates in batches. -->
<queue>
<name>dns-publish</name>
<max-dispatches-per-second>100</max-dispatches-per-second>
<!-- 30 sec backoff increasing linearly up to 30 minutes. -->
<min-backoff>30s</min-backoff>
<max-backoff>1800s</max-backoff>
<max-doublings>0</max-doublings>
</queue>
<!-- Queue for uploading RDE deposits to the escrow provider. -->
<queue>
<name>rde-upload</name>
<max-dispatches-per-second>0.166666667</max-dispatches-per-second>
<max-concurrent-dispatches>5</max-concurrent-dispatches>
<max-retry-duration>14400s</max-retry-duration>
</queue>
<!-- Queue for uploading RDE reports to ICANN. -->
<queue>
<name>rde-report</name>
<max-dispatches-per-second>1</max-dispatches-per-second>
<max-concurrent-dispatches>1</max-concurrent-dispatches>
<max-retry-duration>14400s</max-retry-duration>
</queue>
<!-- Queue for copying BRDA deposits to GCS. -->
<queue>
<name>brda</name>
<max-dispatches-per-second>0.016666667</max-dispatches-per-second>
<max-concurrent-dispatches>10</max-concurrent-dispatches>
<max-retry-duration>82800s</max-retry-duration>
</queue>
<!-- Queue for tasks that trigger domain DNS update upon host rename. -->
<queue>
<name>async-host-rename</name>
<max-dispatches-per-second>1</max-dispatches-per-second>
</queue>
<!-- Queue for tasks that wait for a Beam pipeline to complete (i.e. Spec11 and invoicing). -->
<queue>
<name>beam-reporting</name>
<max-dispatches-per-second>0.016666667</max-dispatches-per-second>
<max-concurrent-dispatches>1</max-concurrent-dispatches>
<max-attempts>5</max-attempts>
<min-backoff>180s</min-backoff>
<max-backoff>180s</max-backoff>
</queue>
<!-- Queue for tasks that communicate with TMCH MarksDB webserver. -->
<queue>
<name>marksdb</name>
<max-dispatches-per-second>0.016666667</max-dispatches-per-second>
<max-concurrent-dispatches>1</max-concurrent-dispatches>
<max-retry-duration>39600s</max-retry-duration> <!-- cron interval minus hour -->
</queue>
<!-- Queue for tasks to produce LORDN CSV reports, populated by a Cloud Scheduler fanout job. -->
<queue>
<name>nordn</name>
<max-dispatches-per-second>1</max-dispatches-per-second>
<max-concurrent-dispatches>10</max-concurrent-dispatches>
<max-retry-duration>39600s</max-retry-duration> <!-- cron interval minus hour -->
</queue>
<!-- Queue for tasks that sync data to Google Spreadsheets. -->
<queue>
<name>sheet</name>
<max-dispatches-per-second>1</max-dispatches-per-second>
<!-- max-concurrent-dispatches is intentionally omitted. -->
<max-retry-duration>3600s</max-retry-duration>
</queue>
<!-- Queue for infrequent cron tasks (i.e. hourly or less often) that should retry three times on failure. -->
<queue>
<name>retryable-cron-tasks</name>
<max-dispatches-per-second>1</max-dispatches-per-second>
<max-attempts>3</max-attempts>
</queue>
<!-- &lt;!&ndash; Queue for async actions that should be run at some point in the future. &ndash;&gt;-->
<queue>
<name>async-actions</name>
<max-dispatches-per-second>1</max-dispatches-per-second>
<max-concurrent-dispatches>5</max-concurrent-dispatches>
</queue>
</entries>

1
core/src/main/java/google/registry/env/common/default/WEB-INF/queue.xml → core/src/main/java/google/registry/env/common/default/WEB-INF/to_be_deleted_queue.xml vendored
View file

@ -1,4 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!-- TODO: @ptkach - Delete once Cloud Api deployer is up and running -->
<queue-entries> <queue-entries>
<!-- Queue for reading DNS update requests and batching them off to the dns-publish queue. --> <!-- Queue for reading DNS update requests and batching them off to the dns-publish queue. -->

4
core/src/main/java/google/registry/env/crash/default/WEB-INF/cloud-scheduler-tasks.xml vendored
View file

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<taskentries> <entries>
<!-- <!--
/cron/fanout params: /cron/fanout params:
@ -148,4 +148,4 @@
</description> </description>
<schedule>7 3 * * *</schedule> <schedule>7 3 * * *</schedule>
</task> </task>
</taskentries> </entries>

4
core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml vendored
View file

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<taskentries> <entries>
<task> <task>
<url>/_dr/task/rdeStaging</url> <url>/_dr/task/rdeStaging</url>
<name>rdeStaging</name> <name>rdeStaging</name>
@ -273,4 +273,4 @@
</description> </description>
<schedule>0 15 * * 1</schedule> <schedule>0 15 * * 1</schedule>
</task> </task>
</taskentries> </entries>

4
core/src/main/java/google/registry/env/qa/default/WEB-INF/cloud-scheduler-tasks.xml vendored
View file

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<taskentries> <entries>
<task> <task>
<url>/_dr/task/rdeStaging</url> <url>/_dr/task/rdeStaging</url>
<name>rdeStaging</name> <name>rdeStaging</name>
@ -68,4 +68,4 @@
</description> </description>
<schedule>7 3 * * 6</schedule> <schedule>7 3 * * 6</schedule>
</task> </task>
</taskentries> </entries>

4
core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cloud-scheduler-tasks.xml vendored
View file

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<taskentries> <entries>
<task> <task>
<url>/_dr/task/rdeStaging</url> <url>/_dr/task/rdeStaging</url>
<name>rdeStaging</name> <name>rdeStaging</name>
@ -162,4 +162,4 @@
</description> </description>
<schedule>0 15 * * 1</schedule> <schedule>0 15 * * 1</schedule>
</task> </task>
</taskentries> </entries>

12
docs/architecture.md
View file

@ -110,12 +110,12 @@ the push queue will, upon execution, poll the corresponding pull queue for a
specified number of tasks and execute them in a batch. This allows the code to specified number of tasks and execute them in a batch. This allows the code to
execute in the background while taking advantage of batch processing. execute in the background while taking advantage of batch processing.
The task queues used by Nomulus are configured in the `queue.xml` file. Note The task queues used by Nomulus are configured in the `cloud-tasks-queue.xml`
that many push queues have a direct one-to-one correspondence with entries in file. Note that many push queues have a direct one-to-one correspondence with
`cron.xml` because they need to be fanned-out on a per-TLD or other basis (see entries in `cloud-scheduler-tasks.xml` because they need to be fanned-out on a
the Cron section below for more explanation). The exact queue that a given cron per-TLD or other basis (see the Cron section below for more explanation).
task will use is passed as the query string parameter "queue" in the url The exact queue that a given cron task will use is passed as the query string
specification for the cron task. parameter "queue" in the url specification for the cron task.
Here are the task queues in use by the system. All are push queues unless Here are the task queues in use by the system. All are push queues unless
explicitly marked as otherwise. explicitly marked as otherwise.

3
docs/configuration.md
View file

@ -39,7 +39,8 @@ The main files of note that come pre-configured in Nomulus are:
* `web.xml` -- Configuration of URL paths on the webserver * `web.xml` -- Configuration of URL paths on the webserver
* `appengine-web.xml` -- Overall App Engine settings including number and type * `appengine-web.xml` -- Overall App Engine settings including number and type
of instances of instances
* `queue.xml` -- Configuration of App Engine task queues * `cloud-scheduler-tasks.xml` -- Configuration of Cloud Scheduler Tasks
* * `cloud-tasks-queue.xml` -- Configuration of Cloud Tasks Queue
* `application.xml` -- Configuration of the application name and its services * `application.xml` -- Configuration of the application name and its services
Cron, web, and queue are covered in more detail in the "App Engine architecture" Cron, web, and queue are covered in more detail in the "App Engine architecture"

14
release/builder/Dockerfile
View file

@ -19,18 +19,18 @@
# 3. Google Cloud SDK for generating the WARs. # 3. Google Cloud SDK for generating the WARs.
# 4. Git to manipulate the private and the merged repos. # 4. Git to manipulate the private and the merged repos.
# 5. Docker to build and push images. # 5. Docker to build and push images.
# 6. cloudSchedulerDeployer for deploying cloud scheduler tasks based on config # 6. deployerForSchedulerAndTasks for deploying cloud scheduler and cloud tasks
FROM golang:1.19 as cloudSchedulerBuilder FROM golang:1.19 as deployCloudSchedulerAndQueueBuilder
WORKDIR /usr/src/cloudSchedulerDeployer WORKDIR /usr/src/deployCloudSchedulerAndQueue
RUN go mod init cloudSchedulerDeployer RUN go mod init deployCloudSchedulerAndQueue
COPY *.go ./ COPY *.go ./
RUN go build -o /cloudSchedulerDeployer RUN go build -o /deployCloudSchedulerAndQueue
FROM marketplace.gcr.io/google/debian10 FROM marketplace.gcr.io/google/debian10
ENV DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF-8 ENV DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF-8
# Add script for cloud scheduler deployer # Add script for cloud scheduler and cloud tasks deployment
COPY --from=cloudSchedulerBuilder /cloudSchedulerDeployer /usr/local/bin/cloudSchedulerDeployer COPY --from=deployCloudSchedulerAndQueueBuilder /deployCloudSchedulerAndQueue /usr/local/bin/deployCloudSchedulerAndQueue
# Add Cloud sql connector # Add Cloud sql connector
ADD https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \ ADD https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \
/usr/local/bin/cloud_sql_proxy /usr/local/bin/cloud_sql_proxy

176
release/builder/cloudSchedulerDeployer.go
View file

@ -1,176 +0,0 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The cloudScheduler tool allows creating, updating and deleting cloud
// scheduler jobs from xml config file
package main
import (
"encoding/json"
"encoding/xml"
"fmt"
"io"
"log"
"os"
"os/exec"
"strings"
)
func main() {
if len(os.Args) < 3 || os.Args[1] == "" || os.Args[2] == "" {
panic("Error - invalid parameters:\nFirst parameter required - config file path;\nSecond parameter required - project name")
}
// Config file path
configFileLocation := os.Args[1]
// Project name where to submit the tasks
projectName := os.Args[2]
log.Default().Println("Filepath " + configFileLocation)
xmlFile, err := os.Open(configFileLocation)
if err != nil {
panic(err)
}
defer xmlFile.Close()
type Task struct {
XMLName xml.Name `xml:"task"`
URL string `xml:"url"`
Description string `xml:"description"`
Schedule string `xml:"schedule"`
Name string `xml:"name"`
}
type Taskentries struct {
XMLName xml.Name `xml:"taskentries"`
Task []Task `xml:"task"`
}
type ServiceAccount struct {
DisplayName string `json:"displayName"`
Email string `json:"email"`
}
type ExistingJob struct {
Name string `json:"name"`
State string `json:"state"`
}
byteValue, _ := io.ReadAll(xmlFile)
var taskEntries Taskentries
if err := xml.Unmarshal(byteValue, &taskEntries); err != nil {
panic("Failed to unmarshal taskentries: " + err.Error())
}
getArgs := func(taskRecord Task, operationType string, serviceAccountEmail string) []string {
// Cloud Schedule doesn't allow description of more than 499 chars and \n
var description string
if len(taskRecord.Description) > 499 {
description = taskRecord.Description[:499]
} else {
description = taskRecord.Description
}
description = strings.ReplaceAll(description, "\n", " ")
return []string{
"--project", projectName,
"scheduler", "jobs", operationType,
"http", taskRecord.Name,
"--location", "us-central1",
"--schedule", taskRecord.Schedule,
"--uri", fmt.Sprintf("https://backend-dot-%s.appspot.com%s", projectName, strings.TrimSpace(taskRecord.URL)),
"--description", description,
"--http-method", "get",
"--oidc-service-account-email", serviceAccountEmail,
"--oidc-token-audience", projectName,
}
}
// Get existing jobs from Cloud Scheduler
var allExistingJobs []ExistingJob
cmdGetExistingList := exec.Command("gcloud", "scheduler", "jobs", "list", "--project="+projectName, "--location=us-central1", "--format=json")
cmdGetExistingListOutput, cmdGetExistingListError := cmdGetExistingList.CombinedOutput()
if cmdGetExistingListError != nil {
panic("Can't obtain existing cloud scheduler jobs for " + projectName)
}
err = json.Unmarshal(cmdGetExistingListOutput, &allExistingJobs)
if err != nil {
panic("Failed to parse existing jobs from cloud schedule: " + err.Error())
}
// Sync deleted jobs
enabledExistingJobs := map[string]bool{}
for i := 0; i < len(allExistingJobs); i++ {
jobName := strings.Split(allExistingJobs[i].Name, "jobs/")[1]
toBeDeleted := true
if allExistingJobs[i].State == "ENABLED" {
enabledExistingJobs[jobName] = true
}
for i := 0; i < len(taskEntries.Task); i++ {
if taskEntries.Task[i].Name == jobName {
toBeDeleted = false
break
}
}
if toBeDeleted {
cmdDelete := exec.Command("gcloud", "scheduler", "jobs", "delete", jobName, "--project="+projectName, "--quiet")
cmdDeleteOutput, cmdDeleteError := cmdDelete.CombinedOutput()
log.Default().Println("Deleting cloud scheduler job " + jobName)
if cmdDeleteError != nil {
panic(string(cmdDeleteOutput))
}
}
}
// Find service account email
var serviceAccounts []ServiceAccount
var serviceAccountEmail string
cmdGetServiceAccounts := exec.Command("gcloud", "iam", "service-accounts", "list", "--project="+projectName, "--format=json")
cmdGetServiceAccountsOutput, cmdGetServiceAccountsError := cmdGetServiceAccounts.CombinedOutput()
if cmdGetServiceAccountsError != nil {
panic(cmdGetServiceAccountsError)
}
err = json.Unmarshal(cmdGetServiceAccountsOutput, &serviceAccounts)
if err != nil {
panic(err.Error())
}
for i := 0; i < len(serviceAccounts); i++ {
if serviceAccounts[i].DisplayName == "cloud-scheduler" {
serviceAccountEmail = serviceAccounts[i].Email
break
}
}
if serviceAccountEmail == "" {
panic("Service account for cloud scheduler is not created for " + projectName)
}
// Sync created and updated jobs
for i := 0; i < len(taskEntries.Task); i++ {
cmdType := "update"
if enabledExistingJobs[taskEntries.Task[i].Name] != true {
cmdType = "create"
}
syncCommand := exec.Command("gcloud", getArgs(taskEntries.Task[i], cmdType, serviceAccountEmail)...)
syncCommandOutput, syncCommandError := syncCommand.CombinedOutput()
log.Default().Println(cmdType + " cloud scheduler job " + taskEntries.Task[i].Name)
if syncCommandError != nil {
panic(string(syncCommandOutput))
}
}
}

339
release/builder/deployCloudSchedulerAndQueue.go Normal file
View file

@ -0,0 +1,339 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The deployerForSchedulerAndTasks tool allows creating, updating and deleting
// cloud scheduler jobs and tasks from xml config file
package main
import (
"encoding/json"
"encoding/xml"
"fmt"
"io"
"log"
"os"
"os/exec"
"strings"
)
var projectName string
const GcpLocation = "us-central1"
type SyncManager[T Task | Queue] interface {
syncCreated(e T) ([]byte, error)
syncDeleted(e string) ([]byte, error)
syncUpdated(e T, v ExistingEntry) ([]byte, error)
fetchExistingRecords() ExistingEntries
getXmlEntries() []T
getEntryName(e T) string
}
type ServiceAccount struct {
DisplayName string `json:"displayName"`
Email string `json:"email"`
}
type Queue struct {
Name string `xml:"name"`
MaxAttempts string `xml:"max-attempts"`
MaxBackoff string `xml:"max-backoff"`
MaxConcurrentDispatches string `xml:"max-concurrent-dispatches"`
MaxDispatchesPerSecond string `xml:"max-dispatches-per-second"`
MaxDoublings string `xml:"max-doublings"`
MaxRetryDuration string `xml:"max-retry-duration"`
MinBackoff string `xml:"min-backoff"`
MaxBurstSize string `xml:"max-burst-size"`
}
type Task struct {
URL string `xml:"url"`
Description string `xml:"description"`
Schedule string `xml:"schedule"`
Name string `xml:"name"`
}
type QueuesSyncManager struct {
Queues []Queue
}
type TasksSyncManager struct {
Tasks []Task
ServiceAccountEmail string
}
type XmlEntries struct {
XMLName xml.Name `xml:"entries"`
Tasks []Task `xml:"task"`
Queues []Queue `xml:"queue"`
}
type ExistingEntry struct {
Name string `json:"name"`
State string `json:"state"`
}
type ExistingEntries struct {
asAMap map[string]ExistingEntry
}
func handleSyncErrors(response []byte, err error) {
if err != nil {
panic(string(response) + " " + err.Error())
}
}
func sync[T Queue | Task](manager SyncManager[T]) {
// State from xml file
xmlEntries := manager.getXmlEntries()
// Fetch tasks currently in GCP
existingTasks := manager.fetchExistingRecords()
// Find and sync deleted tasks
for taskName := range existingTasks.asAMap {
toBeDeleted := true
for j := 0; j < len(xmlEntries); j++ {
if manager.getEntryName(xmlEntries[j]) == taskName {
toBeDeleted = false
break
}
}
if toBeDeleted {
handleSyncErrors(manager.syncDeleted(taskName))
}
}
// Sync created and updated tasks
for j := 0; j < len(xmlEntries); j++ {
if val, ok := existingTasks.asAMap[manager.getEntryName(xmlEntries[j])]; ok {
handleSyncErrors(manager.syncUpdated(xmlEntries[j], val))
} else {
handleSyncErrors(manager.syncCreated(xmlEntries[j]))
}
}
}
var cloudSchedulerServiceAccountEmail string
func getCloudSchedulerServiceAccountEmail() string {
if cloudSchedulerServiceAccountEmail != "" {
return cloudSchedulerServiceAccountEmail
}
var serviceAccounts []ServiceAccount
cmdGetServiceAccounts := exec.Command("gcloud", "iam", "service-accounts", "list", "--project="+projectName, "--format=json")
cmdGetServiceAccountsOutput, cmdGetServiceAccountsError := cmdGetServiceAccounts.CombinedOutput()
if cmdGetServiceAccountsError != nil {
panic("Failed to fetch service accounts " + string(cmdGetServiceAccountsOutput))
}
err := json.Unmarshal(cmdGetServiceAccountsOutput, &serviceAccounts)
if err != nil {
panic("Failed to parse service account response: " + err.Error())
}
for i := 0; i < len(serviceAccounts); i++ {
if serviceAccounts[i].DisplayName == "cloud-scheduler" {
cloudSchedulerServiceAccountEmail = serviceAccounts[i].Email
return serviceAccounts[i].Email
}
}
panic("Service account for cloud scheduler is not created for " + projectName)
}
func (manager TasksSyncManager) getEntryName(task Task) string {
return task.Name
}
func (manager TasksSyncManager) getXmlEntries() []Task {
return manager.Tasks
}
func (manager TasksSyncManager) getArgs(task Task, operationType string) []string {
// Cloud Schedule doesn't allow description of more than 499 chars and \n
var description string
if len(task.Description) > 499 {
log.Default().Println("Task description exceeds the allowed length of " +
"500 characters, clipping the description before submitting the task " +
task.Name)
description = task.Description[:499]
} else {
description = task.Description
}
description = strings.ReplaceAll(description, "\n", " ")
return []string{
"--project", projectName,
"scheduler", "jobs", operationType,
"http", task.Name,
"--location", GcpLocation,
"--schedule", task.Schedule,
"--uri", fmt.Sprintf("https://backend-dot-%s.appspot.com%s", projectName, strings.TrimSpace(task.URL)),
"--description", description,
"--http-method", "get",
"--oidc-service-account-email", getCloudSchedulerServiceAccountEmail(),
"--oidc-token-audience", projectName,
}
}
func (manager TasksSyncManager) fetchExistingRecords() ExistingEntries {
return getExistingEntries(exec.Command("gcloud", "scheduler", "jobs", "list", "--project="+projectName, "--location="+GcpLocation, "--format=json"))
}
func (manager TasksSyncManager) syncDeleted(taskName string) ([]byte, error) {
log.Default().Println("Deleting cloud scheduler task " + taskName)
cmdDelete := exec.Command("gcloud", "scheduler", "jobs", "delete", taskName, "--project="+projectName, "--quiet")
return cmdDelete.CombinedOutput()
}
func (manager TasksSyncManager) syncCreated(task Task) ([]byte, error) {
log.Default().Println("Creating cloud scheduler task " + task.Name)
syncCommand := exec.Command("gcloud", manager.getArgs(task, "create")...)
return syncCommand.CombinedOutput()
}
func (manager TasksSyncManager) syncUpdated(task Task, _ ExistingEntry) ([]byte, error) {
log.Default().Println("Updating cloud scheduler task " + task.Name)
syncCommand := exec.Command("gcloud", manager.getArgs(task, "update")...)
return syncCommand.CombinedOutput()
}
func (manager QueuesSyncManager) getEntryName(queue Queue) string {
return queue.Name
}
func (manager QueuesSyncManager) getXmlEntries() []Queue {
return manager.Queues
}
func (manager QueuesSyncManager) getArgs(queue Queue, operationType string) []string {
args := []string{
"tasks", "queues", operationType, queue.Name, "--project", projectName,
}
if queue.MaxAttempts != "" {
args = append(args, "--max-attempts", queue.MaxAttempts)
}
if queue.MaxBackoff != "" {
args = append(args, "--max-backoff", queue.MaxBackoff)
}
if queue.MaxConcurrentDispatches != "" {
args = append(args, "--max-concurrent-dispatches", queue.MaxConcurrentDispatches)
}
if queue.MaxDispatchesPerSecond != "" {
args = append(args, "--max-dispatches-per-second", queue.MaxDispatchesPerSecond)
}
if queue.MaxDoublings != "" {
args = append(args, "--max-doublings", queue.MaxDoublings)
}
if queue.MaxRetryDuration != "" {
args = append(args, "--max-retry-duration", queue.MaxRetryDuration)
}
if queue.MinBackoff != "" {
args = append(args, "--min-backoff", queue.MinBackoff)
}
if queue.MaxBurstSize != "" {
args = append(args, "--max-burst-size", queue.MaxBurstSize)
}
args = append(args, "--format=json")
return args
}
func (manager QueuesSyncManager) fetchExistingRecords() ExistingEntries {
return getExistingEntries(exec.Command("gcloud", "tasks", "queues", "list", "--project="+projectName, "--location="+GcpLocation, "--format=json"))
}
func (manager QueuesSyncManager) syncDeleted(taskName string) ([]byte, error) {
// Default queue can't be deleted
if taskName == "default" {
return []byte{}, nil
}
log.Default().Println("Pausing cloud tasks queue " + taskName)
cmd := exec.Command("gcloud", "tasks", "queues", "pause", taskName, "--project="+projectName, "--quiet")
return cmd.CombinedOutput()
}
func (manager QueuesSyncManager) syncCreated(queue Queue) ([]byte, error) {
log.Default().Println("Creating cloud tasks queue " + queue.Name)
cmd := exec.Command("gcloud", manager.getArgs(queue, "create")...)
return cmd.CombinedOutput()
}
func (manager QueuesSyncManager) syncUpdated(queue Queue, existingQueue ExistingEntry) ([]byte, error) {
if existingQueue.State == "PAUSED" {
log.Default().Println("Resuming cloud tasks queue " + queue.Name)
cmdResume := exec.Command("gcloud", "tasks", "queues", "resume", queue.Name, "--project="+projectName, "--quiet")
r, err := cmdResume.CombinedOutput()
if err != nil {
return r, err
}
}
log.Default().Println("Updating cloud tasks queue " + queue.Name)
cmd := exec.Command("gcloud", manager.getArgs(queue, "update")...)
return cmd.CombinedOutput()
}
func getExistingEntries(cmd *exec.Cmd) ExistingEntries {
var allExistingEntries []ExistingEntry
cmdGetExistingListOutput, cmdGetExistingListError := cmd.CombinedOutput()
if cmdGetExistingListError != nil {
panic("Failed to load existing entries: " + cmdGetExistingListError.Error())
}
err := json.Unmarshal(cmdGetExistingListOutput, &allExistingEntries)
if err != nil {
panic("Failed to parse existing entries: " + err.Error())
}
e := ExistingEntries{
asAMap: map[string]ExistingEntry{},
}
for i := 0; i < len(allExistingEntries); i++ {
existingEntry := allExistingEntries[i]
e.asAMap[existingEntry.Name[strings.LastIndex(existingEntry.Name, "/")+1:]] = existingEntry
}
return e
}
func main() {
if len(os.Args) < 3 || os.Args[1] == "" || os.Args[2] == "" {
panic("Error - Invalid Parameters.\nRequired params: 1 - config file path; 2 - project name;")
}
// Config file path
configFileLocation := os.Args[1]
// Project name where to submit the tasks
projectName = os.Args[2]
log.Default().Println("Filepath " + configFileLocation)
xmlFile, err := os.Open(configFileLocation)
if err != nil {
panic(err)
}
defer xmlFile.Close()
byteValue, _ := io.ReadAll(xmlFile)
var xmlEntries XmlEntries
if err := xml.Unmarshal(byteValue, &xmlEntries); err != nil {
panic("Failed to parse xml file entries: " + err.Error())
}
if len(xmlEntries.Tasks) > 0 {
log.Default().Println("Syncing cloud scheduler tasks from xml...")
tasksSyncManager := TasksSyncManager{
Tasks: xmlEntries.Tasks,
}
sync[Task](tasksSyncManager)
}
if len(xmlEntries.Queues) > 0 {
log.Default().Println("Syncing cloud task queues from xml...")
queuesSyncManager := QueuesSyncManager{
Queues: xmlEntries.Queues,
}
sync[Queue](queuesSyncManager)
}
}

10
release/cloudbuild-deploy.yaml
View file

@ -28,7 +28,7 @@ steps:
set -e set -e
gcloud secrets versions access latest \ gcloud secrets versions access latest \
--secret nomulus-tool-cloudbuild-credential > tool-credential.json --secret nomulus-tool-cloudbuild-credential > tool-credential.json
# Create/Update cloud scheduler jobs based on a cloud-scheduler-tasks.xml # Create/Update cloud scheduler and cloud tasks based on a cloud-scheduler-tasks.xml
- name: 'gcr.io/$PROJECT_ID/builder:latest' - name: 'gcr.io/$PROJECT_ID/builder:latest'
entrypoint: /bin/bash entrypoint: /bin/bash
args: args:
@ -43,7 +43,8 @@ steps:
fi fi
gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/${_ENV}.tar . gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/${_ENV}.tar .
tar -xvf ${_ENV}.tar tar -xvf ${_ENV}.tar
cloudSchedulerDeployer default/WEB-INF/cloud-scheduler-tasks.xml $project_id deployCloudSchedulerAndQueue default/WEB-INF/cloud-scheduler-tasks.xml $project_id
deployCloudSchedulerAndQueue default/WEB-INF/cloud-tasks-queue.xml $project_id
# Deploy the GAE config files. # Deploy the GAE config files.
# First authorize the gcloud tool to use the credential json file, then # First authorize the gcloud tool to use the credential json file, then
# download and unzip the tarball that contains the relevant config files # download and unzip the tarball that contains the relevant config files
@ -59,10 +60,7 @@ steps:
else else
project_id="domain-registry-${_ENV}" project_id="domain-registry-${_ENV}"
fi fi
for filename in dispatch queue; do gcloud -q --project $project_id app deploy default/WEB-INF/appengine-generated/dispatch.yaml
gcloud -q --project $project_id app deploy \
default/WEB-INF/appengine-generated/$filename.yaml
done
# Save the deployed tag for the current environment on GCS, and update the # Save the deployed tag for the current environment on GCS, and update the
# mappings from Nomulus releases to Appengine versions. # mappings from Nomulus releases to Appengine versions.
- name: 'gcr.io/$PROJECT_ID/builder:latest' - name: 'gcr.io/$PROJECT_ID/builder:latest'