mirror of
https://github.com/google/nomulus.git
synced 2025-04-29 03:27:51 +02:00
Adds cloud scheduler and tasks deployer (#1999)
This commit is contained in:
Pavlo Tkach
GitHub
parent
26efe67211
commit
f173b4fb4e
15 changed files with 496 additions and 214 deletions
|
|
@ -103,7 +103,7 @@ explodeWar.doLast {
|
|||
file("${it.explodedAppDirectory}/WEB-INF/lib/tools.jar").setWritable(true)
|
||||
}
|
||||
|
||||
appengineDeployAll.finalizedBy ':cloudSchedulerDeployer'
|
||||
appengineDeployAll.finalizedBy ':deployCloudSchedulerAndQueue'
|
||||
rootProject.deploy.dependsOn appengineDeployAll
|
||||
rootProject.stage.dependsOn appengineStage
|
||||
tasks['war'].dependsOn ':console-webapp:buildConsoleWebappProd'
|
||||
|
|
|
|||
12
build.gradle
12
build.gradle
|
|
@ -558,17 +558,23 @@ task coreDev {
|
|||
|
||||
javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }
|
||||
|
||||
// Runs the script, which deploys cloud scheduler tasks based on the config
|
||||
task cloudSchedulerDeployer {
|
||||
// Runs the script, which deploys cloud scheduler and tasks based on the config
|
||||
task deployCloudSchedulerAndQueue {
|
||||
doLast {
|
||||
def env = environment
|
||||
if (!prodOrSandboxEnv) {
|
||||
exec {
|
||||
commandLine 'go', 'run',
|
||||
"${rootDir}/release/builder/cloudSchedulerDeployer.go",
|
||||
"${rootDir}/release/builder/deployCloudSchedulerAndQueue.go",
|
||||
"${rootDir}/core/src/main/java/google/registry/env/${env}/default/WEB-INF/cloud-scheduler-tasks.xml",
|
||||
"domain-registry-${env}"
|
||||
}
|
||||
exec {
|
||||
commandLine 'go', 'run',
|
||||
"${rootDir}/release/builder/deployCloudSchedulerAndQueue.go",
|
||||
"${rootDir}/core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml",
|
||||
"domain-registry-${env}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<taskentries>
|
||||
<entries>
|
||||
<task>
|
||||
<url>/_dr/task/rdeStaging</url>
|
||||
<name>rdeStaging</name>
|
||||
|
|
@ -138,4 +138,4 @@
|
|||
</description>
|
||||
<schedule>*/1 * * * *</schedule>
|
||||
</task>
|
||||
</taskentries>
|
||||
</entries>
|
||||
|
|
|
|||
113
core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml
vendored
Normal file
113
core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml
vendored
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<entries>
|
||||
<!-- Queue template with all supported params -->
|
||||
<!-- More information - https://cloud.google.com/sdk/gcloud/reference/tasks/queues/create -->
|
||||
<!--
|
||||
<queue>
|
||||
<name></name>
|
||||
<max-attempts></max-attempts>
|
||||
<max-backoff></max-backoff>
|
||||
<max-concurrent-dispatches></max-concurrent-dispatches>
|
||||
<max-dispatches-per-second></max-dispatches-per-second>
|
||||
<max-doublings></max-doublings>
|
||||
<max-retry-duration></max-retry-duration>
|
||||
<min-backoff></min-backoff>
|
||||
</queue>
|
||||
-->
|
||||
|
||||
<!-- Queue for reading DNS update requests and batching them off to the dns-publish queue. -->
|
||||
<queue>
|
||||
<name>dns-refresh</name>
|
||||
<max-dispatches-per-second>100</max-dispatches-per-second>
|
||||
</queue>
|
||||
|
||||
<!-- Queue for publishing DNS updates in batches. -->
|
||||
<queue>
|
||||
<name>dns-publish</name>
|
||||
<max-dispatches-per-second>100</max-dispatches-per-second>
|
||||
<!-- 30 sec backoff increasing linearly up to 30 minutes. -->
|
||||
<min-backoff>30s</min-backoff>
|
||||
<max-backoff>1800s</max-backoff>
|
||||
<max-doublings>0</max-doublings>
|
||||
</queue>
|
||||
|
||||
<!-- Queue for uploading RDE deposits to the escrow provider. -->
|
||||
<queue>
|
||||
<name>rde-upload</name>
|
||||
<max-dispatches-per-second>0.166666667</max-dispatches-per-second>
|
||||
<max-concurrent-dispatches>5</max-concurrent-dispatches>
|
||||
<max-retry-duration>14400s</max-retry-duration>
|
||||
</queue>
|
||||
|
||||
<!-- Queue for uploading RDE reports to ICANN. -->
|
||||
<queue>
|
||||
<name>rde-report</name>
|
||||
<max-dispatches-per-second>1</max-dispatches-per-second>
|
||||
<max-concurrent-dispatches>1</max-concurrent-dispatches>
|
||||
<max-retry-duration>14400s</max-retry-duration>
|
||||
</queue>
|
||||
|
||||
<!-- Queue for copying BRDA deposits to GCS. -->
|
||||
<queue>
|
||||
<name>brda</name>
|
||||
<max-dispatches-per-second>0.016666667</max-dispatches-per-second>
|
||||
<max-concurrent-dispatches>10</max-concurrent-dispatches>
|
||||
<max-retry-duration>82800s</max-retry-duration>
|
||||
</queue>
|
||||
|
||||
<!-- Queue for tasks that trigger domain DNS update upon host rename. -->
|
||||
<queue>
|
||||
<name>async-host-rename</name>
|
||||
<max-dispatches-per-second>1</max-dispatches-per-second>
|
||||
</queue>
|
||||
|
||||
<!-- Queue for tasks that wait for a Beam pipeline to complete (i.e. Spec11 and invoicing). -->
|
||||
<queue>
|
||||
<name>beam-reporting</name>
|
||||
<max-dispatches-per-second>0.016666667</max-dispatches-per-second>
|
||||
<max-concurrent-dispatches>1</max-concurrent-dispatches>
|
||||
<max-attempts>5</max-attempts>
|
||||
<min-backoff>180s</min-backoff>
|
||||
<max-backoff>180s</max-backoff>
|
||||
</queue>
|
||||
|
||||
|
||||
<!-- Queue for tasks that communicate with TMCH MarksDB webserver. -->
|
||||
<queue>
|
||||
<name>marksdb</name>
|
||||
<max-dispatches-per-second>0.016666667</max-dispatches-per-second>
|
||||
<max-concurrent-dispatches>1</max-concurrent-dispatches>
|
||||
<max-retry-duration>39600s</max-retry-duration> <!-- cron interval minus hour -->
|
||||
</queue>
|
||||
|
||||
<!-- Queue for tasks to produce LORDN CSV reports, populated by a Cloud Scheduler fanout job. -->
|
||||
<queue>
|
||||
<name>nordn</name>
|
||||
<max-dispatches-per-second>1</max-dispatches-per-second>
|
||||
<max-concurrent-dispatches>10</max-concurrent-dispatches>
|
||||
<max-retry-duration>39600s</max-retry-duration> <!-- cron interval minus hour -->
|
||||
</queue>
|
||||
|
||||
<!-- Queue for tasks that sync data to Google Spreadsheets. -->
|
||||
<queue>
|
||||
<name>sheet</name>
|
||||
<max-dispatches-per-second>1</max-dispatches-per-second>
|
||||
<!-- max-concurrent-dispatches is intentionally omitted. -->
|
||||
<max-retry-duration>3600s</max-retry-duration>
|
||||
</queue>
|
||||
|
||||
<!-- Queue for infrequent cron tasks (i.e. hourly or less often) that should retry three times on failure. -->
|
||||
<queue>
|
||||
<name>retryable-cron-tasks</name>
|
||||
<max-dispatches-per-second>1</max-dispatches-per-second>
|
||||
<max-attempts>3</max-attempts>
|
||||
</queue>
|
||||
|
||||
<!-- <!– Queue for async actions that should be run at some point in the future. –>-->
|
||||
<queue>
|
||||
<name>async-actions</name>
|
||||
<max-dispatches-per-second>1</max-dispatches-per-second>
|
||||
<max-concurrent-dispatches>5</max-concurrent-dispatches>
|
||||
</queue>
|
||||
|
||||
</entries>
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- TODO: @ptkach - Delete once Cloud Api deployer is up and running -->
|
||||
<queue-entries>
|
||||
|
||||
<!-- Queue for reading DNS update requests and batching them off to the dns-publish queue. -->
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<taskentries>
|
||||
<entries>
|
||||
|
||||
<!--
|
||||
/cron/fanout params:
|
||||
|
|
@ -148,4 +148,4 @@
|
|||
</description>
|
||||
<schedule>7 3 * * *</schedule>
|
||||
</task>
|
||||
</taskentries>
|
||||
</entries>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<taskentries>
|
||||
<entries>
|
||||
<task>
|
||||
<url>/_dr/task/rdeStaging</url>
|
||||
<name>rdeStaging</name>
|
||||
|
|
@ -273,4 +273,4 @@
|
|||
</description>
|
||||
<schedule>0 15 * * 1</schedule>
|
||||
</task>
|
||||
</taskentries>
|
||||
</entries>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<taskentries>
|
||||
<entries>
|
||||
<task>
|
||||
<url>/_dr/task/rdeStaging</url>
|
||||
<name>rdeStaging</name>
|
||||
|
|
@ -68,4 +68,4 @@
|
|||
</description>
|
||||
<schedule>7 3 * * 6</schedule>
|
||||
</task>
|
||||
</taskentries>
|
||||
</entries>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<taskentries>
|
||||
<entries>
|
||||
<task>
|
||||
<url>/_dr/task/rdeStaging</url>
|
||||
<name>rdeStaging</name>
|
||||
|
|
@ -162,4 +162,4 @@
|
|||
</description>
|
||||
<schedule>0 15 * * 1</schedule>
|
||||
</task>
|
||||
</taskentries>
|
||||
</entries>
|
||||
|
|
|
|||
|
|
@ -110,12 +110,12 @@ the push queue will, upon execution, poll the corresponding pull queue for a
|
|||
specified number of tasks and execute them in a batch. This allows the code to
|
||||
execute in the background while taking advantage of batch processing.
|
||||
|
||||
The task queues used by Nomulus are configured in the `queue.xml` file. Note
|
||||
that many push queues have a direct one-to-one correspondence with entries in
|
||||
`cron.xml` because they need to be fanned-out on a per-TLD or other basis (see
|
||||
the Cron section below for more explanation). The exact queue that a given cron
|
||||
task will use is passed as the query string parameter "queue" in the url
|
||||
specification for the cron task.
|
||||
The task queues used by Nomulus are configured in the `cloud-tasks-queue.xml`
|
||||
file. Note that many push queues have a direct one-to-one correspondence with
|
||||
entries in `cloud-scheduler-tasks.xml` because they need to be fanned-out on a
|
||||
per-TLD or other basis (see the Cron section below for more explanation).
|
||||
The exact queue that a given cron task will use is passed as the query string
|
||||
parameter "queue" in the url specification for the cron task.
|
||||
|
||||
Here are the task queues in use by the system. All are push queues unless
|
||||
explicitly marked as otherwise.
|
||||
|
|
|
|||
|
|
@ -35,12 +35,13 @@ App Engine configuration isn't covered in depth in this document as it is
|
|||
thoroughly documented in the [App Engine configuration docs][app-engine-config].
|
||||
The main files of note that come pre-configured in Nomulus are:
|
||||
|
||||
* `cron.xml` -- Configuration of cronjobs
|
||||
* `web.xml` -- Configuration of URL paths on the webserver
|
||||
* `appengine-web.xml` -- Overall App Engine settings including number and type
|
||||
* `cron.xml` -- Configuration of cronjobs
|
||||
* `web.xml` -- Configuration of URL paths on the webserver
|
||||
* `appengine-web.xml` -- Overall App Engine settings including number and type
|
||||
of instances
|
||||
* `queue.xml` -- Configuration of App Engine task queues
|
||||
* `application.xml` -- Configuration of the application name and its services
|
||||
* `cloud-scheduler-tasks.xml` -- Configuration of Cloud Scheduler Tasks
|
||||
* * `cloud-tasks-queue.xml` -- Configuration of Cloud Tasks Queue
|
||||
* `application.xml` -- Configuration of the application name and its services
|
||||
|
||||
Cron, web, and queue are covered in more detail in the "App Engine architecture"
|
||||
doc, and the rest are covered in the general App Engine documentation.
|
||||
|
|
|
|||
|
|
@ -19,18 +19,18 @@
|
|||
# 3. Google Cloud SDK for generating the WARs.
|
||||
# 4. Git to manipulate the private and the merged repos.
|
||||
# 5. Docker to build and push images.
|
||||
# 6. cloudSchedulerDeployer for deploying cloud scheduler tasks based on config
|
||||
# 6. deployerForSchedulerAndTasks for deploying cloud scheduler and cloud tasks
|
||||
|
||||
FROM golang:1.19 as cloudSchedulerBuilder
|
||||
WORKDIR /usr/src/cloudSchedulerDeployer
|
||||
RUN go mod init cloudSchedulerDeployer
|
||||
FROM golang:1.19 as deployCloudSchedulerAndQueueBuilder
|
||||
WORKDIR /usr/src/deployCloudSchedulerAndQueue
|
||||
RUN go mod init deployCloudSchedulerAndQueue
|
||||
COPY *.go ./
|
||||
RUN go build -o /cloudSchedulerDeployer
|
||||
RUN go build -o /deployCloudSchedulerAndQueue
|
||||
|
||||
FROM marketplace.gcr.io/google/debian10
|
||||
ENV DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF-8
|
||||
# Add script for cloud scheduler deployer
|
||||
COPY --from=cloudSchedulerBuilder /cloudSchedulerDeployer /usr/local/bin/cloudSchedulerDeployer
|
||||
# Add script for cloud scheduler and cloud tasks deployment
|
||||
COPY --from=deployCloudSchedulerAndQueueBuilder /deployCloudSchedulerAndQueue /usr/local/bin/deployCloudSchedulerAndQueue
|
||||
# Add Cloud sql connector
|
||||
ADD https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 \
|
||||
/usr/local/bin/cloud_sql_proxy
|
||||
|
|
|
|||
|
|
@ -1,176 +0,0 @@
|
|||
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The cloudScheduler tool allows creating, updating and deleting cloud
|
||||
// scheduler jobs from xml config file
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 3 || os.Args[1] == "" || os.Args[2] == "" {
|
||||
panic("Error - invalid parameters:\nFirst parameter required - config file path;\nSecond parameter required - project name")
|
||||
}
|
||||
|
||||
// Config file path
|
||||
configFileLocation := os.Args[1]
|
||||
// Project name where to submit the tasks
|
||||
projectName := os.Args[2]
|
||||
|
||||
log.Default().Println("Filepath " + configFileLocation)
|
||||
|
||||
xmlFile, err := os.Open(configFileLocation)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer xmlFile.Close()
|
||||
|
||||
type Task struct {
|
||||
XMLName xml.Name `xml:"task"`
|
||||
URL string `xml:"url"`
|
||||
Description string `xml:"description"`
|
||||
Schedule string `xml:"schedule"`
|
||||
Name string `xml:"name"`
|
||||
}
|
||||
|
||||
type Taskentries struct {
|
||||
XMLName xml.Name `xml:"taskentries"`
|
||||
Task []Task `xml:"task"`
|
||||
}
|
||||
|
||||
type ServiceAccount struct {
|
||||
DisplayName string `json:"displayName"`
|
||||
Email string `json:"email"`
|
||||
}
|
||||
|
||||
type ExistingJob struct {
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
byteValue, _ := io.ReadAll(xmlFile)
|
||||
|
||||
var taskEntries Taskentries
|
||||
|
||||
if err := xml.Unmarshal(byteValue, &taskEntries); err != nil {
|
||||
panic("Failed to unmarshal taskentries: " + err.Error())
|
||||
}
|
||||
|
||||
getArgs := func(taskRecord Task, operationType string, serviceAccountEmail string) []string {
|
||||
// Cloud Schedule doesn't allow description of more than 499 chars and \n
|
||||
var description string
|
||||
if len(taskRecord.Description) > 499 {
|
||||
description = taskRecord.Description[:499]
|
||||
} else {
|
||||
description = taskRecord.Description
|
||||
}
|
||||
description = strings.ReplaceAll(description, "\n", " ")
|
||||
|
||||
return []string{
|
||||
"--project", projectName,
|
||||
"scheduler", "jobs", operationType,
|
||||
"http", taskRecord.Name,
|
||||
"--location", "us-central1",
|
||||
"--schedule", taskRecord.Schedule,
|
||||
"--uri", fmt.Sprintf("https://backend-dot-%s.appspot.com%s", projectName, strings.TrimSpace(taskRecord.URL)),
|
||||
"--description", description,
|
||||
"--http-method", "get",
|
||||
"--oidc-service-account-email", serviceAccountEmail,
|
||||
"--oidc-token-audience", projectName,
|
||||
}
|
||||
}
|
||||
|
||||
// Get existing jobs from Cloud Scheduler
|
||||
var allExistingJobs []ExistingJob
|
||||
cmdGetExistingList := exec.Command("gcloud", "scheduler", "jobs", "list", "--project="+projectName, "--location=us-central1", "--format=json")
|
||||
cmdGetExistingListOutput, cmdGetExistingListError := cmdGetExistingList.CombinedOutput()
|
||||
if cmdGetExistingListError != nil {
|
||||
panic("Can't obtain existing cloud scheduler jobs for " + projectName)
|
||||
}
|
||||
err = json.Unmarshal(cmdGetExistingListOutput, &allExistingJobs)
|
||||
if err != nil {
|
||||
panic("Failed to parse existing jobs from cloud schedule: " + err.Error())
|
||||
}
|
||||
|
||||
// Sync deleted jobs
|
||||
enabledExistingJobs := map[string]bool{}
|
||||
for i := 0; i < len(allExistingJobs); i++ {
|
||||
jobName := strings.Split(allExistingJobs[i].Name, "jobs/")[1]
|
||||
toBeDeleted := true
|
||||
if allExistingJobs[i].State == "ENABLED" {
|
||||
enabledExistingJobs[jobName] = true
|
||||
}
|
||||
for i := 0; i < len(taskEntries.Task); i++ {
|
||||
if taskEntries.Task[i].Name == jobName {
|
||||
toBeDeleted = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if toBeDeleted {
|
||||
cmdDelete := exec.Command("gcloud", "scheduler", "jobs", "delete", jobName, "--project="+projectName, "--quiet")
|
||||
cmdDeleteOutput, cmdDeleteError := cmdDelete.CombinedOutput()
|
||||
log.Default().Println("Deleting cloud scheduler job " + jobName)
|
||||
if cmdDeleteError != nil {
|
||||
panic(string(cmdDeleteOutput))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Find service account email
|
||||
var serviceAccounts []ServiceAccount
|
||||
var serviceAccountEmail string
|
||||
cmdGetServiceAccounts := exec.Command("gcloud", "iam", "service-accounts", "list", "--project="+projectName, "--format=json")
|
||||
cmdGetServiceAccountsOutput, cmdGetServiceAccountsError := cmdGetServiceAccounts.CombinedOutput()
|
||||
if cmdGetServiceAccountsError != nil {
|
||||
panic(cmdGetServiceAccountsError)
|
||||
}
|
||||
err = json.Unmarshal(cmdGetServiceAccountsOutput, &serviceAccounts)
|
||||
if err != nil {
|
||||
panic(err.Error())
|
||||
}
|
||||
for i := 0; i < len(serviceAccounts); i++ {
|
||||
if serviceAccounts[i].DisplayName == "cloud-scheduler" {
|
||||
serviceAccountEmail = serviceAccounts[i].Email
|
||||
break
|
||||
}
|
||||
}
|
||||
if serviceAccountEmail == "" {
|
||||
panic("Service account for cloud scheduler is not created for " + projectName)
|
||||
}
|
||||
|
||||
// Sync created and updated jobs
|
||||
for i := 0; i < len(taskEntries.Task); i++ {
|
||||
cmdType := "update"
|
||||
if enabledExistingJobs[taskEntries.Task[i].Name] != true {
|
||||
cmdType = "create"
|
||||
}
|
||||
|
||||
syncCommand := exec.Command("gcloud", getArgs(taskEntries.Task[i], cmdType, serviceAccountEmail)...)
|
||||
syncCommandOutput, syncCommandError := syncCommand.CombinedOutput()
|
||||
log.Default().Println(cmdType + " cloud scheduler job " + taskEntries.Task[i].Name)
|
||||
if syncCommandError != nil {
|
||||
panic(string(syncCommandOutput))
|
||||
}
|
||||
}
|
||||
}
|
||||
339
release/builder/deployCloudSchedulerAndQueue.go
Normal file
339
release/builder/deployCloudSchedulerAndQueue.go
Normal file
|
|
@ -0,0 +1,339 @@
|
|||
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The deployerForSchedulerAndTasks tool allows creating, updating and deleting
|
||||
// cloud scheduler jobs and tasks from xml config file
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var projectName string
|
||||
|
||||
const GcpLocation = "us-central1"
|
||||
|
||||
type SyncManager[T Task | Queue] interface {
|
||||
syncCreated(e T) ([]byte, error)
|
||||
syncDeleted(e string) ([]byte, error)
|
||||
syncUpdated(e T, v ExistingEntry) ([]byte, error)
|
||||
fetchExistingRecords() ExistingEntries
|
||||
getXmlEntries() []T
|
||||
getEntryName(e T) string
|
||||
}
|
||||
|
||||
type ServiceAccount struct {
|
||||
DisplayName string `json:"displayName"`
|
||||
Email string `json:"email"`
|
||||
}
|
||||
|
||||
type Queue struct {
|
||||
Name string `xml:"name"`
|
||||
MaxAttempts string `xml:"max-attempts"`
|
||||
MaxBackoff string `xml:"max-backoff"`
|
||||
MaxConcurrentDispatches string `xml:"max-concurrent-dispatches"`
|
||||
MaxDispatchesPerSecond string `xml:"max-dispatches-per-second"`
|
||||
MaxDoublings string `xml:"max-doublings"`
|
||||
MaxRetryDuration string `xml:"max-retry-duration"`
|
||||
MinBackoff string `xml:"min-backoff"`
|
||||
MaxBurstSize string `xml:"max-burst-size"`
|
||||
}
|
||||
|
||||
type Task struct {
|
||||
URL string `xml:"url"`
|
||||
Description string `xml:"description"`
|
||||
Schedule string `xml:"schedule"`
|
||||
Name string `xml:"name"`
|
||||
}
|
||||
|
||||
type QueuesSyncManager struct {
|
||||
Queues []Queue
|
||||
}
|
||||
|
||||
type TasksSyncManager struct {
|
||||
Tasks []Task
|
||||
ServiceAccountEmail string
|
||||
}
|
||||
|
||||
type XmlEntries struct {
|
||||
XMLName xml.Name `xml:"entries"`
|
||||
Tasks []Task `xml:"task"`
|
||||
Queues []Queue `xml:"queue"`
|
||||
}
|
||||
|
||||
type ExistingEntry struct {
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
type ExistingEntries struct {
|
||||
asAMap map[string]ExistingEntry
|
||||
}
|
||||
|
||||
func handleSyncErrors(response []byte, err error) {
|
||||
if err != nil {
|
||||
panic(string(response) + " " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func sync[T Queue | Task](manager SyncManager[T]) {
|
||||
// State from xml file
|
||||
xmlEntries := manager.getXmlEntries()
|
||||
// Fetch tasks currently in GCP
|
||||
existingTasks := manager.fetchExistingRecords()
|
||||
// Find and sync deleted tasks
|
||||
for taskName := range existingTasks.asAMap {
|
||||
toBeDeleted := true
|
||||
for j := 0; j < len(xmlEntries); j++ {
|
||||
if manager.getEntryName(xmlEntries[j]) == taskName {
|
||||
toBeDeleted = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if toBeDeleted {
|
||||
handleSyncErrors(manager.syncDeleted(taskName))
|
||||
}
|
||||
}
|
||||
// Sync created and updated tasks
|
||||
for j := 0; j < len(xmlEntries); j++ {
|
||||
if val, ok := existingTasks.asAMap[manager.getEntryName(xmlEntries[j])]; ok {
|
||||
handleSyncErrors(manager.syncUpdated(xmlEntries[j], val))
|
||||
} else {
|
||||
handleSyncErrors(manager.syncCreated(xmlEntries[j]))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var cloudSchedulerServiceAccountEmail string
|
||||
|
||||
func getCloudSchedulerServiceAccountEmail() string {
|
||||
if cloudSchedulerServiceAccountEmail != "" {
|
||||
return cloudSchedulerServiceAccountEmail
|
||||
}
|
||||
|
||||
var serviceAccounts []ServiceAccount
|
||||
cmdGetServiceAccounts := exec.Command("gcloud", "iam", "service-accounts", "list", "--project="+projectName, "--format=json")
|
||||
cmdGetServiceAccountsOutput, cmdGetServiceAccountsError := cmdGetServiceAccounts.CombinedOutput()
|
||||
if cmdGetServiceAccountsError != nil {
|
||||
panic("Failed to fetch service accounts " + string(cmdGetServiceAccountsOutput))
|
||||
}
|
||||
err := json.Unmarshal(cmdGetServiceAccountsOutput, &serviceAccounts)
|
||||
if err != nil {
|
||||
panic("Failed to parse service account response: " + err.Error())
|
||||
}
|
||||
for i := 0; i < len(serviceAccounts); i++ {
|
||||
if serviceAccounts[i].DisplayName == "cloud-scheduler" {
|
||||
cloudSchedulerServiceAccountEmail = serviceAccounts[i].Email
|
||||
return serviceAccounts[i].Email
|
||||
}
|
||||
}
|
||||
panic("Service account for cloud scheduler is not created for " + projectName)
|
||||
}
|
||||
|
||||
func (manager TasksSyncManager) getEntryName(task Task) string {
|
||||
return task.Name
|
||||
}
|
||||
|
||||
func (manager TasksSyncManager) getXmlEntries() []Task {
|
||||
return manager.Tasks
|
||||
}
|
||||
|
||||
func (manager TasksSyncManager) getArgs(task Task, operationType string) []string {
|
||||
// Cloud Schedule doesn't allow description of more than 499 chars and \n
|
||||
var description string
|
||||
if len(task.Description) > 499 {
|
||||
log.Default().Println("Task description exceeds the allowed length of " +
|
||||
"500 characters, clipping the description before submitting the task " +
|
||||
task.Name)
|
||||
description = task.Description[:499]
|
||||
} else {
|
||||
description = task.Description
|
||||
}
|
||||
description = strings.ReplaceAll(description, "\n", " ")
|
||||
|
||||
return []string{
|
||||
"--project", projectName,
|
||||
"scheduler", "jobs", operationType,
|
||||
"http", task.Name,
|
||||
"--location", GcpLocation,
|
||||
"--schedule", task.Schedule,
|
||||
"--uri", fmt.Sprintf("https://backend-dot-%s.appspot.com%s", projectName, strings.TrimSpace(task.URL)),
|
||||
"--description", description,
|
||||
"--http-method", "get",
|
||||
"--oidc-service-account-email", getCloudSchedulerServiceAccountEmail(),
|
||||
"--oidc-token-audience", projectName,
|
||||
}
|
||||
}
|
||||
|
||||
func (manager TasksSyncManager) fetchExistingRecords() ExistingEntries {
|
||||
return getExistingEntries(exec.Command("gcloud", "scheduler", "jobs", "list", "--project="+projectName, "--location="+GcpLocation, "--format=json"))
|
||||
}
|
||||
|
||||
func (manager TasksSyncManager) syncDeleted(taskName string) ([]byte, error) {
|
||||
log.Default().Println("Deleting cloud scheduler task " + taskName)
|
||||
cmdDelete := exec.Command("gcloud", "scheduler", "jobs", "delete", taskName, "--project="+projectName, "--quiet")
|
||||
return cmdDelete.CombinedOutput()
|
||||
}
|
||||
|
||||
func (manager TasksSyncManager) syncCreated(task Task) ([]byte, error) {
|
||||
log.Default().Println("Creating cloud scheduler task " + task.Name)
|
||||
syncCommand := exec.Command("gcloud", manager.getArgs(task, "create")...)
|
||||
return syncCommand.CombinedOutput()
|
||||
}
|
||||
|
||||
func (manager TasksSyncManager) syncUpdated(task Task, _ ExistingEntry) ([]byte, error) {
|
||||
log.Default().Println("Updating cloud scheduler task " + task.Name)
|
||||
syncCommand := exec.Command("gcloud", manager.getArgs(task, "update")...)
|
||||
return syncCommand.CombinedOutput()
|
||||
}
|
||||
|
||||
func (manager QueuesSyncManager) getEntryName(queue Queue) string {
|
||||
return queue.Name
|
||||
}
|
||||
|
||||
func (manager QueuesSyncManager) getXmlEntries() []Queue {
|
||||
return manager.Queues
|
||||
}
|
||||
|
||||
func (manager QueuesSyncManager) getArgs(queue Queue, operationType string) []string {
|
||||
args := []string{
|
||||
"tasks", "queues", operationType, queue.Name, "--project", projectName,
|
||||
}
|
||||
if queue.MaxAttempts != "" {
|
||||
args = append(args, "--max-attempts", queue.MaxAttempts)
|
||||
}
|
||||
if queue.MaxBackoff != "" {
|
||||
args = append(args, "--max-backoff", queue.MaxBackoff)
|
||||
}
|
||||
if queue.MaxConcurrentDispatches != "" {
|
||||
args = append(args, "--max-concurrent-dispatches", queue.MaxConcurrentDispatches)
|
||||
}
|
||||
if queue.MaxDispatchesPerSecond != "" {
|
||||
args = append(args, "--max-dispatches-per-second", queue.MaxDispatchesPerSecond)
|
||||
}
|
||||
if queue.MaxDoublings != "" {
|
||||
args = append(args, "--max-doublings", queue.MaxDoublings)
|
||||
}
|
||||
if queue.MaxRetryDuration != "" {
|
||||
args = append(args, "--max-retry-duration", queue.MaxRetryDuration)
|
||||
}
|
||||
if queue.MinBackoff != "" {
|
||||
args = append(args, "--min-backoff", queue.MinBackoff)
|
||||
}
|
||||
if queue.MaxBurstSize != "" {
|
||||
args = append(args, "--max-burst-size", queue.MaxBurstSize)
|
||||
}
|
||||
args = append(args, "--format=json")
|
||||
return args
|
||||
}
|
||||
|
||||
func (manager QueuesSyncManager) fetchExistingRecords() ExistingEntries {
|
||||
return getExistingEntries(exec.Command("gcloud", "tasks", "queues", "list", "--project="+projectName, "--location="+GcpLocation, "--format=json"))
|
||||
}
|
||||
|
||||
func (manager QueuesSyncManager) syncDeleted(taskName string) ([]byte, error) {
|
||||
// Default queue can't be deleted
|
||||
if taskName == "default" {
|
||||
return []byte{}, nil
|
||||
}
|
||||
log.Default().Println("Pausing cloud tasks queue " + taskName)
|
||||
cmd := exec.Command("gcloud", "tasks", "queues", "pause", taskName, "--project="+projectName, "--quiet")
|
||||
return cmd.CombinedOutput()
|
||||
}
|
||||
|
||||
func (manager QueuesSyncManager) syncCreated(queue Queue) ([]byte, error) {
|
||||
log.Default().Println("Creating cloud tasks queue " + queue.Name)
|
||||
cmd := exec.Command("gcloud", manager.getArgs(queue, "create")...)
|
||||
return cmd.CombinedOutput()
|
||||
}
|
||||
|
||||
func (manager QueuesSyncManager) syncUpdated(queue Queue, existingQueue ExistingEntry) ([]byte, error) {
|
||||
if existingQueue.State == "PAUSED" {
|
||||
log.Default().Println("Resuming cloud tasks queue " + queue.Name)
|
||||
cmdResume := exec.Command("gcloud", "tasks", "queues", "resume", queue.Name, "--project="+projectName, "--quiet")
|
||||
r, err := cmdResume.CombinedOutput()
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
}
|
||||
log.Default().Println("Updating cloud tasks queue " + queue.Name)
|
||||
cmd := exec.Command("gcloud", manager.getArgs(queue, "update")...)
|
||||
return cmd.CombinedOutput()
|
||||
}
|
||||
|
||||
func getExistingEntries(cmd *exec.Cmd) ExistingEntries {
|
||||
var allExistingEntries []ExistingEntry
|
||||
cmdGetExistingListOutput, cmdGetExistingListError := cmd.CombinedOutput()
|
||||
if cmdGetExistingListError != nil {
|
||||
panic("Failed to load existing entries: " + cmdGetExistingListError.Error())
|
||||
}
|
||||
err := json.Unmarshal(cmdGetExistingListOutput, &allExistingEntries)
|
||||
if err != nil {
|
||||
panic("Failed to parse existing entries: " + err.Error())
|
||||
}
|
||||
e := ExistingEntries{
|
||||
asAMap: map[string]ExistingEntry{},
|
||||
}
|
||||
for i := 0; i < len(allExistingEntries); i++ {
|
||||
existingEntry := allExistingEntries[i]
|
||||
e.asAMap[existingEntry.Name[strings.LastIndex(existingEntry.Name, "/")+1:]] = existingEntry
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 3 || os.Args[1] == "" || os.Args[2] == "" {
|
||||
panic("Error - Invalid Parameters.\nRequired params: 1 - config file path; 2 - project name;")
|
||||
}
|
||||
// Config file path
|
||||
configFileLocation := os.Args[1]
|
||||
// Project name where to submit the tasks
|
||||
projectName = os.Args[2]
|
||||
|
||||
log.Default().Println("Filepath " + configFileLocation)
|
||||
xmlFile, err := os.Open(configFileLocation)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer xmlFile.Close()
|
||||
byteValue, _ := io.ReadAll(xmlFile)
|
||||
var xmlEntries XmlEntries
|
||||
if err := xml.Unmarshal(byteValue, &xmlEntries); err != nil {
|
||||
panic("Failed to parse xml file entries: " + err.Error())
|
||||
}
|
||||
|
||||
if len(xmlEntries.Tasks) > 0 {
|
||||
log.Default().Println("Syncing cloud scheduler tasks from xml...")
|
||||
tasksSyncManager := TasksSyncManager{
|
||||
Tasks: xmlEntries.Tasks,
|
||||
}
|
||||
sync[Task](tasksSyncManager)
|
||||
}
|
||||
|
||||
if len(xmlEntries.Queues) > 0 {
|
||||
log.Default().Println("Syncing cloud task queues from xml...")
|
||||
queuesSyncManager := QueuesSyncManager{
|
||||
Queues: xmlEntries.Queues,
|
||||
}
|
||||
sync[Queue](queuesSyncManager)
|
||||
}
|
||||
}
|
||||
|
|
@ -28,7 +28,7 @@ steps:
|
|||
set -e
|
||||
gcloud secrets versions access latest \
|
||||
--secret nomulus-tool-cloudbuild-credential > tool-credential.json
|
||||
# Create/Update cloud scheduler jobs based on a cloud-scheduler-tasks.xml
|
||||
# Create/Update cloud scheduler and cloud tasks based on a cloud-scheduler-tasks.xml
|
||||
- name: 'gcr.io/$PROJECT_ID/builder:latest'
|
||||
entrypoint: /bin/bash
|
||||
args:
|
||||
|
|
@ -43,7 +43,8 @@ steps:
|
|||
fi
|
||||
gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/${_ENV}.tar .
|
||||
tar -xvf ${_ENV}.tar
|
||||
cloudSchedulerDeployer default/WEB-INF/cloud-scheduler-tasks.xml $project_id
|
||||
deployCloudSchedulerAndQueue default/WEB-INF/cloud-scheduler-tasks.xml $project_id
|
||||
deployCloudSchedulerAndQueue default/WEB-INF/cloud-tasks-queue.xml $project_id
|
||||
# Deploy the GAE config files.
|
||||
# First authorize the gcloud tool to use the credential json file, then
|
||||
# download and unzip the tarball that contains the relevant config files
|
||||
|
|
@ -59,10 +60,7 @@ steps:
|
|||
else
|
||||
project_id="domain-registry-${_ENV}"
|
||||
fi
|
||||
for filename in dispatch queue; do
|
||||
gcloud -q --project $project_id app deploy \
|
||||
default/WEB-INF/appengine-generated/$filename.yaml
|
||||
done
|
||||
gcloud -q --project $project_id app deploy default/WEB-INF/appengine-generated/dispatch.yaml
|
||||
# Save the deployed tag for the current environment on GCS, and update the
|
||||
# mappings from Nomulus releases to Appengine versions.
|
||||
- name: 'gcr.io/$PROJECT_ID/builder:latest'
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue