zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-04-30 03:57:51 +02:00
Code Issues Projects Releases Packages Wiki Activity

Make it possible to stage a single Beam pipeline (#1351)

Browse source
This commit is contained in:
Lai Jiang 2021-09-29 18:27:23 -04:00 • committed by GitHub
parent 47a38e8309
commit 0d8f9882e4
3 changed files with 47 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
config/nom_build.py
View file

@ -140,6 +140,8 @@ PROPERTIES = [
'a BEAM pipeline to image. Setting this property to empty string ' 'a BEAM pipeline to image. Setting this property to empty string '
'will disable image generation.', 'will disable image generation.',
'/usr/bin/dot'), '/usr/bin/dot'),
Property('pipeline',
'The name of the Beam pipeline being staged.')
] ]
GRADLE_FLAGS = [ GRADLE_FLAGS = [

80
core/build.gradle
View file

@ -772,50 +772,57 @@ createUberJar('nomulus', 'nomulus', 'google.registry.tools.RegistryTool')
// This packages more code and dependency than necessary. However, without // This packages more code and dependency than necessary. However, without
// restructuring the source tree it is difficult to generate leaner jars. // restructuring the source tree it is difficult to generate leaner jars.
createUberJar( createUberJar(
'beam_pipeline_common', 'beamPipelineCommon',
'beam_pipeline_common', 'beam_pipeline_common',
'') '')
// Create beam staging task if environment is alpha or crash. // Create beam staging task if the environment is alpha. Production, sandbox and
// All other environments use formally released pipelines through CloudBuild. // qa use formally released pipelines through CloudBuild, whereas crash and
// alpha use the pipelines staged on alpha deployment project.
// //
// User should install gcloud and login to GCP before invoking this tasks. // User should install gcloud and login to GCP before invoking this tasks.
if (environment in ['alpha', 'crash']) { if (environment == 'alpha') {
def pipelines = [ def pipelines = [
[ InitSql :
mainClass: 'google.registry.beam.initsql.InitSqlPipeline', [
metaData: 'google/registry/beam/init_sql_pipeline_metadata.json' mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
], metaData : 'google/registry/beam/init_sql_pipeline_metadata.json'
[ ],
mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline', BulkDeleteDatastore:
metaData: 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json' [
], mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
[ metaData : 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
mainClass: 'google.registry.beam.spec11.Spec11Pipeline', ],
metaData: 'google/registry/beam/spec11_pipeline_metadata.json' Spec11 :
], [
[ mainClass: 'google.registry.beam.spec11.Spec11Pipeline',
mainClass: 'google.registry.beam.invoicing.InvoicingPipeline', metaData : 'google/registry/beam/spec11_pipeline_metadata.json'
metaData: 'google/registry/beam/invoicing_pipeline_metadata.json' ],
], Invoicing :
[ [
mainClass: 'google.registry.beam.rde.RdePipeline', mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
metaData: 'google/registry/beam/rde_pipeline_metadata.json' metaData : 'google/registry/beam/invoicing_pipeline_metadata.json'
], ],
Rde :
[
mainClass: 'google.registry.beam.rde.RdePipeline',
metaData : 'google/registry/beam/rde_pipeline_metadata.json'
],
] ]
project.tasks.create("stage_beam_pipelines") { project.tasks.create("stageBeamPipelines") {
doLast { doLast {
pipelines.each { pipelines.each {
def mainClass = it['mainClass'] if (rootProject.pipeline == ''|| rootProject.pipeline == it.key) {
def metaData = it['metaData'] def mainClass = it.value['mainClass']
def pipelineName = CaseFormat.UPPER_CAMEL.to( def metaData = it.value['metaData']
CaseFormat.LOWER_UNDERSCORE, def pipelineName = CaseFormat.UPPER_CAMEL.to(
mainClass.substring(mainClass.lastIndexOf('.') + 1)) CaseFormat.LOWER_UNDERSCORE,
def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}" mainClass.substring(mainClass.lastIndexOf('.') + 1))
def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1) def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
def uberJarName = tasks.beam_pipeline_common.outputs.files.asPath def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
def uberJarName = tasks.beamPipelineCommon.outputs.files.asPath
def command = "\ def command = "\
gcloud dataflow flex-template build \ gcloud dataflow flex-template build \
gs://${gcpProject}-deploy/live/beam/${metaDataBaseName} \ gs://${gcpProject}-deploy/live/beam/${metaDataBaseName} \
--image-gcr-path ${imageName}:live \ --image-gcr-path ${imageName}:live \
@ -825,10 +832,11 @@ if (environment in ['alpha', 'crash']) {
--jar ${uberJarName} \ --jar ${uberJarName} \
--env FLEX_TEMPLATE_JAVA_MAIN_CLASS=${mainClass} \ --env FLEX_TEMPLATE_JAVA_MAIN_CLASS=${mainClass} \
--project ${gcpProject}".toString() --project ${gcpProject}".toString()
rootProject.ext.execInBash(command, '/tmp') rootProject.ext.execInBash(command, '/tmp')
}
} }
} }
}.dependsOn(tasks.beam_pipeline_common) }.dependsOn(tasks.beamPipelineCommon)
} }
// A jar with classes and resources from main sourceSet, excluding internal // A jar with classes and resources from main sourceSet, excluding internal

1
gradle.properties
View file

@ -28,3 +28,4 @@ baseSchemaTag=
schema_version= schema_version=
nomulus_version= nomulus_version=
dot_path=/usr/bin/dot dot_path=/usr/bin/dot
pipeline=