diff --git a/core/build.gradle b/core/build.gradle index 2cf4f4246..c428ca787 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import java.lang.reflect.Constructor +import com.google.common.base.CaseFormat import java.util.Optional plugins { @@ -791,6 +791,49 @@ createUberJar( 'beam_pipeline_common', '') +// Create beam staging task if environment is alpha or crash. +// All other environments use formally released pipelines through CloudBuild. +// +// User should install gcloud and login to GCP before invoking this tasks. +if (environment in ['alpha', 'crash']) { + def pipelines = [ + [ + mainClass: 'google.registry.beam.initsql.InitSqlPipeline', + metaData: 'google/registry/beam/init_sql_pipeline_metadata.json' + ], + [ + mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline', + metaData: 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json' + ], + ] + project.tasks.create("stage_beam_pipelines") { + doLast { + pipelines.each { + def mainClass = it['mainClass'] + def metaData = it['metaData'] + def pipelineName = CaseFormat.UPPER_CAMEL.to( + CaseFormat.LOWER_UNDERSCORE, + mainClass.substring(mainClass.lastIndexOf('.') + 1)) + def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}" + def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1) + def uberJarName = tasks.beam_pipeline_common.outputs.files.asPath + + def command = "\ + gcloud dataflow flex-template build \ + gs://${gcpProject}-deploy/live/beam/${metaDataBaseName} \ + --image-gcr-path ${imageName}:live \ + --sdk-language JAVA \ + --flex-template-base-image JAVA11 \ + --metadata-file ${projectDir}/src/main/resources/${metaData} \ + --jar ${uberJarName} \ + --env FLEX_TEMPLATE_JAVA_MAIN_CLASS=${mainClass} \ + --project ${gcpProject}".toString() + rootProject.ext.execInBash(command, '/tmp') + } + } + }.dependsOn(tasks.beam_pipeline_common) +} + // A jar with classes and resources from main sourceSet, excluding internal // data. See comments on configurations.nomulus_test above for details. // TODO(weiminyu): release process should build this using the public repo to eliminate the need diff --git a/docs/install.md b/docs/install.md index 5b2134474..7d3808d22 100644 --- a/docs/install.md +++ b/docs/install.md @@ -133,3 +133,16 @@ for more information. [app-engine-sdk]: https://cloud.google.com/appengine/docs/java/download [java-jdk11]: https://www.oracle.com/java/technologies/javase-downloads.html + +## Deploy the BEAM Pipelines + +Nomulus is in the middle of migrating all pipelines to use flex-template. For +pipelines already based on flex-template, deployment in the testing environments +(alpha and crash) can be done using the following command: + +```shell +./nom_build :core:stage_beam_pipelines --environment=alpha +``` + +Pipeline deployment in other environments are through CloudBuild. Please refer +to the [release folder](http://github.com/google/nomulus/release) for details.