Add Gradle tasks to stage BEAM pipelines (#1031)

* Add Gradle tasks to stage BEAM pipelines

Add a Gracle task to stage flex-template based pipelines for alpha and
crash environments.

This is a follow up to go/r3pr/1028, which is also under review.

core/build.gradle

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import java.lang.reflect.Constructor
+import com.google.common.base.CaseFormat
 import java.util.Optional
 
 plugins {
@@ -791,6 +791,49 @@ createUberJar(
     'beam_pipeline_common',
     '')
 
+// Create beam staging task if environment is alpha or crash.
+// All other environments use formally released pipelines through CloudBuild.
+//
+// User should install gcloud and login to GCP before invoking this tasks.
+if (environment in ['alpha', 'crash']) {
+  def pipelines = [
+      [
+          mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
+          metaData: 'google/registry/beam/init_sql_pipeline_metadata.json'
+      ],
+      [
+          mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
+          metaData: 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
+      ],
+  ]
+  project.tasks.create("stage_beam_pipelines") {
+    doLast {
+      pipelines.each {
+        def mainClass = it['mainClass']
+        def metaData = it['metaData']
+        def pipelineName = CaseFormat.UPPER_CAMEL.to(
+                CaseFormat.LOWER_UNDERSCORE,
+                mainClass.substring(mainClass.lastIndexOf('.') + 1))
+        def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
+        def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
+        def uberJarName = tasks.beam_pipeline_common.outputs.files.asPath
+
+        def command = "\
+            gcloud dataflow flex-template build \
+            gs://${gcpProject}-deploy/live/beam/${metaDataBaseName} \
+            --image-gcr-path ${imageName}:live \
+            --sdk-language JAVA \
+            --flex-template-base-image JAVA11 \
+            --metadata-file ${projectDir}/src/main/resources/${metaData} \
+            --jar ${uberJarName} \
+            --env FLEX_TEMPLATE_JAVA_MAIN_CLASS=${mainClass} \
+            --project ${gcpProject}".toString()
+        rootProject.ext.execInBash(command, '/tmp')
+      }
+    }
+  }.dependsOn(tasks.beam_pipeline_common)
+}
+
 // A jar with classes and resources from main sourceSet, excluding internal
 // data. See comments on configurations.nomulus_test above for details.
 // TODO(weiminyu): release process should build this using the public repo to eliminate the need

docs/install.md

@@ -133,3 +133,16 @@ for more information.
 
 [app-engine-sdk]: https://cloud.google.com/appengine/docs/java/download
 [java-jdk11]: https://www.oracle.com/java/technologies/javase-downloads.html 
+
+## Deploy the BEAM Pipelines
+
+Nomulus is in the middle of migrating all pipelines to use flex-template. For
+pipelines already based on flex-template, deployment in the testing environments
+(alpha and crash) can be done using the following command:
+
+```shell
+./nom_build :core:stage_beam_pipelines --environment=alpha
+```
+
+Pipeline deployment in other environments are through CloudBuild. Please refer
+to the [release folder](http://github.com/google/nomulus/release) for details.