From b52e289037a72f3d19c3d967e4f47e4b97e22da4 Mon Sep 17 00:00:00 2001 From: Weimin Yu Date: Mon, 11 Oct 2021 17:31:50 -0400 Subject: [PATCH] Update parameter to Datastore wipe pipeline (#1385) * Update parameter to Datastore wipe pipeline Add the newly required RegistryEnvironment parameter to BulkDeleteDatastorePipeline. Remove the nullable annotation for this parameter in options class. Update metadata files regarding this parameter. --- .../google/registry/batch/WipeoutDatastoreAction.java | 7 ++++++- .../registry/beam/common/RegistryPipelineOptions.java | 1 - .../beam/datastore/BulkDeleteDatastorePipeline.java | 6 ++++++ .../beam/bulk_delete_datastore_pipeline_metadata.json | 9 +++++++++ .../google/registry/beam/init_sql_pipeline_metadata.json | 6 +++--- .../registry/beam/invoicing_pipeline_metadata.json | 2 +- .../google/registry/beam/rde_pipeline_metadata.json | 2 +- .../google/registry/beam/spec11_pipeline_metadata.json | 2 +- 8 files changed, 27 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java b/core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java index 6d89f00fa..599922ee8 100644 --- a/core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java +++ b/core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java @@ -92,7 +92,12 @@ public class WipeoutDatastoreAction implements Runnable { .setJobName(createJobName("bulk-delete-datastore-", clock)) .setContainerSpecGcsPath( String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME)) - .setParameters(ImmutableMap.of("kindsToDelete", "*")); + .setParameters( + ImmutableMap.of( + "kindsToDelete", + "*", + "registryEnvironment", + RegistryEnvironment.get().name())); LaunchFlexTemplateResponse launchResponse = dataflow .projects() diff --git a/core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java b/core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java index cda1e33d6..60be2ff6c 100644 --- a/core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java +++ b/core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java @@ -34,7 +34,6 @@ import org.apache.beam.sdk.options.Description; public interface RegistryPipelineOptions extends GcpOptions { @Description("The Registry environment.") - @Nullable RegistryEnvironment getRegistryEnvironment(); void setRegistryEnvironment(RegistryEnvironment environment); diff --git a/core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java b/core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java index e6500e13e..d7e63178a 100644 --- a/core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java +++ b/core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; import com.google.common.flogger.FluentLogger; import com.google.datastore.v1.Entity; +import google.registry.config.RegistryEnvironment; import java.util.Iterator; import java.util.Map; import org.apache.beam.sdk.Pipeline; @@ -308,6 +309,11 @@ public class BulkDeleteDatastorePipeline { public interface BulkDeletePipelineOptions extends GcpOptions { + @Description("The Registry environment.") + RegistryEnvironment getRegistryEnvironment(); + + void setRegistryEnvironment(RegistryEnvironment environment); + @Description( "The Datastore KINDs to be deleted. The format may be:\n" + "\t- The list of kinds to be deleted as a comma-separated string, or\n" diff --git a/core/src/main/resources/google/registry/beam/bulk_delete_datastore_pipeline_metadata.json b/core/src/main/resources/google/registry/beam/bulk_delete_datastore_pipeline_metadata.json index 5ea1d534f..2f2cfb38e 100644 --- a/core/src/main/resources/google/registry/beam/bulk_delete_datastore_pipeline_metadata.json +++ b/core/src/main/resources/google/registry/beam/bulk_delete_datastore_pipeline_metadata.json @@ -2,6 +2,15 @@ "name": "Bulk Delete Cloud Datastore", "description": "An Apache Beam batch pipeline that deletes Cloud Datastore in bulk. This is easier to use than the GCP-provided template.", "parameters": [ + { + "name": "registryEnvironment", + "label": "The Registry environment.", + "helpText": "The Registry environment, required only because the worker initializer demands it.", + "is_optional": false, + "regexes": [ + "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$" + ] + }, { "name": "kindsToDelete", "label": "The data KINDs to delete.", diff --git a/core/src/main/resources/google/registry/beam/init_sql_pipeline_metadata.json b/core/src/main/resources/google/registry/beam/init_sql_pipeline_metadata.json index d3c251b50..64dfc5284 100644 --- a/core/src/main/resources/google/registry/beam/init_sql_pipeline_metadata.json +++ b/core/src/main/resources/google/registry/beam/init_sql_pipeline_metadata.json @@ -5,10 +5,10 @@ { "name": "registryEnvironment", "label": "The Registry environment.", - "helpText": "The Registry environment, required if environment-specific initialization (such as JPA) is needed on worker VMs.", - "is_optional": true, + "helpText": "The Registry environment.", + "is_optional": false, "regexes": [ - "^[0-9A-Z_]+$" + "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$" ] }, { diff --git a/core/src/main/resources/google/registry/beam/invoicing_pipeline_metadata.json b/core/src/main/resources/google/registry/beam/invoicing_pipeline_metadata.json index d531e6bba..ecab86358 100644 --- a/core/src/main/resources/google/registry/beam/invoicing_pipeline_metadata.json +++ b/core/src/main/resources/google/registry/beam/invoicing_pipeline_metadata.json @@ -5,7 +5,7 @@ { "name": "registryEnvironment", "label": "The Registry environment.", - "helpText": "The Registry environment, required if environment-specific initialization (such as JPA) is needed on worker VMs.", + "helpText": "The Registry environment.", "is_optional": false, "regexes": [ "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$" diff --git a/core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json b/core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json index 6128150a0..3b39c5725 100644 --- a/core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json +++ b/core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json @@ -5,7 +5,7 @@ { "name": "registryEnvironment", "label": "The Registry environment.", - "helpText": "The Registry environment, required if environment-specific initialization (such as JPA) is needed on worker VMs.", + "helpText": "The Registry environment.", "is_optional": false, "regexes": [ "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$" diff --git a/core/src/main/resources/google/registry/beam/spec11_pipeline_metadata.json b/core/src/main/resources/google/registry/beam/spec11_pipeline_metadata.json index 4e48a5eb9..4311cc412 100644 --- a/core/src/main/resources/google/registry/beam/spec11_pipeline_metadata.json +++ b/core/src/main/resources/google/registry/beam/spec11_pipeline_metadata.json @@ -5,7 +5,7 @@ { "name": "registryEnvironment", "label": "The Registry environment.", - "helpText": "The Registry environment, required if environment-specific initialization (such as JPA) is needed on worker VMs.", + "helpText": "The Registry environment.", "is_optional": false, "regexes": [ "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"