Add a wipeout action for Datastore in QA (#1064)

* Add a wipeout action for Datastore in QA

core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java

@@ -0,0 +1,115 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/**
+ * Wipes out all Cloud Datastore data in a Nomulus GCP environment.
+ *
+ * <p>This class is created for the QA environment, where migration testing with production data
+ * will happen. A regularly scheduled wipeout is a prerequisite to using production data there.
+ */
+@Action(
+    service = Action.Service.BACKEND,
+    path = "/_dr/task/wipeOutDatastore",
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class WipeoutDatastoreAction implements Runnable {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private static final String PIPELINE_NAME = "bulk_delete_datastore_pipeline";
+
+  // As a short-lived class, hardcode allowed projects here instead of using config files.
+  private static final ImmutableSet<String> ALLOWED_PROJECTS =
+      ImmutableSet.of("domain-registry-qa");
+
+  private final String projectId;
+  private final String jobRegion;
+  private final Response response;
+  private final Dataflow dataflow;
+  private final String stagingBucketUrl;
+
+  @Inject
+  WipeoutDatastoreAction(
+      @Config("projectId") String projectId,
+      @Config("defaultJobRegion") String jobRegion,
+      @Config("beamStagingBucketUrl") String stagingBucketUrl,
+      Response response,
+      Dataflow dataflow) {
+    this.projectId = projectId;
+    this.jobRegion = jobRegion;
+    this.stagingBucketUrl = stagingBucketUrl;
+    this.response = response;
+    this.dataflow = dataflow;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(PLAIN_TEXT_UTF_8);
+
+    if (!ALLOWED_PROJECTS.contains(projectId)) {
+      response.setStatus(SC_FORBIDDEN);
+      response.setPayload("Wipeout is not allowed in " + projectId);
+      return;
+    }
+
+    try {
+      LaunchFlexTemplateParameter parameters =
+          new LaunchFlexTemplateParameter()
+              // Job name must be unique and in [-a-z0-9].
+              .setJobName(
+                  "bulk-delete-datastore-"
+                      + DateTime.now(DateTimeZone.UTC).toString("yyyy-MM-dd'T'HH-mm-ss'Z'"))
+              .setContainerSpecGcsPath(
+                  String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
+              .setParameters(ImmutableMap.of("kindsToDelete", "*"));
+      LaunchFlexTemplateResponse launchResponse =
+          dataflow
+              .projects()
+              .locations()
+              .flexTemplates()
+              .launch(
+                  projectId,
+                  jobRegion,
+                  new LaunchFlexTemplateRequest().setLaunchParameter(parameters))
+              .execute();
+      response.setStatus(SC_OK);
+      response.setPayload("Launched " + launchResponse.getJob().getName());
+    } catch (Exception e) {
+      String msg = String.format("Failed to launch %s.", PIPELINE_NAME);
+      logger.atSevere().withCause(e).log(msg);
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(msg);
+    }
+  }
+}

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -385,6 +385,12 @@
     <url-pattern>/_dr/task/wipeOutCloudSql</url-pattern>
   </servlet-mapping>
 
+  <!-- Action to wipeout Cloud Datastore data -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/wipeOutDatastore</url-pattern>
+  </servlet-mapping>
+
   <!-- Security config -->
   <security-constraint>
     <web-resource-collection>

core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml

@@ -91,4 +91,13 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutDatastore]]></url>
+    <description>
+      This job runs an action that deletes all data in Cloud Datastore.
+    </description>
+    <schedule>every saturday 03:07</schedule>
+    <target>backend</target>
+  </cron>
+
 </cronentries>

core/src/main/java/google/registry/module/backend/BackendRequestComponent.java

@@ -31,6 +31,7 @@ import google.registry.batch.RelockDomainAction;
 import google.registry.batch.ResaveAllEppResourcesAction;
 import google.registry.batch.ResaveEntityAction;
 import google.registry.batch.WipeOutCloudSqlAction;
+import google.registry.batch.WipeoutDatastoreAction;
 import google.registry.cron.CommitLogFanoutAction;
 import google.registry.cron.CronModule;
 import google.registry.cron.TldFanoutAction;
@@ -208,6 +209,8 @@ interface BackendRequestComponent {
 
   WipeOutCloudSqlAction wipeOutCloudSqlAction();
 
+  WipeoutDatastoreAction wipeoutDatastoreAction();
+
   @Subcomponent.Builder
   abstract class Builder implements RequestComponentBuilder<BackendRequestComponent> {
 

core/src/test/java/google/registry/batch/WipeOutDatastoreActionTest.java

@@ -0,0 +1,98 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.truth.Truth.assertThat;
+import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
+import static org.apache.http.HttpStatus.SC_OK;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.model.Job;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
+import google.registry.testing.FakeResponse;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+/** Unit tests for {@link WipeoutDatastoreAction}. */
+@ExtendWith(MockitoExtension.class)
+class WipeOutDatastoreActionTest {
+
+  @Mock private Dataflow dataflow;
+  @Mock private Dataflow.Projects projects;
+  @Mock private Dataflow.Projects.Locations locations;
+  @Mock private Dataflow.Projects.Locations.FlexTemplates flexTemplates;
+  @Mock private Dataflow.Projects.Locations.FlexTemplates.Launch launch;
+  private LaunchFlexTemplateResponse launchResponse =
+      new LaunchFlexTemplateResponse().setJob(new Job());
+
+  private FakeResponse response = new FakeResponse();
+
+  @BeforeEach
+  void beforeEach() throws Exception {
+    lenient().when(dataflow.projects()).thenReturn(projects);
+    lenient().when(projects.locations()).thenReturn(locations);
+    lenient().when(locations.flexTemplates()).thenReturn(flexTemplates);
+    lenient()
+        .when(flexTemplates.launch(anyString(), anyString(), any(LaunchFlexTemplateRequest.class)))
+        .thenReturn(launch);
+    lenient().when(launch.execute()).thenReturn(launchResponse);
+  }
+
+  @Test
+  void run_projectNotAllowed() {
+    WipeoutDatastoreAction action =
+        new WipeoutDatastoreAction(
+            "domain-registry", "us-central1", "gs://some-bucket", response, dataflow);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_FORBIDDEN);
+    verifyNoInteractions(dataflow);
+  }
+
+  @Test
+  void run_projectAllowed() throws Exception {
+    WipeoutDatastoreAction action =
+        new WipeoutDatastoreAction(
+            "domain-registry-qa", "us-central1", "gs://some-bucket", response, dataflow);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_OK);
+    verify(launch, times(1)).execute();
+    verifyNoMoreInteractions(launch);
+  }
+
+  @Test
+  void run_failure() throws Exception {
+    when(launch.execute()).thenThrow(new RuntimeException());
+    WipeoutDatastoreAction action =
+        new WipeoutDatastoreAction(
+            "domain-registry-qa", "us-central1", "gs://some-bucket", response, dataflow);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);
+    verify(launch, times(1)).execute();
+    verifyNoMoreInteractions(launch);
+  }
+}

core/src/test/resources/google/registry/module/backend/backend_routing.txt

@@ -44,3 +44,4 @@ PATH                                   CLASS                              METHOD
 /_dr/task/updateSnapshotView           UpdateSnapshotViewAction           POST     n  INTERNAL,API APP ADMIN
 /_dr/task/uploadDatastoreBackup        UploadDatastoreBackupAction        POST     n  INTERNAL,API APP ADMIN
 /_dr/task/wipeOutCloudSql              WipeOutCloudSqlAction              GET      n  INTERNAL,API APP ADMIN
+/_dr/task/wipeOutDatastore             WipeoutDatastoreAction             GET      n  INTERNAL,API APP ADMIN