diff --git a/core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java b/core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java new file mode 100644 index 000000000..4db43dfb9 --- /dev/null +++ b/core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java @@ -0,0 +1,115 @@ +// Copyright 2021 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.batch; + +import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; +import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_OK; + +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter; +import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest; +import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.FluentLogger; +import google.registry.config.RegistryConfig.Config; +import google.registry.request.Action; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import javax.inject.Inject; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +/** + * Wipes out all Cloud Datastore data in a Nomulus GCP environment. + * + *

This class is created for the QA environment, where migration testing with production data + * will happen. A regularly scheduled wipeout is a prerequisite to using production data there. + */ +@Action( + service = Action.Service.BACKEND, + path = "/_dr/task/wipeOutDatastore", + auth = Auth.AUTH_INTERNAL_OR_ADMIN) +public class WipeoutDatastoreAction implements Runnable { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private static final String PIPELINE_NAME = "bulk_delete_datastore_pipeline"; + + // As a short-lived class, hardcode allowed projects here instead of using config files. + private static final ImmutableSet ALLOWED_PROJECTS = + ImmutableSet.of("domain-registry-qa"); + + private final String projectId; + private final String jobRegion; + private final Response response; + private final Dataflow dataflow; + private final String stagingBucketUrl; + + @Inject + WipeoutDatastoreAction( + @Config("projectId") String projectId, + @Config("defaultJobRegion") String jobRegion, + @Config("beamStagingBucketUrl") String stagingBucketUrl, + Response response, + Dataflow dataflow) { + this.projectId = projectId; + this.jobRegion = jobRegion; + this.stagingBucketUrl = stagingBucketUrl; + this.response = response; + this.dataflow = dataflow; + } + + @Override + public void run() { + response.setContentType(PLAIN_TEXT_UTF_8); + + if (!ALLOWED_PROJECTS.contains(projectId)) { + response.setStatus(SC_FORBIDDEN); + response.setPayload("Wipeout is not allowed in " + projectId); + return; + } + + try { + LaunchFlexTemplateParameter parameters = + new LaunchFlexTemplateParameter() + // Job name must be unique and in [-a-z0-9]. + .setJobName( + "bulk-delete-datastore-" + + DateTime.now(DateTimeZone.UTC).toString("yyyy-MM-dd'T'HH-mm-ss'Z'")) + .setContainerSpecGcsPath( + String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME)) + .setParameters(ImmutableMap.of("kindsToDelete", "*")); + LaunchFlexTemplateResponse launchResponse = + dataflow + .projects() + .locations() + .flexTemplates() + .launch( + projectId, + jobRegion, + new LaunchFlexTemplateRequest().setLaunchParameter(parameters)) + .execute(); + response.setStatus(SC_OK); + response.setPayload("Launched " + launchResponse.getJob().getName()); + } catch (Exception e) { + String msg = String.format("Failed to launch %s.", PIPELINE_NAME); + logger.atSevere().withCause(e).log(msg); + response.setStatus(SC_INTERNAL_SERVER_ERROR); + response.setPayload(msg); + } + } +} diff --git a/core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml b/core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml index 76e88279b..85ddceac8 100644 --- a/core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml +++ b/core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml @@ -385,6 +385,12 @@ /_dr/task/wipeOutCloudSql + + + backend-servlet + /_dr/task/wipeOutDatastore + + diff --git a/core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml b/core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml index 179e2d936..783af1db9 100644 --- a/core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml +++ b/core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml @@ -91,4 +91,13 @@ backend + + + + This job runs an action that deletes all data in Cloud Datastore. + + every saturday 03:07 + backend + + diff --git a/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java b/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java index ef94a30aa..756b90ae7 100644 --- a/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java +++ b/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java @@ -31,6 +31,7 @@ import google.registry.batch.RelockDomainAction; import google.registry.batch.ResaveAllEppResourcesAction; import google.registry.batch.ResaveEntityAction; import google.registry.batch.WipeOutCloudSqlAction; +import google.registry.batch.WipeoutDatastoreAction; import google.registry.cron.CommitLogFanoutAction; import google.registry.cron.CronModule; import google.registry.cron.TldFanoutAction; @@ -208,6 +209,8 @@ interface BackendRequestComponent { WipeOutCloudSqlAction wipeOutCloudSqlAction(); + WipeoutDatastoreAction wipeoutDatastoreAction(); + @Subcomponent.Builder abstract class Builder implements RequestComponentBuilder { diff --git a/core/src/test/java/google/registry/batch/WipeOutDatastoreActionTest.java b/core/src/test/java/google/registry/batch/WipeOutDatastoreActionTest.java new file mode 100644 index 000000000..3af391c2a --- /dev/null +++ b/core/src/test/java/google/registry/batch/WipeOutDatastoreActionTest.java @@ -0,0 +1,98 @@ +// Copyright 2021 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.batch; + +import static com.google.common.truth.Truth.assertThat; +import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; +import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR; +import static org.apache.http.HttpStatus.SC_OK; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.model.Job; +import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest; +import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse; +import google.registry.testing.FakeResponse; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** Unit tests for {@link WipeoutDatastoreAction}. */ +@ExtendWith(MockitoExtension.class) +class WipeOutDatastoreActionTest { + + @Mock private Dataflow dataflow; + @Mock private Dataflow.Projects projects; + @Mock private Dataflow.Projects.Locations locations; + @Mock private Dataflow.Projects.Locations.FlexTemplates flexTemplates; + @Mock private Dataflow.Projects.Locations.FlexTemplates.Launch launch; + private LaunchFlexTemplateResponse launchResponse = + new LaunchFlexTemplateResponse().setJob(new Job()); + + private FakeResponse response = new FakeResponse(); + + @BeforeEach + void beforeEach() throws Exception { + lenient().when(dataflow.projects()).thenReturn(projects); + lenient().when(projects.locations()).thenReturn(locations); + lenient().when(locations.flexTemplates()).thenReturn(flexTemplates); + lenient() + .when(flexTemplates.launch(anyString(), anyString(), any(LaunchFlexTemplateRequest.class))) + .thenReturn(launch); + lenient().when(launch.execute()).thenReturn(launchResponse); + } + + @Test + void run_projectNotAllowed() { + WipeoutDatastoreAction action = + new WipeoutDatastoreAction( + "domain-registry", "us-central1", "gs://some-bucket", response, dataflow); + action.run(); + assertThat(response.getStatus()).isEqualTo(SC_FORBIDDEN); + verifyNoInteractions(dataflow); + } + + @Test + void run_projectAllowed() throws Exception { + WipeoutDatastoreAction action = + new WipeoutDatastoreAction( + "domain-registry-qa", "us-central1", "gs://some-bucket", response, dataflow); + action.run(); + assertThat(response.getStatus()).isEqualTo(SC_OK); + verify(launch, times(1)).execute(); + verifyNoMoreInteractions(launch); + } + + @Test + void run_failure() throws Exception { + when(launch.execute()).thenThrow(new RuntimeException()); + WipeoutDatastoreAction action = + new WipeoutDatastoreAction( + "domain-registry-qa", "us-central1", "gs://some-bucket", response, dataflow); + action.run(); + assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR); + verify(launch, times(1)).execute(); + verifyNoMoreInteractions(launch); + } +} diff --git a/core/src/test/resources/google/registry/module/backend/backend_routing.txt b/core/src/test/resources/google/registry/module/backend/backend_routing.txt index abcc34c4a..bb9c4c485 100644 --- a/core/src/test/resources/google/registry/module/backend/backend_routing.txt +++ b/core/src/test/resources/google/registry/module/backend/backend_routing.txt @@ -44,3 +44,4 @@ PATH CLASS METHOD /_dr/task/updateSnapshotView UpdateSnapshotViewAction POST n INTERNAL,API APP ADMIN /_dr/task/uploadDatastoreBackup UploadDatastoreBackupAction POST n INTERNAL,API APP ADMIN /_dr/task/wipeOutCloudSql WipeOutCloudSqlAction GET n INTERNAL,API APP ADMIN +/_dr/task/wipeOutDatastore WipeoutDatastoreAction GET n INTERNAL,API APP ADMIN