Add SQL wipeout action in QA (#1035)

* Add SQL wipeout action in QA

Added the WipeOutSqlAction that deletes all data in Cloud SQL.

Wipe out is restricted to the QA environment, which will get production
data during migration testing.

Also added a cron job that invokes wipeout on every saturday morning.
This is part of the privacy requirments for using production data in QA.

Tested in QA.

core/src/main/java/google/registry/batch/WipeOutCloudSqlAction.java

@@ -0,0 +1,98 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.persistence.PersistenceModule.SchemaManagerConnection;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.Retrier;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.function.Supplier;
+import javax.inject.Inject;
+import org.flywaydb.core.api.FlywayException;
+
+/**
+ * Wipes out all Cloud SQL data in a Nomulus GCP environment.
+ *
+ * <p>This class is created for the QA environment, where migration testing with production data
+ * will happen. A regularly scheduled wipeout is a prerequisite to using production data there.
+ */
+@Action(
+    service = Action.Service.BACKEND,
+    path = "/_dr/task/wipeOutCloudSql",
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class WipeOutCloudSqlAction implements Runnable {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  // As a short-lived class, hardcode allowed projects here instead of using config files.
+  private static final ImmutableSet<String> ALLOWED_PROJECTS =
+      ImmutableSet.of("domain-registry-qa");
+
+  private final String projectId;
+  private final Supplier<Connection> connectionSupplier;
+  private final Response response;
+  private final Retrier retrier;
+
+  @Inject
+  WipeOutCloudSqlAction(
+      @Config("projectId") String projectId,
+      @SchemaManagerConnection Supplier<Connection> connectionSupplier,
+      Response response,
+      Retrier retrier) {
+    this.projectId = projectId;
+    this.connectionSupplier = connectionSupplier;
+    this.response = response;
+    this.retrier = retrier;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(PLAIN_TEXT_UTF_8);
+
+    if (!ALLOWED_PROJECTS.contains(projectId)) {
+      response.setStatus(SC_FORBIDDEN);
+      response.setPayload("Wipeout is not allowed in " + projectId);
+      return;
+    }
+
+    try {
+      retrier.callWithRetry(
+          () -> {
+            try (Connection conn = connectionSupplier.get();
+                Statement statement = conn.createStatement()) {
+              statement.execute("drop owned by schema_deployer;");
+            }
+            return null;
+          },
+          e -> !(e instanceof FlywayException));
+      response.setStatus(SC_OK);
+      response.setPayload("Wiped out Cloud SQL in " + projectId);
+    } catch (RuntimeException e) {
+      logger.atSevere().withCause(e).log("Failed to wipe out Cloud SQL data.");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload("Failed to wipe out Cloud SQL in " + projectId);
+    }
+  }
+}

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -379,6 +379,12 @@
     <url-pattern>/_dr/task/relockDomain</url-pattern>
   </servlet-mapping>
 
+  <!-- Action to wipeout Cloud SQL data -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/wipeOutCloudSql</url-pattern>
+  </servlet-mapping>
+
   <!-- Security config -->
   <security-constraint>
     <web-resource-collection>

core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml

@@ -82,4 +82,13 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutCloudSql]]></url>
+    <description>
+      This job runs an action that deletes all data in Cloud SQL.
+    </description>
+    <schedule>every saturday 03:07</schedule>
+    <target>backend</target>
+  </cron>
+
 </cronentries>

core/src/main/java/google/registry/module/backend/BackendComponent.java

@@ -36,6 +36,8 @@ import google.registry.keyring.api.KeyModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.module.backend.BackendRequestComponent.BackendRequestComponentModule;
 import google.registry.monitoring.whitebox.StackdriverModule;
+import google.registry.persistence.PersistenceModule;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.rde.JSchModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
@@ -71,6 +73,8 @@ import javax.inject.Singleton;
       KeyringModule.class,
       KmsModule.class,
       NetHttpTransportModule.class,
+      PersistenceModule.class,
+      SecretManagerModule.class,
       ServerTridProviderModule.class,
       SheetsServiceModule.class,
       StackdriverModule.class,

core/src/main/java/google/registry/module/backend/BackendRequestComponent.java

@@ -30,6 +30,7 @@ import google.registry.batch.RefreshDnsOnHostRenameAction;
 import google.registry.batch.RelockDomainAction;
 import google.registry.batch.ResaveAllEppResourcesAction;
 import google.registry.batch.ResaveEntityAction;
+import google.registry.batch.WipeOutCloudSqlAction;
 import google.registry.cron.CommitLogFanoutAction;
 import google.registry.cron.CronModule;
 import google.registry.cron.TldFanoutAction;
@@ -205,6 +206,8 @@ interface BackendRequestComponent {
 
   PublishInvoicesAction uploadInvoicesAction();
 
+  WipeOutCloudSqlAction wipeOutCloudSqlAction();
+
   @Subcomponent.Builder
   abstract class Builder implements RequestComponentBuilder<BackendRequestComponent> {
 

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -44,10 +44,14 @@ import google.registry.tools.AuthModule.CloudSqlClientCredential;
 import google.registry.util.Clock;
 import java.lang.annotation.Documented;
 import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Properties;
+import java.util.function.Supplier;
 import javax.annotation.Nullable;
 import javax.inject.Provider;
 import javax.inject.Qualifier;
@@ -163,6 +167,37 @@ public abstract class PersistenceModule {
     return ImmutableMap.copyOf(overrides);
   }
 
+  /**
+   * Provides a {@link Supplier} of single-use JDBC {@link Connection connections} that can manage
+   * the database DDL schema.
+   */
+  @Provides
+  @Singleton
+  @SchemaManagerConnection
+  static Supplier<Connection> provideSchemaManagerConnectionSupplier(
+      SqlCredentialStore credentialStore,
+      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs) {
+    SqlCredential credential =
+        credentialStore.getCredential(new RobotUser(RobotId.SCHEMA_DEPLOYER));
+    String user = credential.login();
+    String password = credential.password();
+    return () -> createJdbcConnection(user, password, cloudSqlConfigs);
+  }
+
+  private static Connection createJdbcConnection(
+      String user, String password, ImmutableMap<String, String> cloudSqlConfigs) {
+    Properties properties = new Properties();
+    properties.put("user", user);
+    properties.put("password", password);
+    properties.put("cloudSqlInstance", cloudSqlConfigs.get(HIKARI_DS_CLOUD_SQL_INSTANCE));
+    properties.put("socketFactory", cloudSqlConfigs.get(HIKARI_DS_SOCKET_FACTORY));
+    try {
+      return DriverManager.getConnection(cloudSqlConfigs.get(Environment.URL), properties);
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   @Provides
   @Singleton
   @AppEngineJpaTm
@@ -378,6 +413,11 @@ public abstract class PersistenceModule {
     }
   }
 
+  /** Dagger qualifier for JDBC {@link Connection} with schema management privilege. */
+  @Qualifier
+  @Documented
+  public @interface SchemaManagerConnection {}
+
   /** Dagger qualifier for {@link JpaTransactionManager} used for App Engine application. */
   @Qualifier
   @Documented

core/src/test/java/google/registry/batch/WipeOutCloudSqlActionTest.java

@@ -0,0 +1,101 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.truth.Truth.assertThat;
+import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+import google.registry.testing.FakeClock;
+import google.registry.testing.FakeResponse;
+import google.registry.testing.FakeSleeper;
+import google.registry.util.Retrier;
+import java.sql.Connection;
+import java.sql.Statement;
+import org.flywaydb.core.api.FlywayException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+/** Unit tests for {@link WipeOutCloudSqlAction}. */
+@ExtendWith(MockitoExtension.class)
+public class WipeOutCloudSqlActionTest {
+
+  @Mock private Statement stmt;
+  @Mock private Connection conn;
+
+  private FakeResponse response = new FakeResponse();
+  private Retrier retrier = new Retrier(new FakeSleeper(new FakeClock()), 2);
+
+  @BeforeEach
+  void beforeEach() throws Exception {
+    lenient().when(conn.createStatement()).thenReturn(stmt);
+  }
+
+  @Test
+  void run_projectAllowed() throws Exception {
+    WipeOutCloudSqlAction action =
+        new WipeOutCloudSqlAction("domain-registry-qa", () -> conn, response, retrier);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_OK);
+    verify(stmt, times(1)).execute(anyString());
+    verify(stmt, times(1)).close();
+    verifyNoMoreInteractions(stmt);
+  }
+
+  @Test
+  void run_projectNotAllowed() {
+    WipeOutCloudSqlAction action =
+        new WipeOutCloudSqlAction("domain-registry", () -> conn, response, retrier);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_FORBIDDEN);
+    verifyNoInteractions(stmt);
+  }
+
+  @Test
+  void run_nonRetrieableFailure() throws Exception {
+    doThrow(new FlywayException()).when(stmt).execute(anyString());
+    WipeOutCloudSqlAction action =
+        new WipeOutCloudSqlAction("domain-registry-qa", () -> conn, response, retrier);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);
+    verify(stmt, times(1)).execute(anyString());
+    verify(stmt, times(1)).close();
+    verifyNoMoreInteractions(stmt);
+  }
+
+  @Test
+  void run_retrieableFailure() throws Exception {
+    when(stmt.execute(anyString())).thenThrow(new RuntimeException()).thenReturn(true);
+    WipeOutCloudSqlAction action =
+        new WipeOutCloudSqlAction("domain-registry-qa", () -> conn, response, retrier);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_OK);
+    verify(stmt, times(2)).execute(anyString());
+    verify(stmt, times(2)).close();
+    verifyNoMoreInteractions(stmt);
+  }
+}

core/src/test/resources/google/registry/module/backend/backend_routing.txt

@@ -43,3 +43,4 @@ PATH                                   CLASS                              METHOD
 /_dr/task/updateRegistrarRdapBaseUrls  UpdateRegistrarRdapBaseUrlsAction  GET      y  INTERNAL,API APP ADMIN
 /_dr/task/updateSnapshotView           UpdateSnapshotViewAction           POST     n  INTERNAL,API APP ADMIN
 /_dr/task/uploadDatastoreBackup        UploadDatastoreBackupAction        POST     n  INTERNAL,API APP ADMIN
+/_dr/task/wipeOutCloudSql              WipeOutCloudSqlAction              GET      n  INTERNAL,API APP ADMIN