Add a mapreduce to delete all domain applications

This also deletes associated entities including indexes and history entries.

This needs to run as a prerequisite to [] which deletes all domain
application code entirely. The entities themselves need to be deleted first so
that loading DomainBases in the future doesn't accidentally get applications
which the code no longer knows how to handle.

This deletion is safe to perform because the only remaining applications in our
system are historical and we no longer refer to them. Backups will be retained
in BigQuery.

This mapreduce will be deleted at the same time that the DomainApplication code
is.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=227738528

java/google/registry/env/common/tools/WEB-INF/web.xml

@@ -72,6 +72,12 @@
     <url-pattern>/_dr/task/resaveAllHistoryEntries</url-pattern>
   </servlet-mapping>
 
+  <!--  Mapreduce to delete all domain applications. -->
+  <servlet-mapping>
+    <servlet-name>tools-servlet</servlet-name>
+    <url-pattern>/_dr/task/killAllDomainApplications</url-pattern>
+  </servlet-mapping>
+
   <!-- Mapreduce to delete EppResources, children, and indices. -->
   <servlet-mapping>
     <servlet-name>tools-servlet</servlet-name>

java/google/registry/module/tools/ToolsRequestComponent.java

@@ -34,6 +34,7 @@ import google.registry.tools.server.CreatePremiumListAction;
 import google.registry.tools.server.DeleteEntityAction;
 import google.registry.tools.server.GenerateZoneFilesAction;
 import google.registry.tools.server.KillAllCommitLogsAction;
+import google.registry.tools.server.KillAllDomainApplicationsAction;
 import google.registry.tools.server.KillAllEppResourcesAction;
 import google.registry.tools.server.ListDomainsAction;
 import google.registry.tools.server.ListHostsAction;
@@ -69,6 +70,7 @@ interface ToolsRequestComponent {
   FlowComponent.Builder flowComponentBuilder();
   GenerateZoneFilesAction generateZoneFilesAction();
   KillAllCommitLogsAction killAllCommitLogsAction();
+  KillAllDomainApplicationsAction killAllDomainApplicationsAction();
   KillAllEppResourcesAction killAllEppResourcesAction();
   ListDomainsAction listDomainsAction();
   ListHostsAction listHostsAction();

java/google/registry/tools/server/KillAllDomainApplicationsAction.java

@@ -0,0 +1,97 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.server;
+
+import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.request.Action.Method.POST;
+import static google.registry.util.PipelineUtils.createJobPath;
+
+import com.google.appengine.tools.mapreduce.Mapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.flogger.FluentLogger;
+import com.googlecode.objectify.Key;
+import google.registry.mapreduce.MapreduceRunner;
+import google.registry.model.domain.DomainApplication;
+import google.registry.model.index.DomainApplicationIndex;
+import google.registry.model.index.EppResourceIndex;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import javax.inject.Inject;
+
+/**
+ * Deletes all {@link DomainApplication} entities in Datastore.
+ *
+ * <p>This also deletes the corresponding {@link DomainApplicationIndex}, {@link EppResourceIndex},
+ * and descendent {@link HistoryEntry}s.
+ */
+@Action(path = "/_dr/task/killAllDomainApplications", method = POST, auth = Auth.AUTH_INTERNAL_ONLY)
+public class KillAllDomainApplicationsAction implements Runnable {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  @Inject MapreduceRunner mrRunner;
+  @Inject Response response;
+
+  @Inject
+  KillAllDomainApplicationsAction() {}
+
+  @Override
+  public void run() {
+    response.sendJavaScriptRedirect(
+        createJobPath(
+            mrRunner
+                .setJobName("Delete all domain applications and associated entities")
+                .setModuleName("tools")
+                .runMapOnly(
+                    new KillAllDomainApplicationsMapper(),
+                    ImmutableList.of(createEntityInput(DomainApplication.class)))));
+  }
+
+  static class KillAllDomainApplicationsMapper extends Mapper<DomainApplication, Void, Void> {
+
+    private static final long serialVersionUID = 2862967335000340688L;
+
+    @Override
+    public void map(final DomainApplication application) {
+      ofy()
+          .transact(
+              () -> {
+                if (ofy().load().entity(application).now() == null) {
+                  getContext().incrementCounter("applications already deleted");
+                  return;
+                }
+                Key<DomainApplication> applicationKey = Key.create(application);
+                DomainApplicationIndex dai =
+                    ofy().load().key(DomainApplicationIndex.createKey(application)).now();
+                EppResourceIndex eri =
+                    ofy().load().entity(EppResourceIndex.create(applicationKey)).now();
+                if (dai == null || eri == null) {
+                  logger.atSevere().log(
+                      "Missing index(es) for application %s; skipping.", applicationKey);
+                  getContext().incrementCounter("missing indexes");
+                  return;
+                }
+                // Delete the application, its descendents, and the indexes.
+                ofy().delete().keys(ofy().load().ancestor(application).keys());
+                ofy().delete().entities(dai, eri);
+                logger.atInfo().log("Deleted domain application %s.", applicationKey);
+                getContext().incrementCounter("applications deleted");
+              });
+    }
+  }
+}

javatests/google/registry/module/tools/testdata/tools_routing.txt

@@ -1,21 +1,22 @@
-PATH                              CLASS                         METHODS  OK AUTH_METHODS MIN USER_POLICY
+PATH                                CLASS                           METHODS  OK AUTH_METHODS MIN USER_POLICY
-/_dr/admin/createGroups           CreateGroupsAction            POST     n  INTERNAL,API APP ADMIN
+/_dr/admin/createGroups             CreateGroupsAction              POST     n  INTERNAL,API APP ADMIN
-/_dr/admin/createPremiumList      CreatePremiumListAction       POST     n  INTERNAL,API APP ADMIN
+/_dr/admin/createPremiumList        CreatePremiumListAction         POST     n  INTERNAL,API APP ADMIN
-/_dr/admin/deleteEntity           DeleteEntityAction            GET      n  INTERNAL,API APP ADMIN
+/_dr/admin/deleteEntity             DeleteEntityAction              GET      n  INTERNAL,API APP ADMIN
-/_dr/admin/list/domains           ListDomainsAction             GET,POST n  INTERNAL,API APP ADMIN
+/_dr/admin/list/domains             ListDomainsAction               GET,POST n  INTERNAL,API APP ADMIN
-/_dr/admin/list/hosts             ListHostsAction               GET,POST n  INTERNAL,API APP ADMIN
+/_dr/admin/list/hosts               ListHostsAction                 GET,POST n  INTERNAL,API APP ADMIN
-/_dr/admin/list/premiumLists      ListPremiumListsAction        GET,POST n  INTERNAL,API APP ADMIN
+/_dr/admin/list/premiumLists        ListPremiumListsAction          GET,POST n  INTERNAL,API APP ADMIN
-/_dr/admin/list/registrars        ListRegistrarsAction          GET,POST n  INTERNAL,API APP ADMIN
+/_dr/admin/list/registrars          ListRegistrarsAction            GET,POST n  INTERNAL,API APP ADMIN
-/_dr/admin/list/reservedLists     ListReservedListsAction       GET,POST n  INTERNAL,API APP ADMIN
+/_dr/admin/list/reservedLists       ListReservedListsAction         GET,POST n  INTERNAL,API APP ADMIN
-/_dr/admin/list/tlds              ListTldsAction                GET,POST n  INTERNAL,API APP ADMIN
+/_dr/admin/list/tlds                ListTldsAction                  GET,POST n  INTERNAL,API APP ADMIN
-/_dr/admin/updatePremiumList      UpdatePremiumListAction       POST     n  INTERNAL,API APP ADMIN
+/_dr/admin/updatePremiumList        UpdatePremiumListAction         POST     n  INTERNAL,API APP ADMIN
-/_dr/admin/verifyOte              VerifyOteAction               POST     n  INTERNAL,API APP ADMIN
+/_dr/admin/verifyOte                VerifyOteAction                 POST     n  INTERNAL,API APP ADMIN
-/_dr/epptool                      EppToolAction                 POST     n  INTERNAL,API APP ADMIN
+/_dr/epptool                        EppToolAction                   POST     n  INTERNAL,API APP ADMIN
-/_dr/loadtest                     LoadTestAction                POST     y  INTERNAL,API APP ADMIN
+/_dr/loadtest                       LoadTestAction                  POST     y  INTERNAL,API APP ADMIN
-/_dr/task/generateZoneFiles       GenerateZoneFilesAction       POST     n  INTERNAL,API APP ADMIN
+/_dr/task/generateZoneFiles         GenerateZoneFilesAction         POST     n  INTERNAL,API APP ADMIN
-/_dr/task/killAllCommitLogs       KillAllCommitLogsAction       POST     n  INTERNAL     APP IGNORED
+/_dr/task/killAllCommitLogs         KillAllCommitLogsAction         POST     n  INTERNAL     APP IGNORED
-/_dr/task/killAllEppResources     KillAllEppResourcesAction     POST     n  INTERNAL     APP IGNORED
+/_dr/task/killAllDomainApplications KillAllDomainApplicationsAction POST     n  INTERNAL     APP IGNORED
-/_dr/task/pollMapreduce           PollMapreduceAction           POST     n  INTERNAL     APP IGNORED
+/_dr/task/killAllEppResources       KillAllEppResourcesAction       POST     n  INTERNAL     APP IGNORED
-/_dr/task/refreshDnsForAllDomains RefreshDnsForAllDomainsAction GET      n  INTERNAL,API APP ADMIN
+/_dr/task/pollMapreduce             PollMapreduceAction             POST     n  INTERNAL     APP IGNORED
-/_dr/task/resaveAllHistoryEntries ResaveAllHistoryEntriesAction GET      n  INTERNAL,API APP ADMIN
+/_dr/task/refreshDnsForAllDomains   RefreshDnsForAllDomainsAction   GET      n  INTERNAL,API APP ADMIN
-/_dr/task/restoreCommitLogs       RestoreCommitLogsAction       POST     y  INTERNAL,API APP ADMIN
+/_dr/task/resaveAllHistoryEntries   ResaveAllHistoryEntriesAction   GET      n  INTERNAL,API APP ADMIN
+/_dr/task/restoreCommitLogs         RestoreCommitLogsAction         POST     y  INTERNAL,API APP ADMIN

javatests/google/registry/tools/server/KillAllDomainApplicationsActionTest.java

@@ -0,0 +1,94 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.server;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.testing.DatastoreHelper.createTlds;
+import static google.registry.testing.DatastoreHelper.persistActiveContact;
+import static google.registry.testing.DatastoreHelper.persistActiveDomain;
+import static google.registry.testing.DatastoreHelper.persistActiveDomainApplication;
+import static google.registry.testing.DatastoreHelper.persistActiveHost;
+import static google.registry.testing.DatastoreHelper.persistResource;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainApplication;
+import google.registry.model.domain.DomainResource;
+import google.registry.model.host.HostResource;
+import google.registry.model.index.DomainApplicationIndex;
+import google.registry.model.index.EppResourceIndex;
+import google.registry.model.index.ForeignKeyIndex;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.testing.FakeResponse;
+import google.registry.testing.mapreduce.MapreduceTestCase;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link KillAllDomainApplicationsAction}. */
+@RunWith(JUnit4.class)
+public class KillAllDomainApplicationsActionTest
+    extends MapreduceTestCase<KillAllDomainApplicationsAction> {
+
+  private void runMapreduce() throws Exception {
+    action = new KillAllDomainApplicationsAction();
+    action.mrRunner = makeDefaultRunner();
+    action.response = new FakeResponse();
+    action.run();
+    executeTasksUntilEmpty("mapreduce");
+  }
+
+  @Test
+  public void test_deletesOnlyApplicationsAndAssociatedEntities() throws Exception {
+    createTlds("tld1", "tld2");
+
+    DomainResource domain = persistActiveDomain("foo1.tld1");
+    EppResourceIndex domainEri =
+        ofy().load().entity(EppResourceIndex.create(Key.create(domain))).now();
+    ForeignKeyIndex<DomainResource> domainFki =
+        ofy().load().key(ForeignKeyIndex.createKey(domain)).now();
+    HistoryEntry domainHistoryEntry =
+        persistResource(new HistoryEntry.Builder().setParent(domain).build());
+
+    DomainApplication application = persistActiveDomainApplication("foo2.tld1");
+    EppResourceIndex applicationEri =
+        ofy().load().entity(EppResourceIndex.create(Key.create(application))).now();
+    DomainApplicationIndex applicationDai =
+        ofy().load().key(DomainApplicationIndex.createKey(application)).now();
+    HistoryEntry applicationHistoryEntry =
+        persistResource(new HistoryEntry.Builder().setParent(application).build());
+
+    ContactResource contact = persistActiveContact("foo");
+    HostResource host = persistActiveHost("ns.foo.tld1");
+
+    runMapreduce();
+    ofy().clearSessionCache();
+
+    // Check that none of the domain, contact, and host entities were deleted.
+    assertThat(
+            ofy()
+                .load()
+                .entities(domain, domainEri, domainFki, domainHistoryEntry, contact, host)
+                .values())
+        .containsExactly(domain, domainEri, domainFki, domainHistoryEntry, contact, host);
+    // Check that all of the domain application entities were deleted.
+    assertThat(
+            ofy()
+                .load()
+                .entities(application, applicationEri, applicationDai, applicationHistoryEntry))
+        .isEmpty();
+  }
+}