From 577c6f6bc9cf87fe9d85a700a2b0c1066b862e41 Mon Sep 17 00:00:00 2001 From: mcilwain Date: Thu, 3 Jan 2019 13:20:39 -0800 Subject: [PATCH] Add a mapreduce to delete all domain applications This also deletes associated entities including indexes and history entries. This needs to run as a prerequisite to [] which deletes all domain application code entirely. The entities themselves need to be deleted first so that loading DomainBases in the future doesn't accidentally get applications which the code no longer knows how to handle. This deletion is safe to perform because the only remaining applications in our system are historical and we no longer refer to them. Backups will be retained in BigQuery. This mapreduce will be deleted at the same time that the DomainApplication code is. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=227738528 --- .../registry/env/common/tools/WEB-INF/web.xml | 6 ++ .../module/tools/ToolsRequestComponent.java | 2 + .../KillAllDomainApplicationsAction.java | 97 +++++++++++++++++++ .../module/tools/testdata/tools_routing.txt | 43 ++++---- .../KillAllDomainApplicationsActionTest.java | 94 ++++++++++++++++++ 5 files changed, 221 insertions(+), 21 deletions(-) create mode 100644 java/google/registry/tools/server/KillAllDomainApplicationsAction.java create mode 100644 javatests/google/registry/tools/server/KillAllDomainApplicationsActionTest.java diff --git a/java/google/registry/env/common/tools/WEB-INF/web.xml b/java/google/registry/env/common/tools/WEB-INF/web.xml index eb7dde515..0bd9341d1 100644 --- a/java/google/registry/env/common/tools/WEB-INF/web.xml +++ b/java/google/registry/env/common/tools/WEB-INF/web.xml @@ -72,6 +72,12 @@ /_dr/task/resaveAllHistoryEntries + + + tools-servlet + /_dr/task/killAllDomainApplications + + tools-servlet diff --git a/java/google/registry/module/tools/ToolsRequestComponent.java b/java/google/registry/module/tools/ToolsRequestComponent.java index 633a38709..e8e6879c5 100644 --- a/java/google/registry/module/tools/ToolsRequestComponent.java +++ b/java/google/registry/module/tools/ToolsRequestComponent.java @@ -34,6 +34,7 @@ import google.registry.tools.server.CreatePremiumListAction; import google.registry.tools.server.DeleteEntityAction; import google.registry.tools.server.GenerateZoneFilesAction; import google.registry.tools.server.KillAllCommitLogsAction; +import google.registry.tools.server.KillAllDomainApplicationsAction; import google.registry.tools.server.KillAllEppResourcesAction; import google.registry.tools.server.ListDomainsAction; import google.registry.tools.server.ListHostsAction; @@ -69,6 +70,7 @@ interface ToolsRequestComponent { FlowComponent.Builder flowComponentBuilder(); GenerateZoneFilesAction generateZoneFilesAction(); KillAllCommitLogsAction killAllCommitLogsAction(); + KillAllDomainApplicationsAction killAllDomainApplicationsAction(); KillAllEppResourcesAction killAllEppResourcesAction(); ListDomainsAction listDomainsAction(); ListHostsAction listHostsAction(); diff --git a/java/google/registry/tools/server/KillAllDomainApplicationsAction.java b/java/google/registry/tools/server/KillAllDomainApplicationsAction.java new file mode 100644 index 000000000..8b639ba09 --- /dev/null +++ b/java/google/registry/tools/server/KillAllDomainApplicationsAction.java @@ -0,0 +1,97 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.server; + +import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.request.Action.Method.POST; +import static google.registry.util.PipelineUtils.createJobPath; + +import com.google.appengine.tools.mapreduce.Mapper; +import com.google.common.collect.ImmutableList; +import com.google.common.flogger.FluentLogger; +import com.googlecode.objectify.Key; +import google.registry.mapreduce.MapreduceRunner; +import google.registry.model.domain.DomainApplication; +import google.registry.model.index.DomainApplicationIndex; +import google.registry.model.index.EppResourceIndex; +import google.registry.model.reporting.HistoryEntry; +import google.registry.request.Action; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import javax.inject.Inject; + +/** + * Deletes all {@link DomainApplication} entities in Datastore. + * + *

This also deletes the corresponding {@link DomainApplicationIndex}, {@link EppResourceIndex}, + * and descendent {@link HistoryEntry}s. + */ +@Action(path = "/_dr/task/killAllDomainApplications", method = POST, auth = Auth.AUTH_INTERNAL_ONLY) +public class KillAllDomainApplicationsAction implements Runnable { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + @Inject MapreduceRunner mrRunner; + @Inject Response response; + + @Inject + KillAllDomainApplicationsAction() {} + + @Override + public void run() { + response.sendJavaScriptRedirect( + createJobPath( + mrRunner + .setJobName("Delete all domain applications and associated entities") + .setModuleName("tools") + .runMapOnly( + new KillAllDomainApplicationsMapper(), + ImmutableList.of(createEntityInput(DomainApplication.class))))); + } + + static class KillAllDomainApplicationsMapper extends Mapper { + + private static final long serialVersionUID = 2862967335000340688L; + + @Override + public void map(final DomainApplication application) { + ofy() + .transact( + () -> { + if (ofy().load().entity(application).now() == null) { + getContext().incrementCounter("applications already deleted"); + return; + } + Key applicationKey = Key.create(application); + DomainApplicationIndex dai = + ofy().load().key(DomainApplicationIndex.createKey(application)).now(); + EppResourceIndex eri = + ofy().load().entity(EppResourceIndex.create(applicationKey)).now(); + if (dai == null || eri == null) { + logger.atSevere().log( + "Missing index(es) for application %s; skipping.", applicationKey); + getContext().incrementCounter("missing indexes"); + return; + } + // Delete the application, its descendents, and the indexes. + ofy().delete().keys(ofy().load().ancestor(application).keys()); + ofy().delete().entities(dai, eri); + logger.atInfo().log("Deleted domain application %s.", applicationKey); + getContext().incrementCounter("applications deleted"); + }); + } + } +} diff --git a/javatests/google/registry/module/tools/testdata/tools_routing.txt b/javatests/google/registry/module/tools/testdata/tools_routing.txt index 40756e6f4..2d953941c 100644 --- a/javatests/google/registry/module/tools/testdata/tools_routing.txt +++ b/javatests/google/registry/module/tools/testdata/tools_routing.txt @@ -1,21 +1,22 @@ -PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY -/_dr/admin/createGroups CreateGroupsAction POST n INTERNAL,API APP ADMIN -/_dr/admin/createPremiumList CreatePremiumListAction POST n INTERNAL,API APP ADMIN -/_dr/admin/deleteEntity DeleteEntityAction GET n INTERNAL,API APP ADMIN -/_dr/admin/list/domains ListDomainsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/list/hosts ListHostsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/list/premiumLists ListPremiumListsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/list/registrars ListRegistrarsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/list/reservedLists ListReservedListsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/list/tlds ListTldsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/updatePremiumList UpdatePremiumListAction POST n INTERNAL,API APP ADMIN -/_dr/admin/verifyOte VerifyOteAction POST n INTERNAL,API APP ADMIN -/_dr/epptool EppToolAction POST n INTERNAL,API APP ADMIN -/_dr/loadtest LoadTestAction POST y INTERNAL,API APP ADMIN -/_dr/task/generateZoneFiles GenerateZoneFilesAction POST n INTERNAL,API APP ADMIN -/_dr/task/killAllCommitLogs KillAllCommitLogsAction POST n INTERNAL APP IGNORED -/_dr/task/killAllEppResources KillAllEppResourcesAction POST n INTERNAL APP IGNORED -/_dr/task/pollMapreduce PollMapreduceAction POST n INTERNAL APP IGNORED -/_dr/task/refreshDnsForAllDomains RefreshDnsForAllDomainsAction GET n INTERNAL,API APP ADMIN -/_dr/task/resaveAllHistoryEntries ResaveAllHistoryEntriesAction GET n INTERNAL,API APP ADMIN -/_dr/task/restoreCommitLogs RestoreCommitLogsAction POST y INTERNAL,API APP ADMIN +PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY +/_dr/admin/createGroups CreateGroupsAction POST n INTERNAL,API APP ADMIN +/_dr/admin/createPremiumList CreatePremiumListAction POST n INTERNAL,API APP ADMIN +/_dr/admin/deleteEntity DeleteEntityAction GET n INTERNAL,API APP ADMIN +/_dr/admin/list/domains ListDomainsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/list/hosts ListHostsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/list/premiumLists ListPremiumListsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/list/registrars ListRegistrarsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/list/reservedLists ListReservedListsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/list/tlds ListTldsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/updatePremiumList UpdatePremiumListAction POST n INTERNAL,API APP ADMIN +/_dr/admin/verifyOte VerifyOteAction POST n INTERNAL,API APP ADMIN +/_dr/epptool EppToolAction POST n INTERNAL,API APP ADMIN +/_dr/loadtest LoadTestAction POST y INTERNAL,API APP ADMIN +/_dr/task/generateZoneFiles GenerateZoneFilesAction POST n INTERNAL,API APP ADMIN +/_dr/task/killAllCommitLogs KillAllCommitLogsAction POST n INTERNAL APP IGNORED +/_dr/task/killAllDomainApplications KillAllDomainApplicationsAction POST n INTERNAL APP IGNORED +/_dr/task/killAllEppResources KillAllEppResourcesAction POST n INTERNAL APP IGNORED +/_dr/task/pollMapreduce PollMapreduceAction POST n INTERNAL APP IGNORED +/_dr/task/refreshDnsForAllDomains RefreshDnsForAllDomainsAction GET n INTERNAL,API APP ADMIN +/_dr/task/resaveAllHistoryEntries ResaveAllHistoryEntriesAction GET n INTERNAL,API APP ADMIN +/_dr/task/restoreCommitLogs RestoreCommitLogsAction POST y INTERNAL,API APP ADMIN diff --git a/javatests/google/registry/tools/server/KillAllDomainApplicationsActionTest.java b/javatests/google/registry/tools/server/KillAllDomainApplicationsActionTest.java new file mode 100644 index 000000000..102df8adf --- /dev/null +++ b/javatests/google/registry/tools/server/KillAllDomainApplicationsActionTest.java @@ -0,0 +1,94 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.server; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.testing.DatastoreHelper.createTlds; +import static google.registry.testing.DatastoreHelper.persistActiveContact; +import static google.registry.testing.DatastoreHelper.persistActiveDomain; +import static google.registry.testing.DatastoreHelper.persistActiveDomainApplication; +import static google.registry.testing.DatastoreHelper.persistActiveHost; +import static google.registry.testing.DatastoreHelper.persistResource; + +import com.googlecode.objectify.Key; +import google.registry.model.contact.ContactResource; +import google.registry.model.domain.DomainApplication; +import google.registry.model.domain.DomainResource; +import google.registry.model.host.HostResource; +import google.registry.model.index.DomainApplicationIndex; +import google.registry.model.index.EppResourceIndex; +import google.registry.model.index.ForeignKeyIndex; +import google.registry.model.reporting.HistoryEntry; +import google.registry.testing.FakeResponse; +import google.registry.testing.mapreduce.MapreduceTestCase; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link KillAllDomainApplicationsAction}. */ +@RunWith(JUnit4.class) +public class KillAllDomainApplicationsActionTest + extends MapreduceTestCase { + + private void runMapreduce() throws Exception { + action = new KillAllDomainApplicationsAction(); + action.mrRunner = makeDefaultRunner(); + action.response = new FakeResponse(); + action.run(); + executeTasksUntilEmpty("mapreduce"); + } + + @Test + public void test_deletesOnlyApplicationsAndAssociatedEntities() throws Exception { + createTlds("tld1", "tld2"); + + DomainResource domain = persistActiveDomain("foo1.tld1"); + EppResourceIndex domainEri = + ofy().load().entity(EppResourceIndex.create(Key.create(domain))).now(); + ForeignKeyIndex domainFki = + ofy().load().key(ForeignKeyIndex.createKey(domain)).now(); + HistoryEntry domainHistoryEntry = + persistResource(new HistoryEntry.Builder().setParent(domain).build()); + + DomainApplication application = persistActiveDomainApplication("foo2.tld1"); + EppResourceIndex applicationEri = + ofy().load().entity(EppResourceIndex.create(Key.create(application))).now(); + DomainApplicationIndex applicationDai = + ofy().load().key(DomainApplicationIndex.createKey(application)).now(); + HistoryEntry applicationHistoryEntry = + persistResource(new HistoryEntry.Builder().setParent(application).build()); + + ContactResource contact = persistActiveContact("foo"); + HostResource host = persistActiveHost("ns.foo.tld1"); + + runMapreduce(); + ofy().clearSessionCache(); + + // Check that none of the domain, contact, and host entities were deleted. + assertThat( + ofy() + .load() + .entities(domain, domainEri, domainFki, domainHistoryEntry, contact, host) + .values()) + .containsExactly(domain, domainEri, domainFki, domainHistoryEntry, contact, host); + // Check that all of the domain application entities were deleted. + assertThat( + ofy() + .load() + .entities(application, applicationEri, applicationDai, applicationHistoryEntry)) + .isEmpty(); + } +}