diff --git a/java/google/registry/env/common/backend/WEB-INF/web.xml b/java/google/registry/env/common/backend/WEB-INF/web.xml index 0eacc37ec..3f5b1db7a 100644 --- a/java/google/registry/env/common/backend/WEB-INF/web.xml +++ b/java/google/registry/env/common/backend/WEB-INF/web.xml @@ -240,6 +240,12 @@ /_dr/task/exportDomainLists + + + backend-servlet + /_dr/task/deleteProberData + + diff --git a/java/google/registry/env/common/tools/WEB-INF/web.xml b/java/google/registry/env/common/tools/WEB-INF/web.xml index 352219e97..8aa2680dd 100644 --- a/java/google/registry/env/common/tools/WEB-INF/web.xml +++ b/java/google/registry/env/common/tools/WEB-INF/web.xml @@ -66,13 +66,6 @@ /_dr/epptool - - - - tools-servlet - /_dr/task/deleteProberData - - tools-servlet diff --git a/java/google/registry/env/production/default/WEB-INF/cron.xml b/java/google/registry/env/production/default/WEB-INF/cron.xml index df0b7da98..16803bd6d 100644 --- a/java/google/registry/env/production/default/WEB-INF/cron.xml +++ b/java/google/registry/env/production/default/WEB-INF/cron.xml @@ -187,8 +187,7 @@ every monday 14:00 UTC - - tools + backend diff --git a/java/google/registry/env/sandbox/default/WEB-INF/cron.xml b/java/google/registry/env/sandbox/default/WEB-INF/cron.xml index 9eb3a45cc..e33009f02 100644 --- a/java/google/registry/env/sandbox/default/WEB-INF/cron.xml +++ b/java/google/registry/env/sandbox/default/WEB-INF/cron.xml @@ -98,8 +98,7 @@ every monday 14:00 UTC - - tools + backend diff --git a/java/google/registry/module/backend/BUILD b/java/google/registry/module/backend/BUILD index 31c936d01..0ad9172bd 100644 --- a/java/google/registry/module/backend/BUILD +++ b/java/google/registry/module/backend/BUILD @@ -20,6 +20,7 @@ java_library( "//third_party/java/jsr330_inject", "//third_party/java/servlet/servlet_api", "//java/google/registry/backup", + "//java/google/registry/batch", "//java/google/registry/bigquery", "//java/google/registry/billing", "//java/google/registry/config", diff --git a/java/google/registry/module/backend/BackendRequestComponent.java b/java/google/registry/module/backend/BackendRequestComponent.java index e5cea3837..dea79f9fd 100644 --- a/java/google/registry/module/backend/BackendRequestComponent.java +++ b/java/google/registry/module/backend/BackendRequestComponent.java @@ -20,6 +20,7 @@ import google.registry.backup.CommitLogCheckpointAction; import google.registry.backup.DeleteOldCommitLogsAction; import google.registry.backup.ExportCommitLogDiffAction; import google.registry.backup.RestoreCommitLogsAction; +import google.registry.batch.DeleteProberDataAction; import google.registry.billing.ExpandRecurringBillingEventsAction; import google.registry.cron.CommitLogFanoutAction; import google.registry.cron.CronModule; @@ -95,6 +96,7 @@ interface BackendRequestComponent { DeleteContactsAndHostsAction deleteContactsAndHostsAction(); DeleteHostResourceAction deleteHostResourceAction(); DeleteOldCommitLogsAction deleteOldCommitLogsAction(); + DeleteProberDataAction deleteProberDataAction(); DnsRefreshForHostRenameAction dnsRefreshForHostRenameAction(); ExpandRecurringBillingEventsAction expandRecurringBillingEventsAction(); ExportCommitLogDiffAction exportCommitLogDiffAction(); diff --git a/java/google/registry/module/tools/ToolsRequestComponent.java b/java/google/registry/module/tools/ToolsRequestComponent.java index 65b3be5f2..918cb2b5a 100644 --- a/java/google/registry/module/tools/ToolsRequestComponent.java +++ b/java/google/registry/module/tools/ToolsRequestComponent.java @@ -28,7 +28,6 @@ import google.registry.request.RequestScope; import google.registry.tools.server.CreateGroupsAction; import google.registry.tools.server.CreatePremiumListAction; import google.registry.tools.server.DeleteEntityAction; -import google.registry.tools.server.DeleteProberDataAction; import google.registry.tools.server.GenerateZoneFilesAction; import google.registry.tools.server.KillAllCommitLogsAction; import google.registry.tools.server.KillAllEppResourcesAction; @@ -63,7 +62,6 @@ interface ToolsRequestComponent { CreateGroupsAction createGroupsAction(); CreatePremiumListAction createPremiumListAction(); DeleteEntityAction deleteEntityAction(); - DeleteProberDataAction deleteProberDataAction(); EppToolAction eppToolAction(); FlowComponent.Builder flowComponentBuilder(); GenerateZoneFilesAction generateZoneFilesAction(); diff --git a/java/google/registry/tools/server/DeleteProberDataAction.java b/java/google/registry/tools/server/DeleteProberDataAction.java deleted file mode 100644 index 8767c0e2f..000000000 --- a/java/google/registry/tools/server/DeleteProberDataAction.java +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2016 The Domain Registry Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools.server; - -import static com.google.common.base.Verify.verifyNotNull; -import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN; -import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.model.registry.Registries.getTldsOfType; -import static google.registry.request.Action.Method.POST; - -import com.google.appengine.tools.mapreduce.Mapper; -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.base.Splitter; -import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.googlecode.objectify.Key; -import com.googlecode.objectify.Work; -import google.registry.mapreduce.MapreduceRunner; -import google.registry.mapreduce.inputs.EppResourceInputs; -import google.registry.model.domain.DomainApplication; -import google.registry.model.domain.DomainBase; -import google.registry.model.index.EppResourceIndex; -import google.registry.model.index.ForeignKeyIndex; -import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex; -import google.registry.model.registry.Registry; -import google.registry.model.registry.Registry.TldType; -import google.registry.request.Action; -import google.registry.request.Parameter; -import google.registry.request.Response; -import google.registry.util.FormattingLogger; -import google.registry.util.PipelineUtils; -import java.util.List; -import javax.inject.Inject; - -/** - * Deletes all prober DomainResources and their subordinate history entries, poll messages, and - * billing events, along with their ForeignKeyDomainIndex and EppResourceIndex entities. - * - *

See: https://www.youtube.com/watch?v=xuuv0syoHnM - */ -@Action(path = "/_dr/task/deleteProberData", method = POST) -public class DeleteProberDataAction implements Runnable { - - private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); - - @Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun; - @Inject MapreduceRunner mrRunner; - @Inject Response response; - @Inject DeleteProberDataAction() {} - - @Override - public void run() { - response.sendJavaScriptRedirect(PipelineUtils.createJobPath(mrRunner - .setJobName("Delete prober data") - // TODO(b/27309488): maybe move this to the backend module. - .setModuleName("tools") - .runMapOnly( - new DeleteProberDataMapper(getProberRoidSuffixes(), isDryRun), - ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class))))); - } - - private static ImmutableSet getProberRoidSuffixes() { - return FluentIterable.from(getTldsOfType(TldType.TEST)) - .filter(new Predicate() { - @Override - public boolean apply(String tld) { - // Extra sanity check to prevent us from nuking prod data if a real TLD accidentally - // gets set to type TEST. - return tld.endsWith(".test"); - }}) - .transform( - new Function() { - @Override - public String apply(String tld) { - return Registry.get(tld).getRoidSuffix(); - }}) - .toSet(); - } - - /** Provides the map method that runs for each existing DomainBase entity. */ - public static class DeleteProberDataMapper extends Mapper, Void, Void> { - - private static final long serialVersionUID = 1737761271804180412L; - - private final ImmutableSet proberRoidSuffixes; - private final Boolean isDryRun; - - public DeleteProberDataMapper(ImmutableSet proberRoidSuffixes, Boolean isDryRun) { - this.proberRoidSuffixes = proberRoidSuffixes; - this.isDryRun = isDryRun; - } - - @Override - public final void map(Key key) { - try { - String roidSuffix = Iterables.getLast(Splitter.on('-').split(key.getName())); - if (proberRoidSuffixes.contains(roidSuffix)) { - deleteDomain(key); - } else { - getContext().incrementCounter(String.format("skipped, non-prober data")); - } - } catch (Throwable t) { - logger.severefmt(t, "Error while deleting prober data for key %s", key); - getContext().incrementCounter(String.format("error, kind %s", key.getKind())); - } - } - - private void deleteDomain(final Key domainKey) { - final DomainBase domain = ofy().load().key(domainKey).now(); - if (domain == null) { - // Depending on how stale Datastore indexes are, we can get keys to resources that are - // already deleted (e.g. by a recent previous invocation of this mapreduce). So ignore them. - getContext().incrementCounter("already deleted"); - return; - } - if (domain instanceof DomainApplication) { - // Cover the case where we somehow have a domain application with a prober ROID suffix. - getContext().incrementCounter("skipped, domain application"); - return; - } - if (domain.getFullyQualifiedDomainName().equals("nic." + domain.getTld())) { - getContext().incrementCounter("skipped, NIC domain"); - return; - } - int dependentsDeleted = ofy().transact(new Work() { - @Override - public Integer run() { - EppResourceIndex eppIndex = ofy().load().entity(EppResourceIndex.create(domainKey)).now(); - verifyNotNull(eppIndex, "Missing EppResourceIndex for domain %s", domain); - ForeignKeyIndex fki = ofy().load().key(ForeignKeyDomainIndex.createKey(domain)).now(); - verifyNotNull(fki, "Missing ForeignKeyDomainIndex for domain %s", domain); - // This ancestor query selects all descendant HistoryEntries, BillingEvents, and - // PollMessages, as well as the domain itself. - List> domainAndDependentKeys = ofy().load().ancestor(domainKey).keys().list(); - if (isDryRun) { - logger.infofmt( - "Would delete the following entities: %s", - new ImmutableList.Builder() - .add(fki) - .add(eppIndex) - .addAll(domainAndDependentKeys) - .build()); - } else { - ofy().deleteWithoutBackup().keys(domainAndDependentKeys); - ofy().deleteWithoutBackup().entities(eppIndex, fki); - } - return domainAndDependentKeys.size() - 1; - } - }); - getContext().incrementCounter(String.format("deleted, kind %s", domainKey.getKind())); - getContext().incrementCounter("deleted, dependent keys", dependentsDeleted); - } - } -} diff --git a/javatests/google/registry/tools/server/DeleteProberDataActionTest.java b/javatests/google/registry/tools/server/DeleteProberDataActionTest.java deleted file mode 100644 index 6150d0af5..000000000 --- a/javatests/google/registry/tools/server/DeleteProberDataActionTest.java +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2016 The Domain Registry Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools.server; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.testing.DatastoreHelper.createTld; -import static google.registry.testing.DatastoreHelper.persistActiveDomain; -import static google.registry.testing.DatastoreHelper.persistDeletedDomain; -import static google.registry.testing.DatastoreHelper.persistResource; -import static google.registry.testing.DatastoreHelper.persistSimpleResource; -import static google.registry.util.DateTimeUtils.START_OF_TIME; - -import com.google.common.base.Optional; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.mapreduce.MapreduceRunner; -import google.registry.model.ImmutableObject; -import google.registry.model.billing.BillingEvent; -import google.registry.model.billing.BillingEvent.Reason; -import google.registry.model.domain.DomainResource; -import google.registry.model.index.EppResourceIndex; -import google.registry.model.index.ForeignKeyIndex; -import google.registry.model.poll.PollMessage; -import google.registry.model.registry.Registry; -import google.registry.model.registry.Registry.TldType; -import google.registry.model.reporting.HistoryEntry; -import google.registry.testing.ExceptionRule; -import google.registry.testing.FakeResponse; -import google.registry.testing.mapreduce.MapreduceTestCase; -import java.util.Set; -import org.joda.money.Money; -import org.joda.time.DateTime; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link DeleteProberDataAction}. */ -@RunWith(JUnit4.class) -public class DeleteProberDataActionTest extends MapreduceTestCase { - - private static final DateTime DELETION_TIME = DateTime.parse("2010-01-01T00:00:00.000Z"); - - @Rule - public final ExceptionRule thrown = new ExceptionRule(); - - @Before - public void init() { - // Entities in these two should not be touched. - createTld("tld", "TLD"); - // Since "example" doesn't end with .test, its entities won't be deleted even though it is of - // TEST type. - createTld("example", "EXAMPLE"); - persistResource(Registry.get("example").asBuilder().setTldType(TldType.TEST).build()); - - // Entities in these two should be deleted. - createTld("ib-any.test", "IBANYT"); - persistResource(Registry.get("ib-any.test").asBuilder().setTldType(TldType.TEST).build()); - createTld("oa-canary.test", "OACANT"); - persistResource(Registry.get("oa-canary.test").asBuilder().setTldType(TldType.TEST).build()); - - action = new DeleteProberDataAction(); - action.mrRunner = new MapreduceRunner(Optional.of(5), Optional.absent()); - action.response = new FakeResponse(); - action.isDryRun = false; - } - - private void runMapreduce() throws Exception { - action.run(); - executeTasksUntilEmpty("mapreduce"); - } - - @Test - public void test_deletesAllAndOnlyProberData() throws Exception { - Set tldEntities = persistLotsOfDomains("tld"); - Set exampleEntities = persistLotsOfDomains("example"); - Set ibEntities = persistLotsOfDomains("ib-any.test"); - Set oaEntities = persistLotsOfDomains("oa-canary.test"); - runMapreduce(); - assertNotDeleted(tldEntities); - assertNotDeleted(exampleEntities); - assertDeleted(ibEntities); - assertDeleted(oaEntities); - } - - @Test - public void testSuccess_doesntDeleteNicDomainForProbers() throws Exception { - DomainResource nic = persistActiveDomain("nic.ib-any.test"); - ForeignKeyIndex fkiNic = - ForeignKeyIndex.load(DomainResource.class, "nic.ib-any.test", START_OF_TIME); - Set ibEntities = persistLotsOfDomains("ib-any.test"); - runMapreduce(); - assertDeleted(ibEntities); - assertNotDeleted(ImmutableSet.of(nic, fkiNic)); - } - - @Test - public void testSuccess_dryRunDoesntDeleteData() throws Exception { - Set tldEntities = persistLotsOfDomains("tld"); - Set oaEntities = persistLotsOfDomains("oa-canary.test"); - action.isDryRun = true; - assertNotDeleted(tldEntities); - assertNotDeleted(oaEntities); - } - - /** - * Persists and returns a domain and a descendant history entry, billing event, and poll message, - * along with the ForeignKeyIndex and EppResourceIndex. - */ - private static Set persistDomainAndDescendants(String fqdn) { - DomainResource domain = persistDeletedDomain(fqdn, DELETION_TIME); - HistoryEntry historyEntry = persistSimpleResource( - new HistoryEntry.Builder() - .setParent(domain) - .setType(HistoryEntry.Type.DOMAIN_CREATE) - .build()); - BillingEvent.OneTime billingEvent = persistSimpleResource( - new BillingEvent.OneTime.Builder() - .setParent(historyEntry) - .setBillingTime(DELETION_TIME.plusYears(1)) - .setCost(Money.parse("USD 10")) - .setPeriodYears(1) - .setReason(Reason.CREATE) - .setClientId("TheRegistrar") - .setEventTime(DELETION_TIME) - .setTargetId(fqdn) - .build()); - PollMessage.OneTime pollMessage = persistSimpleResource( - new PollMessage.OneTime.Builder() - .setParent(historyEntry) - .setEventTime(DELETION_TIME) - .setClientId("TheRegistrar") - .setMsg("Domain registered") - .build()); - ForeignKeyIndex fki = - ForeignKeyIndex.load(DomainResource.class, fqdn, START_OF_TIME); - EppResourceIndex eppIndex = - ofy().load().entity(EppResourceIndex.create(Key.create(domain))).now(); - return ImmutableSet.of( - domain, historyEntry, billingEvent, pollMessage, fki, eppIndex); - } - - private static Set persistLotsOfDomains(String tld) { - ImmutableSet.Builder persistedObjects = new ImmutableSet.Builder<>(); - for (int i = 0; i < 20; i++) { - persistedObjects.addAll(persistDomainAndDescendants(String.format("domain%d.%s", i, tld))); - } - return persistedObjects.build(); - } - - private static void assertNotDeleted(Iterable entities) { - for (ImmutableObject entity : entities) { - assertThat(ofy().load().entity(entity).now()).isNotNull(); - } - } - - private static void assertDeleted(Iterable entities) { - for (ImmutableObject entity : entities) { - assertThat(ofy().load().entity(entity).now()).isNull(); - } - } -}