diff --git a/java/com/google/domain/registry/env/common/tools/WEB-INF/web.xml b/java/com/google/domain/registry/env/common/tools/WEB-INF/web.xml index 0ec59669f..37adbb9b0 100644 --- a/java/com/google/domain/registry/env/common/tools/WEB-INF/web.xml +++ b/java/com/google/domain/registry/env/common/tools/WEB-INF/web.xml @@ -112,6 +112,12 @@ /_dr/task/killAllCommitLogs + + + tools-servlet + /_dr/task/killAllCrossTld + + mapreduce diff --git a/java/com/google/domain/registry/module/tools/ToolsRequestComponent.java b/java/com/google/domain/registry/module/tools/ToolsRequestComponent.java index 3f2f57a99..94ad919ef 100644 --- a/java/com/google/domain/registry/module/tools/ToolsRequestComponent.java +++ b/java/com/google/domain/registry/module/tools/ToolsRequestComponent.java @@ -26,6 +26,7 @@ import com.google.domain.registry.tools.server.DeleteEntityAction; import com.google.domain.registry.tools.server.DeleteProberDataAction; import com.google.domain.registry.tools.server.GenerateZoneFilesAction; import com.google.domain.registry.tools.server.KillAllCommitLogsAction; +import com.google.domain.registry.tools.server.KillAllCrossTldEntitiesAction; import com.google.domain.registry.tools.server.KillAllEppResourcesAction; import com.google.domain.registry.tools.server.ListDomainsAction; import com.google.domain.registry.tools.server.ListHostsAction; @@ -60,6 +61,7 @@ interface ToolsRequestComponent { DeleteProberDataAction deleteProberDataAction(); GenerateZoneFilesAction generateZoneFilesAction(); KillAllCommitLogsAction killAllCommitLogsAction(); + KillAllCrossTldEntitiesAction killAllCrossTldEntitiesAction(); KillAllEppResourcesAction killAllEppResourcesAction(); ListDomainsAction listDomainsAction(); ListHostsAction listHostsAction(); diff --git a/java/com/google/domain/registry/tools/server/KillAllCommitLogsAction.java b/java/com/google/domain/registry/tools/server/KillAllCommitLogsAction.java index a3cdde0b1..7976103bf 100644 --- a/java/com/google/domain/registry/tools/server/KillAllCommitLogsAction.java +++ b/java/com/google/domain/registry/tools/server/KillAllCommitLogsAction.java @@ -15,16 +15,15 @@ package com.google.domain.registry.tools.server; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.Lists.partition; import static com.google.domain.registry.model.ofy.ObjectifyService.ofy; import static com.google.domain.registry.request.Action.Method.POST; import static com.google.domain.registry.util.PipelineUtils.createJobPath; -import com.google.appengine.tools.mapreduce.Input; import com.google.appengine.tools.mapreduce.Mapper; import com.google.appengine.tools.mapreduce.inputs.InMemoryInput; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import com.google.domain.registry.config.RegistryEnvironment; import com.google.domain.registry.mapreduce.MapreduceAction; import com.google.domain.registry.mapreduce.MapreduceRunner; @@ -48,27 +47,27 @@ public class KillAllCommitLogsAction implements MapreduceAction { @Inject KillAllCommitLogsAction() {} @Override - public void run() { + public final void run() { checkArgument( // safety RegistryEnvironment.get() == RegistryEnvironment.CRASH || RegistryEnvironment.get() == RegistryEnvironment.UNITTEST, "DO NOT RUN ANYWHERE ELSE EXCEPT CRASH OR TESTS."); - // Create a in-memory input, assigning each bucket to its own shard for maximum parallelization, - // with one extra shard for the CommitLogCheckpointRoot. - Input> input = new InMemoryInput<>( - Lists.partition( - FluentIterable - .from(Arrays.>asList(CommitLogCheckpointRoot.getKey())) - .append(CommitLogBucket.getAllBucketKeys()) - .toList(), - 1)); response.sendJavaScriptRedirect(createJobPath(mrRunner - .setJobName("Delete all commit logs") + .setJobName("Delete all commit logs and checkpoints") .setModuleName("tools") .runMapreduce( new KillAllCommitLogsMapper(), new KillAllEntitiesReducer(), - ImmutableList.of(input)))); + // Create a in-memory input, assigning each bucket to its own shard for maximum + // parallelization, with one extra shard for the CommitLogCheckpointRoot. + ImmutableList.of( + new InMemoryInput<>( + partition( + FluentIterable + .from(Arrays.>asList(CommitLogCheckpointRoot.getKey())) + .append(CommitLogBucket.getAllBucketKeys()) + .toList(), + 1)))))); } /** diff --git a/java/com/google/domain/registry/tools/server/KillAllCrossTldEntitiesAction.java b/java/com/google/domain/registry/tools/server/KillAllCrossTldEntitiesAction.java new file mode 100644 index 000000000..9fcb5b28e --- /dev/null +++ b/java/com/google/domain/registry/tools/server/KillAllCrossTldEntitiesAction.java @@ -0,0 +1,105 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.domain.registry.tools.server; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.domain.registry.model.common.EntityGroupRoot.getCrossTldKey; +import static com.google.domain.registry.model.ofy.ObjectifyService.ofy; +import static com.google.domain.registry.request.Action.Method.POST; +import static com.google.domain.registry.util.PipelineUtils.createJobPath; + +import com.google.appengine.tools.mapreduce.Mapper; +import com.google.common.collect.ImmutableList; +import com.google.domain.registry.config.RegistryEnvironment; +import com.google.domain.registry.mapreduce.MapreduceAction; +import com.google.domain.registry.mapreduce.MapreduceRunner; +import com.google.domain.registry.mapreduce.inputs.NullInput; +import com.google.domain.registry.model.common.EntityGroupRoot; +import com.google.domain.registry.request.Action; +import com.google.domain.registry.request.Response; + +import com.googlecode.objectify.Key; + +import javax.inject.Inject; + +/** + * Deletes all cross tld entities in datastore. + * + *

This doesn't really need to be a mapreduce, but doing so makes it consistent with the other + * kill-all actions, and gives us retries, a dashboard and counters for free. + */ +@Action(path = "/_dr/task/killAllCrossTld", method = POST) +public class KillAllCrossTldEntitiesAction implements MapreduceAction { + + @Inject MapreduceRunner mrRunner; + @Inject Response response; + @Inject KillAllCrossTldEntitiesAction() {} + + @Override + public final void run() { + checkArgument( // safety + RegistryEnvironment.get() == RegistryEnvironment.CRASH + || RegistryEnvironment.get() == RegistryEnvironment.UNITTEST, + "DO NOT RUN ANYWHERE ELSE EXCEPT CRASH OR TESTS."); + response.sendJavaScriptRedirect(createJobPath(mrRunner + .setJobName("Delete all cross-tld entities") + .setModuleName("tools") + .runMapreduce( + new KillAllCrossTldEntitiesMapper(), + new KillAllEntitiesReducer(), + ImmutableList.of(new NullInput())))); + } + + /** + * Mapper to delete all descendants of {@link EntityGroupRoot#getCrossTldKey()}. + * + *

This will delete: + *

    + * {@code ClaimsListShard} + * {@code ClaimsListSingleton} + * {@link EntityGroupRoot} + * {@code LogsExportCursor} + * {@code PremiumList} + * {@code PremiumListEntry} + * {@code Registrar} + * {@code RegistrarBillingEntry} + * {@code RegistrarContact} + * {@code RegistrarCredit} + * {@code RegistrarCreditBalance} + * {@code Registry} + * {@code RegistryCursor} + * {@code ReservedList} + * {@code ServerSecret} + * {@code SignedMarkRevocationList} + * {@code TmchCrl} + *
+ */ + static class KillAllCrossTldEntitiesMapper extends Mapper, Key> { + + private static final long serialVersionUID = 8343696167876596542L; + + @Override + public void map(Object ignored) { + // There will be exactly one input to the mapper, and we ignore it. + Key crossTldKey = getCrossTldKey(); + for (Key key : ofy().load().ancestor(crossTldKey).keys()) { + emit(crossTldKey, key); + getContext().incrementCounter("entities emitted"); + getContext().incrementCounter(String.format("%s emitted", key.getKind())); + } + } + } +} + diff --git a/java/com/google/domain/registry/tools/server/KillAllEppResourcesAction.java b/java/com/google/domain/registry/tools/server/KillAllEppResourcesAction.java index 7dd053e49..5da35cc0c 100644 --- a/java/com/google/domain/registry/tools/server/KillAllEppResourcesAction.java +++ b/java/com/google/domain/registry/tools/server/KillAllEppResourcesAction.java @@ -47,7 +47,7 @@ public class KillAllEppResourcesAction implements MapreduceAction { @Inject KillAllEppResourcesAction() {} @Override - public void run() { + public final void run() { checkArgument( // safety RegistryEnvironment.get() == RegistryEnvironment.CRASH || RegistryEnvironment.get() == RegistryEnvironment.UNITTEST, @@ -61,25 +61,26 @@ public class KillAllEppResourcesAction implements MapreduceAction { ImmutableList.of(EppResourceInputs.createIndexInput())))); } + /** + * Mapper to delete an {@link EppResourceIndex}, its referent, all descendants of each referent, + * and the {@link ForeignKeyIndex} or {@link DomainApplicationIndex} of the referent, as + * appropriate. + * + *

This will delete: + *

    + *
  • All {@link ForeignKeyIndex} types + *
  • {@link DomainApplicationIndex} + *
  • {@link EppResourceIndex} + *
  • All {@link EppResource} types + *
  • {@code HistoryEntry} + *
  • All {@code BillingEvent} types + *
  • All {@code PollMessage} types + *
+ */ static class KillAllEppResourcesMapper extends Mapper, Key> { private static final long serialVersionUID = 8205309000002507407L; - /** - * Delete an {@link EppResourceIndex}, its referent, all descendants of each referent, and the - * {@link ForeignKeyIndex} or {@link DomainApplicationIndex} of the referent, as appropriate. - * - *

This will delete: - *

    - *
  • All {@link ForeignKeyIndex} types - *
  • {@link DomainApplicationIndex} - *
  • {@link EppResourceIndex} - *
  • All {@link EppResource} types - *
  • {@code HistoryEntry} - *
  • All {@code BillingEvent} types - *
  • All {@code PollMessage} types - *
- */ @Override public void map(final EppResourceIndex eri) { Key eriKey = Key.create(eri); diff --git a/javatests/com/google/domain/registry/tools/server/KillAllActionTestCase.java b/javatests/com/google/domain/registry/tools/server/KillAllActionTestCase.java new file mode 100644 index 000000000..214252167 --- /dev/null +++ b/javatests/com/google/domain/registry/tools/server/KillAllActionTestCase.java @@ -0,0 +1,240 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.domain.registry.tools.server; + +import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService; +import static com.google.common.base.Predicates.in; +import static com.google.common.base.Predicates.instanceOf; +import static com.google.common.base.Predicates.not; +import static com.google.common.collect.Multimaps.filterKeys; +import static com.google.common.truth.Truth.assertThat; +import static com.google.domain.registry.model.ofy.ObjectifyService.ofy; +import static com.google.domain.registry.model.server.ServerSecret.getServerSecret; +import static com.google.domain.registry.testing.DatastoreHelper.createTld; +import static com.google.domain.registry.testing.DatastoreHelper.newContactResource; +import static com.google.domain.registry.testing.DatastoreHelper.newDomainApplication; +import static com.google.domain.registry.testing.DatastoreHelper.newDomainResource; +import static com.google.domain.registry.testing.DatastoreHelper.newHostResource; +import static com.google.domain.registry.testing.DatastoreHelper.persistPremiumList; +import static com.google.domain.registry.testing.DatastoreHelper.persistReservedList; +import static com.google.domain.registry.testing.DatastoreHelper.persistResource; +import static com.google.domain.registry.testing.DatastoreHelper.persistResourceWithCommitLog; +import static com.google.domain.registry.util.DateTimeUtils.END_OF_TIME; +import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME; +import static java.util.Arrays.asList; +import static org.joda.money.CurrencyUnit.USD; + +import com.google.appengine.api.datastore.Entity; +import com.google.common.base.Predicate; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.domain.registry.mapreduce.MapreduceAction; +import com.google.domain.registry.model.EntityClasses; +import com.google.domain.registry.model.EppResource; +import com.google.domain.registry.model.ImmutableObject; +import com.google.domain.registry.model.annotations.VirtualEntity; +import com.google.domain.registry.model.billing.BillingEvent; +import com.google.domain.registry.model.billing.BillingEvent.Reason; +import com.google.domain.registry.model.billing.RegistrarBillingEntry; +import com.google.domain.registry.model.billing.RegistrarCredit; +import com.google.domain.registry.model.billing.RegistrarCredit.CreditType; +import com.google.domain.registry.model.billing.RegistrarCreditBalance; +import com.google.domain.registry.model.common.EntityGroupRoot; +import com.google.domain.registry.model.common.GaeUserIdConverter; +import com.google.domain.registry.model.export.LogsExportCursor; +import com.google.domain.registry.model.ofy.CommitLogCheckpoint; +import com.google.domain.registry.model.ofy.CommitLogCheckpointRoot; +import com.google.domain.registry.model.poll.PollMessage; +import com.google.domain.registry.model.rde.RdeMode; +import com.google.domain.registry.model.rde.RdeRevision; +import com.google.domain.registry.model.registrar.Registrar; +import com.google.domain.registry.model.registry.Registry; +import com.google.domain.registry.model.registry.RegistryCursor; +import com.google.domain.registry.model.registry.RegistryCursor.CursorType; +import com.google.domain.registry.model.reporting.HistoryEntry; +import com.google.domain.registry.model.server.Lock; +import com.google.domain.registry.model.smd.SignedMarkRevocationList; +import com.google.domain.registry.model.tmch.ClaimsListShard; +import com.google.domain.registry.model.tmch.TmchCrl; +import com.google.domain.registry.testing.mapreduce.MapreduceTestCase; + +import com.googlecode.objectify.Key; +import com.googlecode.objectify.Ref; +import com.googlecode.objectify.VoidWork; + +import org.joda.money.Money; +import org.junit.Test; + +public abstract class KillAllActionTestCase extends MapreduceTestCase{ + + private final ImmutableSet affectedKinds; + + private static final ImmutableSet ALL_PERSISTED_KINDS = FluentIterable + .from(EntityClasses.ALL_CLASSES) + .filter( + new Predicate>() { + @Override + public boolean apply(Class clazz) { + return !clazz.isAnnotationPresent(VirtualEntity.class); + }}) + .transform(EntityClasses.CLASS_TO_KIND_FUNCTION) + .toSet(); + + + public KillAllActionTestCase(ImmutableSet affectedKinds) { + this.affectedKinds = affectedKinds; + } + + /** Create at least one of each type of entity in the schema. */ + void createData() { + createTld("tld1"); + createTld("tld2"); + persistResource(CommitLogCheckpointRoot.create(START_OF_TIME.plusDays(1))); + persistResource( + CommitLogCheckpoint.create( + START_OF_TIME.plusDays(1), + ImmutableMap.of(1, START_OF_TIME.plusDays(2)))); + for (EppResource resource : asList( + newDomainResource("foo.tld1"), + newDomainResource("foo.tld2"), + newDomainApplication("foo.tld1"), + newDomainApplication("foo.tld2"), + newContactResource("foo1"), + newContactResource("foo2"), + newHostResource("ns.foo.tld1"), + newHostResource("ns.foo.tld2"))) { + persistResourceWithCommitLog(resource); + HistoryEntry history = + persistResource(new HistoryEntry.Builder().setParent(resource).build()); + BillingEvent.OneTime oneTime = persistResource(new BillingEvent.OneTime.Builder() + .setParent(history) + .setBillingTime(START_OF_TIME) + .setEventTime(START_OF_TIME) + .setClientId("") + .setTargetId("") + .setReason(Reason.ERROR) + .setCost(Money.of(USD, 1)) + .build()); + for (ImmutableObject descendant : asList( + new PollMessage.OneTime.Builder() + .setParent(history) + .setClientId("") + .setEventTime(START_OF_TIME) + .build(), + new PollMessage.Autorenew.Builder() + .setParent(history) + .setClientId("") + .setEventTime(START_OF_TIME) + .build(), + new BillingEvent.Cancellation.Builder() + .setOneTimeEventRef(Ref.create(oneTime)) + .setParent(history) + .setBillingTime(START_OF_TIME) + .setEventTime(START_OF_TIME) + .setClientId("") + .setTargetId("") + .setReason(Reason.ERROR) + .build(), + new BillingEvent.Modification.Builder() + .setEventRef(Ref.create(oneTime)) + .setParent(history) + .setEventTime(START_OF_TIME) + .setClientId("") + .setTargetId("") + .setReason(Reason.ERROR) + .setCost(Money.of(USD, 1)) + .build(), + new BillingEvent.Recurring.Builder() + .setParent(history) + .setEventTime(START_OF_TIME) + .setClientId("") + .setTargetId("") + .setReason(Reason.ERROR) + .build())) { + persistResource(descendant); + } + } + persistResource(new RegistrarBillingEntry.Builder() + .setParent(Registrar.loadByClientId("TheRegistrar")) + .setCreated(START_OF_TIME) + .setDescription("description") + .setAmount(Money.of(USD, 1)) + .build()); + RegistrarCredit credit = persistResource(new RegistrarCredit.Builder() + .setParent(Registrar.loadByClientId("TheRegistrar")) + .setCreationTime(START_OF_TIME) + .setCurrency(USD) + .setDescription("description") + .setTld("tld1") + .setType(CreditType.AUCTION) + .build()); + persistResource(new RegistrarCreditBalance.Builder() + .setParent(credit) + .setAmount(Money.of(USD, 1)) + .setEffectiveTime(START_OF_TIME) + .setWrittenTime(START_OF_TIME) + .build()); + persistPremiumList("premium", "a,USD 100", "b,USD 200"); + persistReservedList("reserved", "lol,RESERVED_FOR_ANCHOR_TENANT,foobar1"); + getServerSecret(); // Forces persist. + TmchCrl.set("crl content"); + persistResource(new LogsExportCursor.Builder().build()); + persistPremiumList("premium", "a,USD 100", "b,USD 200"); + SignedMarkRevocationList.create( + START_OF_TIME, ImmutableMap.of("a", START_OF_TIME, "b", END_OF_TIME)).save(); + ClaimsListShard.create(START_OF_TIME, ImmutableMap.of("a", "1", "b", "2")).save(); + // These entities must be saved within a transaction. + ofy().transact(new VoidWork() { + @Override + public void vrun() { + RegistryCursor.save(Registry.get("tld1"), CursorType.BRDA, START_OF_TIME); + RdeRevision.saveRevision("tld1", START_OF_TIME, RdeMode.FULL, 0); + }}); + // These entities intentionally don't expose constructors or factory methods. + getDatastoreService().put(new Entity(EntityGroupRoot.getCrossTldKey().getRaw())); + getDatastoreService().put(new Entity(Key.getKind(GaeUserIdConverter.class), 1)); + getDatastoreService().put(new Entity(Key.getKind(Lock.class), 1)); + } + + abstract MapreduceAction createAction(); + + @Test + public void testKill() throws Exception { + createData(); + ImmutableMultimap beforeContents = getDatastoreContents(); + assertThat(beforeContents.keySet()).named("persisted test data") + .containsAllIn(ALL_PERSISTED_KINDS); + MapreduceAction action = createAction(); + action.run(); + executeTasksUntilEmpty("mapreduce"); + ImmutableMultimap afterContents = getDatastoreContents(); + assertThat(afterContents.keySet()).containsNoneIn(affectedKinds); + assertThat(afterContents) + .containsExactlyEntriesIn(filterKeys(beforeContents, not(in(affectedKinds)))); + } + + private ImmutableMultimap getDatastoreContents() { + ofy().clearSessionCache(); + ImmutableMultimap.Builder contentsBuilder = new ImmutableMultimap.Builder<>(); + // Filter out raw Entity objects created by the mapreduce. + for (Object obj : Iterables.filter(ofy().load(), not(instanceOf(Entity.class)))) { + contentsBuilder.put(Key.getKind(obj.getClass()), obj); + } + return contentsBuilder.build(); + } +} diff --git a/javatests/com/google/domain/registry/tools/server/KillAllCommitLogsActionTest.java b/javatests/com/google/domain/registry/tools/server/KillAllCommitLogsActionTest.java index 9c6d0a08c..9fa8ef38a 100644 --- a/javatests/com/google/domain/registry/tools/server/KillAllCommitLogsActionTest.java +++ b/javatests/com/google/domain/registry/tools/server/KillAllCommitLogsActionTest.java @@ -14,89 +14,44 @@ package com.google.domain.registry.tools.server; -import static com.google.common.base.Predicates.instanceOf; -import static com.google.common.base.Predicates.not; -import static com.google.common.collect.Iterables.filter; -import static com.google.common.truth.Truth.assertThat; -import static com.google.domain.registry.model.ofy.ObjectifyService.ofy; -import static com.google.domain.registry.testing.DatastoreHelper.createTld; -import static com.google.domain.registry.testing.DatastoreHelper.newContactResource; -import static com.google.domain.registry.testing.DatastoreHelper.persistResource; -import static com.google.domain.registry.testing.DatastoreHelper.persistResourceWithCommitLog; -import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME; +import static com.google.domain.registry.model.EntityClasses.CLASS_TO_KIND_FUNCTION; import static java.util.Arrays.asList; -import com.google.appengine.api.datastore.Entity; import com.google.common.base.Optional; -import com.google.common.base.Predicate; import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import com.google.domain.registry.mapreduce.MapreduceAction; import com.google.domain.registry.mapreduce.MapreduceRunner; -import com.google.domain.registry.model.ImmutableObject; import com.google.domain.registry.model.ofy.CommitLogBucket; import com.google.domain.registry.model.ofy.CommitLogCheckpoint; import com.google.domain.registry.model.ofy.CommitLogCheckpointRoot; import com.google.domain.registry.model.ofy.CommitLogManifest; import com.google.domain.registry.model.ofy.CommitLogMutation; import com.google.domain.registry.testing.FakeResponse; -import com.google.domain.registry.testing.mapreduce.MapreduceTestCase; -import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import java.util.List; - /** Tests for {@link KillAllCommitLogsAction}.*/ @RunWith(JUnit4.class) -public class KillAllCommitLogsActionTest extends MapreduceTestCase { +public class KillAllCommitLogsActionTest extends KillAllActionTestCase { - static final List> AFFECTED_TYPES = ImmutableList.of( - CommitLogBucket.class, - CommitLogCheckpoint.class, - CommitLogCheckpointRoot.class, - CommitLogMutation.class, - CommitLogManifest.class); + public KillAllCommitLogsActionTest() { + super(FluentIterable + .from(asList( + CommitLogBucket.class, + CommitLogCheckpoint.class, + CommitLogCheckpointRoot.class, + CommitLogMutation.class, + CommitLogManifest.class)) + .transform(CLASS_TO_KIND_FUNCTION) + .toSet()); + } - private void runMapreduce() throws Exception { + @Override + MapreduceAction createAction() { action = new KillAllCommitLogsAction(); action.mrRunner = new MapreduceRunner(Optional.absent(), Optional.absent()); action.response = new FakeResponse(); - action.run(); - executeTasksUntilEmpty("mapreduce"); - } - - @Test - public void testKill() throws Exception { - int nextContactId = 5432; - for (String tld : asList("tld1", "tld2")) { - createTld(tld); - persistResourceWithCommitLog( - newContactResource(String.format("abc%d", nextContactId++))); - } - persistResource(CommitLogCheckpointRoot.create(START_OF_TIME.plusDays(1))); - persistResource( - CommitLogCheckpoint.create( - START_OF_TIME.plusDays(1), - ImmutableMap.of(1, START_OF_TIME.plusDays(2)))); - for (Class clazz : AFFECTED_TYPES) { - assertThat(ofy().load().type(clazz)).named("entities of type " + clazz).isNotEmpty(); - } - ImmutableList otherStuff = FluentIterable.from(ofy().load()) - .filter(new Predicate() { - @Override - public boolean apply(Object obj) { - return !AFFECTED_TYPES.contains(obj.getClass()); - }}) - .toList(); - assertThat(otherStuff).isNotEmpty(); - runMapreduce(); - for (Class clazz : AFFECTED_TYPES) { - assertThat(ofy().load().type(clazz)).named("entities of type " + clazz).isEmpty(); - } - // Filter out raw Entity objects created by the mapreduce. - assertThat(filter(ofy().load(), not(instanceOf(Entity.class)))) - .containsExactlyElementsIn(otherStuff); + return action; } } diff --git a/javatests/com/google/domain/registry/tools/server/KillAllCrossTldEntitiesActionTest.java b/javatests/com/google/domain/registry/tools/server/KillAllCrossTldEntitiesActionTest.java new file mode 100644 index 000000000..86d127e4a --- /dev/null +++ b/javatests/com/google/domain/registry/tools/server/KillAllCrossTldEntitiesActionTest.java @@ -0,0 +1,82 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.domain.registry.tools.server; + +import static com.google.domain.registry.model.EntityClasses.CLASS_TO_KIND_FUNCTION; +import static java.util.Arrays.asList; + +import com.google.common.base.Optional; +import com.google.common.collect.FluentIterable; +import com.google.domain.registry.mapreduce.MapreduceAction; +import com.google.domain.registry.mapreduce.MapreduceRunner; +import com.google.domain.registry.model.billing.RegistrarBillingEntry; +import com.google.domain.registry.model.billing.RegistrarCredit; +import com.google.domain.registry.model.billing.RegistrarCreditBalance; +import com.google.domain.registry.model.common.EntityGroupRoot; +import com.google.domain.registry.model.export.LogsExportCursor; +import com.google.domain.registry.model.registrar.Registrar; +import com.google.domain.registry.model.registrar.RegistrarContact; +import com.google.domain.registry.model.registry.Registry; +import com.google.domain.registry.model.registry.RegistryCursor; +import com.google.domain.registry.model.registry.label.PremiumList; +import com.google.domain.registry.model.registry.label.PremiumList.PremiumListEntry; +import com.google.domain.registry.model.registry.label.ReservedList; +import com.google.domain.registry.model.server.ServerSecret; +import com.google.domain.registry.model.smd.SignedMarkRevocationList; +import com.google.domain.registry.model.tmch.ClaimsListShard; +import com.google.domain.registry.model.tmch.ClaimsListShard.ClaimsListSingleton; +import com.google.domain.registry.model.tmch.TmchCrl; +import com.google.domain.registry.testing.FakeResponse; + +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link KillAllCommitLogsAction}.*/ +@RunWith(JUnit4.class) +public class KillAllCrossTldEntitiesActionTest + extends KillAllActionTestCase { + + public KillAllCrossTldEntitiesActionTest() { + super(FluentIterable + .from(asList( + ClaimsListShard.class, + ClaimsListSingleton.class, + EntityGroupRoot.class, + LogsExportCursor.class, + PremiumList.class, + PremiumListEntry.class, + Registrar.class, + RegistrarBillingEntry.class, + RegistrarContact.class, + RegistrarCredit.class, + RegistrarCreditBalance.class, + Registry.class, + RegistryCursor.class, + ReservedList.class, + ServerSecret.class, + SignedMarkRevocationList.class, + TmchCrl.class)) + .transform(CLASS_TO_KIND_FUNCTION) + .toSet()); + } + + @Override + MapreduceAction createAction() { + action = new KillAllCrossTldEntitiesAction(); + action.mrRunner = new MapreduceRunner(Optional.absent(), Optional.absent()); + action.response = new FakeResponse(); + return action; + } +} diff --git a/javatests/com/google/domain/registry/tools/server/KillAllEppResourcesActionTest.java b/javatests/com/google/domain/registry/tools/server/KillAllEppResourcesActionTest.java index ebaebb9e9..0b9a3bf22 100644 --- a/javatests/com/google/domain/registry/tools/server/KillAllEppResourcesActionTest.java +++ b/javatests/com/google/domain/registry/tools/server/KillAllEppResourcesActionTest.java @@ -14,33 +14,14 @@ package com.google.domain.registry.tools.server; -import static com.google.common.base.Predicates.in; -import static com.google.common.base.Predicates.instanceOf; -import static com.google.common.base.Predicates.not; -import static com.google.common.collect.Multimaps.filterKeys; -import static com.google.common.collect.Sets.difference; -import static com.google.common.truth.Truth.assertThat; import static com.google.domain.registry.model.EntityClasses.CLASS_TO_KIND_FUNCTION; -import static com.google.domain.registry.model.ofy.ObjectifyService.ofy; -import static com.google.domain.registry.testing.DatastoreHelper.createTld; -import static com.google.domain.registry.testing.DatastoreHelper.persistActiveContact; -import static com.google.domain.registry.testing.DatastoreHelper.persistActiveDomain; -import static com.google.domain.registry.testing.DatastoreHelper.persistActiveDomainApplication; -import static com.google.domain.registry.testing.DatastoreHelper.persistActiveHost; -import static com.google.domain.registry.testing.DatastoreHelper.persistResource; -import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME; import static java.util.Arrays.asList; -import com.google.appengine.api.datastore.Entity; import com.google.common.base.Optional; import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.Iterables; +import com.google.domain.registry.mapreduce.MapreduceAction; import com.google.domain.registry.mapreduce.MapreduceRunner; -import com.google.domain.registry.model.EppResource; -import com.google.domain.registry.model.ImmutableObject; import com.google.domain.registry.model.billing.BillingEvent; -import com.google.domain.registry.model.billing.BillingEvent.Reason; import com.google.domain.registry.model.contact.ContactResource; import com.google.domain.registry.model.domain.DomainBase; import com.google.domain.registry.model.host.HostResource; @@ -52,109 +33,41 @@ import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyHostInde import com.google.domain.registry.model.poll.PollMessage; import com.google.domain.registry.model.reporting.HistoryEntry; import com.google.domain.registry.testing.FakeResponse; -import com.google.domain.registry.testing.mapreduce.MapreduceTestCase; -import com.googlecode.objectify.Key; - -import org.joda.money.CurrencyUnit; -import org.joda.money.Money; -import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import java.util.Set; - /** Tests for {@link KillAllEppResourcesAction}.*/ @RunWith(JUnit4.class) -public class KillAllEppResourcesActionTest extends MapreduceTestCase { +public class KillAllEppResourcesActionTest + extends KillAllActionTestCase { - static final Set AFFECTED_KINDS = FluentIterable - .from(asList( - EppResourceIndex.class, - ForeignKeyContactIndex.class, - ForeignKeyDomainIndex.class, - ForeignKeyHostIndex.class, - DomainApplicationIndex.class, - DomainBase.class, - ContactResource.class, - HostResource.class, - HistoryEntry.class, - PollMessage.class, - BillingEvent.OneTime.class, - BillingEvent.Recurring.class)) - .transform(CLASS_TO_KIND_FUNCTION) - .toSet(); + public KillAllEppResourcesActionTest() { + super(FluentIterable + .from(asList( + EppResourceIndex.class, + ForeignKeyContactIndex.class, + ForeignKeyDomainIndex.class, + ForeignKeyHostIndex.class, + DomainApplicationIndex.class, + DomainBase.class, + ContactResource.class, + HostResource.class, + HistoryEntry.class, + PollMessage.class, + BillingEvent.OneTime.class, + BillingEvent.Recurring.class, + BillingEvent.Cancellation.class, + BillingEvent.Modification.class)) + .transform(CLASS_TO_KIND_FUNCTION) + .toSet()); + } - private void runMapreduce() throws Exception { + @Override + MapreduceAction createAction() { action = new KillAllEppResourcesAction(); action.mrRunner = new MapreduceRunner(Optional.absent(), Optional.absent()); action.response = new FakeResponse(); - action.run(); - executeTasksUntilEmpty("mapreduce"); - } - - @Test - public void testKill() throws Exception { - createTld("tld1"); - createTld("tld2"); - for (EppResource resource : asList( - persistActiveDomain("foo.tld1"), - persistActiveDomain("foo.tld2"), - persistActiveDomainApplication("foo.tld1"), - persistActiveDomainApplication("foo.tld2"), - persistActiveContact("foo"), - persistActiveContact("foo"), - persistActiveHost("ns.foo.tld1"), - persistActiveHost("ns.foo.tld2"))) { - HistoryEntry history = new HistoryEntry.Builder().setParent(resource).build(); - for (ImmutableObject descendant : asList( - history, - new PollMessage.OneTime.Builder() - .setParent(history) - .setClientId("") - .setEventTime(START_OF_TIME) - .build(), - new PollMessage.Autorenew.Builder() - .setParent(history) - .setClientId("") - .setEventTime(START_OF_TIME) - .build(), - new BillingEvent.OneTime.Builder() - .setParent(history) - .setBillingTime(START_OF_TIME) - .setEventTime(START_OF_TIME) - .setClientId("") - .setTargetId("") - .setReason(Reason.ERROR) - .setCost(Money.of(CurrencyUnit.USD, 1)) - .build(), - new BillingEvent.Recurring.Builder() - .setParent(history) - .setEventTime(START_OF_TIME) - .setClientId("") - .setTargetId("") - .setReason(Reason.ERROR) - .build())) { - persistResource(descendant); - } - } - ImmutableMultimap beforeContents = getDatastoreContents(); - assertThat(beforeContents.keySet()).containsAllIn(AFFECTED_KINDS); - assertThat(difference(beforeContents.keySet(), AFFECTED_KINDS)).isNotEmpty(); - runMapreduce(); - ofy().clearSessionCache(); - ImmutableMultimap afterContents = getDatastoreContents(); - assertThat(afterContents.keySet()).containsNoneIn(AFFECTED_KINDS); - assertThat(afterContents) - .containsExactlyEntriesIn(filterKeys(beforeContents, not(in(AFFECTED_KINDS)))); - } - - private ImmutableMultimap getDatastoreContents() { - ImmutableMultimap.Builder contentsBuilder = new ImmutableMultimap.Builder<>(); - // Filter out raw Entity objects created by the mapreduce. - for (Object obj : Iterables.filter(ofy().load(), not(instanceOf(Entity.class)))) { - contentsBuilder.put(Key.getKind(obj.getClass()), obj); - } - return contentsBuilder.build(); + return action; } }