mirror of
https://github.com/google/nomulus.git
synced 2025-05-17 17:59:41 +02:00
Action to delete all cross-tld entities.
Plus refactoring of the KillAllXXXActions and tests to share common code. This essentially completes the KillAll []s. We can now reliably clean out everything except for: * Lock - harmless to leave alone or delete from the gae console * GaeUserIdConverter - same * RdeRevision - filed [] to track, but harmless if not cleaned up * ForeignKeyHostIndex of renamed hosts - tracked in [] ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=120781975
This commit is contained in:
cgoldfeder
Justine Tunney
parent
ac2f17e10f
commit
a41677aea1
9 changed files with 508 additions and 205 deletions
|
|
@ -112,6 +112,12 @@
|
|||
<url-pattern>/_dr/task/killAllCommitLogs</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to delete all cross-tld entities. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>tools-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/killAllCrossTld</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- This path serves up the App Engine results page for mapreduce runs. -->
|
||||
<servlet>
|
||||
<servlet-name>mapreduce</servlet-name>
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import com.google.domain.registry.tools.server.DeleteEntityAction;
|
|||
import com.google.domain.registry.tools.server.DeleteProberDataAction;
|
||||
import com.google.domain.registry.tools.server.GenerateZoneFilesAction;
|
||||
import com.google.domain.registry.tools.server.KillAllCommitLogsAction;
|
||||
import com.google.domain.registry.tools.server.KillAllCrossTldEntitiesAction;
|
||||
import com.google.domain.registry.tools.server.KillAllEppResourcesAction;
|
||||
import com.google.domain.registry.tools.server.ListDomainsAction;
|
||||
import com.google.domain.registry.tools.server.ListHostsAction;
|
||||
|
|
@ -60,6 +61,7 @@ interface ToolsRequestComponent {
|
|||
DeleteProberDataAction deleteProberDataAction();
|
||||
GenerateZoneFilesAction generateZoneFilesAction();
|
||||
KillAllCommitLogsAction killAllCommitLogsAction();
|
||||
KillAllCrossTldEntitiesAction killAllCrossTldEntitiesAction();
|
||||
KillAllEppResourcesAction killAllEppResourcesAction();
|
||||
ListDomainsAction listDomainsAction();
|
||||
ListHostsAction listHostsAction();
|
||||
|
|
|
|||
|
|
@ -15,16 +15,15 @@
|
|||
package com.google.domain.registry.tools.server;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.collect.Lists.partition;
|
||||
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static com.google.domain.registry.request.Action.Method.POST;
|
||||
import static com.google.domain.registry.util.PipelineUtils.createJobPath;
|
||||
|
||||
import com.google.appengine.tools.mapreduce.Input;
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.appengine.tools.mapreduce.inputs.InMemoryInput;
|
||||
import com.google.common.collect.FluentIterable;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.domain.registry.config.RegistryEnvironment;
|
||||
import com.google.domain.registry.mapreduce.MapreduceAction;
|
||||
import com.google.domain.registry.mapreduce.MapreduceRunner;
|
||||
|
|
@ -48,27 +47,27 @@ public class KillAllCommitLogsAction implements MapreduceAction {
|
|||
@Inject KillAllCommitLogsAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
public final void run() {
|
||||
checkArgument( // safety
|
||||
RegistryEnvironment.get() == RegistryEnvironment.CRASH
|
||||
|| RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
|
||||
"DO NOT RUN ANYWHERE ELSE EXCEPT CRASH OR TESTS.");
|
||||
// Create a in-memory input, assigning each bucket to its own shard for maximum parallelization,
|
||||
// with one extra shard for the CommitLogCheckpointRoot.
|
||||
Input<Key<?>> input = new InMemoryInput<>(
|
||||
Lists.partition(
|
||||
FluentIterable
|
||||
.from(Arrays.<Key<?>>asList(CommitLogCheckpointRoot.getKey()))
|
||||
.append(CommitLogBucket.getAllBucketKeys())
|
||||
.toList(),
|
||||
1));
|
||||
response.sendJavaScriptRedirect(createJobPath(mrRunner
|
||||
.setJobName("Delete all commit logs")
|
||||
.setJobName("Delete all commit logs and checkpoints")
|
||||
.setModuleName("tools")
|
||||
.runMapreduce(
|
||||
new KillAllCommitLogsMapper(),
|
||||
new KillAllEntitiesReducer(),
|
||||
ImmutableList.of(input))));
|
||||
// Create a in-memory input, assigning each bucket to its own shard for maximum
|
||||
// parallelization, with one extra shard for the CommitLogCheckpointRoot.
|
||||
ImmutableList.of(
|
||||
new InMemoryInput<>(
|
||||
partition(
|
||||
FluentIterable
|
||||
.from(Arrays.<Key<?>>asList(CommitLogCheckpointRoot.getKey()))
|
||||
.append(CommitLogBucket.getAllBucketKeys())
|
||||
.toList(),
|
||||
1))))));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,105 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.tools.server;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.domain.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static com.google.domain.registry.request.Action.Method.POST;
|
||||
import static com.google.domain.registry.util.PipelineUtils.createJobPath;
|
||||
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.domain.registry.config.RegistryEnvironment;
|
||||
import com.google.domain.registry.mapreduce.MapreduceAction;
|
||||
import com.google.domain.registry.mapreduce.MapreduceRunner;
|
||||
import com.google.domain.registry.mapreduce.inputs.NullInput;
|
||||
import com.google.domain.registry.model.common.EntityGroupRoot;
|
||||
import com.google.domain.registry.request.Action;
|
||||
import com.google.domain.registry.request.Response;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* Deletes all cross tld entities in datastore.
|
||||
*
|
||||
* <p>This doesn't really need to be a mapreduce, but doing so makes it consistent with the other
|
||||
* kill-all actions, and gives us retries, a dashboard and counters for free.
|
||||
*/
|
||||
@Action(path = "/_dr/task/killAllCrossTld", method = POST)
|
||||
public class KillAllCrossTldEntitiesAction implements MapreduceAction {
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
@Inject KillAllCrossTldEntitiesAction() {}
|
||||
|
||||
@Override
|
||||
public final void run() {
|
||||
checkArgument( // safety
|
||||
RegistryEnvironment.get() == RegistryEnvironment.CRASH
|
||||
|| RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
|
||||
"DO NOT RUN ANYWHERE ELSE EXCEPT CRASH OR TESTS.");
|
||||
response.sendJavaScriptRedirect(createJobPath(mrRunner
|
||||
.setJobName("Delete all cross-tld entities")
|
||||
.setModuleName("tools")
|
||||
.runMapreduce(
|
||||
new KillAllCrossTldEntitiesMapper(),
|
||||
new KillAllEntitiesReducer(),
|
||||
ImmutableList.of(new NullInput<Object>()))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Mapper to delete all descendants of {@link EntityGroupRoot#getCrossTldKey()}.
|
||||
*
|
||||
* <p>This will delete:
|
||||
* <ul>
|
||||
* <i>{@code ClaimsListShard}
|
||||
* <i>{@code ClaimsListSingleton}
|
||||
* <i>{@link EntityGroupRoot}
|
||||
* <i>{@code LogsExportCursor}
|
||||
* <i>{@code PremiumList}
|
||||
* <i>{@code PremiumListEntry}
|
||||
* <i>{@code Registrar}
|
||||
* <i>{@code RegistrarBillingEntry}
|
||||
* <i>{@code RegistrarContact}
|
||||
* <i>{@code RegistrarCredit}
|
||||
* <i>{@code RegistrarCreditBalance}
|
||||
* <i>{@code Registry}
|
||||
* <i>{@code RegistryCursor}
|
||||
* <i>{@code ReservedList}
|
||||
* <i>{@code ServerSecret}
|
||||
* <i>{@code SignedMarkRevocationList}
|
||||
* <i>{@code TmchCrl}
|
||||
* </ul>
|
||||
*/
|
||||
static class KillAllCrossTldEntitiesMapper extends Mapper<Object, Key<?>, Key<?>> {
|
||||
|
||||
private static final long serialVersionUID = 8343696167876596542L;
|
||||
|
||||
@Override
|
||||
public void map(Object ignored) {
|
||||
// There will be exactly one input to the mapper, and we ignore it.
|
||||
Key<EntityGroupRoot> crossTldKey = getCrossTldKey();
|
||||
for (Key<Object> key : ofy().load().ancestor(crossTldKey).keys()) {
|
||||
emit(crossTldKey, key);
|
||||
getContext().incrementCounter("entities emitted");
|
||||
getContext().incrementCounter(String.format("%s emitted", key.getKind()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -47,7 +47,7 @@ public class KillAllEppResourcesAction implements MapreduceAction {
|
|||
@Inject KillAllEppResourcesAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
public final void run() {
|
||||
checkArgument( // safety
|
||||
RegistryEnvironment.get() == RegistryEnvironment.CRASH
|
||||
|| RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
|
||||
|
|
@ -61,25 +61,26 @@ public class KillAllEppResourcesAction implements MapreduceAction {
|
|||
ImmutableList.of(EppResourceInputs.createIndexInput()))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Mapper to delete an {@link EppResourceIndex}, its referent, all descendants of each referent,
|
||||
* and the {@link ForeignKeyIndex} or {@link DomainApplicationIndex} of the referent, as
|
||||
* appropriate.
|
||||
*
|
||||
* <p>This will delete:
|
||||
* <ul>
|
||||
* <li>All {@link ForeignKeyIndex} types
|
||||
* <li>{@link DomainApplicationIndex}
|
||||
* <li>{@link EppResourceIndex}
|
||||
* <li>All {@link EppResource} types
|
||||
* <li>{@code HistoryEntry}
|
||||
* <li>All {@code BillingEvent} types
|
||||
* <li>All {@code PollMessage} types
|
||||
* </ul>
|
||||
*/
|
||||
static class KillAllEppResourcesMapper extends Mapper<EppResourceIndex, Key<?>, Key<?>> {
|
||||
|
||||
private static final long serialVersionUID = 8205309000002507407L;
|
||||
|
||||
/**
|
||||
* Delete an {@link EppResourceIndex}, its referent, all descendants of each referent, and the
|
||||
* {@link ForeignKeyIndex} or {@link DomainApplicationIndex} of the referent, as appropriate.
|
||||
*
|
||||
* <p>This will delete:
|
||||
* <ul>
|
||||
* <li>All {@link ForeignKeyIndex} types
|
||||
* <li>{@link DomainApplicationIndex}
|
||||
* <li>{@link EppResourceIndex}
|
||||
* <li>All {@link EppResource} types
|
||||
* <li>{@code HistoryEntry}
|
||||
* <li>All {@code BillingEvent} types
|
||||
* <li>All {@code PollMessage} types
|
||||
* </ul>
|
||||
*/
|
||||
@Override
|
||||
public void map(final EppResourceIndex eri) {
|
||||
Key<EppResourceIndex> eriKey = Key.create(eri);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,240 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.tools.server;
|
||||
|
||||
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
|
||||
import static com.google.common.base.Predicates.in;
|
||||
import static com.google.common.base.Predicates.instanceOf;
|
||||
import static com.google.common.base.Predicates.not;
|
||||
import static com.google.common.collect.Multimaps.filterKeys;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static com.google.domain.registry.model.server.ServerSecret.getServerSecret;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.createTld;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.newContactResource;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.newDomainApplication;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.newDomainResource;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.newHostResource;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistPremiumList;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistReservedList;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistResource;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistResourceWithCommitLog;
|
||||
import static com.google.domain.registry.util.DateTimeUtils.END_OF_TIME;
|
||||
import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static java.util.Arrays.asList;
|
||||
import static org.joda.money.CurrencyUnit.USD;
|
||||
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.common.base.Predicate;
|
||||
import com.google.common.collect.FluentIterable;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableMultimap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.domain.registry.mapreduce.MapreduceAction;
|
||||
import com.google.domain.registry.model.EntityClasses;
|
||||
import com.google.domain.registry.model.EppResource;
|
||||
import com.google.domain.registry.model.ImmutableObject;
|
||||
import com.google.domain.registry.model.annotations.VirtualEntity;
|
||||
import com.google.domain.registry.model.billing.BillingEvent;
|
||||
import com.google.domain.registry.model.billing.BillingEvent.Reason;
|
||||
import com.google.domain.registry.model.billing.RegistrarBillingEntry;
|
||||
import com.google.domain.registry.model.billing.RegistrarCredit;
|
||||
import com.google.domain.registry.model.billing.RegistrarCredit.CreditType;
|
||||
import com.google.domain.registry.model.billing.RegistrarCreditBalance;
|
||||
import com.google.domain.registry.model.common.EntityGroupRoot;
|
||||
import com.google.domain.registry.model.common.GaeUserIdConverter;
|
||||
import com.google.domain.registry.model.export.LogsExportCursor;
|
||||
import com.google.domain.registry.model.ofy.CommitLogCheckpoint;
|
||||
import com.google.domain.registry.model.ofy.CommitLogCheckpointRoot;
|
||||
import com.google.domain.registry.model.poll.PollMessage;
|
||||
import com.google.domain.registry.model.rde.RdeMode;
|
||||
import com.google.domain.registry.model.rde.RdeRevision;
|
||||
import com.google.domain.registry.model.registrar.Registrar;
|
||||
import com.google.domain.registry.model.registry.Registry;
|
||||
import com.google.domain.registry.model.registry.RegistryCursor;
|
||||
import com.google.domain.registry.model.registry.RegistryCursor.CursorType;
|
||||
import com.google.domain.registry.model.reporting.HistoryEntry;
|
||||
import com.google.domain.registry.model.server.Lock;
|
||||
import com.google.domain.registry.model.smd.SignedMarkRevocationList;
|
||||
import com.google.domain.registry.model.tmch.ClaimsListShard;
|
||||
import com.google.domain.registry.model.tmch.TmchCrl;
|
||||
import com.google.domain.registry.testing.mapreduce.MapreduceTestCase;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.Ref;
|
||||
import com.googlecode.objectify.VoidWork;
|
||||
|
||||
import org.joda.money.Money;
|
||||
import org.junit.Test;
|
||||
|
||||
public abstract class KillAllActionTestCase<T> extends MapreduceTestCase<T>{
|
||||
|
||||
private final ImmutableSet<String> affectedKinds;
|
||||
|
||||
private static final ImmutableSet<String> ALL_PERSISTED_KINDS = FluentIterable
|
||||
.from(EntityClasses.ALL_CLASSES)
|
||||
.filter(
|
||||
new Predicate<Class<?>>() {
|
||||
@Override
|
||||
public boolean apply(Class<?> clazz) {
|
||||
return !clazz.isAnnotationPresent(VirtualEntity.class);
|
||||
}})
|
||||
.transform(EntityClasses.CLASS_TO_KIND_FUNCTION)
|
||||
.toSet();
|
||||
|
||||
|
||||
public KillAllActionTestCase(ImmutableSet<String> affectedKinds) {
|
||||
this.affectedKinds = affectedKinds;
|
||||
}
|
||||
|
||||
/** Create at least one of each type of entity in the schema. */
|
||||
void createData() {
|
||||
createTld("tld1");
|
||||
createTld("tld2");
|
||||
persistResource(CommitLogCheckpointRoot.create(START_OF_TIME.plusDays(1)));
|
||||
persistResource(
|
||||
CommitLogCheckpoint.create(
|
||||
START_OF_TIME.plusDays(1),
|
||||
ImmutableMap.of(1, START_OF_TIME.plusDays(2))));
|
||||
for (EppResource resource : asList(
|
||||
newDomainResource("foo.tld1"),
|
||||
newDomainResource("foo.tld2"),
|
||||
newDomainApplication("foo.tld1"),
|
||||
newDomainApplication("foo.tld2"),
|
||||
newContactResource("foo1"),
|
||||
newContactResource("foo2"),
|
||||
newHostResource("ns.foo.tld1"),
|
||||
newHostResource("ns.foo.tld2"))) {
|
||||
persistResourceWithCommitLog(resource);
|
||||
HistoryEntry history =
|
||||
persistResource(new HistoryEntry.Builder().setParent(resource).build());
|
||||
BillingEvent.OneTime oneTime = persistResource(new BillingEvent.OneTime.Builder()
|
||||
.setParent(history)
|
||||
.setBillingTime(START_OF_TIME)
|
||||
.setEventTime(START_OF_TIME)
|
||||
.setClientId("")
|
||||
.setTargetId("")
|
||||
.setReason(Reason.ERROR)
|
||||
.setCost(Money.of(USD, 1))
|
||||
.build());
|
||||
for (ImmutableObject descendant : asList(
|
||||
new PollMessage.OneTime.Builder()
|
||||
.setParent(history)
|
||||
.setClientId("")
|
||||
.setEventTime(START_OF_TIME)
|
||||
.build(),
|
||||
new PollMessage.Autorenew.Builder()
|
||||
.setParent(history)
|
||||
.setClientId("")
|
||||
.setEventTime(START_OF_TIME)
|
||||
.build(),
|
||||
new BillingEvent.Cancellation.Builder()
|
||||
.setOneTimeEventRef(Ref.create(oneTime))
|
||||
.setParent(history)
|
||||
.setBillingTime(START_OF_TIME)
|
||||
.setEventTime(START_OF_TIME)
|
||||
.setClientId("")
|
||||
.setTargetId("")
|
||||
.setReason(Reason.ERROR)
|
||||
.build(),
|
||||
new BillingEvent.Modification.Builder()
|
||||
.setEventRef(Ref.create(oneTime))
|
||||
.setParent(history)
|
||||
.setEventTime(START_OF_TIME)
|
||||
.setClientId("")
|
||||
.setTargetId("")
|
||||
.setReason(Reason.ERROR)
|
||||
.setCost(Money.of(USD, 1))
|
||||
.build(),
|
||||
new BillingEvent.Recurring.Builder()
|
||||
.setParent(history)
|
||||
.setEventTime(START_OF_TIME)
|
||||
.setClientId("")
|
||||
.setTargetId("")
|
||||
.setReason(Reason.ERROR)
|
||||
.build())) {
|
||||
persistResource(descendant);
|
||||
}
|
||||
}
|
||||
persistResource(new RegistrarBillingEntry.Builder()
|
||||
.setParent(Registrar.loadByClientId("TheRegistrar"))
|
||||
.setCreated(START_OF_TIME)
|
||||
.setDescription("description")
|
||||
.setAmount(Money.of(USD, 1))
|
||||
.build());
|
||||
RegistrarCredit credit = persistResource(new RegistrarCredit.Builder()
|
||||
.setParent(Registrar.loadByClientId("TheRegistrar"))
|
||||
.setCreationTime(START_OF_TIME)
|
||||
.setCurrency(USD)
|
||||
.setDescription("description")
|
||||
.setTld("tld1")
|
||||
.setType(CreditType.AUCTION)
|
||||
.build());
|
||||
persistResource(new RegistrarCreditBalance.Builder()
|
||||
.setParent(credit)
|
||||
.setAmount(Money.of(USD, 1))
|
||||
.setEffectiveTime(START_OF_TIME)
|
||||
.setWrittenTime(START_OF_TIME)
|
||||
.build());
|
||||
persistPremiumList("premium", "a,USD 100", "b,USD 200");
|
||||
persistReservedList("reserved", "lol,RESERVED_FOR_ANCHOR_TENANT,foobar1");
|
||||
getServerSecret(); // Forces persist.
|
||||
TmchCrl.set("crl content");
|
||||
persistResource(new LogsExportCursor.Builder().build());
|
||||
persistPremiumList("premium", "a,USD 100", "b,USD 200");
|
||||
SignedMarkRevocationList.create(
|
||||
START_OF_TIME, ImmutableMap.of("a", START_OF_TIME, "b", END_OF_TIME)).save();
|
||||
ClaimsListShard.create(START_OF_TIME, ImmutableMap.of("a", "1", "b", "2")).save();
|
||||
// These entities must be saved within a transaction.
|
||||
ofy().transact(new VoidWork() {
|
||||
@Override
|
||||
public void vrun() {
|
||||
RegistryCursor.save(Registry.get("tld1"), CursorType.BRDA, START_OF_TIME);
|
||||
RdeRevision.saveRevision("tld1", START_OF_TIME, RdeMode.FULL, 0);
|
||||
}});
|
||||
// These entities intentionally don't expose constructors or factory methods.
|
||||
getDatastoreService().put(new Entity(EntityGroupRoot.getCrossTldKey().getRaw()));
|
||||
getDatastoreService().put(new Entity(Key.getKind(GaeUserIdConverter.class), 1));
|
||||
getDatastoreService().put(new Entity(Key.getKind(Lock.class), 1));
|
||||
}
|
||||
|
||||
abstract MapreduceAction createAction();
|
||||
|
||||
@Test
|
||||
public void testKill() throws Exception {
|
||||
createData();
|
||||
ImmutableMultimap<String, Object> beforeContents = getDatastoreContents();
|
||||
assertThat(beforeContents.keySet()).named("persisted test data")
|
||||
.containsAllIn(ALL_PERSISTED_KINDS);
|
||||
MapreduceAction action = createAction();
|
||||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce");
|
||||
ImmutableMultimap<String, Object> afterContents = getDatastoreContents();
|
||||
assertThat(afterContents.keySet()).containsNoneIn(affectedKinds);
|
||||
assertThat(afterContents)
|
||||
.containsExactlyEntriesIn(filterKeys(beforeContents, not(in(affectedKinds))));
|
||||
}
|
||||
|
||||
private ImmutableMultimap<String, Object> getDatastoreContents() {
|
||||
ofy().clearSessionCache();
|
||||
ImmutableMultimap.Builder<String, Object> contentsBuilder = new ImmutableMultimap.Builder<>();
|
||||
// Filter out raw Entity objects created by the mapreduce.
|
||||
for (Object obj : Iterables.filter(ofy().load(), not(instanceOf(Entity.class)))) {
|
||||
contentsBuilder.put(Key.getKind(obj.getClass()), obj);
|
||||
}
|
||||
return contentsBuilder.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -14,89 +14,44 @@
|
|||
|
||||
package com.google.domain.registry.tools.server;
|
||||
|
||||
import static com.google.common.base.Predicates.instanceOf;
|
||||
import static com.google.common.base.Predicates.not;
|
||||
import static com.google.common.collect.Iterables.filter;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.createTld;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.newContactResource;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistResource;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistResourceWithCommitLog;
|
||||
import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static com.google.domain.registry.model.EntityClasses.CLASS_TO_KIND_FUNCTION;
|
||||
import static java.util.Arrays.asList;
|
||||
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.common.base.Optional;
|
||||
import com.google.common.base.Predicate;
|
||||
import com.google.common.collect.FluentIterable;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.domain.registry.mapreduce.MapreduceAction;
|
||||
import com.google.domain.registry.mapreduce.MapreduceRunner;
|
||||
import com.google.domain.registry.model.ImmutableObject;
|
||||
import com.google.domain.registry.model.ofy.CommitLogBucket;
|
||||
import com.google.domain.registry.model.ofy.CommitLogCheckpoint;
|
||||
import com.google.domain.registry.model.ofy.CommitLogCheckpointRoot;
|
||||
import com.google.domain.registry.model.ofy.CommitLogManifest;
|
||||
import com.google.domain.registry.model.ofy.CommitLogMutation;
|
||||
import com.google.domain.registry.testing.FakeResponse;
|
||||
import com.google.domain.registry.testing.mapreduce.MapreduceTestCase;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Tests for {@link KillAllCommitLogsAction}.*/
|
||||
@RunWith(JUnit4.class)
|
||||
public class KillAllCommitLogsActionTest extends MapreduceTestCase<KillAllCommitLogsAction> {
|
||||
public class KillAllCommitLogsActionTest extends KillAllActionTestCase<KillAllCommitLogsAction> {
|
||||
|
||||
static final List<Class<? extends ImmutableObject>> AFFECTED_TYPES = ImmutableList.of(
|
||||
CommitLogBucket.class,
|
||||
CommitLogCheckpoint.class,
|
||||
CommitLogCheckpointRoot.class,
|
||||
CommitLogMutation.class,
|
||||
CommitLogManifest.class);
|
||||
public KillAllCommitLogsActionTest() {
|
||||
super(FluentIterable
|
||||
.from(asList(
|
||||
CommitLogBucket.class,
|
||||
CommitLogCheckpoint.class,
|
||||
CommitLogCheckpointRoot.class,
|
||||
CommitLogMutation.class,
|
||||
CommitLogManifest.class))
|
||||
.transform(CLASS_TO_KIND_FUNCTION)
|
||||
.toSet());
|
||||
}
|
||||
|
||||
private void runMapreduce() throws Exception {
|
||||
@Override
|
||||
MapreduceAction createAction() {
|
||||
action = new KillAllCommitLogsAction();
|
||||
action.mrRunner = new MapreduceRunner(Optional.<Integer>absent(), Optional.<Integer>absent());
|
||||
action.response = new FakeResponse();
|
||||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testKill() throws Exception {
|
||||
int nextContactId = 5432;
|
||||
for (String tld : asList("tld1", "tld2")) {
|
||||
createTld(tld);
|
||||
persistResourceWithCommitLog(
|
||||
newContactResource(String.format("abc%d", nextContactId++)));
|
||||
}
|
||||
persistResource(CommitLogCheckpointRoot.create(START_OF_TIME.plusDays(1)));
|
||||
persistResource(
|
||||
CommitLogCheckpoint.create(
|
||||
START_OF_TIME.plusDays(1),
|
||||
ImmutableMap.of(1, START_OF_TIME.plusDays(2))));
|
||||
for (Class<?> clazz : AFFECTED_TYPES) {
|
||||
assertThat(ofy().load().type(clazz)).named("entities of type " + clazz).isNotEmpty();
|
||||
}
|
||||
ImmutableList<?> otherStuff = FluentIterable.from(ofy().load())
|
||||
.filter(new Predicate<Object>() {
|
||||
@Override
|
||||
public boolean apply(Object obj) {
|
||||
return !AFFECTED_TYPES.contains(obj.getClass());
|
||||
}})
|
||||
.toList();
|
||||
assertThat(otherStuff).isNotEmpty();
|
||||
runMapreduce();
|
||||
for (Class<?> clazz : AFFECTED_TYPES) {
|
||||
assertThat(ofy().load().type(clazz)).named("entities of type " + clazz).isEmpty();
|
||||
}
|
||||
// Filter out raw Entity objects created by the mapreduce.
|
||||
assertThat(filter(ofy().load(), not(instanceOf(Entity.class))))
|
||||
.containsExactlyElementsIn(otherStuff);
|
||||
return action;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.tools.server;
|
||||
|
||||
import static com.google.domain.registry.model.EntityClasses.CLASS_TO_KIND_FUNCTION;
|
||||
import static java.util.Arrays.asList;
|
||||
|
||||
import com.google.common.base.Optional;
|
||||
import com.google.common.collect.FluentIterable;
|
||||
import com.google.domain.registry.mapreduce.MapreduceAction;
|
||||
import com.google.domain.registry.mapreduce.MapreduceRunner;
|
||||
import com.google.domain.registry.model.billing.RegistrarBillingEntry;
|
||||
import com.google.domain.registry.model.billing.RegistrarCredit;
|
||||
import com.google.domain.registry.model.billing.RegistrarCreditBalance;
|
||||
import com.google.domain.registry.model.common.EntityGroupRoot;
|
||||
import com.google.domain.registry.model.export.LogsExportCursor;
|
||||
import com.google.domain.registry.model.registrar.Registrar;
|
||||
import com.google.domain.registry.model.registrar.RegistrarContact;
|
||||
import com.google.domain.registry.model.registry.Registry;
|
||||
import com.google.domain.registry.model.registry.RegistryCursor;
|
||||
import com.google.domain.registry.model.registry.label.PremiumList;
|
||||
import com.google.domain.registry.model.registry.label.PremiumList.PremiumListEntry;
|
||||
import com.google.domain.registry.model.registry.label.ReservedList;
|
||||
import com.google.domain.registry.model.server.ServerSecret;
|
||||
import com.google.domain.registry.model.smd.SignedMarkRevocationList;
|
||||
import com.google.domain.registry.model.tmch.ClaimsListShard;
|
||||
import com.google.domain.registry.model.tmch.ClaimsListShard.ClaimsListSingleton;
|
||||
import com.google.domain.registry.model.tmch.TmchCrl;
|
||||
import com.google.domain.registry.testing.FakeResponse;
|
||||
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
|
||||
/** Tests for {@link KillAllCommitLogsAction}.*/
|
||||
@RunWith(JUnit4.class)
|
||||
public class KillAllCrossTldEntitiesActionTest
|
||||
extends KillAllActionTestCase<KillAllCrossTldEntitiesAction> {
|
||||
|
||||
public KillAllCrossTldEntitiesActionTest() {
|
||||
super(FluentIterable
|
||||
.from(asList(
|
||||
ClaimsListShard.class,
|
||||
ClaimsListSingleton.class,
|
||||
EntityGroupRoot.class,
|
||||
LogsExportCursor.class,
|
||||
PremiumList.class,
|
||||
PremiumListEntry.class,
|
||||
Registrar.class,
|
||||
RegistrarBillingEntry.class,
|
||||
RegistrarContact.class,
|
||||
RegistrarCredit.class,
|
||||
RegistrarCreditBalance.class,
|
||||
Registry.class,
|
||||
RegistryCursor.class,
|
||||
ReservedList.class,
|
||||
ServerSecret.class,
|
||||
SignedMarkRevocationList.class,
|
||||
TmchCrl.class))
|
||||
.transform(CLASS_TO_KIND_FUNCTION)
|
||||
.toSet());
|
||||
}
|
||||
|
||||
@Override
|
||||
MapreduceAction createAction() {
|
||||
action = new KillAllCrossTldEntitiesAction();
|
||||
action.mrRunner = new MapreduceRunner(Optional.<Integer>absent(), Optional.<Integer>absent());
|
||||
action.response = new FakeResponse();
|
||||
return action;
|
||||
}
|
||||
}
|
||||
|
|
@ -14,33 +14,14 @@
|
|||
|
||||
package com.google.domain.registry.tools.server;
|
||||
|
||||
import static com.google.common.base.Predicates.in;
|
||||
import static com.google.common.base.Predicates.instanceOf;
|
||||
import static com.google.common.base.Predicates.not;
|
||||
import static com.google.common.collect.Multimaps.filterKeys;
|
||||
import static com.google.common.collect.Sets.difference;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.domain.registry.model.EntityClasses.CLASS_TO_KIND_FUNCTION;
|
||||
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.createTld;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistActiveContact;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistActiveDomain;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistActiveDomainApplication;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistActiveHost;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistResource;
|
||||
import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static java.util.Arrays.asList;
|
||||
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.common.base.Optional;
|
||||
import com.google.common.collect.FluentIterable;
|
||||
import com.google.common.collect.ImmutableMultimap;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.domain.registry.mapreduce.MapreduceAction;
|
||||
import com.google.domain.registry.mapreduce.MapreduceRunner;
|
||||
import com.google.domain.registry.model.EppResource;
|
||||
import com.google.domain.registry.model.ImmutableObject;
|
||||
import com.google.domain.registry.model.billing.BillingEvent;
|
||||
import com.google.domain.registry.model.billing.BillingEvent.Reason;
|
||||
import com.google.domain.registry.model.contact.ContactResource;
|
||||
import com.google.domain.registry.model.domain.DomainBase;
|
||||
import com.google.domain.registry.model.host.HostResource;
|
||||
|
|
@ -52,109 +33,41 @@ import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyHostInde
|
|||
import com.google.domain.registry.model.poll.PollMessage;
|
||||
import com.google.domain.registry.model.reporting.HistoryEntry;
|
||||
import com.google.domain.registry.testing.FakeResponse;
|
||||
import com.google.domain.registry.testing.mapreduce.MapreduceTestCase;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
|
||||
import org.joda.money.CurrencyUnit;
|
||||
import org.joda.money.Money;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
/** Tests for {@link KillAllEppResourcesAction}.*/
|
||||
@RunWith(JUnit4.class)
|
||||
public class KillAllEppResourcesActionTest extends MapreduceTestCase<KillAllEppResourcesAction> {
|
||||
public class KillAllEppResourcesActionTest
|
||||
extends KillAllActionTestCase<KillAllEppResourcesAction> {
|
||||
|
||||
static final Set<String> AFFECTED_KINDS = FluentIterable
|
||||
.from(asList(
|
||||
EppResourceIndex.class,
|
||||
ForeignKeyContactIndex.class,
|
||||
ForeignKeyDomainIndex.class,
|
||||
ForeignKeyHostIndex.class,
|
||||
DomainApplicationIndex.class,
|
||||
DomainBase.class,
|
||||
ContactResource.class,
|
||||
HostResource.class,
|
||||
HistoryEntry.class,
|
||||
PollMessage.class,
|
||||
BillingEvent.OneTime.class,
|
||||
BillingEvent.Recurring.class))
|
||||
.transform(CLASS_TO_KIND_FUNCTION)
|
||||
.toSet();
|
||||
public KillAllEppResourcesActionTest() {
|
||||
super(FluentIterable
|
||||
.from(asList(
|
||||
EppResourceIndex.class,
|
||||
ForeignKeyContactIndex.class,
|
||||
ForeignKeyDomainIndex.class,
|
||||
ForeignKeyHostIndex.class,
|
||||
DomainApplicationIndex.class,
|
||||
DomainBase.class,
|
||||
ContactResource.class,
|
||||
HostResource.class,
|
||||
HistoryEntry.class,
|
||||
PollMessage.class,
|
||||
BillingEvent.OneTime.class,
|
||||
BillingEvent.Recurring.class,
|
||||
BillingEvent.Cancellation.class,
|
||||
BillingEvent.Modification.class))
|
||||
.transform(CLASS_TO_KIND_FUNCTION)
|
||||
.toSet());
|
||||
}
|
||||
|
||||
private void runMapreduce() throws Exception {
|
||||
@Override
|
||||
MapreduceAction createAction() {
|
||||
action = new KillAllEppResourcesAction();
|
||||
action.mrRunner = new MapreduceRunner(Optional.<Integer>absent(), Optional.<Integer>absent());
|
||||
action.response = new FakeResponse();
|
||||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testKill() throws Exception {
|
||||
createTld("tld1");
|
||||
createTld("tld2");
|
||||
for (EppResource resource : asList(
|
||||
persistActiveDomain("foo.tld1"),
|
||||
persistActiveDomain("foo.tld2"),
|
||||
persistActiveDomainApplication("foo.tld1"),
|
||||
persistActiveDomainApplication("foo.tld2"),
|
||||
persistActiveContact("foo"),
|
||||
persistActiveContact("foo"),
|
||||
persistActiveHost("ns.foo.tld1"),
|
||||
persistActiveHost("ns.foo.tld2"))) {
|
||||
HistoryEntry history = new HistoryEntry.Builder().setParent(resource).build();
|
||||
for (ImmutableObject descendant : asList(
|
||||
history,
|
||||
new PollMessage.OneTime.Builder()
|
||||
.setParent(history)
|
||||
.setClientId("")
|
||||
.setEventTime(START_OF_TIME)
|
||||
.build(),
|
||||
new PollMessage.Autorenew.Builder()
|
||||
.setParent(history)
|
||||
.setClientId("")
|
||||
.setEventTime(START_OF_TIME)
|
||||
.build(),
|
||||
new BillingEvent.OneTime.Builder()
|
||||
.setParent(history)
|
||||
.setBillingTime(START_OF_TIME)
|
||||
.setEventTime(START_OF_TIME)
|
||||
.setClientId("")
|
||||
.setTargetId("")
|
||||
.setReason(Reason.ERROR)
|
||||
.setCost(Money.of(CurrencyUnit.USD, 1))
|
||||
.build(),
|
||||
new BillingEvent.Recurring.Builder()
|
||||
.setParent(history)
|
||||
.setEventTime(START_OF_TIME)
|
||||
.setClientId("")
|
||||
.setTargetId("")
|
||||
.setReason(Reason.ERROR)
|
||||
.build())) {
|
||||
persistResource(descendant);
|
||||
}
|
||||
}
|
||||
ImmutableMultimap<String, Object> beforeContents = getDatastoreContents();
|
||||
assertThat(beforeContents.keySet()).containsAllIn(AFFECTED_KINDS);
|
||||
assertThat(difference(beforeContents.keySet(), AFFECTED_KINDS)).isNotEmpty();
|
||||
runMapreduce();
|
||||
ofy().clearSessionCache();
|
||||
ImmutableMultimap<String, Object> afterContents = getDatastoreContents();
|
||||
assertThat(afterContents.keySet()).containsNoneIn(AFFECTED_KINDS);
|
||||
assertThat(afterContents)
|
||||
.containsExactlyEntriesIn(filterKeys(beforeContents, not(in(AFFECTED_KINDS))));
|
||||
}
|
||||
|
||||
private ImmutableMultimap<String, Object> getDatastoreContents() {
|
||||
ImmutableMultimap.Builder<String, Object> contentsBuilder = new ImmutableMultimap.Builder<>();
|
||||
// Filter out raw Entity objects created by the mapreduce.
|
||||
for (Object obj : Iterables.filter(ofy().load(), not(instanceOf(Entity.class)))) {
|
||||
contentsBuilder.put(Key.getKind(obj.getClass()), obj);
|
||||
}
|
||||
return contentsBuilder.build();
|
||||
return action;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue