mirror of
https://github.com/google/nomulus.git
synced 2025-05-20 03:09:33 +02:00
Annihilate the annihilator []
I am become Ben, annihilator of annihilators. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=117275133
This commit is contained in:
mcilwain
Justine Tunney
parent
d82ac03c8c
commit
702c5b9692
4 changed files with 0 additions and 260 deletions
|
|
@ -125,21 +125,6 @@
|
||||||
<url-pattern>/_dr/task/deleteProberData</url-pattern>
|
<url-pattern>/_dr/task/deleteProberData</url-pattern>
|
||||||
</servlet-mapping>
|
</servlet-mapping>
|
||||||
|
|
||||||
<!-- Mapreduce to delete all entities in non-default namespaces. -->
|
|
||||||
<servlet>
|
|
||||||
<description>
|
|
||||||
Deletes entities in non-default namespaces.
|
|
||||||
</description>
|
|
||||||
<display-name>Annihilate Non-Default Namespaces Mapreduce</display-name>
|
|
||||||
<servlet-name>annihilate-non-default-namespaces</servlet-name>
|
|
||||||
<servlet-class>com.google.domain.registry.module.tools.ToolsServlet</servlet-class>
|
|
||||||
<load-on-startup>1</load-on-startup>
|
|
||||||
</servlet>
|
|
||||||
<servlet-mapping>
|
|
||||||
<servlet-name>annihilate-non-default-namespaces</servlet-name>
|
|
||||||
<url-pattern>/_dr/task/annihilateNonDefaultNamespaces</url-pattern>
|
|
||||||
</servlet-mapping>
|
|
||||||
|
|
||||||
<!-- Mapreduce to re-save all EppResources. -->
|
<!-- Mapreduce to re-save all EppResources. -->
|
||||||
<servlet>
|
<servlet>
|
||||||
<description>
|
<description>
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,6 @@ import com.google.domain.registry.tools.server.ListTldsAction;
|
||||||
import com.google.domain.registry.tools.server.ToolsServerModule;
|
import com.google.domain.registry.tools.server.ToolsServerModule;
|
||||||
import com.google.domain.registry.tools.server.UpdatePremiumListAction;
|
import com.google.domain.registry.tools.server.UpdatePremiumListAction;
|
||||||
import com.google.domain.registry.tools.server.VerifyOteAction;
|
import com.google.domain.registry.tools.server.VerifyOteAction;
|
||||||
import com.google.domain.registry.tools.server.javascrap.AnnihilateNonDefaultNamespacesAction;
|
|
||||||
|
|
||||||
import dagger.Subcomponent;
|
import dagger.Subcomponent;
|
||||||
|
|
||||||
|
|
@ -51,7 +50,6 @@ import dagger.Subcomponent;
|
||||||
ToolsServerModule.class,
|
ToolsServerModule.class,
|
||||||
})
|
})
|
||||||
interface ToolsRequestComponent {
|
interface ToolsRequestComponent {
|
||||||
AnnihilateNonDefaultNamespacesAction annihilateNonDefaultNamespacesAction();
|
|
||||||
CreateGroupsAction createGroupsAction();
|
CreateGroupsAction createGroupsAction();
|
||||||
CreatePremiumListAction createPremiumListAction();
|
CreatePremiumListAction createPremiumListAction();
|
||||||
DeleteEntityAction deleteEntityAction();
|
DeleteEntityAction deleteEntityAction();
|
||||||
|
|
|
||||||
|
|
@ -1,136 +0,0 @@
|
||||||
// Copyright 2016 Google Inc. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package com.google.domain.registry.tools.server.javascrap;
|
|
||||||
|
|
||||||
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
|
|
||||||
import static com.google.common.base.Preconditions.checkState;
|
|
||||||
import static com.google.domain.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
|
||||||
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
|
|
||||||
|
|
||||||
import com.google.appengine.api.NamespaceManager;
|
|
||||||
import com.google.appengine.api.datastore.DatastoreService;
|
|
||||||
import com.google.appengine.api.datastore.Entities;
|
|
||||||
import com.google.appengine.api.datastore.Entity;
|
|
||||||
import com.google.appengine.api.datastore.Key;
|
|
||||||
import com.google.appengine.api.datastore.Query;
|
|
||||||
import com.google.appengine.tools.mapreduce.Input;
|
|
||||||
import com.google.appengine.tools.mapreduce.Mapper;
|
|
||||||
import com.google.appengine.tools.mapreduce.inputs.DatastoreKeyInput;
|
|
||||||
import com.google.common.base.Strings;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import com.google.domain.registry.mapreduce.MapreduceAction;
|
|
||||||
import com.google.domain.registry.mapreduce.MapreduceRunner;
|
|
||||||
import com.google.domain.registry.mapreduce.inputs.ChunkingKeyInput;
|
|
||||||
import com.google.domain.registry.request.Action;
|
|
||||||
import com.google.domain.registry.request.Parameter;
|
|
||||||
import com.google.domain.registry.request.Response;
|
|
||||||
import com.google.domain.registry.util.FormattingLogger;
|
|
||||||
import com.google.domain.registry.util.PipelineUtils;
|
|
||||||
|
|
||||||
import com.googlecode.objectify.VoidWork;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import javax.inject.Inject;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A mapreduce that deletes all entities in all namespaces except for the default namespace.
|
|
||||||
*/
|
|
||||||
@Action(path = "/_dr/task/annihilateNonDefaultNamespaces")
|
|
||||||
public class AnnihilateNonDefaultNamespacesAction implements MapreduceAction {
|
|
||||||
|
|
||||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
|
||||||
|
|
||||||
@Inject DatastoreService datastoreService;
|
|
||||||
@Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
|
|
||||||
@Inject MapreduceRunner mrRunner;
|
|
||||||
@Inject Response response;
|
|
||||||
@Inject AnnihilateNonDefaultNamespacesAction() {}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
response.sendJavaScriptRedirect(PipelineUtils.createJobPath(mrRunner
|
|
||||||
.setJobName("Annihilate non-default namespaces")
|
|
||||||
.setModuleName("backend")
|
|
||||||
.runMapOnly(
|
|
||||||
new AnnihilateNonDefaultNamespacesMapper(isDryRun),
|
|
||||||
getInputs())));
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Mapper to delete all entities in non-default namespaces. */
|
|
||||||
public static class AnnihilateNonDefaultNamespacesMapper extends Mapper<List<Key>, Void, Void> {
|
|
||||||
|
|
||||||
private static final long serialVersionUID = 8415923063881853727L;
|
|
||||||
private final boolean isDryRun;
|
|
||||||
|
|
||||||
public AnnihilateNonDefaultNamespacesMapper(boolean isDryRun) {
|
|
||||||
this.isDryRun = isDryRun;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public final void map(final List<Key> keys) {
|
|
||||||
if (isDryRun) {
|
|
||||||
logger.infofmt("Would delete these entities: %s", keys);
|
|
||||||
} else {
|
|
||||||
for (Key key : keys) {
|
|
||||||
// Additional safety check to prevent deleting real data.
|
|
||||||
checkState(
|
|
||||||
!Strings.isNullOrEmpty(key.getNamespace()),
|
|
||||||
"Will not delete key %s is in default namespace",
|
|
||||||
key);
|
|
||||||
}
|
|
||||||
ofy().transact(new VoidWork() {
|
|
||||||
@Override
|
|
||||||
public void vrun() {
|
|
||||||
getDatastoreService().delete(keys);
|
|
||||||
}});
|
|
||||||
}
|
|
||||||
getContext().incrementCounter("entities deleted", keys.size());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Iterable<Input<List<Key>>> getInputs() {
|
|
||||||
ImmutableSet.Builder<String> namespaces = new ImmutableSet.Builder<>();
|
|
||||||
Query namespacesQuery = new Query(Entities.NAMESPACE_METADATA_KIND).setKeysOnly();
|
|
||||||
for (Entity entity : datastoreService.prepare(namespacesQuery).asIterable()) {
|
|
||||||
// Don't delete anything in the default namespace!
|
|
||||||
if (!Strings.isNullOrEmpty(entity.getKey().getName())) {
|
|
||||||
namespaces.add(entity.getKey().getName());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ImmutableSet.Builder<Input<List<Key>>> inputs = new ImmutableSet.Builder<>();
|
|
||||||
for (String namespace : namespaces.build()) {
|
|
||||||
NamespaceManager.set(namespace);
|
|
||||||
ImmutableSet.Builder<String> kindsBuilder = new ImmutableSet.Builder<>();
|
|
||||||
Query kindsQuery = new Query(Entities.KIND_METADATA_KIND).setKeysOnly();
|
|
||||||
for (Entity entity : datastoreService.prepare(kindsQuery).asIterable()) {
|
|
||||||
// Don't delete built-in kinds such as __Stat_* entities.
|
|
||||||
if (!entity.getKey().getName().startsWith("_")) {
|
|
||||||
kindsBuilder.add(entity.getKey().getName());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ImmutableSet<String> kinds = kindsBuilder.build();
|
|
||||||
logger.infofmt("For namespace %s, found kinds: %s", namespace, kinds);
|
|
||||||
for (String kind : kinds) {
|
|
||||||
// Don't try to parallelize here, because Registry 1.0 entities are almost all in a single
|
|
||||||
// entity group.
|
|
||||||
inputs.add(new ChunkingKeyInput(new DatastoreKeyInput(kind, 1, namespace), 20));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
NamespaceManager.set("");
|
|
||||||
return inputs.build();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,107 +0,0 @@
|
||||||
// Copyright 2016 Google Inc. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package com.google.domain.registry.tools.server.javascrap;
|
|
||||||
|
|
||||||
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
|
|
||||||
import static com.google.common.truth.Truth.assertThat;
|
|
||||||
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
|
|
||||||
import static com.google.domain.registry.testing.DatastoreHelper.newRegistry;
|
|
||||||
|
|
||||||
import com.google.appengine.api.NamespaceManager;
|
|
||||||
import com.google.common.base.Optional;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import com.google.domain.registry.mapreduce.MapreduceRunner;
|
|
||||||
import com.google.domain.registry.model.ImmutableObject;
|
|
||||||
import com.google.domain.registry.model.domain.DomainApplication;
|
|
||||||
import com.google.domain.registry.model.host.HostResource;
|
|
||||||
import com.google.domain.registry.model.index.DomainApplicationIndex;
|
|
||||||
import com.google.domain.registry.model.registry.Registry;
|
|
||||||
import com.google.domain.registry.testing.DatastoreHelper;
|
|
||||||
import com.google.domain.registry.testing.FakeResponse;
|
|
||||||
import com.google.domain.registry.testing.mapreduce.MapreduceTestCase;
|
|
||||||
|
|
||||||
import com.googlecode.objectify.Key;
|
|
||||||
import com.googlecode.objectify.Ref;
|
|
||||||
import com.googlecode.objectify.VoidWork;
|
|
||||||
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.runner.RunWith;
|
|
||||||
import org.junit.runners.JUnit4;
|
|
||||||
|
|
||||||
/** Unit tests for {@link AnnihilateNonDefaultNamespacesAction}. */
|
|
||||||
@RunWith(JUnit4.class)
|
|
||||||
public class AnnihilateNonDefaultNamespacesActionTest
|
|
||||||
extends MapreduceTestCase<AnnihilateNonDefaultNamespacesAction> {
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void init() {
|
|
||||||
action = new AnnihilateNonDefaultNamespacesAction();
|
|
||||||
action.datastoreService = getDatastoreService();
|
|
||||||
action.mrRunner = new MapreduceRunner(Optional.<Integer>absent(), Optional.<Integer>absent());
|
|
||||||
action.response = new FakeResponse();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void runMapreduce() throws Exception {
|
|
||||||
action.run();
|
|
||||||
executeTasksUntilEmpty("mapreduce");
|
|
||||||
}
|
|
||||||
|
|
||||||
private void runTest(boolean shouldBeDeleted) throws Exception {
|
|
||||||
NamespaceManager.set("blah1");
|
|
||||||
final DomainApplicationIndex toDelete1 =
|
|
||||||
DomainApplicationIndex.createWithSpecifiedReferences(
|
|
||||||
"foo.bar", ImmutableSet.of(Ref.create(Key.create(DomainApplication.class, 1L))));
|
|
||||||
final Registry toDelete2 = newRegistry("bar", "BAR");
|
|
||||||
ofy().transact(new VoidWork() {
|
|
||||||
@Override
|
|
||||||
public void vrun() {
|
|
||||||
ofy().save().<ImmutableObject>entities(toDelete1, toDelete2).now();
|
|
||||||
}});
|
|
||||||
|
|
||||||
NamespaceManager.set("");
|
|
||||||
final HostResource dontDelete1 = DatastoreHelper.newHostResource("blah.test.bar");
|
|
||||||
final Registry dontDelete2 = newRegistry("baz", "BAZ");
|
|
||||||
ofy().transact(new VoidWork() {
|
|
||||||
@Override
|
|
||||||
public void vrun() {
|
|
||||||
ofy().save().<ImmutableObject>entities(dontDelete1, dontDelete2).now();
|
|
||||||
}});
|
|
||||||
ofy().clearSessionCache();
|
|
||||||
runMapreduce();
|
|
||||||
assertThat(ofy().load().entity(dontDelete1).now()).isNotNull();
|
|
||||||
assertThat(ofy().load().entity(dontDelete2).now()).isNotNull();
|
|
||||||
|
|
||||||
NamespaceManager.set("blah1");
|
|
||||||
if (shouldBeDeleted) {
|
|
||||||
assertThat(ofy().load().entity(toDelete1).now()).isNull();
|
|
||||||
assertThat(ofy().load().entity(toDelete2).now()).isNull();
|
|
||||||
} else {
|
|
||||||
assertThat(ofy().load().entity(toDelete1).now()).isNotNull();
|
|
||||||
assertThat(ofy().load().entity(toDelete2).now()).isNotNull();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void test_deletesOnlyEntitesInNonDefaultNamespace() throws Exception {
|
|
||||||
runTest(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void test_dryRunDoesntDeleteAnything() throws Exception {
|
|
||||||
action.isDryRun = true;
|
|
||||||
runTest(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue