mirror of
https://github.com/google/nomulus.git
synced 2025-05-20 03:09:33 +02:00
Annihilate the annihilator []
I am become Ben, annihilator of annihilators. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=117275133
This commit is contained in:
mcilwain
Justine Tunney
parent
d82ac03c8c
commit
702c5b9692
4 changed files with 0 additions and 260 deletions
|
|
@ -125,21 +125,6 @@
|
|||
<url-pattern>/_dr/task/deleteProberData</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to delete all entities in non-default namespaces. -->
|
||||
<servlet>
|
||||
<description>
|
||||
Deletes entities in non-default namespaces.
|
||||
</description>
|
||||
<display-name>Annihilate Non-Default Namespaces Mapreduce</display-name>
|
||||
<servlet-name>annihilate-non-default-namespaces</servlet-name>
|
||||
<servlet-class>com.google.domain.registry.module.tools.ToolsServlet</servlet-class>
|
||||
<load-on-startup>1</load-on-startup>
|
||||
</servlet>
|
||||
<servlet-mapping>
|
||||
<servlet-name>annihilate-non-default-namespaces</servlet-name>
|
||||
<url-pattern>/_dr/task/annihilateNonDefaultNamespaces</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to re-save all EppResources. -->
|
||||
<servlet>
|
||||
<description>
|
||||
|
|
|
|||
|
|
@ -37,7 +37,6 @@ import com.google.domain.registry.tools.server.ListTldsAction;
|
|||
import com.google.domain.registry.tools.server.ToolsServerModule;
|
||||
import com.google.domain.registry.tools.server.UpdatePremiumListAction;
|
||||
import com.google.domain.registry.tools.server.VerifyOteAction;
|
||||
import com.google.domain.registry.tools.server.javascrap.AnnihilateNonDefaultNamespacesAction;
|
||||
|
||||
import dagger.Subcomponent;
|
||||
|
||||
|
|
@ -51,7 +50,6 @@ import dagger.Subcomponent;
|
|||
ToolsServerModule.class,
|
||||
})
|
||||
interface ToolsRequestComponent {
|
||||
AnnihilateNonDefaultNamespacesAction annihilateNonDefaultNamespacesAction();
|
||||
CreateGroupsAction createGroupsAction();
|
||||
CreatePremiumListAction createPremiumListAction();
|
||||
DeleteEntityAction deleteEntityAction();
|
||||
|
|
|
|||
|
|
@ -1,136 +0,0 @@
|
|||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.tools.server.javascrap;
|
||||
|
||||
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.domain.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
|
||||
|
||||
import com.google.appengine.api.NamespaceManager;
|
||||
import com.google.appengine.api.datastore.DatastoreService;
|
||||
import com.google.appengine.api.datastore.Entities;
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.Key;
|
||||
import com.google.appengine.api.datastore.Query;
|
||||
import com.google.appengine.tools.mapreduce.Input;
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.appengine.tools.mapreduce.inputs.DatastoreKeyInput;
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.domain.registry.mapreduce.MapreduceAction;
|
||||
import com.google.domain.registry.mapreduce.MapreduceRunner;
|
||||
import com.google.domain.registry.mapreduce.inputs.ChunkingKeyInput;
|
||||
import com.google.domain.registry.request.Action;
|
||||
import com.google.domain.registry.request.Parameter;
|
||||
import com.google.domain.registry.request.Response;
|
||||
import com.google.domain.registry.util.FormattingLogger;
|
||||
import com.google.domain.registry.util.PipelineUtils;
|
||||
|
||||
import com.googlecode.objectify.VoidWork;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* A mapreduce that deletes all entities in all namespaces except for the default namespace.
|
||||
*/
|
||||
@Action(path = "/_dr/task/annihilateNonDefaultNamespaces")
|
||||
public class AnnihilateNonDefaultNamespacesAction implements MapreduceAction {
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@Inject DatastoreService datastoreService;
|
||||
@Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
@Inject AnnihilateNonDefaultNamespacesAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
response.sendJavaScriptRedirect(PipelineUtils.createJobPath(mrRunner
|
||||
.setJobName("Annihilate non-default namespaces")
|
||||
.setModuleName("backend")
|
||||
.runMapOnly(
|
||||
new AnnihilateNonDefaultNamespacesMapper(isDryRun),
|
||||
getInputs())));
|
||||
}
|
||||
|
||||
/** Mapper to delete all entities in non-default namespaces. */
|
||||
public static class AnnihilateNonDefaultNamespacesMapper extends Mapper<List<Key>, Void, Void> {
|
||||
|
||||
private static final long serialVersionUID = 8415923063881853727L;
|
||||
private final boolean isDryRun;
|
||||
|
||||
public AnnihilateNonDefaultNamespacesMapper(boolean isDryRun) {
|
||||
this.isDryRun = isDryRun;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void map(final List<Key> keys) {
|
||||
if (isDryRun) {
|
||||
logger.infofmt("Would delete these entities: %s", keys);
|
||||
} else {
|
||||
for (Key key : keys) {
|
||||
// Additional safety check to prevent deleting real data.
|
||||
checkState(
|
||||
!Strings.isNullOrEmpty(key.getNamespace()),
|
||||
"Will not delete key %s is in default namespace",
|
||||
key);
|
||||
}
|
||||
ofy().transact(new VoidWork() {
|
||||
@Override
|
||||
public void vrun() {
|
||||
getDatastoreService().delete(keys);
|
||||
}});
|
||||
}
|
||||
getContext().incrementCounter("entities deleted", keys.size());
|
||||
}
|
||||
}
|
||||
|
||||
private Iterable<Input<List<Key>>> getInputs() {
|
||||
ImmutableSet.Builder<String> namespaces = new ImmutableSet.Builder<>();
|
||||
Query namespacesQuery = new Query(Entities.NAMESPACE_METADATA_KIND).setKeysOnly();
|
||||
for (Entity entity : datastoreService.prepare(namespacesQuery).asIterable()) {
|
||||
// Don't delete anything in the default namespace!
|
||||
if (!Strings.isNullOrEmpty(entity.getKey().getName())) {
|
||||
namespaces.add(entity.getKey().getName());
|
||||
}
|
||||
}
|
||||
|
||||
ImmutableSet.Builder<Input<List<Key>>> inputs = new ImmutableSet.Builder<>();
|
||||
for (String namespace : namespaces.build()) {
|
||||
NamespaceManager.set(namespace);
|
||||
ImmutableSet.Builder<String> kindsBuilder = new ImmutableSet.Builder<>();
|
||||
Query kindsQuery = new Query(Entities.KIND_METADATA_KIND).setKeysOnly();
|
||||
for (Entity entity : datastoreService.prepare(kindsQuery).asIterable()) {
|
||||
// Don't delete built-in kinds such as __Stat_* entities.
|
||||
if (!entity.getKey().getName().startsWith("_")) {
|
||||
kindsBuilder.add(entity.getKey().getName());
|
||||
}
|
||||
}
|
||||
ImmutableSet<String> kinds = kindsBuilder.build();
|
||||
logger.infofmt("For namespace %s, found kinds: %s", namespace, kinds);
|
||||
for (String kind : kinds) {
|
||||
// Don't try to parallelize here, because Registry 1.0 entities are almost all in a single
|
||||
// entity group.
|
||||
inputs.add(new ChunkingKeyInput(new DatastoreKeyInput(kind, 1, namespace), 20));
|
||||
}
|
||||
}
|
||||
NamespaceManager.set("");
|
||||
return inputs.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.tools.server.javascrap;
|
||||
|
||||
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.newRegistry;
|
||||
|
||||
import com.google.appengine.api.NamespaceManager;
|
||||
import com.google.common.base.Optional;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.domain.registry.mapreduce.MapreduceRunner;
|
||||
import com.google.domain.registry.model.ImmutableObject;
|
||||
import com.google.domain.registry.model.domain.DomainApplication;
|
||||
import com.google.domain.registry.model.host.HostResource;
|
||||
import com.google.domain.registry.model.index.DomainApplicationIndex;
|
||||
import com.google.domain.registry.model.registry.Registry;
|
||||
import com.google.domain.registry.testing.DatastoreHelper;
|
||||
import com.google.domain.registry.testing.FakeResponse;
|
||||
import com.google.domain.registry.testing.mapreduce.MapreduceTestCase;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.Ref;
|
||||
import com.googlecode.objectify.VoidWork;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
|
||||
/** Unit tests for {@link AnnihilateNonDefaultNamespacesAction}. */
|
||||
@RunWith(JUnit4.class)
|
||||
public class AnnihilateNonDefaultNamespacesActionTest
|
||||
extends MapreduceTestCase<AnnihilateNonDefaultNamespacesAction> {
|
||||
|
||||
@Before
|
||||
public void init() {
|
||||
action = new AnnihilateNonDefaultNamespacesAction();
|
||||
action.datastoreService = getDatastoreService();
|
||||
action.mrRunner = new MapreduceRunner(Optional.<Integer>absent(), Optional.<Integer>absent());
|
||||
action.response = new FakeResponse();
|
||||
}
|
||||
|
||||
private void runMapreduce() throws Exception {
|
||||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce");
|
||||
}
|
||||
|
||||
private void runTest(boolean shouldBeDeleted) throws Exception {
|
||||
NamespaceManager.set("blah1");
|
||||
final DomainApplicationIndex toDelete1 =
|
||||
DomainApplicationIndex.createWithSpecifiedReferences(
|
||||
"foo.bar", ImmutableSet.of(Ref.create(Key.create(DomainApplication.class, 1L))));
|
||||
final Registry toDelete2 = newRegistry("bar", "BAR");
|
||||
ofy().transact(new VoidWork() {
|
||||
@Override
|
||||
public void vrun() {
|
||||
ofy().save().<ImmutableObject>entities(toDelete1, toDelete2).now();
|
||||
}});
|
||||
|
||||
NamespaceManager.set("");
|
||||
final HostResource dontDelete1 = DatastoreHelper.newHostResource("blah.test.bar");
|
||||
final Registry dontDelete2 = newRegistry("baz", "BAZ");
|
||||
ofy().transact(new VoidWork() {
|
||||
@Override
|
||||
public void vrun() {
|
||||
ofy().save().<ImmutableObject>entities(dontDelete1, dontDelete2).now();
|
||||
}});
|
||||
ofy().clearSessionCache();
|
||||
runMapreduce();
|
||||
assertThat(ofy().load().entity(dontDelete1).now()).isNotNull();
|
||||
assertThat(ofy().load().entity(dontDelete2).now()).isNotNull();
|
||||
|
||||
NamespaceManager.set("blah1");
|
||||
if (shouldBeDeleted) {
|
||||
assertThat(ofy().load().entity(toDelete1).now()).isNull();
|
||||
assertThat(ofy().load().entity(toDelete2).now()).isNull();
|
||||
} else {
|
||||
assertThat(ofy().load().entity(toDelete1).now()).isNotNull();
|
||||
assertThat(ofy().load().entity(toDelete2).now()).isNotNull();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_deletesOnlyEntitesInNonDefaultNamespace() throws Exception {
|
||||
runTest(true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_dryRunDoesntDeleteAnything() throws Exception {
|
||||
action.isDryRun = true;
|
||||
runTest(false);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue