diff --git a/java/com/google/domain/registry/env/common/backend/WEB-INF/web.xml b/java/com/google/domain/registry/env/common/backend/WEB-INF/web.xml
index c03734075..179562033 100644
--- a/java/com/google/domain/registry/env/common/backend/WEB-INF/web.xml
+++ b/java/com/google/domain/registry/env/common/backend/WEB-INF/web.xml
@@ -119,6 +119,12 @@
/_dr/dnsRefresh
+
+
+ backend-servlet
+ /_dr/task/verifyEntityIntegrity
+
+
Exports a datastore backup snapshot to GCS.Export snapshot to GCS
diff --git a/java/com/google/domain/registry/module/backend/BUILD b/java/com/google/domain/registry/module/backend/BUILD
index 15fafa579..84471d7d3 100644
--- a/java/com/google/domain/registry/module/backend/BUILD
+++ b/java/com/google/domain/registry/module/backend/BUILD
@@ -25,6 +25,7 @@ java_library(
"//java/com/google/domain/registry/keyring/api",
"//java/com/google/domain/registry/mapreduce",
"//java/com/google/domain/registry/model",
+ "//java/com/google/domain/registry/monitoring/whitebox",
"//java/com/google/domain/registry/rde",
"//java/com/google/domain/registry/request",
"//java/com/google/domain/registry/request:modules",
diff --git a/java/com/google/domain/registry/module/backend/BackendRequestComponent.java b/java/com/google/domain/registry/module/backend/BackendRequestComponent.java
index 5ac5d0b39..a99a94884 100644
--- a/java/com/google/domain/registry/module/backend/BackendRequestComponent.java
+++ b/java/com/google/domain/registry/module/backend/BackendRequestComponent.java
@@ -39,6 +39,7 @@ import com.google.domain.registry.flows.async.DeleteContactResourceAction;
import com.google.domain.registry.flows.async.DeleteHostResourceAction;
import com.google.domain.registry.flows.async.DnsRefreshForHostRenameAction;
import com.google.domain.registry.mapreduce.MapreduceModule;
+import com.google.domain.registry.monitoring.whitebox.VerifyEntityIntegrityAction;
import com.google.domain.registry.rde.BrdaCopyAction;
import com.google.domain.registry.rde.RdeModule;
import com.google.domain.registry.rde.RdeReportAction;
@@ -101,4 +102,5 @@ interface BackendRequestComponent {
TmchDnlAction tmchDnlAction();
TmchSmdrlAction tmchSmdrlAction();
WriteDnsAction writeDnsAction();
+ VerifyEntityIntegrityAction verifyEntityIntegrityAction();
}
diff --git a/java/com/google/domain/registry/monitoring/whitebox/BUILD b/java/com/google/domain/registry/monitoring/whitebox/BUILD
index 6c2ae6268..09a7d60ab 100644
--- a/java/com/google/domain/registry/monitoring/whitebox/BUILD
+++ b/java/com/google/domain/registry/monitoring/whitebox/BUILD
@@ -19,12 +19,15 @@ java_library(
"//java/com/google/domain/registry/bigquery",
"//java/com/google/domain/registry/config",
"//java/com/google/domain/registry/mapreduce",
+ "//java/com/google/domain/registry/mapreduce/inputs",
"//java/com/google/domain/registry/model",
+ "//java/com/google/domain/registry/request",
"//java/com/google/domain/registry/util",
"//third_party/java/appengine:appengine-api",
"//third_party/java/appengine_mapreduce2:appengine_mapreduce",
"//third_party/java/joda_time",
"//third_party/java/jsr305_annotations",
+ "//third_party/java/jsr330_inject",
"//third_party/java/objectify:objectify-v4_1",
"//third_party/java/servlet/servlet_api",
],
diff --git a/java/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityAction.java b/java/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityAction.java
new file mode 100644
index 000000000..78583854e
--- /dev/null
+++ b/java/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityAction.java
@@ -0,0 +1,511 @@
+// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.domain.registry.monitoring.whitebox;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.Iterables.getOnlyElement;
+import static com.google.domain.registry.model.EppResourceUtils.isActive;
+import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
+import static com.google.domain.registry.util.DateTimeUtils.END_OF_TIME;
+import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME;
+import static com.google.domain.registry.util.DateTimeUtils.earliestOf;
+import static com.google.domain.registry.util.DateTimeUtils.isBeforeOrAt;
+import static com.google.domain.registry.util.DateTimeUtils.latestOf;
+import static com.google.domain.registry.util.FormattingLogger.getLoggerForCallerClass;
+import static com.google.domain.registry.util.PipelineUtils.createJobPath;
+import static com.googlecode.objectify.Key.getKind;
+import static org.joda.time.DateTimeZone.UTC;
+
+import com.google.appengine.tools.mapreduce.Input;
+import com.google.appengine.tools.mapreduce.Mapper;
+import com.google.appengine.tools.mapreduce.Reducer;
+import com.google.appengine.tools.mapreduce.ReducerInput;
+import com.google.appengine.tools.mapreduce.inputs.DatastoreKeyInput;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.domain.registry.mapreduce.MapreduceRunner;
+import com.google.domain.registry.mapreduce.inputs.EppResourceInputs;
+import com.google.domain.registry.model.EppResource;
+import com.google.domain.registry.model.ImmutableObject;
+import com.google.domain.registry.model.contact.ContactResource;
+import com.google.domain.registry.model.domain.DomainApplication;
+import com.google.domain.registry.model.domain.DomainBase;
+import com.google.domain.registry.model.domain.DomainResource;
+import com.google.domain.registry.model.domain.GracePeriod;
+import com.google.domain.registry.model.domain.ReferenceUnion;
+import com.google.domain.registry.model.host.HostResource;
+import com.google.domain.registry.model.index.DomainApplicationIndex;
+import com.google.domain.registry.model.index.EppResourceIndex;
+import com.google.domain.registry.model.index.ForeignKeyIndex;
+import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyContactIndex;
+import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
+import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
+import com.google.domain.registry.model.transfer.TransferData.TransferServerApproveEntity;
+import com.google.domain.registry.request.Action;
+import com.google.domain.registry.request.Response;
+import com.google.domain.registry.util.FormattingLogger;
+
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.Ref;
+
+import org.joda.time.DateTime;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+
+/**
+ * A mapreduce to verify integrity of entities in Datastore.
+ *
+ *
Specifically this validates all of the following system invariants that are expected to hold
+ * true for all {@link EppResource} entities and their related indexes:
+ *
+ *
All {@link Key} and {@link Ref} fields (including nested ones) point to entities that
+ * exist.
+ *
There is exactly one {@link EppResourceIndex} pointing to each {@link EppResource}.
+ *
All contacts, hosts, and domains, when grouped by foreign key, have at most one active
+ * resource, and exactly one {@link ForeignKeyIndex} of the appropriate type, which points to
+ * the active resource if one exists, or to the most recently deleted resource if not. The
+ * foreignKey and deletionTime fields on the index must also match the respective resource(s).
+ *
All domain applications, when grouped by foreign key, have exactly one
+ * {@link DomainApplicationIndex} that links to all of them, and has a matching
+ * fullyQualifiedDomainName.
+ *
+ */
+@Action(path = "/_dr/task/verifyEntityIntegrity")
+public class VerifyEntityIntegrityAction implements Runnable {
+
+ @VisibleForTesting
+ static final FormattingLogger logger = getLoggerForCallerClass();
+ private static final int NUM_SHARDS = 200;
+ private static final ImmutableSet> RESOURCE_CLASSES =
+ ImmutableSet.>of(
+ ForeignKeyDomainIndex.class,
+ DomainApplicationIndex.class,
+ ForeignKeyHostIndex.class,
+ ForeignKeyContactIndex.class,
+ DomainBase.class,
+ HostResource.class,
+ ContactResource.class);
+
+ static final String KIND_CONTACT_RESOURCE = getKind(ContactResource.class);
+ static final String KIND_CONTACT_INDEX = getKind(ForeignKeyContactIndex.class);
+ static final String KIND_DOMAIN_APPLICATION_INDEX = getKind(DomainApplicationIndex.class);
+ static final String KIND_DOMAIN_BASE_RESOURCE = getKind(DomainBase.class);
+ static final String KIND_DOMAIN_INDEX = getKind(ForeignKeyDomainIndex.class);
+ static final String KIND_EPPRESOURCE_INDEX = getKind(EppResourceIndex.class);
+ static final String KIND_HOST_RESOURCE = getKind(HostResource.class);
+ static final String KIND_HOST_INDEX = getKind(ForeignKeyHostIndex.class);
+
+ @Inject MapreduceRunner mrRunner;
+ @Inject Response response;
+ @Inject VerifyEntityIntegrityAction() {}
+
+ @Override
+ public void run() {
+ response.sendJavaScriptRedirect(createJobPath(mrRunner
+ .setJobName("Verify entity integrity")
+ .setModuleName("backend")
+ .setDefaultReduceShards(NUM_SHARDS)
+ .runMapreduce(
+ new VerifyEntityIntegrityMapper(),
+ new VerifyEntityIntegrityReducer(),
+ getInputs())));
+ }
+
+ private static ImmutableSet> getInputs() {
+ ImmutableSet.Builder> builder =
+ new ImmutableSet.Builder>()
+ .add(EppResourceInputs.createIndexInput());
+ for (Class> clazz : RESOURCE_CLASSES) {
+ builder.add(new DatastoreKeyInput(getKind(clazz), NUM_SHARDS));
+ }
+ return builder.build();
+ }
+
+ private static enum EntityKind {
+ DOMAIN,
+ APPLICATION,
+ CONTACT,
+ HOST
+ }
+
+ private static class FkAndKind implements Serializable {
+
+ private static final long serialVersionUID = -8466899721968889534L;
+
+ public String foreignKey;
+ public EntityKind kind;
+
+ public static FkAndKind create(EntityKind kind, String foreignKey) {
+ FkAndKind instance = new FkAndKind();
+ instance.kind = kind;
+ instance.foreignKey = foreignKey;
+ return instance;
+ }
+ }
+
+ /**
+ * Mapper that checks validity of references on all resources and outputs key/value pairs used to
+ * check integrity of foreign key entities.
+ */
+ public static class VerifyEntityIntegrityMapper
+ extends Mapper