diff --git a/java/com/google/domain/registry/env/common/backend/WEB-INF/web.xml b/java/com/google/domain/registry/env/common/backend/WEB-INF/web.xml index c03734075..179562033 100644 --- a/java/com/google/domain/registry/env/common/backend/WEB-INF/web.xml +++ b/java/com/google/domain/registry/env/common/backend/WEB-INF/web.xml @@ -119,6 +119,12 @@ /_dr/dnsRefresh + + + backend-servlet + /_dr/task/verifyEntityIntegrity + + Exports a datastore backup snapshot to GCS. Export snapshot to GCS diff --git a/java/com/google/domain/registry/module/backend/BUILD b/java/com/google/domain/registry/module/backend/BUILD index 15fafa579..84471d7d3 100644 --- a/java/com/google/domain/registry/module/backend/BUILD +++ b/java/com/google/domain/registry/module/backend/BUILD @@ -25,6 +25,7 @@ java_library( "//java/com/google/domain/registry/keyring/api", "//java/com/google/domain/registry/mapreduce", "//java/com/google/domain/registry/model", + "//java/com/google/domain/registry/monitoring/whitebox", "//java/com/google/domain/registry/rde", "//java/com/google/domain/registry/request", "//java/com/google/domain/registry/request:modules", diff --git a/java/com/google/domain/registry/module/backend/BackendRequestComponent.java b/java/com/google/domain/registry/module/backend/BackendRequestComponent.java index 5ac5d0b39..a99a94884 100644 --- a/java/com/google/domain/registry/module/backend/BackendRequestComponent.java +++ b/java/com/google/domain/registry/module/backend/BackendRequestComponent.java @@ -39,6 +39,7 @@ import com.google.domain.registry.flows.async.DeleteContactResourceAction; import com.google.domain.registry.flows.async.DeleteHostResourceAction; import com.google.domain.registry.flows.async.DnsRefreshForHostRenameAction; import com.google.domain.registry.mapreduce.MapreduceModule; +import com.google.domain.registry.monitoring.whitebox.VerifyEntityIntegrityAction; import com.google.domain.registry.rde.BrdaCopyAction; import com.google.domain.registry.rde.RdeModule; import com.google.domain.registry.rde.RdeReportAction; @@ -101,4 +102,5 @@ interface BackendRequestComponent { TmchDnlAction tmchDnlAction(); TmchSmdrlAction tmchSmdrlAction(); WriteDnsAction writeDnsAction(); + VerifyEntityIntegrityAction verifyEntityIntegrityAction(); } diff --git a/java/com/google/domain/registry/monitoring/whitebox/BUILD b/java/com/google/domain/registry/monitoring/whitebox/BUILD index 6c2ae6268..09a7d60ab 100644 --- a/java/com/google/domain/registry/monitoring/whitebox/BUILD +++ b/java/com/google/domain/registry/monitoring/whitebox/BUILD @@ -19,12 +19,15 @@ java_library( "//java/com/google/domain/registry/bigquery", "//java/com/google/domain/registry/config", "//java/com/google/domain/registry/mapreduce", + "//java/com/google/domain/registry/mapreduce/inputs", "//java/com/google/domain/registry/model", + "//java/com/google/domain/registry/request", "//java/com/google/domain/registry/util", "//third_party/java/appengine:appengine-api", "//third_party/java/appengine_mapreduce2:appengine_mapreduce", "//third_party/java/joda_time", "//third_party/java/jsr305_annotations", + "//third_party/java/jsr330_inject", "//third_party/java/objectify:objectify-v4_1", "//third_party/java/servlet/servlet_api", ], diff --git a/java/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityAction.java b/java/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityAction.java new file mode 100644 index 000000000..78583854e --- /dev/null +++ b/java/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityAction.java @@ -0,0 +1,511 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.domain.registry.monitoring.whitebox; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.Iterables.getOnlyElement; +import static com.google.domain.registry.model.EppResourceUtils.isActive; +import static com.google.domain.registry.model.ofy.ObjectifyService.ofy; +import static com.google.domain.registry.util.DateTimeUtils.END_OF_TIME; +import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME; +import static com.google.domain.registry.util.DateTimeUtils.earliestOf; +import static com.google.domain.registry.util.DateTimeUtils.isBeforeOrAt; +import static com.google.domain.registry.util.DateTimeUtils.latestOf; +import static com.google.domain.registry.util.FormattingLogger.getLoggerForCallerClass; +import static com.google.domain.registry.util.PipelineUtils.createJobPath; +import static com.googlecode.objectify.Key.getKind; +import static org.joda.time.DateTimeZone.UTC; + +import com.google.appengine.tools.mapreduce.Input; +import com.google.appengine.tools.mapreduce.Mapper; +import com.google.appengine.tools.mapreduce.Reducer; +import com.google.appengine.tools.mapreduce.ReducerInput; +import com.google.appengine.tools.mapreduce.inputs.DatastoreKeyInput; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.domain.registry.mapreduce.MapreduceRunner; +import com.google.domain.registry.mapreduce.inputs.EppResourceInputs; +import com.google.domain.registry.model.EppResource; +import com.google.domain.registry.model.ImmutableObject; +import com.google.domain.registry.model.contact.ContactResource; +import com.google.domain.registry.model.domain.DomainApplication; +import com.google.domain.registry.model.domain.DomainBase; +import com.google.domain.registry.model.domain.DomainResource; +import com.google.domain.registry.model.domain.GracePeriod; +import com.google.domain.registry.model.domain.ReferenceUnion; +import com.google.domain.registry.model.host.HostResource; +import com.google.domain.registry.model.index.DomainApplicationIndex; +import com.google.domain.registry.model.index.EppResourceIndex; +import com.google.domain.registry.model.index.ForeignKeyIndex; +import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyContactIndex; +import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex; +import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex; +import com.google.domain.registry.model.transfer.TransferData.TransferServerApproveEntity; +import com.google.domain.registry.request.Action; +import com.google.domain.registry.request.Response; +import com.google.domain.registry.util.FormattingLogger; + +import com.googlecode.objectify.Key; +import com.googlecode.objectify.Ref; + +import org.joda.time.DateTime; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; +import javax.inject.Inject; + +/** + * A mapreduce to verify integrity of entities in Datastore. + * + *

Specifically this validates all of the following system invariants that are expected to hold + * true for all {@link EppResource} entities and their related indexes: + *

+ */ +@Action(path = "/_dr/task/verifyEntityIntegrity") +public class VerifyEntityIntegrityAction implements Runnable { + + @VisibleForTesting + static final FormattingLogger logger = getLoggerForCallerClass(); + private static final int NUM_SHARDS = 200; + private static final ImmutableSet> RESOURCE_CLASSES = + ImmutableSet.>of( + ForeignKeyDomainIndex.class, + DomainApplicationIndex.class, + ForeignKeyHostIndex.class, + ForeignKeyContactIndex.class, + DomainBase.class, + HostResource.class, + ContactResource.class); + + static final String KIND_CONTACT_RESOURCE = getKind(ContactResource.class); + static final String KIND_CONTACT_INDEX = getKind(ForeignKeyContactIndex.class); + static final String KIND_DOMAIN_APPLICATION_INDEX = getKind(DomainApplicationIndex.class); + static final String KIND_DOMAIN_BASE_RESOURCE = getKind(DomainBase.class); + static final String KIND_DOMAIN_INDEX = getKind(ForeignKeyDomainIndex.class); + static final String KIND_EPPRESOURCE_INDEX = getKind(EppResourceIndex.class); + static final String KIND_HOST_RESOURCE = getKind(HostResource.class); + static final String KIND_HOST_INDEX = getKind(ForeignKeyHostIndex.class); + + @Inject MapreduceRunner mrRunner; + @Inject Response response; + @Inject VerifyEntityIntegrityAction() {} + + @Override + public void run() { + response.sendJavaScriptRedirect(createJobPath(mrRunner + .setJobName("Verify entity integrity") + .setModuleName("backend") + .setDefaultReduceShards(NUM_SHARDS) + .runMapreduce( + new VerifyEntityIntegrityMapper(), + new VerifyEntityIntegrityReducer(), + getInputs()))); + } + + private static ImmutableSet> getInputs() { + ImmutableSet.Builder> builder = + new ImmutableSet.Builder>() + .add(EppResourceInputs.createIndexInput()); + for (Class clazz : RESOURCE_CLASSES) { + builder.add(new DatastoreKeyInput(getKind(clazz), NUM_SHARDS)); + } + return builder.build(); + } + + private static enum EntityKind { + DOMAIN, + APPLICATION, + CONTACT, + HOST + } + + private static class FkAndKind implements Serializable { + + private static final long serialVersionUID = -8466899721968889534L; + + public String foreignKey; + public EntityKind kind; + + public static FkAndKind create(EntityKind kind, String foreignKey) { + FkAndKind instance = new FkAndKind(); + instance.kind = kind; + instance.foreignKey = foreignKey; + return instance; + } + } + + /** + * Mapper that checks validity of references on all resources and outputs key/value pairs used to + * check integrity of foreign key entities. + */ + public static class VerifyEntityIntegrityMapper + extends Mapper> { + + private static final long serialVersionUID = -8881987421971102016L; + + public VerifyEntityIntegrityMapper() {} + + @Override + public final void map(Object keyOrEntity) { + try { + // We use different inputs, some that return keys and some that return entities. Load any + // keys that we get so we're dealing only with entities. + if (keyOrEntity instanceof com.google.appengine.api.datastore.Key) { + Key key = Key.create((com.google.appengine.api.datastore.Key) keyOrEntity); + keyOrEntity = ofy().load().key(key).now(); + } + mapEntity(keyOrEntity); + } catch (Exception e) { + // Log and swallow so that the mapreduce doesn't abort on first error. + logger.severefmt(e, "Integrity error found while parsing entity: %s", keyOrEntity); + } + } + + private void mapEntity(Object entity) { + if (entity instanceof EppResource) { + mapEppResource((EppResource) entity); + } else if (entity instanceof ForeignKeyIndex) { + mapForeignKeyIndex((ForeignKeyIndex) entity); + } else if (entity instanceof DomainApplicationIndex) { + mapDomainApplicationIndex((DomainApplicationIndex) entity); + } else if (entity instanceof EppResourceIndex) { + mapEppResourceIndex((EppResourceIndex) entity); + } else { + throw new IllegalStateException(String.format("Unknown entity in mapper: %s", entity)); + } + } + + private void mapEppResource(EppResource resource) { + if (resource instanceof DomainBase) { + DomainBase domainBase = (DomainBase) resource; + Key key = Key.create(domainBase); + verifyExistence(key, bustUnions(domainBase.getReferencedContacts())); + verifyExistence(key, bustUnions(domainBase.getNameservers())); + verifyExistence(key, domainBase.getTransferData().getServerApproveAutorenewEvent()); + verifyExistence(key, domainBase.getTransferData().getServerApproveAutorenewPollMessage()); + verifyExistence(key, domainBase.getTransferData().getServerApproveBillingEvent()); + verifyExistence(key, FluentIterable + .from(domainBase.getTransferData().getServerApproveEntities()) + .transform( + new Function, + Key>() { + @SuppressWarnings("unchecked") + @Override + public Key apply( + Key key) { + return (Key) key; + }}) + .toSet()); + if (domainBase instanceof DomainApplication) { + getContext().incrementCounter("domain applications"); + DomainApplication application = (DomainApplication) domainBase; + emit( + FkAndKind.create(EntityKind.APPLICATION, application.getFullyQualifiedDomainName()), + Key.create(application)); + } else if (domainBase instanceof DomainResource) { + getContext().incrementCounter("domain resources"); + DomainResource domain = (DomainResource) domainBase; + verifyExistence(key, domain.getApplication()); + verifyExistence(key, domain.getAutorenewBillingEvent()); + verifyExistence(key, domain.getAutorenewPollMessage()); + for (GracePeriod gracePeriod : domain.getGracePeriods()) { + verifyExistence(key, gracePeriod.getOneTimeBillingEvent()); + verifyExistence(key, gracePeriod.getRecurringBillingEvent()); + } + emit( + FkAndKind.create(EntityKind.DOMAIN, domain.getFullyQualifiedDomainName()), + Key.create(domain)); + } + } else if (resource instanceof ContactResource) { + getContext().incrementCounter("contact resources"); + ContactResource contact = (ContactResource) resource; + emit( + FkAndKind.create(EntityKind.CONTACT, contact.getContactId()), + Key.create(contact)); + } else if (resource instanceof HostResource) { + getContext().incrementCounter("host resources"); + HostResource host = (HostResource) resource; + verifyExistence(Key.create(host), host.getSuperordinateDomain()); + emit( + FkAndKind.create(EntityKind.HOST, host.getFullyQualifiedHostName()), + Key.create(host)); + } else { + throw new IllegalStateException( + String.format("EppResource with unknown type: %s", resource)); + } + } + + private void mapForeignKeyIndex(ForeignKeyIndex fki) { + @SuppressWarnings("cast") + EppResource resource = verifyExistence(Key.create(fki), fki.getReference()); + checkState( + fki.getForeignKey().equals(resource.getForeignKey()), + "Foreign key index %s points to EppResource with different foreign key: %s", + fki, + resource); + if (resource instanceof DomainResource) { + getContext().incrementCounter("domain foreign key indexes"); + emit(FkAndKind.create(EntityKind.DOMAIN, resource.getForeignKey()), Key.create(fki)); + } else if (resource instanceof ContactResource) { + getContext().incrementCounter("contact foreign key indexes"); + emit(FkAndKind.create(EntityKind.CONTACT, resource.getForeignKey()), Key.create(fki)); + } else if (resource instanceof HostResource) { + getContext().incrementCounter("host foreign key indexes"); + emit(FkAndKind.create(EntityKind.HOST, resource.getForeignKey()), Key.create(fki)); + } else { + throw new IllegalStateException( + String.format( + "Foreign key index %s points to EppResource of unknown type: %s", fki, resource)); + } + } + + private void mapDomainApplicationIndex(DomainApplicationIndex dai) { + getContext().incrementCounter("domain application indexes"); + for (Ref ref : dai.getReferences()) { + DomainApplication application = verifyExistence(Key.create(dai), ref); + checkState( + dai.getFullyQualifiedDomainName().equals(application.getFullyQualifiedDomainName()), + "Domain application index %s points to application with different domain name: %s", + dai, + application); + emit( + FkAndKind.create(EntityKind.APPLICATION, application.getFullyQualifiedDomainName()), + Key.create(application)); + } + } + + private void mapEppResourceIndex(EppResourceIndex eri) { + @SuppressWarnings("cast") + EppResource resource = verifyExistence(Key.create(eri), eri.getReference()); + if (resource instanceof DomainResource) { + getContext().incrementCounter("domain EPP resource indexes"); + emit(FkAndKind.create(EntityKind.DOMAIN, resource.getForeignKey()), Key.create(eri)); + } else if (resource instanceof ContactResource) { + getContext().incrementCounter("contact EPP resource indexes"); + emit( + FkAndKind.create(EntityKind.CONTACT, resource.getForeignKey()), Key.create(eri)); + } else if (resource instanceof HostResource) { + getContext().incrementCounter("host EPP resource indexes"); + emit(FkAndKind.create(EntityKind.HOST, resource.getForeignKey()), Key.create(eri)); + } else { + throw new IllegalStateException( + String.format( + "EPP resource index %s points to resource of unknown type: %s", eri, resource)); + } + } + + private static void verifyExistence(Key source, Set> keys) { + Set> missingEntityKeys = Sets.difference(keys, ofy().load().keys(keys).keySet()); + checkState( + missingEntityKeys.isEmpty(), + "Existing entity %s referenced entities that do not exist: %s", + source, + missingEntityKeys); + } + + @Nullable + private static E verifyExistence(Key source, @Nullable Ref target) { + if (target == null) { + return null; + } + return verifyExistence(source, target.getKey()); + } + + @Nullable + private static E verifyExistence(Key source, @Nullable Key target) { + if (target == null) { + return null; + } + E entity = ofy().load().key(target).now(); + checkState(entity != null, + "Existing entity %s referenced entity that does not exist: %s", + source, + target); + return entity; + } + + private static Set> bustUnions( + Iterable> unions) { + return FluentIterable + .from(unions) + .transform(new Function, Key>() { + @Override + public Key apply(ReferenceUnion union) { + return union.getLinked().getKey(); + }}) + .toSet(); + } + } + + /** Reducer that checks integrity of foreign key entities. */ + public static class VerifyEntityIntegrityReducer + extends Reducer, Void> { + + private static final long serialVersionUID = -8531280188397051521L; + + @SuppressWarnings("unchecked") + @Override + public void reduce(FkAndKind fkAndKind, ReducerInput> keys) { + try { + reduceKeys(fkAndKind, keys); + } catch (Exception e) { + // Log and swallow so that the mapreduce doesn't abort on first error. + logger.severefmt( + e, "Integrity error found while checking foreign key contraints for: %s", fkAndKind); + } + } + + private void reduceKeys( + FkAndKind fkAndKind, ReducerInput> keys) { + switch (fkAndKind.kind) { + case APPLICATION: + getContext().incrementCounter("domain applications"); + checkIndexes( + keys, + fkAndKind.foreignKey, + KIND_DOMAIN_BASE_RESOURCE, + KIND_DOMAIN_APPLICATION_INDEX, + false); + break; + case CONTACT: + getContext().incrementCounter("contact resources"); + checkIndexes(keys, fkAndKind.foreignKey, KIND_CONTACT_RESOURCE, KIND_CONTACT_INDEX, true); + break; + case DOMAIN: + getContext().incrementCounter("domain resources"); + checkIndexes( + keys, fkAndKind.foreignKey, KIND_DOMAIN_BASE_RESOURCE, KIND_DOMAIN_INDEX, true); + break; + case HOST: + getContext().incrementCounter("host resources"); + checkIndexes(keys, fkAndKind.foreignKey, KIND_HOST_RESOURCE, KIND_HOST_INDEX, true); + break; + default: + throw new IllegalStateException( + String.format("Unknown type of foreign key %s", fkAndKind.kind)); + } + } + + @SuppressWarnings("unchecked") + private void checkIndexes( + Iterator> keys, + String foreignKey, + String resourceKind, + String foreignKeyIndexKind, + boolean thereCanBeOnlyOne) { + List> resources = new ArrayList<>(); + List> foreignKeyIndexes = new ArrayList<>(); + List> eppResourceIndexes = new ArrayList<>(); + while (keys.hasNext()) { + Key key = keys.next(); + if (key.getKind().equals(resourceKind)) { + resources.add((Key) key); + } else if (key.getKind().equals(foreignKeyIndexKind)) { + foreignKeyIndexes.add((Key) key); + } else if (key.getKind().equals(KIND_EPPRESOURCE_INDEX)) { + eppResourceIndexes.add((Key) key); + } else { + throw new IllegalStateException( + String.format( + "While processing links to foreign key %s of type %s, found unknown key: %s", + foreignKey, + resourceKind, + key)); + } + } + checkState( + foreignKeyIndexes.size() == 1, + String.format( + "Should have found exactly 1 foreign key index for %s, instead found %d: %s", + foreignKey, + foreignKeyIndexes.size(), + foreignKeyIndexes)); + checkState( + !resources.isEmpty(), + "Foreign key index %s exists, but no matching EPP resources found", + getOnlyElement(foreignKeyIndexes)); + checkState(eppResourceIndexes.size() == 1, + "Should have found exactly 1 EPP resource index for %s, instead found: %s", + foreignKey, + eppResourceIndexes); + if (thereCanBeOnlyOne) { + verifyOnlyOneActiveResource(foreignKey, resources, foreignKeyIndexes); + } + } + + private void verifyOnlyOneActiveResource( + String foreignKey, List> resources, List> foreignKeyIndexes) { + DateTime now = DateTime.now(UTC); + DateTime oldestActive = END_OF_TIME; + DateTime mostRecentInactive = START_OF_TIME; + List activeResources = new ArrayList(); + Collection allResources = ofy().load().keys(resources).values(); + ForeignKeyIndex fki = + (ForeignKeyIndex) ofy().load().key(getOnlyElement(foreignKeyIndexes)).now(); + for (R resource : allResources) { + if (isActive(resource, now)) { + activeResources.add(resource); + oldestActive = earliestOf(oldestActive, resource.getCreationTime()); + } else { + mostRecentInactive = latestOf(mostRecentInactive, resource.getDeletionTime()); + } + } + if (activeResources.isEmpty()) { + checkState( + fki.getDeletionTime().isEqual(mostRecentInactive), + "Deletion time on foreign key index %s doesn't match" + + " most recently deleted resource from: %s", + fki, + allResources); + } else { + checkState( + activeResources.size() <= 1, + "Found multiple active resources with foreign key %s: %s", + foreignKey, + activeResources); + checkState( + fki.getDeletionTime().isEqual(END_OF_TIME), + "Deletion time on foreign key index %s doesn't match active resource: %s", + fki, + getOnlyElement(activeResources)); + checkState( + isBeforeOrAt(mostRecentInactive, oldestActive), + "Found inactive resource that is more recent than active resource in: %s", + allResources); + } + + } + } +} diff --git a/java/com/google/domain/registry/util/FormattingLogger.java b/java/com/google/domain/registry/util/FormattingLogger.java index 82b60ca31..bb5f7a392 100644 --- a/java/com/google/domain/registry/util/FormattingLogger.java +++ b/java/com/google/domain/registry/util/FormattingLogger.java @@ -19,6 +19,7 @@ import static java.util.Arrays.asList; import com.google.common.base.Predicate; import com.google.common.collect.FluentIterable; +import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.Logger; @@ -102,4 +103,12 @@ public class FormattingLogger { public void severefmt(Throwable cause, String fmt, Object... args) { log(Level.SEVERE, cause, String.format(fmt, args)); } + + public void addHandler(Handler handler) { + logger.addHandler(handler); + } + + public void removeHandler(Handler handler) { + logger.removeHandler(handler); + } } diff --git a/javatests/com/google/domain/registry/monitoring/whitebox/BUILD b/javatests/com/google/domain/registry/monitoring/whitebox/BUILD index 7683244d1..a56189e8d 100644 --- a/javatests/com/google/domain/registry/monitoring/whitebox/BUILD +++ b/javatests/com/google/domain/registry/monitoring/whitebox/BUILD @@ -15,11 +15,14 @@ java_library( "//java/com/google/common/base", "//java/com/google/common/collect", "//java/com/google/common/net", + "//java/com/google/common/testing", "//java/com/google/domain/registry/bigquery", "//java/com/google/domain/registry/config", + "//java/com/google/domain/registry/mapreduce", "//java/com/google/domain/registry/model", "//java/com/google/domain/registry/monitoring/whitebox", "//javatests/com/google/domain/registry/testing", + "//javatests/com/google/domain/registry/testing/mapreduce", "//third_party/java/appengine:appengine-api-testonly", "//third_party/java/appengine:appengine-stubs", "//third_party/java/appengine:appengine-testing", diff --git a/javatests/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityActionTest.java b/javatests/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityActionTest.java new file mode 100644 index 000000000..8786ccb80 --- /dev/null +++ b/javatests/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityActionTest.java @@ -0,0 +1,87 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.domain.registry.monitoring.whitebox; + +import static com.google.domain.registry.testing.DatastoreHelper.createTld; +import static com.google.domain.registry.testing.DatastoreHelper.deleteResource; +import static com.google.domain.registry.testing.DatastoreHelper.persistActiveDomain; +import static com.google.domain.registry.testing.LogsSubject.assertAboutLogs; +import static java.util.logging.Level.SEVERE; + +import com.google.common.base.Optional; +import com.google.common.testing.TestLogHandler; +import com.google.domain.registry.mapreduce.MapreduceRunner; +import com.google.domain.registry.model.domain.DomainResource; +import com.google.domain.registry.model.index.ForeignKeyIndex; +import com.google.domain.registry.testing.FakeResponse; +import com.google.domain.registry.testing.mapreduce.MapreduceTestCase; + +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link VerifyEntityIntegrityAction}. */ +@RunWith(JUnit4.class) +public class VerifyEntityIntegrityActionTest + extends MapreduceTestCase { + + private TestLogHandler handler; + + @Before + public void before() { + createTld("tld"); + + action = new VerifyEntityIntegrityAction(); + handler = new TestLogHandler(); + VerifyEntityIntegrityAction.logger.addHandler(handler); + action.mrRunner = new MapreduceRunner(Optional.of(2), Optional.of(2)); + action.response = new FakeResponse(); + } + + @After + public void after() { + VerifyEntityIntegrityAction.logger.removeHandler(handler); + } + + private void runMapreduce() throws Exception { + action.run(); + executeTasksUntilEmpty("mapreduce"); + } + + @Test + public void test_singleDomain_noBadInvariants() throws Exception { + persistActiveDomain("ninetails.tld"); + runMapreduce(); + assertAboutLogs().that(handler).hasNoLogsAtLevel(SEVERE); + } + + @Test + public void test_singleDomain_missingFki() throws Exception { + persistActiveDomain("ninetails.tld"); + ForeignKeyIndex fki = + ForeignKeyIndex.load(DomainResource.class, "ninetails.tld", DateTime.now(DateTimeZone.UTC)); + deleteResource(fki); + runMapreduce(); + // TODO(mcilwain): Check for exception message here. + assertAboutLogs() + .that(handler) + .hasLogAtLevelWithMessage( + SEVERE, "Integrity error found while checking foreign key contraints"); + } +} diff --git a/javatests/com/google/domain/registry/testing/BUILD b/javatests/com/google/domain/registry/testing/BUILD index 0025c6e11..f2c4f2ded 100644 --- a/javatests/com/google/domain/registry/testing/BUILD +++ b/javatests/com/google/domain/registry/testing/BUILD @@ -24,6 +24,7 @@ java_library( "//java/com/google/common/collect", "//java/com/google/common/io", "//java/com/google/common/net", + "//java/com/google/common/testing", "//java/com/google/common/util/concurrent", "//java/com/google/domain/registry/config", "//java/com/google/domain/registry/dns:constants", diff --git a/javatests/com/google/domain/registry/testing/LogsSubject.java b/javatests/com/google/domain/registry/testing/LogsSubject.java new file mode 100644 index 000000000..cb498c845 --- /dev/null +++ b/javatests/com/google/domain/registry/testing/LogsSubject.java @@ -0,0 +1,67 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.domain.registry.testing; + +import static com.google.common.truth.Truth.assertAbout; + +import com.google.common.testing.TestLogHandler; +import com.google.common.truth.AbstractVerb.DelegatedVerb; +import com.google.common.truth.FailureStrategy; +import com.google.common.truth.Subject; +import com.google.domain.registry.testing.TruthChainer.And; + +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +/** Utility methods for asserting things about logging {@link Handler} instances. */ +public class LogsSubject extends Subject { + + /** A factory for instances of this subject. */ + private static class SubjectFactory + extends ReflectiveSubjectFactory {} + + public LogsSubject(FailureStrategy strategy, TestLogHandler subject) { + super(strategy, subject); + } + + public And hasNoLogsAtLevel(Level level) { + for (LogRecord log : getSubject().getStoredLogRecords()) { + if (log.getLevel() == level) { + failWithRawMessage( + "Not true that there are no logs at level %s. Found <%s>.", level, log.getMessage()); + } + } + return new And<>(this); + } + + public And hasLogAtLevelWithMessage(Level level, String message) { + boolean found = false; + for (LogRecord log : getSubject().getStoredLogRecords()) { + if (log.getLevel() == level && log.getMessage().contains(message)) { + found = true; + break; + } + } + if (!found) { + failWithRawMessage("Found no logs at level %s with message %s.", level, message); + } + return new And<>(this); + } + + public static DelegatedVerb assertAboutLogs() { + return assertAbout(new SubjectFactory()); + } +}