mirror of
https://github.com/google/nomulus.git
synced 2025-05-13 16:07:15 +02:00
Delete the verify entity integrity mapreduce
We never really used it and it'll be obsolete come Registry 3.0 anyway. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=213274520
This commit is contained in:
mcilwain
Ben McIlwain
parent
f4d392eed9
commit
1d134cdd3f
11 changed files with 0 additions and 1220 deletions
|
|
@ -1,38 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.batch;
|
|
||||||
|
|
||||||
import dagger.Component;
|
|
||||||
import google.registry.bigquery.BigqueryModule;
|
|
||||||
import google.registry.config.RegistryConfig.ConfigModule;
|
|
||||||
import google.registry.monitoring.whitebox.WhiteboxModule;
|
|
||||||
import google.registry.request.Modules.DatastoreServiceModule;
|
|
||||||
import google.registry.util.SystemSleeper.SystemSleeperModule;
|
|
||||||
import javax.inject.Singleton;
|
|
||||||
|
|
||||||
/** Dagger component with instance lifetime for batch package. */
|
|
||||||
@Singleton
|
|
||||||
@Component(
|
|
||||||
modules = {
|
|
||||||
BatchModule.class,
|
|
||||||
BigqueryModule.class,
|
|
||||||
ConfigModule.class,
|
|
||||||
DatastoreServiceModule.class,
|
|
||||||
SystemSleeperModule.class,
|
|
||||||
WhiteboxModule.class
|
|
||||||
})
|
|
||||||
interface BatchComponent {
|
|
||||||
VerifyEntityIntegrityStreamerFactory verifyEntityIntegrityStreamerFactory();
|
|
||||||
}
|
|
||||||
|
|
@ -24,14 +24,10 @@ import static google.registry.request.RequestParameters.extractRequiredDatetimeP
|
||||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||||
import static google.registry.request.RequestParameters.extractSetOfDatetimeParameters;
|
import static google.registry.request.RequestParameters.extractSetOfDatetimeParameters;
|
||||||
|
|
||||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
|
||||||
import com.google.common.collect.ImmutableList;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
import com.google.common.collect.ImmutableSet;
|
||||||
import com.googlecode.objectify.Key;
|
import com.googlecode.objectify.Key;
|
||||||
import dagger.Module;
|
import dagger.Module;
|
||||||
import dagger.Provides;
|
import dagger.Provides;
|
||||||
import dagger.multibindings.IntoMap;
|
|
||||||
import dagger.multibindings.StringKey;
|
|
||||||
import google.registry.model.ImmutableObject;
|
import google.registry.model.ImmutableObject;
|
||||||
import google.registry.request.Parameter;
|
import google.registry.request.Parameter;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
|
@ -44,13 +40,6 @@ import org.joda.time.DateTime;
|
||||||
@Module
|
@Module
|
||||||
public class BatchModule {
|
public class BatchModule {
|
||||||
|
|
||||||
@Provides
|
|
||||||
@IntoMap
|
|
||||||
@StringKey(EntityIntegrityAlertsSchema.TABLE_ID)
|
|
||||||
static ImmutableList<TableFieldSchema> provideEntityIntegrityAlertsSchema() {
|
|
||||||
return EntityIntegrityAlertsSchema.SCHEMA_FIELDS;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter("jobName")
|
@Parameter("jobName")
|
||||||
static Optional<String> provideJobName(HttpServletRequest req) {
|
static Optional<String> provideJobName(HttpServletRequest req) {
|
||||||
|
|
|
||||||
|
|
@ -1,42 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.batch;
|
|
||||||
|
|
||||||
import static google.registry.bigquery.BigqueryUtils.FieldType.STRING;
|
|
||||||
import static google.registry.bigquery.BigqueryUtils.FieldType.TIMESTAMP;
|
|
||||||
|
|
||||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
|
||||||
import com.google.common.collect.ImmutableList;
|
|
||||||
|
|
||||||
/** The Bigquery schema for the entity integrity alerts table. */
|
|
||||||
final class EntityIntegrityAlertsSchema {
|
|
||||||
|
|
||||||
static final String DATASET = "entity_integrity";
|
|
||||||
static final String TABLE_ID = "alerts";
|
|
||||||
static final String FIELD_SCANTIME = "scanTime";
|
|
||||||
static final String FIELD_SOURCE = "source";
|
|
||||||
static final String FIELD_TARGET = "target";
|
|
||||||
static final String FIELD_MESSAGE = "message";
|
|
||||||
|
|
||||||
static final ImmutableList<TableFieldSchema> SCHEMA_FIELDS =
|
|
||||||
ImmutableList.of(
|
|
||||||
new TableFieldSchema().setName(FIELD_SCANTIME).setType(TIMESTAMP.name()),
|
|
||||||
new TableFieldSchema().setName(FIELD_SOURCE).setType(STRING.name()),
|
|
||||||
new TableFieldSchema().setName(FIELD_TARGET).setType(STRING.name()),
|
|
||||||
new TableFieldSchema().setName(FIELD_MESSAGE).setType(STRING.name()));
|
|
||||||
|
|
||||||
private EntityIntegrityAlertsSchema() {}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
@ -1,579 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.batch;
|
|
||||||
|
|
||||||
import static com.google.common.base.Preconditions.checkState;
|
|
||||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
|
||||||
import static com.google.common.collect.Iterables.getOnlyElement;
|
|
||||||
import static com.googlecode.objectify.Key.getKind;
|
|
||||||
import static google.registry.model.EppResourceUtils.isActive;
|
|
||||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
|
||||||
import static google.registry.request.Action.Method.POST;
|
|
||||||
import static google.registry.util.DateTimeUtils.END_OF_TIME;
|
|
||||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
|
||||||
import static google.registry.util.DateTimeUtils.earliestOf;
|
|
||||||
import static google.registry.util.DateTimeUtils.isAtOrAfter;
|
|
||||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
|
||||||
import static google.registry.util.PipelineUtils.createJobPath;
|
|
||||||
import static org.joda.time.DateTimeZone.UTC;
|
|
||||||
|
|
||||||
import com.google.appengine.tools.mapreduce.Input;
|
|
||||||
import com.google.appengine.tools.mapreduce.Mapper;
|
|
||||||
import com.google.appengine.tools.mapreduce.Reducer;
|
|
||||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
|
||||||
import com.google.appengine.tools.mapreduce.inputs.DatastoreKeyInput;
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import com.google.common.collect.Sets;
|
|
||||||
import com.google.common.flogger.FluentLogger;
|
|
||||||
import com.googlecode.objectify.Key;
|
|
||||||
import google.registry.mapreduce.MapreduceRunner;
|
|
||||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
|
||||||
import google.registry.model.EppResource;
|
|
||||||
import google.registry.model.ImmutableObject;
|
|
||||||
import google.registry.model.contact.ContactResource;
|
|
||||||
import google.registry.model.domain.DomainApplication;
|
|
||||||
import google.registry.model.domain.DomainBase;
|
|
||||||
import google.registry.model.domain.DomainResource;
|
|
||||||
import google.registry.model.domain.GracePeriod;
|
|
||||||
import google.registry.model.host.HostResource;
|
|
||||||
import google.registry.model.index.DomainApplicationIndex;
|
|
||||||
import google.registry.model.index.EppResourceIndex;
|
|
||||||
import google.registry.model.index.ForeignKeyIndex;
|
|
||||||
import google.registry.model.index.ForeignKeyIndex.ForeignKeyContactIndex;
|
|
||||||
import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
|
|
||||||
import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
|
|
||||||
import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
|
|
||||||
import google.registry.request.Action;
|
|
||||||
import google.registry.request.Response;
|
|
||||||
import google.registry.request.auth.Auth;
|
|
||||||
import google.registry.util.NonFinalForTesting;
|
|
||||||
import java.io.Serializable;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Iterator;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
|
||||||
import javax.annotation.Nullable;
|
|
||||||
import javax.inject.Inject;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A mapreduce to verify integrity of entities in Datastore.
|
|
||||||
*
|
|
||||||
* <p>Specifically this validates all of the following system invariants that are expected to hold
|
|
||||||
* true for all {@link EppResource} entities and their related indexes:
|
|
||||||
* <ul>
|
|
||||||
* <li>All {@link Key} fields (including nested ones) point to entities that exist.
|
|
||||||
* <li>There is exactly one {@link EppResourceIndex} pointing to each {@link EppResource}.
|
|
||||||
* <li>All contacts, hosts, and domains, when grouped by foreign key, have at most one active
|
|
||||||
* resource, and exactly one {@link ForeignKeyIndex} of the appropriate type, which points to
|
|
||||||
* the active resource if one exists, or to the most recently deleted resource if not. The
|
|
||||||
* foreignKey and deletionTime fields on the index must also match the respective resource(s).
|
|
||||||
* <li>All domain applications, when grouped by foreign key, have exactly one
|
|
||||||
* {@link DomainApplicationIndex} that links to all of them, and has a matching
|
|
||||||
* fullyQualifiedDomainName.
|
|
||||||
* </ul>
|
|
||||||
*/
|
|
||||||
@Action(
|
|
||||||
path = "/_dr/task/verifyEntityIntegrity",
|
|
||||||
method = POST,
|
|
||||||
auth = Auth.AUTH_INTERNAL_ONLY
|
|
||||||
)
|
|
||||||
public class VerifyEntityIntegrityAction implements Runnable {
|
|
||||||
|
|
||||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
|
||||||
private static final int NUM_SHARDS = 200;
|
|
||||||
@NonFinalForTesting
|
|
||||||
@VisibleForTesting
|
|
||||||
static BatchComponent component = DaggerBatchComponent.create();
|
|
||||||
private static final ImmutableSet<Class<?>> RESOURCE_CLASSES =
|
|
||||||
ImmutableSet.of(
|
|
||||||
ForeignKeyDomainIndex.class,
|
|
||||||
DomainApplicationIndex.class,
|
|
||||||
ForeignKeyHostIndex.class,
|
|
||||||
ForeignKeyContactIndex.class,
|
|
||||||
DomainBase.class,
|
|
||||||
HostResource.class,
|
|
||||||
ContactResource.class);
|
|
||||||
|
|
||||||
static final String KIND_CONTACT_RESOURCE = getKind(ContactResource.class);
|
|
||||||
static final String KIND_CONTACT_INDEX = getKind(ForeignKeyContactIndex.class);
|
|
||||||
static final String KIND_DOMAIN_APPLICATION_INDEX = getKind(DomainApplicationIndex.class);
|
|
||||||
static final String KIND_DOMAIN_BASE_RESOURCE = getKind(DomainBase.class);
|
|
||||||
static final String KIND_DOMAIN_INDEX = getKind(ForeignKeyDomainIndex.class);
|
|
||||||
static final String KIND_EPPRESOURCE_INDEX = getKind(EppResourceIndex.class);
|
|
||||||
static final String KIND_HOST_RESOURCE = getKind(HostResource.class);
|
|
||||||
static final String KIND_HOST_INDEX = getKind(ForeignKeyHostIndex.class);
|
|
||||||
|
|
||||||
@Inject MapreduceRunner mrRunner;
|
|
||||||
@Inject Response response;
|
|
||||||
@Inject VerifyEntityIntegrityAction() {}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
DateTime scanTime = DateTime.now(UTC);
|
|
||||||
response.sendJavaScriptRedirect(createJobPath(mrRunner
|
|
||||||
.setJobName("Verify entity integrity")
|
|
||||||
.setModuleName("backend")
|
|
||||||
.setDefaultReduceShards(NUM_SHARDS)
|
|
||||||
.runMapreduce(
|
|
||||||
new VerifyEntityIntegrityMapper(scanTime),
|
|
||||||
new VerifyEntityIntegrityReducer(scanTime),
|
|
||||||
getInputs())));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ImmutableSet<Input<?>> getInputs() {
|
|
||||||
ImmutableSet.Builder<Input<?>> builder =
|
|
||||||
new ImmutableSet.Builder<Input<?>>().add(EppResourceInputs.createIndexInput());
|
|
||||||
RESOURCE_CLASSES
|
|
||||||
.stream()
|
|
||||||
.map(clazz -> new DatastoreKeyInput(getKind(clazz), NUM_SHARDS))
|
|
||||||
.forEach(builder::add);
|
|
||||||
return builder.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The mapreduce key that the mapper outputs. Each {@link EppResource} has two different
|
|
||||||
* mapreduce keys that are output for it: one for its specific type (domain, application, host, or
|
|
||||||
* contact), which is used to check {@link ForeignKeyIndex} constraints, and one that is common
|
|
||||||
* for all EppResources, to check {@link EppResourceIndex} constraints.
|
|
||||||
*/
|
|
||||||
private enum EntityKind {
|
|
||||||
DOMAIN,
|
|
||||||
APPLICATION,
|
|
||||||
CONTACT,
|
|
||||||
HOST,
|
|
||||||
/**
|
|
||||||
* Used to verify 1-to-1 constraints between all types of EPP resources and their indexes.
|
|
||||||
*/
|
|
||||||
EPP_RESOURCE
|
|
||||||
}
|
|
||||||
|
|
||||||
private static class MapperKey implements Serializable {
|
|
||||||
|
|
||||||
private static final long serialVersionUID = 3222302549441420932L;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The relevant id for this mapper key, which is either the foreign key of the EppResource (for
|
|
||||||
* verifying foreign key indexes) or its repoId (for verifying EppResourceIndexes).
|
|
||||||
*/
|
|
||||||
public String id;
|
|
||||||
public EntityKind kind;
|
|
||||||
|
|
||||||
public static MapperKey create(EntityKind kind, String id) {
|
|
||||||
MapperKey instance = new MapperKey();
|
|
||||||
instance.kind = kind;
|
|
||||||
instance.id = id;
|
|
||||||
return instance;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Mapper that checks validity of references on all resources and outputs key/value pairs used to
|
|
||||||
* check integrity of foreign key entities.
|
|
||||||
*/
|
|
||||||
public static class VerifyEntityIntegrityMapper
|
|
||||||
extends Mapper<Object, MapperKey, Key<? extends ImmutableObject>> {
|
|
||||||
|
|
||||||
private static final long serialVersionUID = -5413882340475018051L;
|
|
||||||
private final DateTime scanTime;
|
|
||||||
|
|
||||||
private transient VerifyEntityIntegrityStreamer integrityStreamer;
|
|
||||||
|
|
||||||
// The integrityStreamer field must be marked as transient so that instances of the Mapper class
|
|
||||||
// can be serialized by the MapReduce framework. Thus, every time is used, lazily construct it
|
|
||||||
// if it doesn't exist yet.
|
|
||||||
private VerifyEntityIntegrityStreamer integrity() {
|
|
||||||
if (integrityStreamer == null) {
|
|
||||||
integrityStreamer = component.verifyEntityIntegrityStreamerFactory().create(scanTime);
|
|
||||||
}
|
|
||||||
return integrityStreamer;
|
|
||||||
}
|
|
||||||
|
|
||||||
public VerifyEntityIntegrityMapper(DateTime scanTime) {
|
|
||||||
this.scanTime = scanTime;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public final void map(Object keyOrEntity) {
|
|
||||||
try {
|
|
||||||
// We use different inputs, some that return keys and some that return entities. Load any
|
|
||||||
// keys that we get so we're dealing only with entities.
|
|
||||||
if (keyOrEntity instanceof com.google.appengine.api.datastore.Key) {
|
|
||||||
Key<?> key = Key.create((com.google.appengine.api.datastore.Key) keyOrEntity);
|
|
||||||
keyOrEntity = ofy().load().key(key).now();
|
|
||||||
}
|
|
||||||
mapEntity(keyOrEntity);
|
|
||||||
} catch (Throwable e) {
|
|
||||||
// Log and swallow so that the mapreduce doesn't abort on first error.
|
|
||||||
logger.atSevere().withCause(e).log(
|
|
||||||
"Exception while checking integrity of entity: %s", keyOrEntity);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void mapEntity(Object entity) {
|
|
||||||
if (entity instanceof EppResource) {
|
|
||||||
mapEppResource((EppResource) entity);
|
|
||||||
} else if (entity instanceof ForeignKeyIndex<?>) {
|
|
||||||
mapForeignKeyIndex((ForeignKeyIndex<?>) entity);
|
|
||||||
} else if (entity instanceof DomainApplicationIndex) {
|
|
||||||
mapDomainApplicationIndex((DomainApplicationIndex) entity);
|
|
||||||
} else if (entity instanceof EppResourceIndex) {
|
|
||||||
mapEppResourceIndex((EppResourceIndex) entity);
|
|
||||||
} else {
|
|
||||||
throw new IllegalStateException(
|
|
||||||
String.format("Unknown entity in integrity mapper: %s", entity));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void mapEppResource(EppResource resource) {
|
|
||||||
emit(MapperKey.create(EntityKind.EPP_RESOURCE, resource.getRepoId()), Key.create(resource));
|
|
||||||
if (resource instanceof DomainBase) {
|
|
||||||
DomainBase domainBase = (DomainBase) resource;
|
|
||||||
Key<?> key = Key.create(domainBase);
|
|
||||||
verifyExistence(key, domainBase.getReferencedContacts());
|
|
||||||
verifyExistence(key, domainBase.getNameservers());
|
|
||||||
if (domainBase instanceof DomainApplication) {
|
|
||||||
getContext().incrementCounter("domain applications");
|
|
||||||
DomainApplication application = (DomainApplication) domainBase;
|
|
||||||
emit(
|
|
||||||
MapperKey.create(EntityKind.APPLICATION, application.getFullyQualifiedDomainName()),
|
|
||||||
Key.create(application));
|
|
||||||
} else if (domainBase instanceof DomainResource) {
|
|
||||||
getContext().incrementCounter("domain resources");
|
|
||||||
DomainResource domain = (DomainResource) domainBase;
|
|
||||||
verifyExistence(key, domain.getTransferData().getServerApproveAutorenewEvent());
|
|
||||||
verifyExistence(key, domain.getTransferData().getServerApproveAutorenewPollMessage());
|
|
||||||
verifyExistence(key, domain.getTransferData().getServerApproveBillingEvent());
|
|
||||||
verifyExistence(
|
|
||||||
key,
|
|
||||||
domain
|
|
||||||
.getTransferData()
|
|
||||||
.getServerApproveEntities()
|
|
||||||
.stream()
|
|
||||||
.map(VerifyEntityIntegrityMapper::castTransferServerApproveEntityKey)
|
|
||||||
.collect(toImmutableSet()));
|
|
||||||
verifyExistence(key, domain.getApplication());
|
|
||||||
verifyExistence(key, domain.getAutorenewBillingEvent());
|
|
||||||
for (GracePeriod gracePeriod : domain.getGracePeriods()) {
|
|
||||||
verifyExistence(key, gracePeriod.getOneTimeBillingEvent());
|
|
||||||
verifyExistence(key, gracePeriod.getRecurringBillingEvent());
|
|
||||||
}
|
|
||||||
emit(
|
|
||||||
MapperKey.create(EntityKind.DOMAIN, domain.getFullyQualifiedDomainName()),
|
|
||||||
Key.create(domain));
|
|
||||||
}
|
|
||||||
} else if (resource instanceof ContactResource) {
|
|
||||||
getContext().incrementCounter("contact resources");
|
|
||||||
ContactResource contact = (ContactResource) resource;
|
|
||||||
emit(
|
|
||||||
MapperKey.create(EntityKind.CONTACT, contact.getContactId()),
|
|
||||||
Key.create(contact));
|
|
||||||
} else if (resource instanceof HostResource) {
|
|
||||||
getContext().incrementCounter("host resources");
|
|
||||||
HostResource host = (HostResource) resource;
|
|
||||||
verifyExistence(Key.create(host), host.getSuperordinateDomain());
|
|
||||||
emit(
|
|
||||||
MapperKey.create(EntityKind.HOST, host.getFullyQualifiedHostName()),
|
|
||||||
Key.create(host));
|
|
||||||
} else {
|
|
||||||
throw new IllegalStateException(
|
|
||||||
String.format("EppResource with unknown type in integrity mapper: %s", resource));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
private static Key<TransferServerApproveEntity> castTransferServerApproveEntityKey(
|
|
||||||
Key<? extends TransferServerApproveEntity> key) {
|
|
||||||
return (Key<TransferServerApproveEntity>) key;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void mapForeignKeyIndex(ForeignKeyIndex<?> fki) {
|
|
||||||
Key<ForeignKeyIndex<?>> fkiKey = Key.create(fki);
|
|
||||||
@SuppressWarnings("cast")
|
|
||||||
EppResource resource = verifyExistence(fkiKey, fki.getResourceKey());
|
|
||||||
if (resource != null) {
|
|
||||||
// TODO(user): Traverse the chain of pointers to old FKIs instead once they are written.
|
|
||||||
if (isAtOrAfter(fki.getDeletionTime(), resource.getDeletionTime())) {
|
|
||||||
integrity().check(
|
|
||||||
fki.getForeignKey().equals(resource.getForeignKey()),
|
|
||||||
fkiKey,
|
|
||||||
Key.create(resource),
|
|
||||||
"Foreign key index points to EppResource with different foreign key");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (fki instanceof ForeignKeyDomainIndex) {
|
|
||||||
getContext().incrementCounter("domain foreign key indexes");
|
|
||||||
emit(MapperKey.create(EntityKind.DOMAIN, fki.getForeignKey()), fkiKey);
|
|
||||||
} else if (fki instanceof ForeignKeyContactIndex) {
|
|
||||||
getContext().incrementCounter("contact foreign key indexes");
|
|
||||||
emit(MapperKey.create(EntityKind.CONTACT, fki.getForeignKey()), fkiKey);
|
|
||||||
} else if (fki instanceof ForeignKeyHostIndex) {
|
|
||||||
getContext().incrementCounter("host foreign key indexes");
|
|
||||||
emit(MapperKey.create(EntityKind.HOST, fki.getForeignKey()), fkiKey);
|
|
||||||
} else {
|
|
||||||
throw new IllegalStateException(
|
|
||||||
String.format("Foreign key index is of unknown type: %s", fki));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void mapDomainApplicationIndex(DomainApplicationIndex dai) {
|
|
||||||
getContext().incrementCounter("domain application indexes");
|
|
||||||
Key<DomainApplicationIndex> daiKey = Key.create(dai);
|
|
||||||
for (Key<DomainApplication> key : dai.getKeys()) {
|
|
||||||
DomainApplication application = verifyExistence(daiKey, key);
|
|
||||||
if (application != null) {
|
|
||||||
integrity().check(
|
|
||||||
dai.getFullyQualifiedDomainName().equals(application.getFullyQualifiedDomainName()),
|
|
||||||
daiKey,
|
|
||||||
Key.create(application),
|
|
||||||
"Domain application index points to application with different domain name");
|
|
||||||
}
|
|
||||||
emit(
|
|
||||||
MapperKey.create(EntityKind.APPLICATION, dai.getFullyQualifiedDomainName()),
|
|
||||||
daiKey);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void mapEppResourceIndex(EppResourceIndex eri) {
|
|
||||||
Key<EppResourceIndex> eriKey = Key.create(eri);
|
|
||||||
String eriRepoId = Key.create(eri.getId()).getName();
|
|
||||||
integrity().check(
|
|
||||||
eriRepoId.equals(eri.getKey().getName()),
|
|
||||||
eriKey,
|
|
||||||
eri.getKey(),
|
|
||||||
"EPP resource index id does not match repoId of reference");
|
|
||||||
verifyExistence(eriKey, eri.getKey());
|
|
||||||
emit(MapperKey.create(EntityKind.EPP_RESOURCE, eriRepoId), eriKey);
|
|
||||||
getContext().incrementCounter("EPP resource indexes to " + eri.getKind());
|
|
||||||
}
|
|
||||||
|
|
||||||
private <E> void verifyExistence(Key<?> source, Set<Key<E>> targets) {
|
|
||||||
Set<Key<E>> missingEntityKeys =
|
|
||||||
Sets.difference(targets, ofy().load().keys(targets).keySet());
|
|
||||||
integrity().checkOneToMany(
|
|
||||||
missingEntityKeys.isEmpty(),
|
|
||||||
source,
|
|
||||||
targets,
|
|
||||||
"Target entity does not exist");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Nullable
|
|
||||||
private <E> E verifyExistence(Key<?> source, @Nullable Key<E> target) {
|
|
||||||
if (target == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
E entity = ofy().load().key(target).now();
|
|
||||||
integrity().check(entity != null, source, target, "Target entity does not exist");
|
|
||||||
return entity;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Reducer that checks integrity of foreign key entities. */
|
|
||||||
public static class VerifyEntityIntegrityReducer
|
|
||||||
extends Reducer<MapperKey, Key<? extends ImmutableObject>, Void> {
|
|
||||||
|
|
||||||
private static final long serialVersionUID = -151271247606894783L;
|
|
||||||
|
|
||||||
private final DateTime scanTime;
|
|
||||||
|
|
||||||
private transient VerifyEntityIntegrityStreamer integrityStreamer;
|
|
||||||
|
|
||||||
// The integrityStreamer field must be marked as transient so that instances of the Reducer
|
|
||||||
// class can be serialized by the MapReduce framework. Thus, every time is used, lazily
|
|
||||||
// construct it if it doesn't exist yet.
|
|
||||||
private VerifyEntityIntegrityStreamer integrity() {
|
|
||||||
if (integrityStreamer == null) {
|
|
||||||
integrityStreamer = component.verifyEntityIntegrityStreamerFactory().create(scanTime);
|
|
||||||
}
|
|
||||||
return integrityStreamer;
|
|
||||||
}
|
|
||||||
|
|
||||||
public VerifyEntityIntegrityReducer(DateTime scanTime) {
|
|
||||||
this.scanTime = scanTime;
|
|
||||||
}
|
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
@Override
|
|
||||||
public void reduce(MapperKey mapperKey, ReducerInput<Key<? extends ImmutableObject>> keys) {
|
|
||||||
try {
|
|
||||||
reduceKeys(mapperKey, keys);
|
|
||||||
} catch (Throwable e) {
|
|
||||||
// Log and swallow so that the mapreduce doesn't abort on first error.
|
|
||||||
logger.atSevere().withCause(e).log(
|
|
||||||
"Exception while checking foreign key integrity constraints for: %s", mapperKey);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void reduceKeys(
|
|
||||||
MapperKey mapperKey, ReducerInput<Key<? extends ImmutableObject>> keys) {
|
|
||||||
getContext().incrementCounter("reduced resources " + mapperKey.kind);
|
|
||||||
switch (mapperKey.kind) {
|
|
||||||
case EPP_RESOURCE:
|
|
||||||
checkEppResourceIndexes(keys, mapperKey.id);
|
|
||||||
break;
|
|
||||||
case APPLICATION:
|
|
||||||
checkIndexes(
|
|
||||||
keys,
|
|
||||||
mapperKey.id,
|
|
||||||
KIND_DOMAIN_BASE_RESOURCE,
|
|
||||||
KIND_DOMAIN_APPLICATION_INDEX,
|
|
||||||
false);
|
|
||||||
break;
|
|
||||||
case CONTACT:
|
|
||||||
checkIndexes(keys, mapperKey.id, KIND_CONTACT_RESOURCE, KIND_CONTACT_INDEX, true);
|
|
||||||
break;
|
|
||||||
case DOMAIN:
|
|
||||||
checkIndexes(
|
|
||||||
keys, mapperKey.id, KIND_DOMAIN_BASE_RESOURCE, KIND_DOMAIN_INDEX, true);
|
|
||||||
break;
|
|
||||||
case HOST:
|
|
||||||
checkIndexes(keys, mapperKey.id, KIND_HOST_RESOURCE, KIND_HOST_INDEX, true);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
throw new IllegalStateException(
|
|
||||||
String.format("Unknown type of foreign key %s", mapperKey.kind));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
private void checkEppResourceIndexes(
|
|
||||||
Iterator<Key<? extends ImmutableObject>> keys, String repoId) {
|
|
||||||
List<Key<EppResource>> resources = new ArrayList<>();
|
|
||||||
List<Key<EppResourceIndex>> eppResourceIndexes = new ArrayList<>();
|
|
||||||
while (keys.hasNext()) {
|
|
||||||
Key<?> key = keys.next();
|
|
||||||
String kind = key.getKind();
|
|
||||||
if (kind.equals(KIND_EPPRESOURCE_INDEX)) {
|
|
||||||
eppResourceIndexes.add((Key<EppResourceIndex>) key);
|
|
||||||
} else if (kind.equals(KIND_DOMAIN_BASE_RESOURCE)
|
|
||||||
|| kind.equals(KIND_CONTACT_RESOURCE)
|
|
||||||
|| kind.equals(KIND_HOST_RESOURCE)) {
|
|
||||||
resources.add((Key<EppResource>) key);
|
|
||||||
} else {
|
|
||||||
throw new IllegalStateException(
|
|
||||||
String.format(
|
|
||||||
"While verifying EppResourceIndexes for repoId %s, found key of unknown type: %s",
|
|
||||||
repoId,
|
|
||||||
key));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// This is a checkState and not an integrity check because the Datastore schema ensures that
|
|
||||||
// there can't be multiple EppResources with the same repoId.
|
|
||||||
checkState(
|
|
||||||
resources.size() == 1,
|
|
||||||
String.format("Found more than one EppResource for repoId %s: %s", repoId, resources));
|
|
||||||
if (integrity().check(
|
|
||||||
!eppResourceIndexes.isEmpty(),
|
|
||||||
null,
|
|
||||||
getOnlyElement(resources),
|
|
||||||
"Missing EPP resource index for EPP resource")) {
|
|
||||||
integrity().checkManyToOne(
|
|
||||||
eppResourceIndexes.size() == 1,
|
|
||||||
eppResourceIndexes,
|
|
||||||
getOnlyElement(resources),
|
|
||||||
"Duplicate EPP resource indexes pointing to same resource");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
private <R extends EppResource, I> void checkIndexes(
|
|
||||||
Iterator<Key<? extends ImmutableObject>> keys,
|
|
||||||
String foreignKey,
|
|
||||||
String resourceKind,
|
|
||||||
String foreignKeyIndexKind,
|
|
||||||
boolean thereCanBeOnlyOne) {
|
|
||||||
List<Key<R>> resources = new ArrayList<>();
|
|
||||||
List<Key<I>> foreignKeyIndexes = new ArrayList<>();
|
|
||||||
while (keys.hasNext()) {
|
|
||||||
Key<?> key = keys.next();
|
|
||||||
if (key.getKind().equals(resourceKind)) {
|
|
||||||
resources.add((Key<R>) key);
|
|
||||||
} else if (key.getKind().equals(foreignKeyIndexKind)) {
|
|
||||||
foreignKeyIndexes.add((Key<I>) key);
|
|
||||||
} else {
|
|
||||||
throw new IllegalStateException(
|
|
||||||
String.format(
|
|
||||||
"While processing links to foreign key %s of type %s, found unknown key: %s",
|
|
||||||
foreignKey,
|
|
||||||
resourceKind,
|
|
||||||
key));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// This is a checkState and not an integrity check because it should truly be impossible to
|
|
||||||
// have multiple foreign key indexes for the same foreign key because of the Datastore schema.
|
|
||||||
checkState(
|
|
||||||
foreignKeyIndexes.size() <= 1,
|
|
||||||
String.format(
|
|
||||||
"Found more than one foreign key index for %s: %s", foreignKey, foreignKeyIndexes));
|
|
||||||
integrity().check(
|
|
||||||
!foreignKeyIndexes.isEmpty(),
|
|
||||||
foreignKey,
|
|
||||||
resourceKind,
|
|
||||||
"Missing foreign key index for EppResource");
|
|
||||||
// Skip the case where no resources were found because entity exceptions are already thrown in
|
|
||||||
// the mapper in invalid situations where FKIs point to non-existent entities.
|
|
||||||
if (thereCanBeOnlyOne && !resources.isEmpty()) {
|
|
||||||
verifyOnlyOneActiveResource(resources, getOnlyElement(foreignKeyIndexes));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private <R extends EppResource, I> void verifyOnlyOneActiveResource(
|
|
||||||
List<Key<R>> resources, Key<I> fkiKey) {
|
|
||||||
DateTime oldestActive = END_OF_TIME;
|
|
||||||
DateTime mostRecentInactive = START_OF_TIME;
|
|
||||||
Key<R> mostRecentInactiveKey = null;
|
|
||||||
List<Key<R>> activeResources = new ArrayList<>();
|
|
||||||
Map<Key<R>, R> allResources = ofy().load().keys(resources);
|
|
||||||
ForeignKeyIndex<?> fki = (ForeignKeyIndex<?>) ofy().load().key(fkiKey).now();
|
|
||||||
for (Map.Entry<Key<R>, R> entry : allResources.entrySet()) {
|
|
||||||
R resource = entry.getValue();
|
|
||||||
if (isActive(resource, scanTime)) {
|
|
||||||
activeResources.add(entry.getKey());
|
|
||||||
oldestActive = earliestOf(oldestActive, resource.getCreationTime());
|
|
||||||
} else {
|
|
||||||
if (resource.getDeletionTime().isAfter(mostRecentInactive)) {
|
|
||||||
mostRecentInactive = resource.getDeletionTime();
|
|
||||||
mostRecentInactiveKey = entry.getKey();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (activeResources.isEmpty()) {
|
|
||||||
integrity().check(
|
|
||||||
fki.getDeletionTime().isEqual(mostRecentInactive),
|
|
||||||
fkiKey,
|
|
||||||
mostRecentInactiveKey,
|
|
||||||
"Foreign key index deletion time not equal to that of most recently deleted resource");
|
|
||||||
} else {
|
|
||||||
integrity().checkOneToMany(
|
|
||||||
activeResources.size() == 1,
|
|
||||||
fkiKey,
|
|
||||||
activeResources,
|
|
||||||
"Multiple active EppResources with same foreign key");
|
|
||||||
integrity().check(
|
|
||||||
fki.getDeletionTime().isEqual(END_OF_TIME),
|
|
||||||
fkiKey,
|
|
||||||
null,
|
|
||||||
"Foreign key index has deletion time but active resource exists");
|
|
||||||
integrity().check(
|
|
||||||
isBeforeOrAt(mostRecentInactive, oldestActive),
|
|
||||||
fkiKey,
|
|
||||||
mostRecentInactiveKey,
|
|
||||||
"Found inactive resource deleted more recently than when active resource was created");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,210 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.batch;
|
|
||||||
|
|
||||||
import static com.google.api.client.util.Data.NULL_STRING;
|
|
||||||
import static google.registry.batch.EntityIntegrityAlertsSchema.DATASET;
|
|
||||||
import static google.registry.batch.EntityIntegrityAlertsSchema.FIELD_MESSAGE;
|
|
||||||
import static google.registry.batch.EntityIntegrityAlertsSchema.FIELD_SCANTIME;
|
|
||||||
import static google.registry.batch.EntityIntegrityAlertsSchema.FIELD_SOURCE;
|
|
||||||
import static google.registry.batch.EntityIntegrityAlertsSchema.FIELD_TARGET;
|
|
||||||
import static google.registry.batch.EntityIntegrityAlertsSchema.TABLE_ID;
|
|
||||||
import static java.util.stream.Collectors.joining;
|
|
||||||
|
|
||||||
import com.google.api.services.bigquery.Bigquery;
|
|
||||||
import com.google.api.services.bigquery.Bigquery.Tabledata.InsertAll;
|
|
||||||
import com.google.api.services.bigquery.model.TableDataInsertAllRequest;
|
|
||||||
import com.google.api.services.bigquery.model.TableDataInsertAllRequest.Rows;
|
|
||||||
import com.google.api.services.bigquery.model.TableDataInsertAllResponse;
|
|
||||||
import com.google.auto.factory.AutoFactory;
|
|
||||||
import com.google.auto.factory.Provided;
|
|
||||||
import com.google.common.base.Supplier;
|
|
||||||
import com.google.common.collect.ImmutableList;
|
|
||||||
import com.google.common.collect.ImmutableMap;
|
|
||||||
import google.registry.bigquery.BigqueryFactory;
|
|
||||||
import google.registry.config.RegistryConfig.Config;
|
|
||||||
import google.registry.util.Retrier;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.Callable;
|
|
||||||
import javax.annotation.Nullable;
|
|
||||||
import javax.inject.Named;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An injected utility class used to check entity integrity and stream violations to BigQuery.
|
|
||||||
*/
|
|
||||||
@AutoFactory(allowSubclasses = true)
|
|
||||||
public class VerifyEntityIntegrityStreamer {
|
|
||||||
|
|
||||||
private final String projectId;
|
|
||||||
private final BigqueryFactory bigqueryFactory;
|
|
||||||
private final Supplier<String> idGenerator;
|
|
||||||
private final Retrier retrier;
|
|
||||||
private final DateTime scanTime;
|
|
||||||
private Bigquery bigquery;
|
|
||||||
|
|
||||||
public VerifyEntityIntegrityStreamer(
|
|
||||||
@Provided @Config("projectId") String projectId,
|
|
||||||
@Provided BigqueryFactory bigqueryFactory,
|
|
||||||
@Provided Retrier retrier,
|
|
||||||
@Provided @Named("insertIdGenerator") Supplier<String> idGenerator,
|
|
||||||
DateTime scanTime) {
|
|
||||||
this.projectId = projectId;
|
|
||||||
this.bigqueryFactory = bigqueryFactory;
|
|
||||||
this.retrier = retrier;
|
|
||||||
this.idGenerator = idGenerator;
|
|
||||||
this.scanTime = scanTime;
|
|
||||||
}
|
|
||||||
|
|
||||||
// This is lazily loaded so we only construct the connection when needed, as in a healthy
|
|
||||||
// Datastore almost every single check should short-circuit return and won't output anything to
|
|
||||||
// BigQuery.
|
|
||||||
private Bigquery getBigquery() throws IOException {
|
|
||||||
if (bigquery == null) {
|
|
||||||
bigquery = bigqueryFactory.create(projectId, DATASET, TABLE_ID);
|
|
||||||
}
|
|
||||||
return bigquery;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check that the given conditional holds, and if not, stream the supplied source, target, and
|
|
||||||
* message information to BigQuery.
|
|
||||||
*
|
|
||||||
* @return Whether the check succeeded.
|
|
||||||
*/
|
|
||||||
boolean check(
|
|
||||||
boolean conditional,
|
|
||||||
@Nullable Object source,
|
|
||||||
@Nullable Object target,
|
|
||||||
@Nullable String message) {
|
|
||||||
return checkOneToMany(
|
|
||||||
conditional, source, ImmutableList.of((target == null) ? NULL_STRING : target), message);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check that the given conditional holds, and if not, stream a separate row to BigQuery for each
|
|
||||||
* supplied target (the source and message will be the same for each).
|
|
||||||
*
|
|
||||||
* @return Whether the check succeeded.
|
|
||||||
*/
|
|
||||||
<T> boolean checkOneToMany(
|
|
||||||
boolean conditional,
|
|
||||||
@Nullable Object source,
|
|
||||||
Iterable<T> targets,
|
|
||||||
@Nullable String message) {
|
|
||||||
return checkManyToMany(
|
|
||||||
conditional, ImmutableList.of((source == null) ? NULL_STRING : source), targets, message);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check that the given conditional holds, and if not, stream a separate row to BigQuery for each
|
|
||||||
* supplied target (the source and message will be the same for each).
|
|
||||||
*
|
|
||||||
* @return Whether the check succeeded.
|
|
||||||
*/
|
|
||||||
<S> boolean checkManyToOne(
|
|
||||||
boolean conditional,
|
|
||||||
Iterable<S> sources,
|
|
||||||
@Nullable Object target,
|
|
||||||
@Nullable String message) {
|
|
||||||
return checkManyToMany(
|
|
||||||
conditional, sources, ImmutableList.of((target == null) ? NULL_STRING : target), message);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check that the given conditional holds, and if not, stream a separate row to BigQuery for the
|
|
||||||
* cross product of every supplied target and source (the message will be the same for each).
|
|
||||||
* This is used in preference to records (repeated fields) in BigQuery because they are
|
|
||||||
* significantly harder to work with.
|
|
||||||
*
|
|
||||||
* @return Whether the check succeeded.
|
|
||||||
*/
|
|
||||||
private <S, T> boolean checkManyToMany(
|
|
||||||
boolean conditional,
|
|
||||||
Iterable<S> sources,
|
|
||||||
Iterable<T> targets,
|
|
||||||
@Nullable String message) {
|
|
||||||
if (conditional) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
ImmutableList.Builder<Rows> rows = new ImmutableList.Builder<>();
|
|
||||||
for (S source : sources) {
|
|
||||||
for (T target : targets) {
|
|
||||||
Map<String, Object> rowData =
|
|
||||||
new ImmutableMap.Builder<String, Object>()
|
|
||||||
.put(
|
|
||||||
FIELD_SCANTIME,
|
|
||||||
new com.google.api.client.util.DateTime(scanTime.toDate()))
|
|
||||||
.put(
|
|
||||||
FIELD_SOURCE,
|
|
||||||
source.toString())
|
|
||||||
.put(
|
|
||||||
FIELD_TARGET,
|
|
||||||
target.toString())
|
|
||||||
.put(
|
|
||||||
FIELD_MESSAGE,
|
|
||||||
(message == null) ? NULL_STRING : message)
|
|
||||||
.build();
|
|
||||||
rows.add(
|
|
||||||
new TableDataInsertAllRequest.Rows().setJson(rowData).setInsertId(idGenerator.get()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
streamToBigqueryWithRetry(rows.build());
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void streamToBigqueryWithRetry(List<Rows> rows) {
|
|
||||||
try {
|
|
||||||
final InsertAll request =
|
|
||||||
getBigquery()
|
|
||||||
.tabledata()
|
|
||||||
.insertAll(
|
|
||||||
projectId,
|
|
||||||
DATASET,
|
|
||||||
TABLE_ID,
|
|
||||||
new TableDataInsertAllRequest().setRows(rows));
|
|
||||||
|
|
||||||
Callable<Void> callable =
|
|
||||||
() -> {
|
|
||||||
TableDataInsertAllResponse response = request.execute();
|
|
||||||
// Turn errors on the response object into RuntimeExceptions that the retrier will
|
|
||||||
// retry.
|
|
||||||
if (response.getInsertErrors() != null && !response.getInsertErrors().isEmpty()) {
|
|
||||||
throw new RuntimeException(
|
|
||||||
response
|
|
||||||
.getInsertErrors()
|
|
||||||
.stream()
|
|
||||||
.map(
|
|
||||||
error -> {
|
|
||||||
try {
|
|
||||||
return error.toPrettyString();
|
|
||||||
} catch (IOException e) {
|
|
||||||
return error.toString();
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect(joining("\n")));
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
retrier.callWithRetry(callable, RuntimeException.class);
|
|
||||||
} catch (IOException e) {
|
|
||||||
throw new RuntimeException("Error sending integrity error to BigQuery", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
@ -174,12 +174,6 @@
|
||||||
<url-pattern>/_dr/dnsRefresh</url-pattern>
|
<url-pattern>/_dr/dnsRefresh</url-pattern>
|
||||||
</servlet-mapping>
|
</servlet-mapping>
|
||||||
|
|
||||||
<!-- Verifies integrity of database invariants. -->
|
|
||||||
<servlet-mapping>
|
|
||||||
<servlet-name>backend-servlet</servlet-name>
|
|
||||||
<url-pattern>/_dr/task/verifyEntityIntegrity</url-pattern>
|
|
||||||
</servlet-mapping>
|
|
||||||
|
|
||||||
<!-- Exports a Datastore backup snapshot to GCS. -->
|
<!-- Exports a Datastore backup snapshot to GCS. -->
|
||||||
<servlet-mapping>
|
<servlet-mapping>
|
||||||
<servlet-name>backend-servlet</servlet-name>
|
<servlet-name>backend-servlet</servlet-name>
|
||||||
|
|
|
||||||
|
|
@ -210,17 +210,4 @@
|
||||||
<target>backend</target>
|
<target>backend</target>
|
||||||
</cron>
|
</cron>
|
||||||
|
|
||||||
<!--
|
|
||||||
TODO(b/71607184): Consider re-enabling once load-testing-related entities are cleared out.
|
|
||||||
<cron>
|
|
||||||
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/verifyEntityIntegrity&runInEmpty]]></url>
|
|
||||||
<description>
|
|
||||||
This job verifies entity integrity and runs once daily.
|
|
||||||
</description>
|
|
||||||
<schedule>every day 06:30</schedule>
|
|
||||||
<timezone>UTC</timezone>
|
|
||||||
<target>backend</target>
|
|
||||||
</cron>
|
|
||||||
-->
|
|
||||||
|
|
||||||
</cronentries>
|
</cronentries>
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,6 @@ import google.registry.batch.ExpandRecurringBillingEventsAction;
|
||||||
import google.registry.batch.RefreshDnsOnHostRenameAction;
|
import google.registry.batch.RefreshDnsOnHostRenameAction;
|
||||||
import google.registry.batch.ResaveAllEppResourcesAction;
|
import google.registry.batch.ResaveAllEppResourcesAction;
|
||||||
import google.registry.batch.ResaveEntityAction;
|
import google.registry.batch.ResaveEntityAction;
|
||||||
import google.registry.batch.VerifyEntityIntegrityAction;
|
|
||||||
import google.registry.cron.CommitLogFanoutAction;
|
import google.registry.cron.CommitLogFanoutAction;
|
||||||
import google.registry.cron.CronModule;
|
import google.registry.cron.CronModule;
|
||||||
import google.registry.cron.TldFanoutAction;
|
import google.registry.cron.TldFanoutAction;
|
||||||
|
|
@ -163,7 +162,6 @@ interface BackendRequestComponent {
|
||||||
TmchSmdrlAction tmchSmdrlAction();
|
TmchSmdrlAction tmchSmdrlAction();
|
||||||
UpdateSnapshotViewAction updateSnapshotViewAction();
|
UpdateSnapshotViewAction updateSnapshotViewAction();
|
||||||
PublishInvoicesAction uploadInvoicesAction();
|
PublishInvoicesAction uploadInvoicesAction();
|
||||||
VerifyEntityIntegrityAction verifyEntityIntegrityAction();
|
|
||||||
|
|
||||||
@Subcomponent.Builder
|
@Subcomponent.Builder
|
||||||
abstract class Builder implements RequestComponentBuilder<BackendRequestComponent> {
|
abstract class Builder implements RequestComponentBuilder<BackendRequestComponent> {
|
||||||
|
|
|
||||||
|
|
@ -1,317 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.batch;
|
|
||||||
|
|
||||||
import static com.google.common.truth.Truth.assertThat;
|
|
||||||
import static google.registry.testing.DatastoreHelper.createTld;
|
|
||||||
import static google.registry.testing.DatastoreHelper.deleteResource;
|
|
||||||
import static google.registry.testing.DatastoreHelper.newContactResource;
|
|
||||||
import static google.registry.testing.DatastoreHelper.newDomainResource;
|
|
||||||
import static google.registry.testing.DatastoreHelper.persistActiveContact;
|
|
||||||
import static google.registry.testing.DatastoreHelper.persistActiveDomain;
|
|
||||||
import static google.registry.testing.DatastoreHelper.persistActiveHost;
|
|
||||||
import static google.registry.testing.DatastoreHelper.persistDomainAsDeleted;
|
|
||||||
import static google.registry.testing.DatastoreHelper.persistResource;
|
|
||||||
import static google.registry.testing.DatastoreHelper.persistSimpleResource;
|
|
||||||
import static org.joda.time.DateTimeZone.UTC;
|
|
||||||
import static org.mockito.Matchers.any;
|
|
||||||
import static org.mockito.Matchers.anyString;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.verifyZeroInteractions;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import com.google.api.client.util.Data;
|
|
||||||
import com.google.api.services.bigquery.Bigquery;
|
|
||||||
import com.google.api.services.bigquery.model.TableDataInsertAllRequest;
|
|
||||||
import com.google.api.services.bigquery.model.TableDataInsertAllRequest.Rows;
|
|
||||||
import com.google.api.services.bigquery.model.TableDataInsertAllResponse;
|
|
||||||
import com.google.common.base.Suppliers;
|
|
||||||
import com.google.common.collect.ImmutableList;
|
|
||||||
import com.google.common.collect.ImmutableMap;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import com.googlecode.objectify.Key;
|
|
||||||
import google.registry.bigquery.BigqueryFactory;
|
|
||||||
import google.registry.mapreduce.MapreduceRunner;
|
|
||||||
import google.registry.model.contact.ContactResource;
|
|
||||||
import google.registry.model.domain.DomainResource;
|
|
||||||
import google.registry.model.host.HostResource;
|
|
||||||
import google.registry.model.index.EppResourceIndex;
|
|
||||||
import google.registry.model.index.ForeignKeyIndex;
|
|
||||||
import google.registry.testing.FakeClock;
|
|
||||||
import google.registry.testing.FakeResponse;
|
|
||||||
import google.registry.testing.FakeSleeper;
|
|
||||||
import google.registry.testing.InjectRule;
|
|
||||||
import google.registry.testing.mapreduce.MapreduceTestCase;
|
|
||||||
import google.registry.util.Retrier;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Optional;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
import org.joda.time.DateTimeZone;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.runner.RunWith;
|
|
||||||
import org.junit.runners.JUnit4;
|
|
||||||
import org.mockito.ArgumentCaptor;
|
|
||||||
|
|
||||||
/** Unit tests for {@link VerifyEntityIntegrityAction}. */
|
|
||||||
@RunWith(JUnit4.class)
|
|
||||||
public class VerifyEntityIntegrityActionTest
|
|
||||||
extends MapreduceTestCase<VerifyEntityIntegrityAction> {
|
|
||||||
|
|
||||||
@Rule
|
|
||||||
public final InjectRule inject = new InjectRule();
|
|
||||||
|
|
||||||
private VerifyEntityIntegrityStreamer integrity;
|
|
||||||
private ArgumentCaptor<TableDataInsertAllRequest> rowsCaptor;
|
|
||||||
private final DateTime now = DateTime.parse("2012-01-02T03:04:05Z");
|
|
||||||
|
|
||||||
private final Bigquery bigquery = mock(Bigquery.class);
|
|
||||||
private final Bigquery.Tabledata bigqueryTableData = mock(Bigquery.Tabledata.class);
|
|
||||||
private final Bigquery.Tabledata.InsertAll bigqueryInsertAll =
|
|
||||||
mock(Bigquery.Tabledata.InsertAll.class);
|
|
||||||
private final BigqueryFactory bigqueryFactory = mock(BigqueryFactory.class);
|
|
||||||
private final VerifyEntityIntegrityStreamerFactory streamerFactory =
|
|
||||||
mock(VerifyEntityIntegrityStreamerFactory.class);
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void before() throws Exception {
|
|
||||||
createTld("tld");
|
|
||||||
|
|
||||||
action = new VerifyEntityIntegrityAction();
|
|
||||||
action.mrRunner = new MapreduceRunner(Optional.of(2), Optional.of(2));
|
|
||||||
action.response = new FakeResponse();
|
|
||||||
BatchComponent component = mock(BatchComponent.class);
|
|
||||||
inject.setStaticField(VerifyEntityIntegrityAction.class, "component", component);
|
|
||||||
integrity =
|
|
||||||
new VerifyEntityIntegrityStreamer(
|
|
||||||
"project-id",
|
|
||||||
bigqueryFactory,
|
|
||||||
new Retrier(new FakeSleeper(new FakeClock()), 1),
|
|
||||||
Suppliers.ofInstance("rowid"),
|
|
||||||
now);
|
|
||||||
when(bigqueryFactory.create(anyString(), anyString(), anyString())).thenReturn(bigquery);
|
|
||||||
when(component.verifyEntityIntegrityStreamerFactory()).thenReturn(streamerFactory);
|
|
||||||
when(streamerFactory.create(any(DateTime.class))).thenReturn(integrity);
|
|
||||||
when(bigquery.tabledata()).thenReturn(bigqueryTableData);
|
|
||||||
rowsCaptor = ArgumentCaptor.forClass(TableDataInsertAllRequest.class);
|
|
||||||
when(bigqueryTableData.insertAll(anyString(), anyString(), anyString(), rowsCaptor.capture()))
|
|
||||||
.thenReturn(bigqueryInsertAll);
|
|
||||||
when(bigqueryInsertAll.execute()).thenReturn(new TableDataInsertAllResponse());
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private void runMapreduce() throws Exception {
|
|
||||||
action.run();
|
|
||||||
executeTasksUntilEmpty("mapreduce");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void test_singleDomain_noBadInvariants() throws Exception {
|
|
||||||
persistActiveDomain("ninetails.tld");
|
|
||||||
runMapreduce();
|
|
||||||
verifyZeroInteractions(bigquery);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void test_lotsOfData_noBadInvariants() throws Exception {
|
|
||||||
createTld("march");
|
|
||||||
ContactResource contact = persistActiveContact("longbottom");
|
|
||||||
persistResource(newDomainResource("ninetails.tld", contact));
|
|
||||||
persistResource(newDomainResource("tentails.tld", contact));
|
|
||||||
persistDomainAsDeleted(newDomainResource("long.march", contact), now.minusMonths(4));
|
|
||||||
persistResource(
|
|
||||||
newDomainResource("long.march", contact)
|
|
||||||
.asBuilder()
|
|
||||||
.setCreationTimeForTest(now.minusMonths(3))
|
|
||||||
.build());
|
|
||||||
DateTime now = DateTime.now(UTC);
|
|
||||||
persistResource(
|
|
||||||
newContactResource("ricketycricket")
|
|
||||||
.asBuilder()
|
|
||||||
.setCreationTimeForTest(now.minusDays(10))
|
|
||||||
.setDeletionTime(now.minusDays(9))
|
|
||||||
.build());
|
|
||||||
persistResource(
|
|
||||||
newContactResource("ricketycricket")
|
|
||||||
.asBuilder()
|
|
||||||
.setCreationTimeForTest(now.minusDays(7))
|
|
||||||
.setDeletionTime(now.minusDays(6))
|
|
||||||
.build());
|
|
||||||
persistResource(
|
|
||||||
newContactResource("ricketycricket")
|
|
||||||
.asBuilder()
|
|
||||||
.setCreationTimeForTest(now.minusDays(4))
|
|
||||||
.setDeletionTime(now.minusDays(3))
|
|
||||||
.build());
|
|
||||||
persistResource(
|
|
||||||
newContactResource("ricketycricket")
|
|
||||||
.asBuilder()
|
|
||||||
.setCreationTimeForTest(now.minusDays(1))
|
|
||||||
.build());
|
|
||||||
persistActiveHost("ns9001.example.net");
|
|
||||||
runMapreduce();
|
|
||||||
verifyZeroInteractions(bigquery);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void test_missingFki() throws Exception {
|
|
||||||
persistActiveDomain("ninetails.tld");
|
|
||||||
ForeignKeyIndex<DomainResource> fki =
|
|
||||||
ForeignKeyIndex.load(DomainResource.class, "ninetails.tld", DateTime.now(DateTimeZone.UTC));
|
|
||||||
deleteResource(fki);
|
|
||||||
runMapreduce();
|
|
||||||
assertIntegrityErrors(IntegrityError.create(
|
|
||||||
"ninetails.tld", "DomainBase", "Missing foreign key index for EppResource"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void test_missingEppResourceIndex() throws Exception {
|
|
||||||
Key<ContactResource> cooperKey = Key.create(persistActiveContact("cooper"));
|
|
||||||
deleteResource(EppResourceIndex.create(cooperKey));
|
|
||||||
runMapreduce();
|
|
||||||
assertIntegrityErrors(IntegrityError.create(
|
|
||||||
Data.NULL_STRING, cooperKey, "Missing EPP resource index for EPP resource"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void test_referencesToHostsThatDontExist() throws Exception {
|
|
||||||
Key<HostResource> missingHost1 = Key.create(HostResource.class, "DEADBEEF-ROID");
|
|
||||||
Key<HostResource> missingHost2 = Key.create(HostResource.class, "123ABC-ROID");
|
|
||||||
Key<HostResource> missingHost3 = Key.create(HostResource.class, "FADDACA-ROID");
|
|
||||||
DomainResource domain =
|
|
||||||
persistResource(
|
|
||||||
newDomainResource("blah.tld")
|
|
||||||
.asBuilder()
|
|
||||||
.setNameservers(ImmutableSet.of(missingHost1, missingHost2, missingHost3))
|
|
||||||
.build());
|
|
||||||
Key<DomainResource> domainKey = Key.create(domain);
|
|
||||||
runMapreduce();
|
|
||||||
assertIntegrityErrors(
|
|
||||||
IntegrityError.create(domainKey, missingHost1, "Target entity does not exist"),
|
|
||||||
IntegrityError.create(domainKey, missingHost2, "Target entity does not exist"),
|
|
||||||
IntegrityError.create(domainKey, missingHost3, "Target entity does not exist"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void test_overlappingActivePeriods() throws Exception {
|
|
||||||
ContactResource contact123 = persistActiveContact("contact123");
|
|
||||||
// These two have overlapping active periods because they will have both been created at
|
|
||||||
// START_OF_TIME.
|
|
||||||
DomainResource domain1 =
|
|
||||||
persistDomainAsDeleted(newDomainResource("penny.tld", contact123), now.minusYears(2));
|
|
||||||
DomainResource domain2 = persistActiveDomain("penny.tld");
|
|
||||||
runMapreduce();
|
|
||||||
assertIntegrityErrors(
|
|
||||||
IntegrityError.create(
|
|
||||||
ForeignKeyIndex.createKey(domain2),
|
|
||||||
Key.create(domain1),
|
|
||||||
"Found inactive resource deleted more recently than when active resource was created"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void test_multipleActiveContactsWithSameContactId() throws Exception {
|
|
||||||
ContactResource contact1 = persistActiveContact("dupeid");
|
|
||||||
ContactResource contact2 = persistActiveContact("dupeid");
|
|
||||||
runMapreduce();
|
|
||||||
assertIntegrityErrors(
|
|
||||||
IntegrityError.create(
|
|
||||||
ForeignKeyIndex.createKey(contact1),
|
|
||||||
Key.create(contact1),
|
|
||||||
"Multiple active EppResources with same foreign key"),
|
|
||||||
IntegrityError.create(
|
|
||||||
ForeignKeyIndex.createKey(contact2),
|
|
||||||
Key.create(contact2),
|
|
||||||
"Multiple active EppResources with same foreign key"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void test_deletedFkiWithOldHostnameIsntAnError() throws Exception {
|
|
||||||
HostResource hostLosing = persistActiveHost("losing.example.tld");
|
|
||||||
persistResource(
|
|
||||||
hostLosing.asBuilder().setFullyQualifiedHostName("gaining.example.tld").build());
|
|
||||||
persistSimpleResource(
|
|
||||||
ForeignKeyIndex.create(hostLosing, DateTime.parse("2010-01-01T00:00:00Z")));
|
|
||||||
// The old FKI with a primary key of "losing.example.tld" still exists, and it points to the
|
|
||||||
// resource which has a hostname of "gaining.example.tld", but this isn't an error because the
|
|
||||||
// FKI is soft-deleted.
|
|
||||||
runMapreduce();
|
|
||||||
verifyZeroInteractions(bigquery);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void test_fkiWithDifferentHostnameDeletedMoreRecentlyIsAnError() throws Exception {
|
|
||||||
HostResource hostLosing = persistActiveHost("losing.example.tld");
|
|
||||||
HostResource hostGaining = persistResource(
|
|
||||||
hostLosing
|
|
||||||
.asBuilder()
|
|
||||||
.setFullyQualifiedHostName("gaining.example.tld")
|
|
||||||
.setDeletionTime(DateTime.parse("2013-01-01T00:00:00Z"))
|
|
||||||
.build());
|
|
||||||
ForeignKeyIndex<?> fki = persistSimpleResource(
|
|
||||||
ForeignKeyIndex.create(hostLosing, DateTime.parse("2014-01-01T00:00:00Z")));
|
|
||||||
// The old FKI is pointing to the old name but has a deletion time more recent than the deleted
|
|
||||||
// host, so it should have the same foreign key.
|
|
||||||
runMapreduce();
|
|
||||||
assertIntegrityErrors(
|
|
||||||
IntegrityError.create(
|
|
||||||
Key.create(fki),
|
|
||||||
Key.create(hostGaining),
|
|
||||||
"Foreign key index points to EppResource with different foreign key"));
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Encapsulates the data representing a single integrity error. */
|
|
||||||
private static class IntegrityError {
|
|
||||||
String source;
|
|
||||||
String target;
|
|
||||||
String message;
|
|
||||||
|
|
||||||
static IntegrityError create(Object source, Object target, String message) {
|
|
||||||
IntegrityError instance = new IntegrityError();
|
|
||||||
instance.source = source.toString();
|
|
||||||
instance.target = target.toString();
|
|
||||||
instance.message = message;
|
|
||||||
return instance;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns a Map representing the JSON blob corresponding to the BigQuery output for this
|
|
||||||
* integrity violation at the given scan time.
|
|
||||||
*/
|
|
||||||
Map<String, Object> toMap(DateTime scanTime) {
|
|
||||||
return new ImmutableMap.Builder<String, Object>()
|
|
||||||
.put("scanTime", new com.google.api.client.util.DateTime(scanTime.toDate()))
|
|
||||||
.put("source", source)
|
|
||||||
.put("target", target)
|
|
||||||
.put("message", message)
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
private IntegrityError() {}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Asserts that the given integrity errors, and no others, were logged to BigQuery. */
|
|
||||||
private void assertIntegrityErrors(IntegrityError... errors) {
|
|
||||||
ImmutableList.Builder<Rows> expected = new ImmutableList.Builder<>();
|
|
||||||
for (IntegrityError error : errors) {
|
|
||||||
expected.add(new Rows().setInsertId("rowid").setJson(error.toMap(now)));
|
|
||||||
}
|
|
||||||
ImmutableList.Builder<Rows> allRows = new ImmutableList.Builder<>();
|
|
||||||
for (TableDataInsertAllRequest req : rowsCaptor.getAllValues()) {
|
|
||||||
allRows.addAll(req.getRows());
|
|
||||||
}
|
|
||||||
assertThat(allRows.build()).containsExactlyElementsIn(expected.build());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -45,4 +45,3 @@ PATH CLASS METHOD
|
||||||
/_dr/task/tmchDnl TmchDnlAction POST y INTERNAL APP IGNORED
|
/_dr/task/tmchDnl TmchDnlAction POST y INTERNAL APP IGNORED
|
||||||
/_dr/task/tmchSmdrl TmchSmdrlAction POST y INTERNAL APP IGNORED
|
/_dr/task/tmchSmdrl TmchSmdrlAction POST y INTERNAL APP IGNORED
|
||||||
/_dr/task/updateSnapshotView UpdateSnapshotViewAction POST n INTERNAL APP IGNORED
|
/_dr/task/updateSnapshotView UpdateSnapshotViewAction POST n INTERNAL APP IGNORED
|
||||||
/_dr/task/verifyEntityIntegrity VerifyEntityIntegrityAction POST n INTERNAL APP IGNORED
|
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,6 @@ java_library(
|
||||||
|
|
||||||
GenTestRules(
|
GenTestRules(
|
||||||
name = "GeneratedTestRules",
|
name = "GeneratedTestRules",
|
||||||
medium_tests = ["VerifyEntityIntegrityActionTest"],
|
|
||||||
test_files = glob(["*Test.java"]),
|
test_files = glob(["*Test.java"]),
|
||||||
deps = [":whitebox"],
|
deps = [":whitebox"],
|
||||||
)
|
)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue