Stop writing ClaimsList to Datastore (#1169)

* Stop writing ClaimsList to Datastore

* Fix some failing tests

* Rename ClaimsListShard to ClaimsList
This commit is contained in:
sarahcaseybot 2021-05-20 15:44:40 -04:00 committed by GitHub
parent 38fa08b930
commit c27e0f4118
24 changed files with 94 additions and 583 deletions

View file

@ -45,7 +45,7 @@ import google.registry.model.eppinput.ResourceCommand;
import google.registry.model.eppoutput.EppResponse; import google.registry.model.eppoutput.EppResponse;
import google.registry.model.registry.Registry; import google.registry.model.registry.Registry;
import google.registry.model.reporting.IcannReportingTypes.ActivityReportField; import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
import google.registry.model.tmch.ClaimsListDualDatabaseDao; import google.registry.model.tmch.ClaimsListDao;
import google.registry.util.Clock; import google.registry.util.Clock;
import java.util.HashSet; import java.util.HashSet;
import java.util.Optional; import java.util.Optional;
@ -104,8 +104,7 @@ public final class DomainClaimsCheckFlow implements Flow {
verifyClaimsPeriodNotEnded(registry, now); verifyClaimsPeriodNotEnded(registry, now);
} }
} }
Optional<String> claimKey = Optional<String> claimKey = ClaimsListDao.get().getClaimKey(parsedDomain.parts().get(0));
ClaimsListDualDatabaseDao.get().getClaimKey(parsedDomain.parts().get(0));
launchChecksBuilder.add( launchChecksBuilder.add(
LaunchCheck.create( LaunchCheck.create(
LaunchCheckName.create(claimKey.isPresent(), domainName), claimKey.orElse(null))); LaunchCheckName.create(claimKey.isPresent(), domainName), claimKey.orElse(null)));

View file

@ -126,7 +126,7 @@ import google.registry.model.registry.label.ReservedList;
import google.registry.model.reporting.DomainTransactionRecord; import google.registry.model.reporting.DomainTransactionRecord;
import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField; import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
import google.registry.model.reporting.HistoryEntry; import google.registry.model.reporting.HistoryEntry;
import google.registry.model.tmch.ClaimsListDualDatabaseDao; import google.registry.model.tmch.ClaimsListDao;
import google.registry.persistence.VKey; import google.registry.persistence.VKey;
import google.registry.tldconfig.idn.IdnLabelValidator; import google.registry.tldconfig.idn.IdnLabelValidator;
import google.registry.util.Idn; import google.registry.util.Idn;
@ -990,8 +990,7 @@ public class DomainFlowUtils {
static void verifyClaimsNoticeIfAndOnlyIfNeeded( static void verifyClaimsNoticeIfAndOnlyIfNeeded(
InternetDomainName domainName, boolean hasSignedMarks, boolean hasClaimsNotice) InternetDomainName domainName, boolean hasSignedMarks, boolean hasClaimsNotice)
throws EppException { throws EppException {
boolean isInClaimsList = boolean isInClaimsList = ClaimsListDao.get().getClaimKey(domainName.parts().get(0)).isPresent();
ClaimsListDualDatabaseDao.get().getClaimKey(domainName.parts().get(0)).isPresent();
if (hasClaimsNotice && !isInClaimsList) { if (hasClaimsNotice && !isInClaimsList) {
throw new UnexpectedClaimsNoticeException(domainName.toString()); throw new UnexpectedClaimsNoticeException(domainName.toString());
} }

View file

@ -47,9 +47,9 @@ import google.registry.model.server.KmsSecret;
import google.registry.model.server.KmsSecretRevision; import google.registry.model.server.KmsSecretRevision;
import google.registry.model.server.Lock; import google.registry.model.server.Lock;
import google.registry.model.server.ServerSecret; import google.registry.model.server.ServerSecret;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListShard.ClaimsListRevision; import google.registry.model.tmch.ClaimsList.ClaimsListRevision;
import google.registry.model.tmch.ClaimsListShard.ClaimsListSingleton; import google.registry.model.tmch.ClaimsList.ClaimsListSingleton;
import google.registry.model.tmch.TmchCrl; import google.registry.model.tmch.TmchCrl;
import google.registry.schema.replay.LastSqlTransaction; import google.registry.schema.replay.LastSqlTransaction;
@ -64,7 +64,7 @@ public final class EntityClasses {
BillingEvent.Modification.class, BillingEvent.Modification.class,
BillingEvent.OneTime.class, BillingEvent.OneTime.class,
BillingEvent.Recurring.class, BillingEvent.Recurring.class,
ClaimsListShard.class, ClaimsList.class,
ClaimsListRevision.class, ClaimsListRevision.class,
ClaimsListSingleton.class, ClaimsListSingleton.class,
CommitLogBucket.class, CommitLogBucket.class,

View file

@ -38,7 +38,7 @@ import google.registry.model.common.TimedTransitionProperty.TimedTransition;
import google.registry.model.registry.label.PremiumList; import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.ReservedList; import google.registry.model.registry.label.ReservedList;
import google.registry.model.smd.SignedMarkRevocationList; import google.registry.model.smd.SignedMarkRevocationList;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsList;
import google.registry.persistence.VKey; import google.registry.persistence.VKey;
import google.registry.schema.replay.DatastoreOnlyEntity; import google.registry.schema.replay.DatastoreOnlyEntity;
import java.util.Optional; import java.util.Optional;
@ -61,7 +61,7 @@ public class DatabaseTransitionSchedule extends ImmutableObject implements Datas
/** The id of the transition schedule. */ /** The id of the transition schedule. */
public enum TransitionId { public enum TransitionId {
/** The schedule for migration of {@link ClaimsListShard} entities. */ /** The schedule for migration of {@link ClaimsList} entities. */
CLAIMS_LIST, CLAIMS_LIST,
/** The schedule for the migration of {@link PremiumList} and {@link ReservedList}. */ /** The schedule for the migration of {@link PremiumList} and {@link ReservedList}. */
DOMAIN_LABEL_LISTS, DOMAIN_LABEL_LISTS,

View file

@ -16,22 +16,16 @@ package google.registry.model.tmch;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Throwables.throwIfUnchecked;
import static com.google.common.base.Verify.verify;
import static google.registry.model.ofy.ObjectifyService.allocateId; import static google.registry.model.ofy.ObjectifyService.allocateId;
import static google.registry.model.ofy.ObjectifyService.auditedOfy; import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.UncheckedExecutionException;
import com.googlecode.objectify.Key; import com.googlecode.objectify.Key;
import com.googlecode.objectify.annotation.EmbedMap; import com.googlecode.objectify.annotation.EmbedMap;
import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Entity;
import com.googlecode.objectify.annotation.Id; import com.googlecode.objectify.annotation.Id;
import com.googlecode.objectify.annotation.Ignore; import com.googlecode.objectify.annotation.Ignore;
import com.googlecode.objectify.annotation.OnSave;
import com.googlecode.objectify.annotation.Parent; import com.googlecode.objectify.annotation.Parent;
import google.registry.model.CreateAutoTimestamp; import google.registry.model.CreateAutoTimestamp;
import google.registry.model.ImmutableObject; import google.registry.model.ImmutableObject;
@ -42,12 +36,6 @@ import google.registry.model.annotations.VirtualEntity;
import google.registry.model.common.CrossTldSingleton; import google.registry.model.common.CrossTldSingleton;
import google.registry.schema.replay.DatastoreOnlyEntity; import google.registry.schema.replay.DatastoreOnlyEntity;
import google.registry.schema.replay.NonReplicatedEntity; import google.registry.schema.replay.NonReplicatedEntity;
import google.registry.util.CollectionUtils;
import google.registry.util.Concurrent;
import google.registry.util.Retrier;
import google.registry.util.SystemSleeper;
import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import javax.annotation.Nullable; import javax.annotation.Nullable;
@ -65,36 +53,20 @@ import org.joda.time.DateTime;
/** /**
* A list of TMCH claims labels and their associated claims keys. * A list of TMCH claims labels and their associated claims keys.
* *
* <p>The claims list is actually sharded into multiple {@link ClaimsListShard} entities to work
* around the Datastore limitation of 1M max size per entity. However, when calling {@link
* #getFromDatastore} all of the shards are recombined into one {@link ClaimsListShard} object.
*
* <p>ClaimsList shards are tied to a specific revision and are persisted individually, then the
* entire claims list is atomically shifted over to using the new shards by persisting the new
* revision object and updating the {@link ClaimsListSingleton} pointing to it. This bypasses the
* 10MB per transaction limit.
*
* <p>Therefore, it is never OK to save an instance of this class directly to Datastore. Instead you
* must use the {@link #saveToDatastore} method to do it for you.
*
* <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by * <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
* the database. So, if a retry of insertion happens after the previous attempt unexpectedly * the database. So, if a retry of insertion happens after the previous attempt unexpectedly
* succeeds, we will end up with having two exact same claims list with only different {@link * succeeds, we will end up with having two exact same claims list with only different {@link
* #revisionId}. However, this is not an actual problem because we only use the claims list with * #revisionId}. However, this is not an actual problem because we only use the claims list with
* highest {@link #revisionId}. * highest {@link #revisionId}.
* *
* <p>TODO(b/162007765): Rename the class to ClaimsList and remove Datastore related fields and * <p>TODO(b/162007765): Remove Datastore related fields and methods.
* methods.
*/ */
@Entity @Entity
@NotBackedUp(reason = Reason.EXTERNALLY_SOURCED) @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
@javax.persistence.Entity(name = "ClaimsList") @javax.persistence.Entity(name = "ClaimsList")
@Table @Table
@InCrossTld @InCrossTld
public class ClaimsListShard extends ImmutableObject implements NonReplicatedEntity { public class ClaimsList extends ImmutableObject implements NonReplicatedEntity {
/** The number of claims list entries to store per shard. */
private static final int SHARD_SIZE = 10000;
@Transient @Id long id; @Transient @Id long id;
@ -132,64 +104,6 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
@Column(name = "claimKey", nullable = false) @Column(name = "claimKey", nullable = false)
Map<String, String> labelsToKeys; Map<String, String> labelsToKeys;
/** Indicates that this is a shard rather than a "full" list. */
@Ignore @Transient boolean isShard = false;
private static final Retrier LOADER_RETRIER = new Retrier(new SystemSleeper(), 2);
private static Optional<ClaimsListShard> loadClaimsListShard() {
// Find the most recent revision.
Key<ClaimsListRevision> revisionKey = getCurrentRevision();
if (revisionKey == null) {
return Optional.empty();
}
Map<String, String> combinedLabelsToKeys = new HashMap<>();
DateTime creationTime = START_OF_TIME;
// Grab all of the keys for the shards that belong to the current revision.
final List<Key<ClaimsListShard>> shardKeys =
auditedOfy().load().type(ClaimsListShard.class).ancestor(revisionKey).keys().list();
List<ClaimsListShard> shards;
try {
// Load all of the shards concurrently, each in a separate transaction.
shards =
Concurrent.transform(
shardKeys,
key ->
ofyTm()
.transactNewReadOnly(
() -> {
ClaimsListShard claimsListShard = auditedOfy().load().key(key).now();
checkState(
claimsListShard != null,
"Key not found when loading claims list shards.");
return claimsListShard;
}));
} catch (UncheckedExecutionException e) {
// We retry on IllegalStateException. However, there's a checkState inside the
// Concurrent.transform, so if it's thrown it'll be wrapped in an
// UncheckedExecutionException. We want to unwrap it so it's caught by the retrier.
if (e.getCause() != null) {
throwIfUnchecked(e.getCause());
}
throw e;
}
// Combine the shards together and return the concatenated ClaimsList.
if (!shards.isEmpty()) {
creationTime = shards.get(0).creationTime;
for (ClaimsListShard shard : shards) {
combinedLabelsToKeys.putAll(shard.labelsToKeys);
checkState(
creationTime.equals(shard.creationTime),
"Inconsistent claims list shard creation times.");
}
}
return Optional.of(create(creationTime, ImmutableMap.copyOf(combinedLabelsToKeys)));
}
/** Returns the revision id of this claims list, or throws exception if it is null. */ /** Returns the revision id of this claims list, or throws exception if it is null. */
public Long getRevisionId() { public Long getRevisionId() {
checkState( checkState(
@ -227,80 +141,14 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
return labelsToKeys.size(); return labelsToKeys.size();
} }
/** public static ClaimsList create(DateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
* Save the Claims list to Datastore by writing the new shards in a series of transactions, ClaimsList instance = new ClaimsList();
* switching over to using them atomically, then deleting the old ones.
*/
void saveToDatastore() {
saveToDatastore(SHARD_SIZE);
}
@VisibleForTesting
void saveToDatastore(int shardSize) {
// Figure out what the next versionId should be based on which ones already exist.
final Key<ClaimsListRevision> oldRevision = getCurrentRevision();
final Key<ClaimsListRevision> parentKey = ClaimsListRevision.createKey();
// Save the ClaimsList shards in separate transactions.
Concurrent.transform(
CollectionUtils.partitionMap(labelsToKeys, shardSize),
(final ImmutableMap<String, String> labelsToKeysShard) ->
ofyTm()
.transact(
() -> {
ClaimsListShard shard = create(creationTime, labelsToKeysShard);
shard.isShard = true;
shard.parent = parentKey;
auditedOfy().saveWithoutBackup().entity(shard);
return shard;
}));
// Persist the new revision, thus causing the newly created shards to go live.
ofyTm()
.transact(
() -> {
verify(
(getCurrentRevision() == null && oldRevision == null)
|| getCurrentRevision().equals(oldRevision),
"Registries' ClaimsList was updated by someone else while attempting to update.");
auditedOfy().saveWithoutBackup().entity(ClaimsListSingleton.create(parentKey));
// Delete the old ClaimsListShard entities.
if (oldRevision != null) {
auditedOfy()
.deleteWithoutBackup()
.keys(
auditedOfy()
.load()
.type(ClaimsListShard.class)
.ancestor(oldRevision)
.keys());
}
});
}
public static ClaimsListShard create(
DateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
ClaimsListShard instance = new ClaimsListShard();
instance.id = allocateId(); instance.id = allocateId();
instance.creationTime = checkNotNull(tmdbGenerationTime); instance.creationTime = checkNotNull(tmdbGenerationTime);
instance.labelsToKeys = checkNotNull(labelsToKeys); instance.labelsToKeys = checkNotNull(labelsToKeys);
return instance; return instance;
} }
/** Return a single logical instance that combines all Datastore shards. */
static Optional<ClaimsListShard> getFromDatastore() {
return LOADER_RETRIER.callWithRetry(
ClaimsListShard::loadClaimsListShard, IllegalStateException.class);
}
/** As a safety mechanism, fail if someone tries to save this class directly. */
@OnSave
void disallowUnshardedSaves() {
if (!isShard) {
throw new UnshardedSaveException();
}
}
/** Virtual parent entity for claims list shards of a specific revision. */ /** Virtual parent entity for claims list shards of a specific revision. */
@Entity @Entity
@VirtualEntity @VirtualEntity
@ -354,6 +202,6 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
return singleton == null ? null : singleton.activeRevision; return singleton == null ? null : singleton.activeRevision;
} }
/** Exception when trying to directly save a {@link ClaimsListShard} without sharding. */ /** Exception when trying to directly save a {@link ClaimsList} without sharding. */
public static class UnshardedSaveException extends RuntimeException {} public static class UnshardedSaveException extends RuntimeException {}
} }

View file

@ -15,22 +15,23 @@
package google.registry.model.tmch; package google.registry.model.tmch;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import java.util.Optional; import com.google.common.collect.ImmutableMap;
/** Data access object for {@link ClaimsListShard}. */ /** Data access object for {@link ClaimsList}. */
public class ClaimsListSqlDao { public class ClaimsListDao {
/** Saves the given {@link ClaimsListShard} to Cloud SQL. */ /** Saves the given {@link ClaimsList} to Cloud SQL. */
static void save(ClaimsListShard claimsList) { public static void save(ClaimsList claimsList) {
jpaTm().transact(() -> jpaTm().insert(claimsList)); jpaTm().transact(() -> jpaTm().insert(claimsList));
} }
/** /**
* Returns the most recent revision of the {@link ClaimsListShard} in SQL or an empty list if it * Returns the most recent revision of the {@link ClaimsList} in SQL or an empty list if it
* doesn't exist. * doesn't exist.
*/ */
static Optional<ClaimsListShard> get() { public static ClaimsList get() {
return jpaTm() return jpaTm()
.transact( .transact(
() -> { () -> {
@ -42,12 +43,13 @@ public class ClaimsListSqlDao {
.query( .query(
"FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId =" "FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId ="
+ " :revisionId", + " :revisionId",
ClaimsListShard.class) ClaimsList.class)
.setParameter("revisionId", revisionId) .setParameter("revisionId", revisionId)
.getResultStream() .getResultStream()
.findFirst(); .findFirst();
}); })
.orElse(ClaimsList.create(START_OF_TIME, ImmutableMap.of()));
} }
private ClaimsListSqlDao() {} private ClaimsListDao() {}
} }

View file

@ -1,128 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.tmch;
import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
import static google.registry.model.CacheUtils.tryMemoizeWithExpiration;
import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
import google.registry.util.NonFinalForTesting;
import java.util.Optional;
/**
* DAO for {@link ClaimsListShard} objects that handles the branching paths for SQL and Datastore.
*
* <p>For write actions, this class will perform the action against Cloud SQL then, after that
* success or failure, against Datastore. If Datastore fails, an error is logged (but not thrown).
*
* <p>For read actions, we will log if the two databases have different values (or if the retrieval
* from Datastore fails).
*/
public class ClaimsListDualDatabaseDao {
/** In-memory cache for claims list. */
@NonFinalForTesting
private static Supplier<ClaimsListShard> claimsListCache =
tryMemoizeWithExpiration(
getDomainLabelListCacheDuration(), ClaimsListDualDatabaseDao::getUncached);
/**
* Saves the given {@link ClaimsListShard} to both the primary and secondary databases, logging
* and skipping errors in Datastore.
*/
public static void save(ClaimsListShard claimsList) {
ClaimsListSqlDao.save(claimsList);
suppressExceptionUnlessInTest(
claimsList::saveToDatastore, "Error saving ClaimsListShard to Datastore.");
}
/** Returns the most recent revision of the {@link ClaimsListShard}, from cache. */
public static ClaimsListShard get() {
return claimsListCache.get();
}
/** Retrieves and compares the latest revision from the databases. */
private static ClaimsListShard getUncached() {
Optional<ClaimsListShard> cloudSqlResult = ClaimsListSqlDao.get();
suppressExceptionUnlessInTest(
() -> {
Optional<ClaimsListShard> datastoreResult = ClaimsListShard.getFromDatastore();
compareClaimsLists(cloudSqlResult, datastoreResult);
},
"Error loading ClaimsListShard from Datastore.");
return cloudSqlResult.orElse(ClaimsListShard.create(START_OF_TIME, ImmutableMap.of()));
}
private static void compareClaimsLists(
Optional<ClaimsListShard> maybeCloudSql, Optional<ClaimsListShard> maybeDatastore) {
if (maybeCloudSql.isPresent() && !maybeDatastore.isPresent()) {
throw new IllegalStateException("Claims list found in Cloud SQL but not in Datastore.");
}
if (!maybeCloudSql.isPresent() && maybeDatastore.isPresent()) {
throw new IllegalStateException("Claims list found in Datastore but not in Cloud SQL.");
}
if (!maybeCloudSql.isPresent()) {
return;
}
ClaimsListShard sqlList = maybeCloudSql.get();
ClaimsListShard datastoreList = maybeDatastore.get();
MapDifference<String, String> diff =
Maps.difference(sqlList.labelsToKeys, datastoreList.getLabelsToKeys());
if (!diff.areEqual()) {
if (diff.entriesDiffering().size()
+ diff.entriesOnlyOnRight().size()
+ diff.entriesOnlyOnLeft().size()
> 10) {
throw new IllegalStateException(
String.format(
"Unequal claims lists detected, Datastore list with revision id %d has %d"
+ " different records than the current Cloud SQL list.",
datastoreList.getRevisionId(), diff.entriesDiffering().size()));
} else {
StringBuilder diffMessage = new StringBuilder("Unequal claims lists detected:\n");
diff.entriesDiffering()
.forEach(
(label, valueDiff) ->
diffMessage.append(
String.format(
"Domain label %s has key %s in Cloud SQL and key %s "
+ "in Datastore.\n",
label, valueDiff.leftValue(), valueDiff.rightValue())));
diff.entriesOnlyOnLeft()
.forEach(
(label, valueDiff) ->
diffMessage.append(
String.format(
"Domain label %s with key %s only appears in Cloud SQL.\n",
label, valueDiff)));
diff.entriesOnlyOnRight()
.forEach(
(label, valueDiff) ->
diffMessage.append(
String.format(
"Domain label %s with key %s only appears in Datastore.\n",
label, valueDiff)));
throw new IllegalStateException(diffMessage.toString());
}
}
}
private ClaimsListDualDatabaseDao() {}
}

View file

@ -37,7 +37,7 @@ import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex; import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
import google.registry.model.ofy.DatastoreTransactionManager; import google.registry.model.ofy.DatastoreTransactionManager;
import google.registry.model.server.KmsSecret; import google.registry.model.server.KmsSecret;
import google.registry.model.tmch.ClaimsListShard.ClaimsListSingleton; import google.registry.model.tmch.ClaimsList.ClaimsListSingleton;
import google.registry.persistence.JpaRetries; import google.registry.persistence.JpaRetries;
import google.registry.persistence.VKey; import google.registry.persistence.VKey;
import google.registry.util.Clock; import google.registry.util.Clock;

View file

@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.base.Splitter; import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsList;
import java.util.List; import java.util.List;
import org.joda.time.DateTime; import org.joda.time.DateTime;
@ -34,11 +34,11 @@ import org.joda.time.DateTime;
public class ClaimsListParser { public class ClaimsListParser {
/** /**
* Converts the lines from the DNL CSV file into a {@link ClaimsListShard} object. * Converts the lines from the DNL CSV file into a {@link ClaimsList} object.
* *
* <p>Please note that this does <b>not</b> insert the object into Datastore. * <p>Please note that this does <b>not</b> insert the object into Datastore.
*/ */
public static ClaimsListShard parse(List<String> lines) { public static ClaimsList parse(List<String> lines) {
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>(); ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
// First line: <version>,<DNL List creation datetime> // First line: <version>,<DNL List creation datetime>
@ -74,6 +74,6 @@ public class ClaimsListParser {
builder.put(label, lookupKey); builder.put(label, lookupKey);
} }
return ClaimsListShard.create(creationTime, builder.build()); return ClaimsList.create(creationTime, builder.build());
} }
} }

View file

@ -18,8 +18,8 @@ import static google.registry.request.Action.Method.POST;
import com.google.common.flogger.FluentLogger; import com.google.common.flogger.FluentLogger;
import google.registry.keyring.api.KeyModule.Key; import google.registry.keyring.api.KeyModule.Key;
import google.registry.model.tmch.ClaimsListDualDatabaseDao; import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsListDao;
import google.registry.request.Action; import google.registry.request.Action;
import google.registry.request.auth.Auth; import google.registry.request.auth.Auth;
import java.io.IOException; import java.io.IOException;
@ -55,8 +55,8 @@ public final class TmchDnlAction implements Runnable {
} catch (SignatureException | IOException | PGPException e) { } catch (SignatureException | IOException | PGPException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
ClaimsListShard claims = ClaimsListParser.parse(lines); ClaimsList claims = ClaimsListParser.parse(lines);
ClaimsListDualDatabaseDao.save(claims); ClaimsListDao.save(claims);
logger.atInfo().log( logger.atInfo().log(
"Inserted %,d claims into the DB(s), created at %s", "Inserted %,d claims into the DB(s), created at %s",
claims.size(), claims.getTmdbGenerationTime()); claims.size(), claims.getTmdbGenerationTime());

View file

@ -21,8 +21,8 @@ import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters; import com.beust.jcommander.Parameters;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import com.google.common.io.Files; import com.google.common.io.Files;
import google.registry.model.tmch.ClaimsListDualDatabaseDao; import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsListDao;
import google.registry.tools.params.PathParameter; import google.registry.tools.params.PathParameter;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
@ -44,7 +44,7 @@ final class GetClaimsListCommand implements CommandWithRemoteApi {
@Override @Override
public void run() throws Exception { public void run() throws Exception {
ClaimsListShard cl = checkNotNull(ClaimsListDualDatabaseDao.get(), "Couldn't load ClaimsList"); ClaimsList cl = checkNotNull(ClaimsListDao.get(), "Couldn't load ClaimsList");
String csv = Joiner.on('\n').withKeyValueSeparator(",").join(cl.getLabelsToKeys()) + "\n"; String csv = Joiner.on('\n').withKeyValueSeparator(",").join(cl.getLabelsToKeys()) + "\n";
Files.asCharSink(output.toFile(), UTF_8).write(csv); Files.asCharSink(output.toFile(), UTF_8).write(csv);
} }

View file

@ -21,15 +21,15 @@ import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters; import com.beust.jcommander.Parameters;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import com.google.common.io.Files; import com.google.common.io.Files;
import google.registry.model.tmch.ClaimsListDualDatabaseDao; import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsListDao;
import google.registry.tmch.ClaimsListParser; import google.registry.tmch.ClaimsListParser;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
/** A command to upload a {@link ClaimsListShard}. */ /** A command to upload a {@link ClaimsList}. */
@Parameters(separators = " =", commandDescription = "Manually upload a new claims list file") @Parameters(separators = " =", commandDescription = "Manually upload a new claims list file")
final class UploadClaimsListCommand extends ConfirmingCommand implements CommandWithRemoteApi { final class UploadClaimsListCommand extends ConfirmingCommand implements CommandWithRemoteApi {
@ -38,7 +38,7 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
private String claimsListFilename; private String claimsListFilename;
private ClaimsListShard claimsList; private ClaimsList claimsList;
@Override @Override
protected void init() throws IOException { protected void init() throws IOException {
@ -57,7 +57,7 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
@Override @Override
public String execute() { public String execute() {
ClaimsListDualDatabaseDao.save(claimsList); ClaimsListDao.save(claimsList);
return String.format("Successfully uploaded claims list %s", claimsListFilename); return String.format("Successfully uploaded claims list %s", claimsListFilename);
} }
} }

View file

@ -68,7 +68,7 @@
<class>google.registry.model.server.Lock</class> <class>google.registry.model.server.Lock</class>
<class>google.registry.model.server.ServerSecret</class> <class>google.registry.model.server.ServerSecret</class>
<class>google.registry.model.smd.SignedMarkRevocationList</class> <class>google.registry.model.smd.SignedMarkRevocationList</class>
<class>google.registry.model.tmch.ClaimsListShard</class> <class>google.registry.model.tmch.ClaimsList</class>
<class>google.registry.model.tmch.TmchCrl</class> <class>google.registry.model.tmch.TmchCrl</class>
<class>google.registry.persistence.transaction.TransactionEntity</class> <class>google.registry.persistence.transaction.TransactionEntity</class>
<class>google.registry.schema.domain.RegistryLock</class> <class>google.registry.schema.domain.RegistryLock</class>

View file

@ -55,7 +55,7 @@ import google.registry.model.registrar.RegistrarContact;
import google.registry.model.registry.label.PremiumList; import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.ReservedList; import google.registry.model.registry.label.ReservedList;
import google.registry.model.server.Lock; import google.registry.model.server.Lock;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsList;
import google.registry.model.translators.VKeyTranslatorFactory; import google.registry.model.translators.VKeyTranslatorFactory;
import google.registry.persistence.VKey; import google.registry.persistence.VKey;
import google.registry.persistence.transaction.JpaTransactionManager; import google.registry.persistence.transaction.JpaTransactionManager;
@ -405,7 +405,7 @@ public class ReplayCommitLogsToSqlActionTest {
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1))); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
// Save a couple deletes that aren't propagated to SQL (the objects deleted are irrelevant) // Save a couple deletes that aren't propagated to SQL (the objects deleted are irrelevant)
Key<ClaimsListShard> claimsListKey = Key.create(ClaimsListShard.class, 1L); Key<ClaimsList> claimsListKey = Key.create(ClaimsList.class, 1L);
saveDiffFile( saveDiffFile(
gcsService, gcsService,
createCheckpoint(now.minusMinutes(1)), createCheckpoint(now.minusMinutes(1)),

View file

@ -47,8 +47,8 @@ import google.registry.model.host.HostHistory;
import google.registry.model.index.EppResourceIndex; import google.registry.model.index.EppResourceIndex;
import google.registry.model.index.EppResourceIndexBucket; import google.registry.model.index.EppResourceIndexBucket;
import google.registry.model.reporting.HistoryEntry; import google.registry.model.reporting.HistoryEntry;
import google.registry.model.tmch.ClaimsListDualDatabaseDao; import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsListDao;
import google.registry.testing.DatabaseHelper; import google.registry.testing.DatabaseHelper;
import google.registry.testing.TaskQueueHelper.TaskMatcher; import google.registry.testing.TaskQueueHelper.TaskMatcher;
import google.registry.util.TypeUtils.TypeInstantiator; import google.registry.util.TypeUtils.TypeInstantiator;
@ -115,9 +115,9 @@ public abstract class ResourceFlowTestCase<F extends Flow, R extends EppResource
return new TypeInstantiator<R>(getClass()) {}.getExactType(); return new TypeInstantiator<R>(getClass()) {}.getExactType();
} }
/** Persists a testing claims list to Datastore that contains a single shard. */ /** Persists a testing claims list to Cloud SQL. */
protected void persistClaimsList(ImmutableMap<String, String> labelsToKeys) { protected void persistClaimsList(ImmutableMap<String, String> labelsToKeys) {
ClaimsListDualDatabaseDao.save(ClaimsListShard.create(clock.nowUtc(), labelsToKeys)); ClaimsListDao.save(ClaimsList.create(clock.nowUtc(), labelsToKeys));
} }
@Test @Test

View file

@ -18,7 +18,6 @@ import static com.google.common.truth.Truth.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.truth.Truth8;
import google.registry.persistence.transaction.JpaTestRules; import google.registry.persistence.transaction.JpaTestRules;
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension; import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension;
import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.DatastoreEntityExtension;
@ -28,8 +27,8 @@ import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.extension.RegisterExtension;
/** Unit tests for {@link ClaimsListSqlDao}. */ /** Unit tests for {@link ClaimsListDao}. */
public class ClaimsListSqlDaoTest { public class ClaimsListDaoTest {
private final FakeClock fakeClock = new FakeClock(); private final FakeClock fakeClock = new FakeClock();
@ -43,55 +42,51 @@ public class ClaimsListSqlDaoTest {
@Test @Test
void save_insertsClaimsListSuccessfully() { void save_insertsClaimsListSuccessfully() {
ClaimsListShard claimsList = ClaimsList claimsList =
ClaimsListShard.create( ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2")); ClaimsListDao.save(claimsList);
ClaimsListSqlDao.save(claimsList); ClaimsList insertedClaimsList = ClaimsListDao.get();
ClaimsListShard insertedClaimsList = ClaimsListSqlDao.get().get();
assertClaimsListEquals(claimsList, insertedClaimsList); assertClaimsListEquals(claimsList, insertedClaimsList);
assertThat(insertedClaimsList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc()); assertThat(insertedClaimsList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc());
} }
@Test @Test
void save_fail_duplicateId() { void save_fail_duplicateId() {
ClaimsListShard claimsList = ClaimsList claimsList =
ClaimsListShard.create( ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2")); ClaimsListDao.save(claimsList);
ClaimsListSqlDao.save(claimsList); ClaimsList insertedClaimsList = ClaimsListDao.get();
ClaimsListShard insertedClaimsList = ClaimsListSqlDao.get().get();
assertClaimsListEquals(claimsList, insertedClaimsList); assertClaimsListEquals(claimsList, insertedClaimsList);
// Save ClaimsList with existing revisionId should fail because revisionId is the primary key. // Save ClaimsList with existing revisionId should fail because revisionId is the primary key.
assertThrows(PersistenceException.class, () -> ClaimsListSqlDao.save(insertedClaimsList)); assertThrows(PersistenceException.class, () -> ClaimsListDao.save(insertedClaimsList));
} }
@Test @Test
void save_claimsListWithNoEntries() { void save_claimsListWithNoEntries() {
ClaimsListShard claimsList = ClaimsListShard.create(fakeClock.nowUtc(), ImmutableMap.of()); ClaimsList claimsList = ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of());
ClaimsListSqlDao.save(claimsList); ClaimsListDao.save(claimsList);
ClaimsListShard insertedClaimsList = ClaimsListSqlDao.get().get(); ClaimsList insertedClaimsList = ClaimsListDao.get();
assertClaimsListEquals(claimsList, insertedClaimsList); assertClaimsListEquals(claimsList, insertedClaimsList);
assertThat(insertedClaimsList.getLabelsToKeys()).isEmpty(); assertThat(insertedClaimsList.getLabelsToKeys()).isEmpty();
} }
@Test @Test
void getCurrent_returnsEmptyListIfTableIsEmpty() { void getCurrent_returnsEmptyListIfTableIsEmpty() {
Truth8.assertThat(ClaimsListSqlDao.get()).isEmpty(); assertThat(ClaimsListDao.get().labelsToKeys).isEmpty();
} }
@Test @Test
void getCurrent_returnsLatestClaims() { void getCurrent_returnsLatestClaims() {
ClaimsListShard oldClaimsList = ClaimsList oldClaimsList =
ClaimsListShard.create( ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2")); ClaimsList newClaimsList =
ClaimsListShard newClaimsList = ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4"));
ClaimsListShard.create( ClaimsListDao.save(oldClaimsList);
fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4")); ClaimsListDao.save(newClaimsList);
ClaimsListSqlDao.save(oldClaimsList); assertClaimsListEquals(newClaimsList, ClaimsListDao.get());
ClaimsListSqlDao.save(newClaimsList);
assertClaimsListEquals(newClaimsList, ClaimsListSqlDao.get().get());
} }
private void assertClaimsListEquals(ClaimsListShard left, ClaimsListShard right) { private void assertClaimsListEquals(ClaimsList left, ClaimsList right) {
assertThat(left.getRevisionId()).isEqualTo(right.getRevisionId()); assertThat(left.getRevisionId()).isEqualTo(right.getRevisionId());
assertThat(left.getTmdbGenerationTime()).isEqualTo(right.getTmdbGenerationTime()); assertThat(left.getTmdbGenerationTime()).isEqualTo(right.getTmdbGenerationTime());
assertThat(left.getLabelsToKeys()).isEqualTo(right.getLabelsToKeys()); assertThat(left.getLabelsToKeys()).isEqualTo(right.getLabelsToKeys());

View file

@ -1,86 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.tmch;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static org.junit.jupiter.api.Assertions.assertThrows;
import com.google.common.collect.ImmutableMap;
import google.registry.config.RegistryEnvironment;
import google.registry.model.EntityTestCase;
import google.registry.testing.SystemPropertyExtension;
import org.joda.time.Duration;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
/** Unit tests for {@link ClaimsListDualDatabaseDao}. */
public class ClaimsListDualDatabaseDaoTest extends EntityTestCase {
@RegisterExtension
@Order(value = Integer.MAX_VALUE)
final SystemPropertyExtension systemPropertyExtension = new SystemPropertyExtension();
@Test
void testGetList_missingOfy() {
ClaimsListSqlDao.save(createClaimsList());
assertThat(assertThrows(IllegalStateException.class, ClaimsListDualDatabaseDao::get))
.hasMessageThat()
.isEqualTo("Claims list found in Cloud SQL but not in Datastore.");
}
@Test
void testGetList_fromSql_different() {
ClaimsListShard.create(fakeClock.nowUtc(), ImmutableMap.of("foo", "bar")).saveToDatastore();
ClaimsListSqlDao.save(createClaimsList());
assertThat(assertThrows(IllegalStateException.class, ClaimsListDualDatabaseDao::get))
.hasMessageThat()
.isEqualTo(
"Unequal claims lists detected:\n"
+ "Domain label label1 with key key1 only appears in Cloud SQL.\n"
+ "Domain label label2 with key key2 only appears in Cloud SQL.\n"
+ "Domain label foo with key bar only appears in Datastore.\n");
}
@Test
void testSaveAndGet() {
tm().transact(() -> ClaimsListDualDatabaseDao.save(createClaimsList()));
assertAboutImmutableObjects()
.that(ClaimsListDualDatabaseDao.get())
.isEqualExceptFields(createClaimsList(), "id", "revisionId", "creationTimestamp");
}
@Test
void testGet_empty() {
assertThat(tm().transact(ClaimsListDualDatabaseDao::get).getLabelsToKeys()).isEmpty();
}
@Test
void testGetList_missingOfy_notInTest() {
RegistryEnvironment.PRODUCTION.setup(systemPropertyExtension);
fakeClock.advanceBy(Duration.standardDays(5));
ClaimsListSqlDao.save(createClaimsList());
// Shouldn't fail in production
assertThat(ClaimsListDualDatabaseDao.get().getLabelsToKeys())
.isEqualTo(createClaimsList().getLabelsToKeys());
}
private ClaimsListShard createClaimsList() {
return ClaimsListShard.create(
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
}
}

View file

@ -1,116 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.tmch;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static org.joda.time.DateTimeZone.UTC;
import static org.junit.jupiter.api.Assertions.assertThrows;
import com.google.common.collect.ImmutableMap;
import com.googlecode.objectify.Key;
import google.registry.model.tmch.ClaimsListShard.ClaimsListRevision;
import google.registry.model.tmch.ClaimsListShard.UnshardedSaveException;
import google.registry.testing.AppEngineExtension;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.joda.time.DateTime;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
/** Unit tests for {@link ClaimsListShard}. */
public class ClaimsListShardTest {
@RegisterExtension
public final AppEngineExtension appEngine =
AppEngineExtension.builder().withDatastoreAndCloudSql().build();
private final int shardSize = 10;
@Test
void test_unshardedSaveFails() {
assertThrows(
UnshardedSaveException.class,
() ->
tm().transact(
() -> {
ClaimsListShard claimsList =
ClaimsListShard.create(
tm().getTransactionTime(), ImmutableMap.of("a", "b"));
claimsList.id = 1; // Without an id this won't save anyways.
claimsList.parent = ClaimsListRevision.createKey();
auditedOfy().saveWithoutBackup().entity(claimsList).now();
}));
}
@Test
void testGet_safelyLoadsEmptyClaimsList_whenNoShardsExist() {
assertThat(ClaimsListShard.getFromDatastore()).isEmpty();
}
@Test
void test_savesAndGets_withSharding() {
// Create a ClaimsList that will need 4 shards to save.
Map<String, String> labelsToKeys = new HashMap<>();
for (int i = 0; i <= shardSize * 3; i++) {
labelsToKeys.put(Integer.toString(i), Integer.toString(i));
}
DateTime now = DateTime.now(UTC);
// Save it with sharding, and make sure that reloading it works.
ClaimsListShard unsharded = ClaimsListShard.create(now, ImmutableMap.copyOf(labelsToKeys));
unsharded.saveToDatastore(shardSize);
assertThat(ClaimsListShard.getFromDatastore().get().labelsToKeys)
.isEqualTo(unsharded.labelsToKeys);
List<ClaimsListShard> shards1 = auditedOfy().load().type(ClaimsListShard.class).list();
assertThat(shards1).hasSize(4);
assertThat(ClaimsListShard.getFromDatastore().get().getClaimKey("1")).hasValue("1");
assertThat(ClaimsListShard.getFromDatastore().get().getClaimKey("a")).isEmpty();
assertThat(ClaimsListShard.getCurrentRevision()).isEqualTo(shards1.get(0).parent);
// Create a smaller ClaimsList that will need only 2 shards to save.
labelsToKeys = new HashMap<>();
for (int i = 0; i <= shardSize; i++) {
labelsToKeys.put(Integer.toString(i), Integer.toString(i));
}
unsharded = ClaimsListShard.create(now.plusDays(1), ImmutableMap.copyOf(labelsToKeys));
unsharded.saveToDatastore(shardSize);
auditedOfy().clearSessionCache();
assertThat(ClaimsListShard.getFromDatastore().get().labelsToKeys)
.hasSize(unsharded.labelsToKeys.size());
assertThat(ClaimsListShard.getFromDatastore().get().labelsToKeys)
.isEqualTo(unsharded.labelsToKeys);
List<ClaimsListShard> shards2 = auditedOfy().load().type(ClaimsListShard.class).list();
assertThat(shards2).hasSize(2);
// Expect that the old revision is deleted.
assertThat(ClaimsListShard.getCurrentRevision()).isEqualTo(shards2.get(0).parent);
}
/**
* Returns a created claims list shard with the specified parent key for testing purposes only.
*/
public static ClaimsListShard createTestClaimsListShard(
DateTime creationTime,
ImmutableMap<String, String> labelsToKeys,
Key<ClaimsListRevision> revision) {
ClaimsListShard claimsList = ClaimsListShard.create(creationTime, labelsToKeys);
claimsList.isShard = true;
claimsList.parent = revision;
return claimsList;
}
}

View file

@ -34,7 +34,7 @@ import google.registry.model.server.KmsSecretRevisionSqlDaoTest;
import google.registry.model.server.LockTest; import google.registry.model.server.LockTest;
import google.registry.model.server.ServerSecretTest; import google.registry.model.server.ServerSecretTest;
import google.registry.model.smd.SignedMarkRevocationListDaoTest; import google.registry.model.smd.SignedMarkRevocationListDaoTest;
import google.registry.model.tmch.ClaimsListSqlDaoTest; import google.registry.model.tmch.ClaimsListDaoTest;
import google.registry.model.tmch.TmchCrlTest; import google.registry.model.tmch.TmchCrlTest;
import google.registry.persistence.transaction.JpaEntityCoverageExtension; import google.registry.persistence.transaction.JpaEntityCoverageExtension;
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension; import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension;
@ -82,7 +82,7 @@ import org.junit.runner.RunWith;
BeforeSuiteTest.class, BeforeSuiteTest.class,
AllocationTokenTest.class, AllocationTokenTest.class,
BillingEventTest.class, BillingEventTest.class,
ClaimsListSqlDaoTest.class, ClaimsListDaoTest.class,
ContactHistoryTest.class, ContactHistoryTest.class,
ContactResourceTest.class, ContactResourceTest.class,
CursorTest.class, CursorTest.class,

View file

@ -20,8 +20,8 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import google.registry.model.tmch.ClaimsListDualDatabaseDao; import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsListDao;
import java.util.Optional; import java.util.Optional;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -38,7 +38,7 @@ class TmchDnlActionTest extends TmchActionTestCase {
@Test @Test
void testDnl() throws Exception { void testDnl() throws Exception {
assertThat(ClaimsListDualDatabaseDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty(); assertThat(ClaimsListDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty();
when(httpResponse.getContent()) when(httpResponse.getContent())
.thenReturn(TmchTestData.loadBytes("dnl-latest.csv").read()) .thenReturn(TmchTestData.loadBytes("dnl-latest.csv").read())
.thenReturn(TmchTestData.loadBytes("dnl-latest.sig").read()); .thenReturn(TmchTestData.loadBytes("dnl-latest.sig").read());
@ -50,7 +50,7 @@ class TmchDnlActionTest extends TmchActionTestCase {
.isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.sig"); .isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.sig");
// Make sure the contents of testdata/dnl-latest.csv got inserted into the database. // Make sure the contents of testdata/dnl-latest.csv got inserted into the database.
ClaimsListShard claimsList = ClaimsListDualDatabaseDao.get(); ClaimsList claimsList = ClaimsListDao.get();
assertThat(claimsList.getTmdbGenerationTime()) assertThat(claimsList.getTmdbGenerationTime())
.isEqualTo(DateTime.parse("2013-11-24T23:15:37.4Z")); .isEqualTo(DateTime.parse("2013-11-24T23:15:37.4Z"));
assertThat(claimsList.getClaimKey("xn----7sbejwbn3axu3d")) assertThat(claimsList.getClaimKey("xn----7sbejwbn3axu3d"))

View file

@ -20,8 +20,8 @@ import static java.nio.file.Files.readAllLines;
import static org.joda.time.DateTimeZone.UTC; import static org.joda.time.DateTimeZone.UTC;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import google.registry.model.tmch.ClaimsListDualDatabaseDao; import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsListDao;
import java.io.File; import java.io.File;
import java.nio.file.Files; import java.nio.file.Files;
import org.joda.time.DateTime; import org.joda.time.DateTime;
@ -32,8 +32,7 @@ class GetClaimsListCommandTest extends CommandTestCase<GetClaimsListCommand> {
@Test @Test
void testSuccess_getWorks() throws Exception { void testSuccess_getWorks() throws Exception {
ClaimsListDualDatabaseDao.save( ClaimsListDao.save(ClaimsList.create(DateTime.now(UTC), ImmutableMap.of("a", "1", "b", "2")));
ClaimsListShard.create(DateTime.now(UTC), ImmutableMap.of("a", "1", "b", "2")));
File output = tmpDir.resolve("claims.txt").toFile(); File output = tmpDir.resolve("claims.txt").toFile();
runCommand("--output=" + output.getAbsolutePath()); runCommand("--output=" + output.getAbsolutePath());
assertThat(readAllLines(output.toPath(), UTF_8)).containsExactly("a,1", "b,2"); assertThat(readAllLines(output.toPath(), UTF_8)).containsExactly("a,1", "b,2");
@ -41,8 +40,7 @@ class GetClaimsListCommandTest extends CommandTestCase<GetClaimsListCommand> {
@Test @Test
void testSuccess_endsWithNewline() throws Exception { void testSuccess_endsWithNewline() throws Exception {
ClaimsListDualDatabaseDao.save( ClaimsListDao.save(ClaimsList.create(DateTime.now(UTC), ImmutableMap.of("a", "1")));
ClaimsListShard.create(DateTime.now(UTC), ImmutableMap.of("a", "1")));
File output = tmpDir.resolve("claims.txt").toFile(); File output = tmpDir.resolve("claims.txt").toFile();
runCommand("--output=" + output.getAbsolutePath()); runCommand("--output=" + output.getAbsolutePath());
assertThat(new String(Files.readAllBytes(output.toPath()), UTF_8)).endsWith("\n"); assertThat(new String(Files.readAllBytes(output.toPath()), UTF_8)).endsWith("\n");

View file

@ -18,8 +18,8 @@ import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat; import static com.google.common.truth.Truth8.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import google.registry.model.tmch.ClaimsListDualDatabaseDao; import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsListDao;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -37,7 +37,7 @@ class UploadClaimsListCommandTest extends CommandTestCase<UploadClaimsListComman
"anotherexample,2013041500/A/C/7/rHdC4wnrWRvPY6nneCVtQhFj0000000003,2011-08-16T12:00:00.0Z"); "anotherexample,2013041500/A/C/7/rHdC4wnrWRvPY6nneCVtQhFj0000000003,2011-08-16T12:00:00.0Z");
runCommand("--force", filename); runCommand("--force", filename);
ClaimsListShard claimsList = ClaimsListDualDatabaseDao.get(); ClaimsList claimsList = ClaimsListDao.get();
assertThat(claimsList.getTmdbGenerationTime()) assertThat(claimsList.getTmdbGenerationTime())
.isEqualTo(DateTime.parse("2012-08-16T00:00:00.0Z")); .isEqualTo(DateTime.parse("2012-08-16T00:00:00.0Z"));
assertThat(claimsList.getClaimKey("example")) assertThat(claimsList.getClaimKey("example"))

View file

@ -1,4 +1,4 @@
ClaimsListShard ClaimsList
ClaimsListSingleton ClaimsListSingleton
Cursor Cursor
DatabaseTransitionSchedule DatabaseTransitionSchedule

View file

@ -847,20 +847,20 @@ class google.registry.model.server.ServerSecret {
long leastSignificant; long leastSignificant;
long mostSignificant; long mostSignificant;
} }
class google.registry.model.tmch.ClaimsListShard { class google.registry.model.tmch.ClaimsList {
@Id long id; @Id long id;
@Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsListShard$ClaimsListRevision> parent; @Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsList$ClaimsListRevision> parent;
java.util.Map<java.lang.String, java.lang.String> labelsToKeys; java.util.Map<java.lang.String, java.lang.String> labelsToKeys;
org.joda.time.DateTime creationTime; org.joda.time.DateTime creationTime;
} }
class google.registry.model.tmch.ClaimsListShard$ClaimsListRevision { class google.registry.model.tmch.ClaimsList$ClaimsListRevision {
@Id long versionId; @Id long versionId;
@Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsListShard$ClaimsListSingleton> parent; @Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsList$ClaimsListSingleton> parent;
} }
class google.registry.model.tmch.ClaimsListShard$ClaimsListSingleton { class google.registry.model.tmch.ClaimsList$ClaimsListSingleton {
@Id long id; @Id long id;
@Parent com.googlecode.objectify.Key<google.registry.model.common.EntityGroupRoot> parent; @Parent com.googlecode.objectify.Key<google.registry.model.common.EntityGroupRoot> parent;
com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsListShard$ClaimsListRevision> activeRevision; com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsList$ClaimsListRevision> activeRevision;
} }
class google.registry.model.tmch.TmchCrl { class google.registry.model.tmch.TmchCrl {
@Id long id; @Id long id;