mirror of
https://github.com/google/nomulus.git
synced 2025-08-05 09:21:49 +02:00
Stop writing ClaimsList to Datastore (#1169)
* Stop writing ClaimsList to Datastore * Fix some failing tests * Rename ClaimsListShard to ClaimsList
This commit is contained in:
sarahcaseybot
GitHub
parent
38fa08b930
commit
c27e0f4118
24 changed files with 94 additions and 583 deletions
|
|
@ -45,7 +45,7 @@ import google.registry.model.eppinput.ResourceCommand;
|
|||
import google.registry.model.eppoutput.EppResponse;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.HashSet;
|
||||
import java.util.Optional;
|
||||
|
|
@ -104,8 +104,7 @@ public final class DomainClaimsCheckFlow implements Flow {
|
|||
verifyClaimsPeriodNotEnded(registry, now);
|
||||
}
|
||||
}
|
||||
Optional<String> claimKey =
|
||||
ClaimsListDualDatabaseDao.get().getClaimKey(parsedDomain.parts().get(0));
|
||||
Optional<String> claimKey = ClaimsListDao.get().getClaimKey(parsedDomain.parts().get(0));
|
||||
launchChecksBuilder.add(
|
||||
LaunchCheck.create(
|
||||
LaunchCheckName.create(claimKey.isPresent(), domainName), claimKey.orElse(null)));
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ import google.registry.model.registry.label.ReservedList;
|
|||
import google.registry.model.reporting.DomainTransactionRecord;
|
||||
import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.tldconfig.idn.IdnLabelValidator;
|
||||
import google.registry.util.Idn;
|
||||
|
|
@ -990,8 +990,7 @@ public class DomainFlowUtils {
|
|||
static void verifyClaimsNoticeIfAndOnlyIfNeeded(
|
||||
InternetDomainName domainName, boolean hasSignedMarks, boolean hasClaimsNotice)
|
||||
throws EppException {
|
||||
boolean isInClaimsList =
|
||||
ClaimsListDualDatabaseDao.get().getClaimKey(domainName.parts().get(0)).isPresent();
|
||||
boolean isInClaimsList = ClaimsListDao.get().getClaimKey(domainName.parts().get(0)).isPresent();
|
||||
if (hasClaimsNotice && !isInClaimsList) {
|
||||
throw new UnexpectedClaimsNoticeException(domainName.toString());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,9 +47,9 @@ import google.registry.model.server.KmsSecret;
|
|||
import google.registry.model.server.KmsSecretRevision;
|
||||
import google.registry.model.server.Lock;
|
||||
import google.registry.model.server.ServerSecret;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsListShard.ClaimsListRevision;
|
||||
import google.registry.model.tmch.ClaimsListShard.ClaimsListSingleton;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsList.ClaimsListRevision;
|
||||
import google.registry.model.tmch.ClaimsList.ClaimsListSingleton;
|
||||
import google.registry.model.tmch.TmchCrl;
|
||||
import google.registry.schema.replay.LastSqlTransaction;
|
||||
|
||||
|
|
@ -64,7 +64,7 @@ public final class EntityClasses {
|
|||
BillingEvent.Modification.class,
|
||||
BillingEvent.OneTime.class,
|
||||
BillingEvent.Recurring.class,
|
||||
ClaimsListShard.class,
|
||||
ClaimsList.class,
|
||||
ClaimsListRevision.class,
|
||||
ClaimsListSingleton.class,
|
||||
CommitLogBucket.class,
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ import google.registry.model.common.TimedTransitionProperty.TimedTransition;
|
|||
import google.registry.model.registry.label.PremiumList;
|
||||
import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.model.smd.SignedMarkRevocationList;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.schema.replay.DatastoreOnlyEntity;
|
||||
import java.util.Optional;
|
||||
|
|
@ -61,7 +61,7 @@ public class DatabaseTransitionSchedule extends ImmutableObject implements Datas
|
|||
|
||||
/** The id of the transition schedule. */
|
||||
public enum TransitionId {
|
||||
/** The schedule for migration of {@link ClaimsListShard} entities. */
|
||||
/** The schedule for migration of {@link ClaimsList} entities. */
|
||||
CLAIMS_LIST,
|
||||
/** The schedule for the migration of {@link PremiumList} and {@link ReservedList}. */
|
||||
DOMAIN_LABEL_LISTS,
|
||||
|
|
|
|||
|
|
@ -16,22 +16,16 @@ package google.registry.model.tmch;
|
|||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Throwables.throwIfUnchecked;
|
||||
import static com.google.common.base.Verify.verify;
|
||||
import static google.registry.model.ofy.ObjectifyService.allocateId;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.EmbedMap;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import com.googlecode.objectify.annotation.Ignore;
|
||||
import com.googlecode.objectify.annotation.OnSave;
|
||||
import com.googlecode.objectify.annotation.Parent;
|
||||
import google.registry.model.CreateAutoTimestamp;
|
||||
import google.registry.model.ImmutableObject;
|
||||
|
|
@ -42,12 +36,6 @@ import google.registry.model.annotations.VirtualEntity;
|
|||
import google.registry.model.common.CrossTldSingleton;
|
||||
import google.registry.schema.replay.DatastoreOnlyEntity;
|
||||
import google.registry.schema.replay.NonReplicatedEntity;
|
||||
import google.registry.util.CollectionUtils;
|
||||
import google.registry.util.Concurrent;
|
||||
import google.registry.util.Retrier;
|
||||
import google.registry.util.SystemSleeper;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import javax.annotation.Nullable;
|
||||
|
|
@ -65,36 +53,20 @@ import org.joda.time.DateTime;
|
|||
/**
|
||||
* A list of TMCH claims labels and their associated claims keys.
|
||||
*
|
||||
* <p>The claims list is actually sharded into multiple {@link ClaimsListShard} entities to work
|
||||
* around the Datastore limitation of 1M max size per entity. However, when calling {@link
|
||||
* #getFromDatastore} all of the shards are recombined into one {@link ClaimsListShard} object.
|
||||
*
|
||||
* <p>ClaimsList shards are tied to a specific revision and are persisted individually, then the
|
||||
* entire claims list is atomically shifted over to using the new shards by persisting the new
|
||||
* revision object and updating the {@link ClaimsListSingleton} pointing to it. This bypasses the
|
||||
* 10MB per transaction limit.
|
||||
*
|
||||
* <p>Therefore, it is never OK to save an instance of this class directly to Datastore. Instead you
|
||||
* must use the {@link #saveToDatastore} method to do it for you.
|
||||
*
|
||||
* <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
|
||||
* the database. So, if a retry of insertion happens after the previous attempt unexpectedly
|
||||
* succeeds, we will end up with having two exact same claims list with only different {@link
|
||||
* #revisionId}. However, this is not an actual problem because we only use the claims list with
|
||||
* highest {@link #revisionId}.
|
||||
*
|
||||
* <p>TODO(b/162007765): Rename the class to ClaimsList and remove Datastore related fields and
|
||||
* methods.
|
||||
* <p>TODO(b/162007765): Remove Datastore related fields and methods.
|
||||
*/
|
||||
@Entity
|
||||
@NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
|
||||
@javax.persistence.Entity(name = "ClaimsList")
|
||||
@Table
|
||||
@InCrossTld
|
||||
public class ClaimsListShard extends ImmutableObject implements NonReplicatedEntity {
|
||||
|
||||
/** The number of claims list entries to store per shard. */
|
||||
private static final int SHARD_SIZE = 10000;
|
||||
public class ClaimsList extends ImmutableObject implements NonReplicatedEntity {
|
||||
|
||||
@Transient @Id long id;
|
||||
|
||||
|
|
@ -132,64 +104,6 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
|
|||
@Column(name = "claimKey", nullable = false)
|
||||
Map<String, String> labelsToKeys;
|
||||
|
||||
/** Indicates that this is a shard rather than a "full" list. */
|
||||
@Ignore @Transient boolean isShard = false;
|
||||
|
||||
private static final Retrier LOADER_RETRIER = new Retrier(new SystemSleeper(), 2);
|
||||
|
||||
private static Optional<ClaimsListShard> loadClaimsListShard() {
|
||||
// Find the most recent revision.
|
||||
Key<ClaimsListRevision> revisionKey = getCurrentRevision();
|
||||
if (revisionKey == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
Map<String, String> combinedLabelsToKeys = new HashMap<>();
|
||||
DateTime creationTime = START_OF_TIME;
|
||||
// Grab all of the keys for the shards that belong to the current revision.
|
||||
final List<Key<ClaimsListShard>> shardKeys =
|
||||
auditedOfy().load().type(ClaimsListShard.class).ancestor(revisionKey).keys().list();
|
||||
|
||||
List<ClaimsListShard> shards;
|
||||
try {
|
||||
// Load all of the shards concurrently, each in a separate transaction.
|
||||
shards =
|
||||
Concurrent.transform(
|
||||
shardKeys,
|
||||
key ->
|
||||
ofyTm()
|
||||
.transactNewReadOnly(
|
||||
() -> {
|
||||
ClaimsListShard claimsListShard = auditedOfy().load().key(key).now();
|
||||
checkState(
|
||||
claimsListShard != null,
|
||||
"Key not found when loading claims list shards.");
|
||||
return claimsListShard;
|
||||
}));
|
||||
} catch (UncheckedExecutionException e) {
|
||||
// We retry on IllegalStateException. However, there's a checkState inside the
|
||||
// Concurrent.transform, so if it's thrown it'll be wrapped in an
|
||||
// UncheckedExecutionException. We want to unwrap it so it's caught by the retrier.
|
||||
if (e.getCause() != null) {
|
||||
throwIfUnchecked(e.getCause());
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Combine the shards together and return the concatenated ClaimsList.
|
||||
if (!shards.isEmpty()) {
|
||||
creationTime = shards.get(0).creationTime;
|
||||
for (ClaimsListShard shard : shards) {
|
||||
combinedLabelsToKeys.putAll(shard.labelsToKeys);
|
||||
checkState(
|
||||
creationTime.equals(shard.creationTime),
|
||||
"Inconsistent claims list shard creation times.");
|
||||
}
|
||||
}
|
||||
|
||||
return Optional.of(create(creationTime, ImmutableMap.copyOf(combinedLabelsToKeys)));
|
||||
}
|
||||
|
||||
/** Returns the revision id of this claims list, or throws exception if it is null. */
|
||||
public Long getRevisionId() {
|
||||
checkState(
|
||||
|
|
@ -227,80 +141,14 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
|
|||
return labelsToKeys.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the Claims list to Datastore by writing the new shards in a series of transactions,
|
||||
* switching over to using them atomically, then deleting the old ones.
|
||||
*/
|
||||
void saveToDatastore() {
|
||||
saveToDatastore(SHARD_SIZE);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
void saveToDatastore(int shardSize) {
|
||||
// Figure out what the next versionId should be based on which ones already exist.
|
||||
final Key<ClaimsListRevision> oldRevision = getCurrentRevision();
|
||||
final Key<ClaimsListRevision> parentKey = ClaimsListRevision.createKey();
|
||||
|
||||
// Save the ClaimsList shards in separate transactions.
|
||||
Concurrent.transform(
|
||||
CollectionUtils.partitionMap(labelsToKeys, shardSize),
|
||||
(final ImmutableMap<String, String> labelsToKeysShard) ->
|
||||
ofyTm()
|
||||
.transact(
|
||||
() -> {
|
||||
ClaimsListShard shard = create(creationTime, labelsToKeysShard);
|
||||
shard.isShard = true;
|
||||
shard.parent = parentKey;
|
||||
auditedOfy().saveWithoutBackup().entity(shard);
|
||||
return shard;
|
||||
}));
|
||||
|
||||
// Persist the new revision, thus causing the newly created shards to go live.
|
||||
ofyTm()
|
||||
.transact(
|
||||
() -> {
|
||||
verify(
|
||||
(getCurrentRevision() == null && oldRevision == null)
|
||||
|| getCurrentRevision().equals(oldRevision),
|
||||
"Registries' ClaimsList was updated by someone else while attempting to update.");
|
||||
auditedOfy().saveWithoutBackup().entity(ClaimsListSingleton.create(parentKey));
|
||||
// Delete the old ClaimsListShard entities.
|
||||
if (oldRevision != null) {
|
||||
auditedOfy()
|
||||
.deleteWithoutBackup()
|
||||
.keys(
|
||||
auditedOfy()
|
||||
.load()
|
||||
.type(ClaimsListShard.class)
|
||||
.ancestor(oldRevision)
|
||||
.keys());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public static ClaimsListShard create(
|
||||
DateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
|
||||
ClaimsListShard instance = new ClaimsListShard();
|
||||
public static ClaimsList create(DateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
|
||||
ClaimsList instance = new ClaimsList();
|
||||
instance.id = allocateId();
|
||||
instance.creationTime = checkNotNull(tmdbGenerationTime);
|
||||
instance.labelsToKeys = checkNotNull(labelsToKeys);
|
||||
return instance;
|
||||
}
|
||||
|
||||
/** Return a single logical instance that combines all Datastore shards. */
|
||||
static Optional<ClaimsListShard> getFromDatastore() {
|
||||
return LOADER_RETRIER.callWithRetry(
|
||||
ClaimsListShard::loadClaimsListShard, IllegalStateException.class);
|
||||
}
|
||||
|
||||
/** As a safety mechanism, fail if someone tries to save this class directly. */
|
||||
@OnSave
|
||||
void disallowUnshardedSaves() {
|
||||
if (!isShard) {
|
||||
throw new UnshardedSaveException();
|
||||
}
|
||||
}
|
||||
|
||||
/** Virtual parent entity for claims list shards of a specific revision. */
|
||||
@Entity
|
||||
@VirtualEntity
|
||||
|
|
@ -354,6 +202,6 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
|
|||
return singleton == null ? null : singleton.activeRevision;
|
||||
}
|
||||
|
||||
/** Exception when trying to directly save a {@link ClaimsListShard} without sharding. */
|
||||
/** Exception when trying to directly save a {@link ClaimsList} without sharding. */
|
||||
public static class UnshardedSaveException extends RuntimeException {}
|
||||
}
|
||||
|
|
@ -15,22 +15,23 @@
|
|||
package google.registry.model.tmch;
|
||||
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import java.util.Optional;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
/** Data access object for {@link ClaimsListShard}. */
|
||||
public class ClaimsListSqlDao {
|
||||
/** Data access object for {@link ClaimsList}. */
|
||||
public class ClaimsListDao {
|
||||
|
||||
/** Saves the given {@link ClaimsListShard} to Cloud SQL. */
|
||||
static void save(ClaimsListShard claimsList) {
|
||||
/** Saves the given {@link ClaimsList} to Cloud SQL. */
|
||||
public static void save(ClaimsList claimsList) {
|
||||
jpaTm().transact(() -> jpaTm().insert(claimsList));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the most recent revision of the {@link ClaimsListShard} in SQL or an empty list if it
|
||||
* Returns the most recent revision of the {@link ClaimsList} in SQL or an empty list if it
|
||||
* doesn't exist.
|
||||
*/
|
||||
static Optional<ClaimsListShard> get() {
|
||||
public static ClaimsList get() {
|
||||
return jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
|
|
@ -42,12 +43,13 @@ public class ClaimsListSqlDao {
|
|||
.query(
|
||||
"FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId ="
|
||||
+ " :revisionId",
|
||||
ClaimsListShard.class)
|
||||
ClaimsList.class)
|
||||
.setParameter("revisionId", revisionId)
|
||||
.getResultStream()
|
||||
.findFirst();
|
||||
});
|
||||
})
|
||||
.orElse(ClaimsList.create(START_OF_TIME, ImmutableMap.of()));
|
||||
}
|
||||
|
||||
private ClaimsListSqlDao() {}
|
||||
private ClaimsListDao() {}
|
||||
}
|
||||
|
|
@ -1,128 +0,0 @@
|
|||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.tmch;
|
||||
|
||||
import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
|
||||
import static google.registry.model.CacheUtils.tryMemoizeWithExpiration;
|
||||
import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.google.common.base.Supplier;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.MapDifference;
|
||||
import com.google.common.collect.Maps;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* DAO for {@link ClaimsListShard} objects that handles the branching paths for SQL and Datastore.
|
||||
*
|
||||
* <p>For write actions, this class will perform the action against Cloud SQL then, after that
|
||||
* success or failure, against Datastore. If Datastore fails, an error is logged (but not thrown).
|
||||
*
|
||||
* <p>For read actions, we will log if the two databases have different values (or if the retrieval
|
||||
* from Datastore fails).
|
||||
*/
|
||||
public class ClaimsListDualDatabaseDao {
|
||||
|
||||
/** In-memory cache for claims list. */
|
||||
@NonFinalForTesting
|
||||
private static Supplier<ClaimsListShard> claimsListCache =
|
||||
tryMemoizeWithExpiration(
|
||||
getDomainLabelListCacheDuration(), ClaimsListDualDatabaseDao::getUncached);
|
||||
|
||||
/**
|
||||
* Saves the given {@link ClaimsListShard} to both the primary and secondary databases, logging
|
||||
* and skipping errors in Datastore.
|
||||
*/
|
||||
public static void save(ClaimsListShard claimsList) {
|
||||
ClaimsListSqlDao.save(claimsList);
|
||||
suppressExceptionUnlessInTest(
|
||||
claimsList::saveToDatastore, "Error saving ClaimsListShard to Datastore.");
|
||||
}
|
||||
|
||||
/** Returns the most recent revision of the {@link ClaimsListShard}, from cache. */
|
||||
public static ClaimsListShard get() {
|
||||
return claimsListCache.get();
|
||||
}
|
||||
|
||||
/** Retrieves and compares the latest revision from the databases. */
|
||||
private static ClaimsListShard getUncached() {
|
||||
Optional<ClaimsListShard> cloudSqlResult = ClaimsListSqlDao.get();
|
||||
suppressExceptionUnlessInTest(
|
||||
() -> {
|
||||
Optional<ClaimsListShard> datastoreResult = ClaimsListShard.getFromDatastore();
|
||||
compareClaimsLists(cloudSqlResult, datastoreResult);
|
||||
},
|
||||
"Error loading ClaimsListShard from Datastore.");
|
||||
return cloudSqlResult.orElse(ClaimsListShard.create(START_OF_TIME, ImmutableMap.of()));
|
||||
}
|
||||
|
||||
private static void compareClaimsLists(
|
||||
Optional<ClaimsListShard> maybeCloudSql, Optional<ClaimsListShard> maybeDatastore) {
|
||||
if (maybeCloudSql.isPresent() && !maybeDatastore.isPresent()) {
|
||||
throw new IllegalStateException("Claims list found in Cloud SQL but not in Datastore.");
|
||||
}
|
||||
if (!maybeCloudSql.isPresent() && maybeDatastore.isPresent()) {
|
||||
throw new IllegalStateException("Claims list found in Datastore but not in Cloud SQL.");
|
||||
}
|
||||
if (!maybeCloudSql.isPresent()) {
|
||||
return;
|
||||
}
|
||||
ClaimsListShard sqlList = maybeCloudSql.get();
|
||||
ClaimsListShard datastoreList = maybeDatastore.get();
|
||||
MapDifference<String, String> diff =
|
||||
Maps.difference(sqlList.labelsToKeys, datastoreList.getLabelsToKeys());
|
||||
if (!diff.areEqual()) {
|
||||
if (diff.entriesDiffering().size()
|
||||
+ diff.entriesOnlyOnRight().size()
|
||||
+ diff.entriesOnlyOnLeft().size()
|
||||
> 10) {
|
||||
throw new IllegalStateException(
|
||||
String.format(
|
||||
"Unequal claims lists detected, Datastore list with revision id %d has %d"
|
||||
+ " different records than the current Cloud SQL list.",
|
||||
datastoreList.getRevisionId(), diff.entriesDiffering().size()));
|
||||
} else {
|
||||
StringBuilder diffMessage = new StringBuilder("Unequal claims lists detected:\n");
|
||||
diff.entriesDiffering()
|
||||
.forEach(
|
||||
(label, valueDiff) ->
|
||||
diffMessage.append(
|
||||
String.format(
|
||||
"Domain label %s has key %s in Cloud SQL and key %s "
|
||||
+ "in Datastore.\n",
|
||||
label, valueDiff.leftValue(), valueDiff.rightValue())));
|
||||
diff.entriesOnlyOnLeft()
|
||||
.forEach(
|
||||
(label, valueDiff) ->
|
||||
diffMessage.append(
|
||||
String.format(
|
||||
"Domain label %s with key %s only appears in Cloud SQL.\n",
|
||||
label, valueDiff)));
|
||||
diff.entriesOnlyOnRight()
|
||||
.forEach(
|
||||
(label, valueDiff) ->
|
||||
diffMessage.append(
|
||||
String.format(
|
||||
"Domain label %s with key %s only appears in Datastore.\n",
|
||||
label, valueDiff)));
|
||||
throw new IllegalStateException(diffMessage.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private ClaimsListDualDatabaseDao() {}
|
||||
}
|
||||
|
|
@ -37,7 +37,7 @@ import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
|
|||
import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
|
||||
import google.registry.model.ofy.DatastoreTransactionManager;
|
||||
import google.registry.model.server.KmsSecret;
|
||||
import google.registry.model.tmch.ClaimsListShard.ClaimsListSingleton;
|
||||
import google.registry.model.tmch.ClaimsList.ClaimsListSingleton;
|
||||
import google.registry.persistence.JpaRetries;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.util.Clock;
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
|
|||
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import java.util.List;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
|
|
@ -34,11 +34,11 @@ import org.joda.time.DateTime;
|
|||
public class ClaimsListParser {
|
||||
|
||||
/**
|
||||
* Converts the lines from the DNL CSV file into a {@link ClaimsListShard} object.
|
||||
* Converts the lines from the DNL CSV file into a {@link ClaimsList} object.
|
||||
*
|
||||
* <p>Please note that this does <b>not</b> insert the object into Datastore.
|
||||
*/
|
||||
public static ClaimsListShard parse(List<String> lines) {
|
||||
public static ClaimsList parse(List<String> lines) {
|
||||
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
|
||||
|
||||
// First line: <version>,<DNL List creation datetime>
|
||||
|
|
@ -74,6 +74,6 @@ public class ClaimsListParser {
|
|||
builder.put(label, lookupKey);
|
||||
}
|
||||
|
||||
return ClaimsListShard.create(creationTime, builder.build());
|
||||
return ClaimsList.create(creationTime, builder.build());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ import static google.registry.request.Action.Method.POST;
|
|||
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.keyring.api.KeyModule.Key;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.auth.Auth;
|
||||
import java.io.IOException;
|
||||
|
|
@ -55,8 +55,8 @@ public final class TmchDnlAction implements Runnable {
|
|||
} catch (SignatureException | IOException | PGPException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
ClaimsListShard claims = ClaimsListParser.parse(lines);
|
||||
ClaimsListDualDatabaseDao.save(claims);
|
||||
ClaimsList claims = ClaimsListParser.parse(lines);
|
||||
ClaimsListDao.save(claims);
|
||||
logger.atInfo().log(
|
||||
"Inserted %,d claims into the DB(s), created at %s",
|
||||
claims.size(), claims.getTmdbGenerationTime());
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ import com.beust.jcommander.Parameter;
|
|||
import com.beust.jcommander.Parameters;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.io.Files;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.tools.params.PathParameter;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
|
|
@ -44,7 +44,7 @@ final class GetClaimsListCommand implements CommandWithRemoteApi {
|
|||
|
||||
@Override
|
||||
public void run() throws Exception {
|
||||
ClaimsListShard cl = checkNotNull(ClaimsListDualDatabaseDao.get(), "Couldn't load ClaimsList");
|
||||
ClaimsList cl = checkNotNull(ClaimsListDao.get(), "Couldn't load ClaimsList");
|
||||
String csv = Joiner.on('\n').withKeyValueSeparator(",").join(cl.getLabelsToKeys()) + "\n";
|
||||
Files.asCharSink(output.toFile(), UTF_8).write(csv);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,15 +21,15 @@ import com.beust.jcommander.Parameter;
|
|||
import com.beust.jcommander.Parameters;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.io.Files;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.tmch.ClaimsListParser;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/** A command to upload a {@link ClaimsListShard}. */
|
||||
/** A command to upload a {@link ClaimsList}. */
|
||||
@Parameters(separators = " =", commandDescription = "Manually upload a new claims list file")
|
||||
final class UploadClaimsListCommand extends ConfirmingCommand implements CommandWithRemoteApi {
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
|
|||
|
||||
private String claimsListFilename;
|
||||
|
||||
private ClaimsListShard claimsList;
|
||||
private ClaimsList claimsList;
|
||||
|
||||
@Override
|
||||
protected void init() throws IOException {
|
||||
|
|
@ -57,7 +57,7 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
|
|||
|
||||
@Override
|
||||
public String execute() {
|
||||
ClaimsListDualDatabaseDao.save(claimsList);
|
||||
ClaimsListDao.save(claimsList);
|
||||
return String.format("Successfully uploaded claims list %s", claimsListFilename);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@
|
|||
<class>google.registry.model.server.Lock</class>
|
||||
<class>google.registry.model.server.ServerSecret</class>
|
||||
<class>google.registry.model.smd.SignedMarkRevocationList</class>
|
||||
<class>google.registry.model.tmch.ClaimsListShard</class>
|
||||
<class>google.registry.model.tmch.ClaimsList</class>
|
||||
<class>google.registry.model.tmch.TmchCrl</class>
|
||||
<class>google.registry.persistence.transaction.TransactionEntity</class>
|
||||
<class>google.registry.schema.domain.RegistryLock</class>
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ import google.registry.model.registrar.RegistrarContact;
|
|||
import google.registry.model.registry.label.PremiumList;
|
||||
import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.model.server.Lock;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.translators.VKeyTranslatorFactory;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.persistence.transaction.JpaTransactionManager;
|
||||
|
|
@ -405,7 +405,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
// Save a couple deletes that aren't propagated to SQL (the objects deleted are irrelevant)
|
||||
Key<ClaimsListShard> claimsListKey = Key.create(ClaimsListShard.class, 1L);
|
||||
Key<ClaimsList> claimsListKey = Key.create(ClaimsList.class, 1L);
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
|
|
|
|||
|
|
@ -47,8 +47,8 @@ import google.registry.model.host.HostHistory;
|
|||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.EppResourceIndexBucket;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.testing.DatabaseHelper;
|
||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||
import google.registry.util.TypeUtils.TypeInstantiator;
|
||||
|
|
@ -115,9 +115,9 @@ public abstract class ResourceFlowTestCase<F extends Flow, R extends EppResource
|
|||
return new TypeInstantiator<R>(getClass()) {}.getExactType();
|
||||
}
|
||||
|
||||
/** Persists a testing claims list to Datastore that contains a single shard. */
|
||||
/** Persists a testing claims list to Cloud SQL. */
|
||||
protected void persistClaimsList(ImmutableMap<String, String> labelsToKeys) {
|
||||
ClaimsListDualDatabaseDao.save(ClaimsListShard.create(clock.nowUtc(), labelsToKeys));
|
||||
ClaimsListDao.save(ClaimsList.create(clock.nowUtc(), labelsToKeys));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ import static com.google.common.truth.Truth.assertThat;
|
|||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.truth.Truth8;
|
||||
import google.registry.persistence.transaction.JpaTestRules;
|
||||
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension;
|
||||
import google.registry.testing.DatastoreEntityExtension;
|
||||
|
|
@ -28,8 +27,8 @@ import org.junit.jupiter.api.Order;
|
|||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
|
||||
/** Unit tests for {@link ClaimsListSqlDao}. */
|
||||
public class ClaimsListSqlDaoTest {
|
||||
/** Unit tests for {@link ClaimsListDao}. */
|
||||
public class ClaimsListDaoTest {
|
||||
|
||||
private final FakeClock fakeClock = new FakeClock();
|
||||
|
||||
|
|
@ -43,55 +42,51 @@ public class ClaimsListSqlDaoTest {
|
|||
|
||||
@Test
|
||||
void save_insertsClaimsListSuccessfully() {
|
||||
ClaimsListShard claimsList =
|
||||
ClaimsListShard.create(
|
||||
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
|
||||
ClaimsListSqlDao.save(claimsList);
|
||||
ClaimsListShard insertedClaimsList = ClaimsListSqlDao.get().get();
|
||||
ClaimsList claimsList =
|
||||
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
|
||||
ClaimsListDao.save(claimsList);
|
||||
ClaimsList insertedClaimsList = ClaimsListDao.get();
|
||||
assertClaimsListEquals(claimsList, insertedClaimsList);
|
||||
assertThat(insertedClaimsList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc());
|
||||
}
|
||||
|
||||
@Test
|
||||
void save_fail_duplicateId() {
|
||||
ClaimsListShard claimsList =
|
||||
ClaimsListShard.create(
|
||||
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
|
||||
ClaimsListSqlDao.save(claimsList);
|
||||
ClaimsListShard insertedClaimsList = ClaimsListSqlDao.get().get();
|
||||
ClaimsList claimsList =
|
||||
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
|
||||
ClaimsListDao.save(claimsList);
|
||||
ClaimsList insertedClaimsList = ClaimsListDao.get();
|
||||
assertClaimsListEquals(claimsList, insertedClaimsList);
|
||||
// Save ClaimsList with existing revisionId should fail because revisionId is the primary key.
|
||||
assertThrows(PersistenceException.class, () -> ClaimsListSqlDao.save(insertedClaimsList));
|
||||
assertThrows(PersistenceException.class, () -> ClaimsListDao.save(insertedClaimsList));
|
||||
}
|
||||
|
||||
@Test
|
||||
void save_claimsListWithNoEntries() {
|
||||
ClaimsListShard claimsList = ClaimsListShard.create(fakeClock.nowUtc(), ImmutableMap.of());
|
||||
ClaimsListSqlDao.save(claimsList);
|
||||
ClaimsListShard insertedClaimsList = ClaimsListSqlDao.get().get();
|
||||
ClaimsList claimsList = ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of());
|
||||
ClaimsListDao.save(claimsList);
|
||||
ClaimsList insertedClaimsList = ClaimsListDao.get();
|
||||
assertClaimsListEquals(claimsList, insertedClaimsList);
|
||||
assertThat(insertedClaimsList.getLabelsToKeys()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void getCurrent_returnsEmptyListIfTableIsEmpty() {
|
||||
Truth8.assertThat(ClaimsListSqlDao.get()).isEmpty();
|
||||
assertThat(ClaimsListDao.get().labelsToKeys).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void getCurrent_returnsLatestClaims() {
|
||||
ClaimsListShard oldClaimsList =
|
||||
ClaimsListShard.create(
|
||||
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
|
||||
ClaimsListShard newClaimsList =
|
||||
ClaimsListShard.create(
|
||||
fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4"));
|
||||
ClaimsListSqlDao.save(oldClaimsList);
|
||||
ClaimsListSqlDao.save(newClaimsList);
|
||||
assertClaimsListEquals(newClaimsList, ClaimsListSqlDao.get().get());
|
||||
ClaimsList oldClaimsList =
|
||||
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
|
||||
ClaimsList newClaimsList =
|
||||
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4"));
|
||||
ClaimsListDao.save(oldClaimsList);
|
||||
ClaimsListDao.save(newClaimsList);
|
||||
assertClaimsListEquals(newClaimsList, ClaimsListDao.get());
|
||||
}
|
||||
|
||||
private void assertClaimsListEquals(ClaimsListShard left, ClaimsListShard right) {
|
||||
private void assertClaimsListEquals(ClaimsList left, ClaimsList right) {
|
||||
assertThat(left.getRevisionId()).isEqualTo(right.getRevisionId());
|
||||
assertThat(left.getTmdbGenerationTime()).isEqualTo(right.getTmdbGenerationTime());
|
||||
assertThat(left.getLabelsToKeys()).isEqualTo(right.getLabelsToKeys());
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.tmch;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.model.EntityTestCase;
|
||||
import google.registry.testing.SystemPropertyExtension;
|
||||
import org.joda.time.Duration;
|
||||
import org.junit.jupiter.api.Order;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
|
||||
/** Unit tests for {@link ClaimsListDualDatabaseDao}. */
|
||||
public class ClaimsListDualDatabaseDaoTest extends EntityTestCase {
|
||||
|
||||
@RegisterExtension
|
||||
@Order(value = Integer.MAX_VALUE)
|
||||
final SystemPropertyExtension systemPropertyExtension = new SystemPropertyExtension();
|
||||
|
||||
@Test
|
||||
void testGetList_missingOfy() {
|
||||
ClaimsListSqlDao.save(createClaimsList());
|
||||
assertThat(assertThrows(IllegalStateException.class, ClaimsListDualDatabaseDao::get))
|
||||
.hasMessageThat()
|
||||
.isEqualTo("Claims list found in Cloud SQL but not in Datastore.");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGetList_fromSql_different() {
|
||||
ClaimsListShard.create(fakeClock.nowUtc(), ImmutableMap.of("foo", "bar")).saveToDatastore();
|
||||
ClaimsListSqlDao.save(createClaimsList());
|
||||
assertThat(assertThrows(IllegalStateException.class, ClaimsListDualDatabaseDao::get))
|
||||
.hasMessageThat()
|
||||
.isEqualTo(
|
||||
"Unequal claims lists detected:\n"
|
||||
+ "Domain label label1 with key key1 only appears in Cloud SQL.\n"
|
||||
+ "Domain label label2 with key key2 only appears in Cloud SQL.\n"
|
||||
+ "Domain label foo with key bar only appears in Datastore.\n");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSaveAndGet() {
|
||||
tm().transact(() -> ClaimsListDualDatabaseDao.save(createClaimsList()));
|
||||
assertAboutImmutableObjects()
|
||||
.that(ClaimsListDualDatabaseDao.get())
|
||||
.isEqualExceptFields(createClaimsList(), "id", "revisionId", "creationTimestamp");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGet_empty() {
|
||||
assertThat(tm().transact(ClaimsListDualDatabaseDao::get).getLabelsToKeys()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGetList_missingOfy_notInTest() {
|
||||
RegistryEnvironment.PRODUCTION.setup(systemPropertyExtension);
|
||||
fakeClock.advanceBy(Duration.standardDays(5));
|
||||
ClaimsListSqlDao.save(createClaimsList());
|
||||
// Shouldn't fail in production
|
||||
assertThat(ClaimsListDualDatabaseDao.get().getLabelsToKeys())
|
||||
.isEqualTo(createClaimsList().getLabelsToKeys());
|
||||
}
|
||||
|
||||
private ClaimsListShard createClaimsList() {
|
||||
return ClaimsListShard.create(
|
||||
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,116 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.tmch;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth8.assertThat;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.tmch.ClaimsListShard.ClaimsListRevision;
|
||||
import google.registry.model.tmch.ClaimsListShard.UnshardedSaveException;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
|
||||
/** Unit tests for {@link ClaimsListShard}. */
|
||||
public class ClaimsListShardTest {
|
||||
|
||||
@RegisterExtension
|
||||
public final AppEngineExtension appEngine =
|
||||
AppEngineExtension.builder().withDatastoreAndCloudSql().build();
|
||||
|
||||
private final int shardSize = 10;
|
||||
|
||||
@Test
|
||||
void test_unshardedSaveFails() {
|
||||
assertThrows(
|
||||
UnshardedSaveException.class,
|
||||
() ->
|
||||
tm().transact(
|
||||
() -> {
|
||||
ClaimsListShard claimsList =
|
||||
ClaimsListShard.create(
|
||||
tm().getTransactionTime(), ImmutableMap.of("a", "b"));
|
||||
claimsList.id = 1; // Without an id this won't save anyways.
|
||||
claimsList.parent = ClaimsListRevision.createKey();
|
||||
auditedOfy().saveWithoutBackup().entity(claimsList).now();
|
||||
}));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGet_safelyLoadsEmptyClaimsList_whenNoShardsExist() {
|
||||
assertThat(ClaimsListShard.getFromDatastore()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_savesAndGets_withSharding() {
|
||||
// Create a ClaimsList that will need 4 shards to save.
|
||||
Map<String, String> labelsToKeys = new HashMap<>();
|
||||
for (int i = 0; i <= shardSize * 3; i++) {
|
||||
labelsToKeys.put(Integer.toString(i), Integer.toString(i));
|
||||
}
|
||||
DateTime now = DateTime.now(UTC);
|
||||
// Save it with sharding, and make sure that reloading it works.
|
||||
ClaimsListShard unsharded = ClaimsListShard.create(now, ImmutableMap.copyOf(labelsToKeys));
|
||||
unsharded.saveToDatastore(shardSize);
|
||||
assertThat(ClaimsListShard.getFromDatastore().get().labelsToKeys)
|
||||
.isEqualTo(unsharded.labelsToKeys);
|
||||
List<ClaimsListShard> shards1 = auditedOfy().load().type(ClaimsListShard.class).list();
|
||||
assertThat(shards1).hasSize(4);
|
||||
assertThat(ClaimsListShard.getFromDatastore().get().getClaimKey("1")).hasValue("1");
|
||||
assertThat(ClaimsListShard.getFromDatastore().get().getClaimKey("a")).isEmpty();
|
||||
assertThat(ClaimsListShard.getCurrentRevision()).isEqualTo(shards1.get(0).parent);
|
||||
|
||||
// Create a smaller ClaimsList that will need only 2 shards to save.
|
||||
labelsToKeys = new HashMap<>();
|
||||
for (int i = 0; i <= shardSize; i++) {
|
||||
labelsToKeys.put(Integer.toString(i), Integer.toString(i));
|
||||
}
|
||||
unsharded = ClaimsListShard.create(now.plusDays(1), ImmutableMap.copyOf(labelsToKeys));
|
||||
unsharded.saveToDatastore(shardSize);
|
||||
auditedOfy().clearSessionCache();
|
||||
assertThat(ClaimsListShard.getFromDatastore().get().labelsToKeys)
|
||||
.hasSize(unsharded.labelsToKeys.size());
|
||||
assertThat(ClaimsListShard.getFromDatastore().get().labelsToKeys)
|
||||
.isEqualTo(unsharded.labelsToKeys);
|
||||
List<ClaimsListShard> shards2 = auditedOfy().load().type(ClaimsListShard.class).list();
|
||||
assertThat(shards2).hasSize(2);
|
||||
|
||||
// Expect that the old revision is deleted.
|
||||
assertThat(ClaimsListShard.getCurrentRevision()).isEqualTo(shards2.get(0).parent);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a created claims list shard with the specified parent key for testing purposes only.
|
||||
*/
|
||||
public static ClaimsListShard createTestClaimsListShard(
|
||||
DateTime creationTime,
|
||||
ImmutableMap<String, String> labelsToKeys,
|
||||
Key<ClaimsListRevision> revision) {
|
||||
ClaimsListShard claimsList = ClaimsListShard.create(creationTime, labelsToKeys);
|
||||
claimsList.isShard = true;
|
||||
claimsList.parent = revision;
|
||||
return claimsList;
|
||||
}
|
||||
}
|
||||
|
|
@ -34,7 +34,7 @@ import google.registry.model.server.KmsSecretRevisionSqlDaoTest;
|
|||
import google.registry.model.server.LockTest;
|
||||
import google.registry.model.server.ServerSecretTest;
|
||||
import google.registry.model.smd.SignedMarkRevocationListDaoTest;
|
||||
import google.registry.model.tmch.ClaimsListSqlDaoTest;
|
||||
import google.registry.model.tmch.ClaimsListDaoTest;
|
||||
import google.registry.model.tmch.TmchCrlTest;
|
||||
import google.registry.persistence.transaction.JpaEntityCoverageExtension;
|
||||
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension;
|
||||
|
|
@ -82,7 +82,7 @@ import org.junit.runner.RunWith;
|
|||
BeforeSuiteTest.class,
|
||||
AllocationTokenTest.class,
|
||||
BillingEventTest.class,
|
||||
ClaimsListSqlDaoTest.class,
|
||||
ClaimsListDaoTest.class,
|
||||
ContactHistoryTest.class,
|
||||
ContactResourceTest.class,
|
||||
CursorTest.class,
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ import static org.mockito.Mockito.times;
|
|||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import java.util.Optional;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
|
@ -38,7 +38,7 @@ class TmchDnlActionTest extends TmchActionTestCase {
|
|||
|
||||
@Test
|
||||
void testDnl() throws Exception {
|
||||
assertThat(ClaimsListDualDatabaseDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty();
|
||||
assertThat(ClaimsListDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty();
|
||||
when(httpResponse.getContent())
|
||||
.thenReturn(TmchTestData.loadBytes("dnl-latest.csv").read())
|
||||
.thenReturn(TmchTestData.loadBytes("dnl-latest.sig").read());
|
||||
|
|
@ -50,7 +50,7 @@ class TmchDnlActionTest extends TmchActionTestCase {
|
|||
.isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.sig");
|
||||
|
||||
// Make sure the contents of testdata/dnl-latest.csv got inserted into the database.
|
||||
ClaimsListShard claimsList = ClaimsListDualDatabaseDao.get();
|
||||
ClaimsList claimsList = ClaimsListDao.get();
|
||||
assertThat(claimsList.getTmdbGenerationTime())
|
||||
.isEqualTo(DateTime.parse("2013-11-24T23:15:37.4Z"));
|
||||
assertThat(claimsList.getClaimKey("xn----7sbejwbn3axu3d"))
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ import static java.nio.file.Files.readAllLines;
|
|||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import java.io.File;
|
||||
import java.nio.file.Files;
|
||||
import org.joda.time.DateTime;
|
||||
|
|
@ -32,8 +32,7 @@ class GetClaimsListCommandTest extends CommandTestCase<GetClaimsListCommand> {
|
|||
|
||||
@Test
|
||||
void testSuccess_getWorks() throws Exception {
|
||||
ClaimsListDualDatabaseDao.save(
|
||||
ClaimsListShard.create(DateTime.now(UTC), ImmutableMap.of("a", "1", "b", "2")));
|
||||
ClaimsListDao.save(ClaimsList.create(DateTime.now(UTC), ImmutableMap.of("a", "1", "b", "2")));
|
||||
File output = tmpDir.resolve("claims.txt").toFile();
|
||||
runCommand("--output=" + output.getAbsolutePath());
|
||||
assertThat(readAllLines(output.toPath(), UTF_8)).containsExactly("a,1", "b,2");
|
||||
|
|
@ -41,8 +40,7 @@ class GetClaimsListCommandTest extends CommandTestCase<GetClaimsListCommand> {
|
|||
|
||||
@Test
|
||||
void testSuccess_endsWithNewline() throws Exception {
|
||||
ClaimsListDualDatabaseDao.save(
|
||||
ClaimsListShard.create(DateTime.now(UTC), ImmutableMap.of("a", "1")));
|
||||
ClaimsListDao.save(ClaimsList.create(DateTime.now(UTC), ImmutableMap.of("a", "1")));
|
||||
File output = tmpDir.resolve("claims.txt").toFile();
|
||||
runCommand("--output=" + output.getAbsolutePath());
|
||||
assertThat(new String(Files.readAllBytes(output.toPath()), UTF_8)).endsWith("\n");
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ import static com.google.common.truth.Truth.assertThat;
|
|||
import static com.google.common.truth.Truth8.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import java.io.FileNotFoundException;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
|
@ -37,7 +37,7 @@ class UploadClaimsListCommandTest extends CommandTestCase<UploadClaimsListComman
|
|||
"anotherexample,2013041500/A/C/7/rHdC4wnrWRvPY6nneCVtQhFj0000000003,2011-08-16T12:00:00.0Z");
|
||||
runCommand("--force", filename);
|
||||
|
||||
ClaimsListShard claimsList = ClaimsListDualDatabaseDao.get();
|
||||
ClaimsList claimsList = ClaimsListDao.get();
|
||||
assertThat(claimsList.getTmdbGenerationTime())
|
||||
.isEqualTo(DateTime.parse("2012-08-16T00:00:00.0Z"));
|
||||
assertThat(claimsList.getClaimKey("example"))
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
ClaimsListShard
|
||||
ClaimsList
|
||||
ClaimsListSingleton
|
||||
Cursor
|
||||
DatabaseTransitionSchedule
|
||||
|
|
|
|||
|
|
@ -847,20 +847,20 @@ class google.registry.model.server.ServerSecret {
|
|||
long leastSignificant;
|
||||
long mostSignificant;
|
||||
}
|
||||
class google.registry.model.tmch.ClaimsListShard {
|
||||
class google.registry.model.tmch.ClaimsList {
|
||||
@Id long id;
|
||||
@Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsListShard$ClaimsListRevision> parent;
|
||||
@Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsList$ClaimsListRevision> parent;
|
||||
java.util.Map<java.lang.String, java.lang.String> labelsToKeys;
|
||||
org.joda.time.DateTime creationTime;
|
||||
}
|
||||
class google.registry.model.tmch.ClaimsListShard$ClaimsListRevision {
|
||||
class google.registry.model.tmch.ClaimsList$ClaimsListRevision {
|
||||
@Id long versionId;
|
||||
@Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsListShard$ClaimsListSingleton> parent;
|
||||
@Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsList$ClaimsListSingleton> parent;
|
||||
}
|
||||
class google.registry.model.tmch.ClaimsListShard$ClaimsListSingleton {
|
||||
class google.registry.model.tmch.ClaimsList$ClaimsListSingleton {
|
||||
@Id long id;
|
||||
@Parent com.googlecode.objectify.Key<google.registry.model.common.EntityGroupRoot> parent;
|
||||
com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsListShard$ClaimsListRevision> activeRevision;
|
||||
com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsList$ClaimsListRevision> activeRevision;
|
||||
}
|
||||
class google.registry.model.tmch.TmchCrl {
|
||||
@Id long id;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue