Merge ClaimsList into ClaimsListShard (#694)

* Merge ClaimsList into ClaimsListShard

* Add a TODO to rename the class

* Rebase on HEAD

* Improve javadoc
This commit is contained in:
Shicong Huang 2020-09-03 11:18:40 -04:00 committed by GitHub
parent ecafebdc3d
commit cdf2c7f7cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 130 additions and 190 deletions

View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package google.registry.schema.tmch; package google.registry.model.tmch;
import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration; import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
import static google.registry.model.CacheUtils.tryMemoizeWithExpiration; import static google.registry.model.CacheUtils.tryMemoizeWithExpiration;
@ -24,28 +24,28 @@ import google.registry.util.NonFinalForTesting;
import java.util.Optional; import java.util.Optional;
import javax.persistence.EntityManager; import javax.persistence.EntityManager;
/** Data access object for {@link ClaimsList}. */ /** Data access object for {@link ClaimsListShard}. */
public class ClaimsListDao { public class ClaimsListDao {
private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final FluentLogger logger = FluentLogger.forEnclosingClass();
/** In-memory cache for claims list. */ /** In-memory cache for claims list. */
@NonFinalForTesting @NonFinalForTesting
private static Supplier<Optional<ClaimsList>> cacheClaimsList = private static Supplier<Optional<ClaimsListShard>> cacheClaimsList =
tryMemoizeWithExpiration(getDomainLabelListCacheDuration(), ClaimsListDao::getLatestRevision); tryMemoizeWithExpiration(getDomainLabelListCacheDuration(), ClaimsListDao::getLatestRevision);
private static void save(ClaimsList claimsList) { private static void save(ClaimsListShard claimsList) {
jpaTm().transact(() -> jpaTm().getEntityManager().persist(claimsList)); jpaTm().transact(() -> jpaTm().getEntityManager().persist(claimsList));
} }
/** /**
* Try to save the given {@link ClaimsList} into Cloud SQL. If the save fails, the error will be * Try to save the given {@link ClaimsListShard} into Cloud SQL. If the save fails, the error will
* logged but no exception will be thrown. * be logged but no exception will be thrown.
* *
* <p>This method is used during the dual-write phase of database migration as Datastore is still * <p>This method is used during the dual-write phase of database migration as Datastore is still
* the authoritative database. * the authoritative database.
*/ */
public static void trySave(ClaimsList claimsList) { static void trySave(ClaimsListShard claimsList) {
try { try {
ClaimsListDao.save(claimsList); ClaimsListDao.save(claimsList);
logger.atInfo().log( logger.atInfo().log(
@ -57,12 +57,12 @@ public class ClaimsListDao {
} }
/** /**
* Returns the most recent revision of the {@link ClaimsList} in Cloud SQL, if it exists. * Returns the most recent revision of the {@link ClaimsListShard} in Cloud SQL, if it exists.
* TODO(shicong): Change this method to package level access after dual-read phase. * TODO(shicong): Change this method to package level access after dual-read phase.
* ClaimsListShard uses this method to retrieve claims list in Cloud SQL for the comparison, and * ClaimsListShard uses this method to retrieve claims list in Cloud SQL for the comparison, and
* ClaimsListShard is not in this package. * ClaimsListShard is not in this package.
*/ */
public static Optional<ClaimsList> getLatestRevision() { public static Optional<ClaimsListShard> getLatestRevision() {
return jpaTm() return jpaTm()
.transact( .transact(
() -> { () -> {
@ -73,15 +73,15 @@ public class ClaimsListDao {
return em.createQuery( return em.createQuery(
"FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId =" "FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId ="
+ " :revisionId", + " :revisionId",
ClaimsList.class) ClaimsListShard.class)
.setParameter("revisionId", revisionId) .setParameter("revisionId", revisionId)
.getResultStream() .getResultStream()
.findFirst(); .findFirst();
}); });
} }
/** Returns the most recent revision of the {@link ClaimsList}, from cache. */ /** Returns the most recent revision of the {@link ClaimsListShard}, from cache. */
public static Optional<ClaimsList> getLatestRevisionCached() { public static Optional<ClaimsListShard> getLatestRevisionCached() {
return cacheClaimsList.get(); return cacheClaimsList.get();
} }

View file

@ -40,6 +40,7 @@ import com.googlecode.objectify.annotation.Id;
import com.googlecode.objectify.annotation.Ignore; import com.googlecode.objectify.annotation.Ignore;
import com.googlecode.objectify.annotation.OnSave; import com.googlecode.objectify.annotation.OnSave;
import com.googlecode.objectify.annotation.Parent; import com.googlecode.objectify.annotation.Parent;
import google.registry.model.CreateAutoTimestamp;
import google.registry.model.ImmutableObject; import google.registry.model.ImmutableObject;
import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp;
import google.registry.model.annotations.NotBackedUp.Reason; import google.registry.model.annotations.NotBackedUp.Reason;
@ -47,8 +48,6 @@ import google.registry.model.annotations.VirtualEntity;
import google.registry.model.common.CrossTldSingleton; import google.registry.model.common.CrossTldSingleton;
import google.registry.schema.replay.DatastoreEntity; import google.registry.schema.replay.DatastoreEntity;
import google.registry.schema.replay.SqlEntity; import google.registry.schema.replay.SqlEntity;
import google.registry.schema.tmch.ClaimsList;
import google.registry.schema.tmch.ClaimsListDao;
import google.registry.util.CollectionUtils; import google.registry.util.CollectionUtils;
import google.registry.util.Concurrent; import google.registry.util.Concurrent;
import google.registry.util.Retrier; import google.registry.util.Retrier;
@ -59,6 +58,15 @@ import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import javax.persistence.CollectionTable;
import javax.persistence.Column;
import javax.persistence.ElementCollection;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.JoinColumn;
import javax.persistence.MapKeyColumn;
import javax.persistence.Table;
import javax.persistence.Transient;
import org.joda.time.DateTime; import org.joda.time.DateTime;
/** /**
@ -74,10 +82,21 @@ import org.joda.time.DateTime;
* 10MB per transaction limit. * 10MB per transaction limit.
* *
* <p>Therefore, it is never OK to save an instance of this class directly to Datastore. Instead you * <p>Therefore, it is never OK to save an instance of this class directly to Datastore. Instead you
* must use the {@link #save} method to do it for you. * must use the {@link #saveToDatastore} method to do it for you.
*
* <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
* the database. So, if a retry of insertion happens after the previous attempt unexpectedly
* succeeds, we will end up with having two exact same claims list with only different {@link
* #revisionId}. However, this is not an actual problem because we only use the claims list with
* highest {@link #revisionId}.
*
* <p>TODO(b/162007765): Rename the class to ClaimsList and remove Datastore related fields and
* methods.
*/ */
@Entity @Entity
@NotBackedUp(reason = Reason.EXTERNALLY_SOURCED) @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
@javax.persistence.Entity(name = "ClaimsList")
@Table
public class ClaimsListShard extends ImmutableObject implements DatastoreEntity { public class ClaimsListShard extends ImmutableObject implements DatastoreEntity {
private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@ -85,22 +104,44 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
/** The number of claims list entries to store per shard. */ /** The number of claims list entries to store per shard. */
private static final int SHARD_SIZE = 10000; private static final int SHARD_SIZE = 10000;
@Id @Transient @Id long id;
long id;
@Parent @Transient @Parent Key<ClaimsListRevision> parent;
Key<ClaimsListRevision> parent;
/** When the claims list was last updated. */ @Ignore
@javax.persistence.Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
Long revisionId;
@Ignore
@Column(nullable = false)
CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
/**
* When the claims list was last updated.
*
* <p>Note that the value of this field is parsed from the claims list file(See this <a
* href="https://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.1">RFC</>), it is
* the DNL List creation datetime from the rfc. Since this field has been used by Datastore, we
* cannot change its name until we finish the migration.
*
* <p>TODO(b/166784536): Rename this field to tmdbGenerationTime.
*/
@Column(name = "tmdb_generation_time", nullable = false)
DateTime creationTime; DateTime creationTime;
/** A map from labels to claims keys. */ /** A map from labels to claims keys. */
@EmbedMap @EmbedMap
@ElementCollection
@CollectionTable(
name = "ClaimsEntry",
joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
@MapKeyColumn(name = "domainLabel", nullable = false)
@Column(name = "claimKey", nullable = false)
Map<String, String> labelsToKeys; Map<String, String> labelsToKeys;
/** Indicates that this is a shard rather than a "full" list. */ /** Indicates that this is a shard rather than a "full" list. */
@Ignore @Ignore @Transient boolean isShard = false;
boolean isShard = false;
private static final Retrier LOADER_RETRIER = new Retrier(new SystemSleeper(), 2); private static final Retrier LOADER_RETRIER = new Retrier(new SystemSleeper(), 2);
@ -164,10 +205,10 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
return datastoreList; return datastoreList;
}; };
private static final void loadAndCompareCloudSqlList(ClaimsListShard datastoreList) { private static void loadAndCompareCloudSqlList(ClaimsListShard datastoreList) {
Optional<ClaimsList> maybeCloudSqlList = ClaimsListDao.getLatestRevision(); Optional<ClaimsListShard> maybeCloudSqlList = ClaimsListDao.getLatestRevision();
if (maybeCloudSqlList.isPresent()) { if (maybeCloudSqlList.isPresent()) {
ClaimsList cloudSqlList = maybeCloudSqlList.get(); ClaimsListShard cloudSqlList = maybeCloudSqlList.get();
MapDifference<String, String> diff = MapDifference<String, String> diff =
Maps.difference(datastoreList.labelsToKeys, cloudSqlList.getLabelsToKeys()); Maps.difference(datastoreList.labelsToKeys, cloudSqlList.getLabelsToKeys());
if (!diff.areEqual()) { if (!diff.areEqual()) {
@ -206,15 +247,34 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
memoizeWithShortExpiration( memoizeWithShortExpiration(
() -> LOADER_RETRIER.callWithRetry(LOADER_CALLABLE, IllegalStateException.class)); () -> LOADER_RETRIER.callWithRetry(LOADER_CALLABLE, IllegalStateException.class));
public DateTime getCreationTime() { /** Returns the revision id of this claims list, or throws exception if it is null. */
public Long getRevisionId() {
checkState(
revisionId != null, "revisionId is null because it is not persisted in the database");
return revisionId;
}
/**
* Returns the time when the external TMDB service generated this revision of the claims list.
*
* @see <a href="https://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.1">DNL List
* creation datetime</a>
*/
public DateTime getTmdbGenerationTime() {
return creationTime; return creationTime;
} }
/** Returns the creation time of this claims list. */
public DateTime getCreationTimestamp() {
return creationTimestamp.getTimestamp();
}
/** Returns the claim key for a given domain if there is one, empty otherwise. */ /** Returns the claim key for a given domain if there is one, empty otherwise. */
public Optional<String> getClaimKey(String label) { public Optional<String> getClaimKey(String label) {
return Optional.ofNullable(labelsToKeys.get(label)); return Optional.ofNullable(labelsToKeys.get(label));
} }
/** Returns an {@link Map} mapping domain label to its lookup key. */
public ImmutableMap<String, String> getLabelsToKeys() { public ImmutableMap<String, String> getLabelsToKeys() {
return ImmutableMap.copyOf(labelsToKeys); return ImmutableMap.copyOf(labelsToKeys);
} }
@ -229,11 +289,12 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
* switching over to using them atomically, then deleting the old ones. * switching over to using them atomically, then deleting the old ones.
*/ */
public void save() { public void save() {
save(SHARD_SIZE); saveToDatastore(SHARD_SIZE);
ClaimsListDao.trySave(this);
} }
@VisibleForTesting @VisibleForTesting
void save(int shardSize) { void saveToDatastore(int shardSize) {
// Figure out what the next versionId should be based on which ones already exist. // Figure out what the next versionId should be based on which ones already exist.
final Key<ClaimsListRevision> oldRevision = getCurrentRevision(); final Key<ClaimsListRevision> oldRevision = getCurrentRevision();
final Key<ClaimsListRevision> parentKey = ClaimsListRevision.createKey(); final Key<ClaimsListRevision> parentKey = ClaimsListRevision.createKey();
@ -270,10 +331,11 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
}); });
} }
public static ClaimsListShard create(DateTime creationTime, Map<String, String> labelsToKeys) { public static ClaimsListShard create(
DateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
ClaimsListShard instance = new ClaimsListShard(); ClaimsListShard instance = new ClaimsListShard();
instance.id = allocateId(); instance.id = allocateId();
instance.creationTime = checkNotNull(creationTime); instance.creationTime = checkNotNull(tmdbGenerationTime);
instance.labelsToKeys = checkNotNull(labelsToKeys); instance.labelsToKeys = checkNotNull(labelsToKeys);
return instance; return instance;
} }

View file

@ -1,116 +0,0 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.schema.tmch;
import static com.google.common.base.Preconditions.checkState;
import static google.registry.util.DateTimeUtils.toJodaDateTime;
import static google.registry.util.DateTimeUtils.toZonedDateTime;
import com.google.common.collect.ImmutableList;
import google.registry.model.CreateAutoTimestamp;
import google.registry.model.ImmutableObject;
import google.registry.schema.replay.DatastoreEntity;
import google.registry.schema.replay.SqlEntity;
import java.time.ZonedDateTime;
import java.util.Map;
import java.util.Optional;
import javax.persistence.CollectionTable;
import javax.persistence.Column;
import javax.persistence.ElementCollection;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.MapKeyColumn;
import javax.persistence.Table;
import org.joda.time.DateTime;
/**
* A list of TMCH claims labels and their associated claims keys.
*
* <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
* the database. So, if a retry of insertion happens after the previous attempt unexpectedly
* succeeds, we will end up with having two exact same claims list with only different {@link
* #revisionId}. However, this is not an actual problem because we only use the claims list with
* highest {@link #revisionId}.
*/
@Entity
@Table
public class ClaimsList extends ImmutableObject implements SqlEntity {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@Column
private Long revisionId;
@Column(nullable = false)
private CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
@Column(nullable = false)
private ZonedDateTime tmdbGenerationTime;
@ElementCollection
@CollectionTable(
name = "ClaimsEntry",
joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
@MapKeyColumn(name = "domainLabel", nullable = false)
@Column(name = "claimKey", nullable = false)
private Map<String, String> labelsToKeys;
private ClaimsList(ZonedDateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
this.tmdbGenerationTime = tmdbGenerationTime;
this.labelsToKeys = labelsToKeys;
}
// Hibernate requires this default constructor.
private ClaimsList() {}
/** Constructs a {@link ClaimsList} object. */
public static ClaimsList create(DateTime creationTimestamp, Map<String, String> labelsToKeys) {
return new ClaimsList(toZonedDateTime(creationTimestamp), labelsToKeys);
}
/** Returns the revision id of this claims list, or throws exception if it is null. */
public Long getRevisionId() {
checkState(
revisionId != null, "revisionId is null because it is not persisted in the database");
return revisionId;
}
/** Returns the TMDB generation time of this claims list. */
public DateTime getTmdbGenerationTime() {
return toJodaDateTime(tmdbGenerationTime);
}
/** Returns the creation time of this claims list. */
public DateTime getCreationTimestamp() {
return creationTimestamp.getTimestamp();
}
/** Returns an {@link Map} mapping domain label to its lookup key. */
public Map<String, String> getLabelsToKeys() {
return labelsToKeys;
}
/** Returns the claim key for a given domain if there is one, empty otherwise. */
public Optional<String> getClaimKey(String label) {
return Optional.ofNullable(labelsToKeys.get(label));
}
@Override
public ImmutableList<DatastoreEntity> toDatastoreEntities() {
return ImmutableList.of(); // ClaimsList is dual-written
}
}

View file

@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.base.Splitter; import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import google.registry.schema.tmch.ClaimsList; import google.registry.model.tmch.ClaimsListShard;
import java.util.List; import java.util.List;
import org.joda.time.DateTime; import org.joda.time.DateTime;
@ -34,11 +34,11 @@ import org.joda.time.DateTime;
public class ClaimsListParser { public class ClaimsListParser {
/** /**
* Converts the lines from the DNL CSV file into a {@link ClaimsList} object. * Converts the lines from the DNL CSV file into a {@link ClaimsListShard} object.
* *
* <p>Please note that this does <b>not</b> insert the object into Datastore. * <p>Please note that this does <b>not</b> insert the object into Datastore.
*/ */
public static ClaimsList parse(List<String> lines) { public static ClaimsListShard parse(List<String> lines) {
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>(); ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
// First line: <version>,<DNL List creation datetime> // First line: <version>,<DNL List creation datetime>
@ -74,6 +74,6 @@ public class ClaimsListParser {
builder.put(label, lookupKey); builder.put(label, lookupKey);
} }
return ClaimsList.create(creationTime, builder.build()); return ClaimsListShard.create(creationTime, builder.build());
} }
} }

View file

@ -21,8 +21,6 @@ import google.registry.keyring.api.KeyModule.Key;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsListShard;
import google.registry.request.Action; import google.registry.request.Action;
import google.registry.request.auth.Auth; import google.registry.request.auth.Auth;
import google.registry.schema.tmch.ClaimsList;
import google.registry.schema.tmch.ClaimsListDao;
import java.io.IOException; import java.io.IOException;
import java.security.SignatureException; import java.security.SignatureException;
import java.util.List; import java.util.List;
@ -56,14 +54,10 @@ public final class TmchDnlAction implements Runnable {
} catch (SignatureException | IOException | PGPException e) { } catch (SignatureException | IOException | PGPException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
ClaimsList claims = ClaimsListParser.parse(lines); ClaimsListShard claims = ClaimsListParser.parse(lines);
ClaimsListShard claimsListShard = claims.save();
ClaimsListShard.create(claims.getTmdbGenerationTime(), claims.getLabelsToKeys());
claimsListShard.save();
logger.atInfo().log( logger.atInfo().log(
"Inserted %,d claims into Datastore, created at %s", "Inserted %,d claims into Datastore, created at %s",
claimsListShard.size(), claimsListShard.getCreationTime()); claims.size(), claims.getTmdbGenerationTime());
ClaimsListDao.trySave(claims);
} }
} }

View file

@ -22,8 +22,6 @@ import com.beust.jcommander.Parameters;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import com.google.common.io.Files; import com.google.common.io.Files;
import google.registry.model.tmch.ClaimsListShard; import google.registry.model.tmch.ClaimsListShard;
import google.registry.schema.tmch.ClaimsList;
import google.registry.schema.tmch.ClaimsListDao;
import google.registry.tmch.ClaimsListParser; import google.registry.tmch.ClaimsListParser;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
@ -39,7 +37,7 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
private String claimsListFilename; private String claimsListFilename;
private ClaimsList claimsList; private ClaimsListShard claimsList;
@Override @Override
protected void init() throws IOException { protected void init() throws IOException {
@ -58,8 +56,7 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
@Override @Override
public String execute() { public String execute() {
ClaimsListShard.create(claimsList.getTmdbGenerationTime(), claimsList.getLabelsToKeys()).save(); claimsList.save();
ClaimsListDao.trySave(claimsList);
return String.format("Successfully uploaded claims list %s", claimsListFilename); return String.format("Successfully uploaded claims list %s", claimsListFilename);
} }
} }

View file

@ -53,8 +53,8 @@
<class>google.registry.model.registry.label.PremiumList</class> <class>google.registry.model.registry.label.PremiumList</class>
<class>google.registry.model.reporting.Spec11ThreatMatch</class> <class>google.registry.model.reporting.Spec11ThreatMatch</class>
<class>google.registry.persistence.transaction.TransactionEntity</class> <class>google.registry.persistence.transaction.TransactionEntity</class>
<class>google.registry.model.tmch.ClaimsListShard</class>
<class>google.registry.schema.domain.RegistryLock</class> <class>google.registry.schema.domain.RegistryLock</class>
<class>google.registry.schema.tmch.ClaimsList</class>
<class>google.registry.schema.cursor.Cursor</class> <class>google.registry.schema.cursor.Cursor</class>
<class>google.registry.schema.server.Lock</class> <class>google.registry.schema.server.Lock</class>
<class>google.registry.schema.tld.PremiumEntry</class> <class>google.registry.schema.tld.PremiumEntry</class>

View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package google.registry.schema.tmch; package google.registry.model.tmch;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
@ -40,20 +40,22 @@ public class ClaimsListDaoTest {
@Test @Test
void trySave_insertsClaimsListSuccessfully() { void trySave_insertsClaimsListSuccessfully() {
ClaimsList claimsList = ClaimsListShard claimsList =
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2")); ClaimsListShard.create(
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
ClaimsListDao.trySave(claimsList); ClaimsListDao.trySave(claimsList);
ClaimsList insertedClaimsList = ClaimsListDao.getLatestRevision().get(); ClaimsListShard insertedClaimsList = ClaimsListDao.getLatestRevision().get();
assertClaimsListEquals(claimsList, insertedClaimsList); assertClaimsListEquals(claimsList, insertedClaimsList);
assertThat(insertedClaimsList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc()); assertThat(insertedClaimsList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc());
} }
@Test @Test
void trySave_noExceptionThrownWhenSaveFail() { void trySave_noExceptionThrownWhenSaveFail() {
ClaimsList claimsList = ClaimsListShard claimsList =
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2")); ClaimsListShard.create(
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
ClaimsListDao.trySave(claimsList); ClaimsListDao.trySave(claimsList);
ClaimsList insertedClaimsList = ClaimsListDao.getLatestRevision().get(); ClaimsListShard insertedClaimsList = ClaimsListDao.getLatestRevision().get();
assertClaimsListEquals(claimsList, insertedClaimsList); assertClaimsListEquals(claimsList, insertedClaimsList);
// Save ClaimsList with existing revisionId should fail because revisionId is the primary key. // Save ClaimsList with existing revisionId should fail because revisionId is the primary key.
ClaimsListDao.trySave(insertedClaimsList); ClaimsListDao.trySave(insertedClaimsList);
@ -61,9 +63,9 @@ public class ClaimsListDaoTest {
@Test @Test
void trySave_claimsListWithNoEntries() { void trySave_claimsListWithNoEntries() {
ClaimsList claimsList = ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of()); ClaimsListShard claimsList = ClaimsListShard.create(fakeClock.nowUtc(), ImmutableMap.of());
ClaimsListDao.trySave(claimsList); ClaimsListDao.trySave(claimsList);
ClaimsList insertedClaimsList = ClaimsListDao.getLatestRevision().get(); ClaimsListShard insertedClaimsList = ClaimsListDao.getLatestRevision().get();
assertClaimsListEquals(claimsList, insertedClaimsList); assertClaimsListEquals(claimsList, insertedClaimsList);
assertThat(insertedClaimsList.getLabelsToKeys()).isEmpty(); assertThat(insertedClaimsList.getLabelsToKeys()).isEmpty();
} }
@ -75,16 +77,18 @@ public class ClaimsListDaoTest {
@Test @Test
void getCurrent_returnsLatestClaims() { void getCurrent_returnsLatestClaims() {
ClaimsList oldClaimsList = ClaimsListShard oldClaimsList =
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2")); ClaimsListShard.create(
ClaimsList newClaimsList = fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4")); ClaimsListShard newClaimsList =
ClaimsListShard.create(
fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4"));
ClaimsListDao.trySave(oldClaimsList); ClaimsListDao.trySave(oldClaimsList);
ClaimsListDao.trySave(newClaimsList); ClaimsListDao.trySave(newClaimsList);
assertClaimsListEquals(newClaimsList, ClaimsListDao.getLatestRevision().get()); assertClaimsListEquals(newClaimsList, ClaimsListDao.getLatestRevision().get());
} }
private void assertClaimsListEquals(ClaimsList left, ClaimsList right) { private void assertClaimsListEquals(ClaimsListShard left, ClaimsListShard right) {
assertThat(left.getRevisionId()).isEqualTo(right.getRevisionId()); assertThat(left.getRevisionId()).isEqualTo(right.getRevisionId());
assertThat(left.getTmdbGenerationTime()).isEqualTo(right.getTmdbGenerationTime()); assertThat(left.getTmdbGenerationTime()).isEqualTo(right.getTmdbGenerationTime());
assertThat(left.getLabelsToKeys()).isEqualTo(right.getLabelsToKeys()); assertThat(left.getLabelsToKeys()).isEqualTo(right.getLabelsToKeys());

View file

@ -76,7 +76,7 @@ public class ClaimsListShardTest {
DateTime now = DateTime.now(UTC); DateTime now = DateTime.now(UTC);
// Save it with sharding, and make sure that reloading it works. // Save it with sharding, and make sure that reloading it works.
ClaimsListShard unsharded = ClaimsListShard.create(now, ImmutableMap.copyOf(labelsToKeys)); ClaimsListShard unsharded = ClaimsListShard.create(now, ImmutableMap.copyOf(labelsToKeys));
unsharded.save(shardSize); unsharded.saveToDatastore(shardSize);
assertThat(ClaimsListShard.get().labelsToKeys).isEqualTo(unsharded.labelsToKeys); assertThat(ClaimsListShard.get().labelsToKeys).isEqualTo(unsharded.labelsToKeys);
List<ClaimsListShard> shards1 = ofy().load().type(ClaimsListShard.class).list(); List<ClaimsListShard> shards1 = ofy().load().type(ClaimsListShard.class).list();
assertThat(shards1).hasSize(4); assertThat(shards1).hasSize(4);
@ -90,7 +90,7 @@ public class ClaimsListShardTest {
labelsToKeys.put(Integer.toString(i), Integer.toString(i)); labelsToKeys.put(Integer.toString(i), Integer.toString(i));
} }
unsharded = ClaimsListShard.create(now.plusDays(1), ImmutableMap.copyOf(labelsToKeys)); unsharded = ClaimsListShard.create(now.plusDays(1), ImmutableMap.copyOf(labelsToKeys));
unsharded.save(shardSize); unsharded.saveToDatastore(shardSize);
ofy().clearSessionCache(); ofy().clearSessionCache();
assertThat(ClaimsListShard.get().labelsToKeys).hasSize(unsharded.labelsToKeys.size()); assertThat(ClaimsListShard.get().labelsToKeys).hasSize(unsharded.labelsToKeys.size());
assertThat(ClaimsListShard.get().labelsToKeys).isEqualTo(unsharded.labelsToKeys); assertThat(ClaimsListShard.get().labelsToKeys).isEqualTo(unsharded.labelsToKeys);

View file

@ -20,7 +20,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import google.registry.model.ImmutableObject; import google.registry.model.ImmutableObject;
import google.registry.persistence.transaction.JpaTestRules.JpaUnitTestExtension; import google.registry.persistence.transaction.JpaTestRules.JpaUnitTestExtension;
import google.registry.schema.tmch.ClaimsList;
import java.util.List; import java.util.List;
import javax.persistence.Entity; import javax.persistence.Entity;
import javax.persistence.Id; import javax.persistence.Id;
@ -36,9 +35,7 @@ public class JpaTransactionManagerRuleTest {
@RegisterExtension @RegisterExtension
public final JpaUnitTestExtension jpaExtension = public final JpaUnitTestExtension jpaExtension =
new JpaTestRules.Builder() new JpaTestRules.Builder().withEntityClass(TestEntity.class).buildUnitTestRule();
.withEntityClass(ClaimsList.class, TestEntity.class)
.buildUnitTestRule();
@Test @Test
void verifiesRuleWorks() { void verifiesRuleWorks() {
@ -58,7 +55,7 @@ public class JpaTransactionManagerRuleTest {
List results = List results =
jpaTm() jpaTm()
.getEntityManager() .getEntityManager()
.createNativeQuery("SELECT * FROM \"ClaimsList\"") .createNativeQuery("SELECT * FROM \"TestEntity\"")
.getResultList(); .getResultList();
assertThat(results).isEmpty(); assertThat(results).isEmpty();
}); });

View file

@ -27,6 +27,7 @@ import google.registry.model.poll.PollMessageTest;
import google.registry.model.registry.RegistryLockDaoTest; import google.registry.model.registry.RegistryLockDaoTest;
import google.registry.model.registry.label.ReservedListSqlDaoTest; import google.registry.model.registry.label.ReservedListSqlDaoTest;
import google.registry.model.reporting.Spec11ThreatMatchTest; import google.registry.model.reporting.Spec11ThreatMatchTest;
import google.registry.model.tmch.ClaimsListDaoTest;
import google.registry.persistence.transaction.JpaEntityCoverageExtension; import google.registry.persistence.transaction.JpaEntityCoverageExtension;
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension; import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension;
import google.registry.schema.cursor.CursorDaoTest; import google.registry.schema.cursor.CursorDaoTest;
@ -35,7 +36,6 @@ import google.registry.schema.integration.SqlIntegrationTestSuite.BeforeSuiteTes
import google.registry.schema.registrar.RegistrarDaoTest; import google.registry.schema.registrar.RegistrarDaoTest;
import google.registry.schema.server.LockDaoTest; import google.registry.schema.server.LockDaoTest;
import google.registry.schema.tld.PremiumListDaoTest; import google.registry.schema.tld.PremiumListDaoTest;
import google.registry.schema.tmch.ClaimsListDaoTest;
import google.registry.testing.AppEngineExtension; import google.registry.testing.AppEngineExtension;
import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;

View file

@ -50,7 +50,8 @@ class TmchDnlActionTest extends TmchActionTestCase {
// Make sure the contents of testdata/dnl-latest.csv got inserted into the database. // Make sure the contents of testdata/dnl-latest.csv got inserted into the database.
ClaimsListShard claimsList = ClaimsListShard.get(); ClaimsListShard claimsList = ClaimsListShard.get();
assertThat(claimsList.getCreationTime()).isEqualTo(DateTime.parse("2013-11-24T23:15:37.4Z")); assertThat(claimsList.getTmdbGenerationTime())
.isEqualTo(DateTime.parse("2013-11-24T23:15:37.4Z"));
assertThat(claimsList.getClaimKey("xn----7sbejwbn3axu3d")) assertThat(claimsList.getClaimKey("xn----7sbejwbn3axu3d"))
.hasValue("2013112500/7/4/8/dIHW0DiuybvhdP8kIz"); .hasValue("2013112500/7/4/8/dIHW0DiuybvhdP8kIz");
assertThat(claimsList.getClaimKey("lolcat")).isEmpty(); assertThat(claimsList.getClaimKey("lolcat")).isEmpty();

View file

@ -37,7 +37,8 @@ class UploadClaimsListCommandTest extends CommandTestCase<UploadClaimsListComman
runCommand("--force", filename); runCommand("--force", filename);
ClaimsListShard claimsList = ClaimsListShard.get(); ClaimsListShard claimsList = ClaimsListShard.get();
assertThat(claimsList.getCreationTime()).isEqualTo(DateTime.parse("2012-08-16T00:00:00.0Z")); assertThat(claimsList.getTmdbGenerationTime())
.isEqualTo(DateTime.parse("2012-08-16T00:00:00.0Z"));
assertThat(claimsList.getClaimKey("example")) assertThat(claimsList.getClaimKey("example"))
.hasValue("2013041500/2/6/9/rJ1NrDO92vDsAzf7EQzgjX4R0000000001"); .hasValue("2013041500/2/6/9/rJ1NrDO92vDsAzf7EQzgjX4R0000000001");
assertThat(claimsList.getClaimKey("another-example")) assertThat(claimsList.getClaimKey("another-example"))

View file

@ -86,8 +86,8 @@ create sequence history_id_sequence start 1 increment 50;
create table "ClaimsList" ( create table "ClaimsList" (
revision_id bigserial not null, revision_id bigserial not null,
creation_timestamp timestamptz not null,
tmdb_generation_time timestamptz not null, tmdb_generation_time timestamptz not null,
creation_timestamp timestamptz not null,
primary key (revision_id) primary key (revision_id)
); );