TODO: We can remove the sharding once we have converted entirely to Cloud SQL storage during
+ * the Registry 3.0 migration. Then, the entire table will be stored conceptually as one entity (in
+ * fact in SignedMarkRevocationList and SignedMarkRevocationEntry tables).
+ *
* @see google.registry.tmch.SmdrlCsvParser
- * @see
- * TMCH functional specifications - SMD Revocation List
+ * @see TMCH
+ * functional specifications - SMD Revocation List
*/
@Entity
+@javax.persistence.Entity
@NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
-public class SignedMarkRevocationList extends ImmutableObject {
+public class SignedMarkRevocationList extends ImmutableObject
+ implements DatastoreEntity, SqlEntity {
- @VisibleForTesting
- static final int SHARD_SIZE = 10000;
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ @VisibleForTesting static final int SHARD_SIZE = 10000;
/** Common ancestor for queries. */
- @Parent
- Key parent = getCrossTldKey();
+ @Parent @Transient Key parent = getCrossTldKey();
/** ID for the sharded entity. */
- @Id
- long id;
+ @Id @Transient long id;
+
+ @Ignore
+ @javax.persistence.Id
+ @GeneratedValue(strategy = GenerationType.IDENTITY)
+ Long revisionId;
/** Time when this list was last updated, as specified in the first line of the CSV file. */
DateTime creationTime;
/** A map from SMD IDs to revocation time. */
@EmbedMap
+ @ElementCollection
+ @CollectionTable(
+ name = "SignedMarkRevocationEntry",
+ joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
+ @MapKeyColumn(name = "smdId")
+ @Column(name = "revocationTime", nullable = false)
Map*@MatchesPattern("[0-9]+-[0-9]+")*/ String, DateTime> revokes;
/** Indicates that this is a shard rather than a "full" list. */
- @Ignore
- boolean isShard;
+ @Ignore @Transient boolean isShard;
/**
* A cached supplier that fetches the SMDRL shards from Datastore and recombines them into a
@@ -92,32 +122,16 @@ public class SignedMarkRevocationList extends ImmutableObject {
*/
private static final Supplier CACHE =
memoizeWithShortExpiration(
- () ->
- tm()
- .transactNewReadOnly(
- () -> {
- Iterable shards =
- ofy()
- .load()
- .type(SignedMarkRevocationList.class)
- .ancestor(getCrossTldKey());
- DateTime creationTime =
- isEmpty(shards)
- ? START_OF_TIME
- : checkNotNull(
- Iterables.get(shards, 0).creationTime, "creationTime");
- ImmutableMap.Builder revokes =
- new ImmutableMap.Builder<>();
- for (SignedMarkRevocationList shard : shards) {
- revokes.putAll(shard.revokes);
- checkState(
- creationTime.equals(shard.creationTime),
- "Inconsistent creation times: %s vs. %s",
- creationTime,
- shard.creationTime);
- }
- return create(creationTime, revokes.build());
- }));
+ () -> {
+ SignedMarkRevocationList datastoreList = loadFromDatastore();
+ // Also load the list from Cloud SQL, compare the two lists, and log if different.
+ try {
+ loadAndCompareCloudSqlList(datastoreList);
+ } catch (Throwable t) {
+ logger.atSevere().withCause(t).log("Error comparing signed mark revocation lists.");
+ }
+ return datastoreList;
+ });
/** Return a single logical instance that combines all Datastore shards. */
public static SignedMarkRevocationList get() {
@@ -149,10 +163,39 @@ public class SignedMarkRevocationList extends ImmutableObject {
return revokes.size();
}
- /** Save this list to Datastore in sharded form. Returns {@code this}. */
+ /** Save this list to Datastore in sharded form and to Cloud SQL. Returns {@code this}. */
public SignedMarkRevocationList save() {
- tm()
- .transact(
+ saveToDatastore();
+ SignedMarkRevocationListDao.trySave(this);
+ return this;
+ }
+
+ /** Loads the shards from Datastore and combines them into one list. */
+ private static SignedMarkRevocationList loadFromDatastore() {
+ return tm().transactNewReadOnly(
+ () -> {
+ Iterable shards =
+ ofy().load().type(SignedMarkRevocationList.class).ancestor(getCrossTldKey());
+ DateTime creationTime =
+ isEmpty(shards)
+ ? START_OF_TIME
+ : checkNotNull(Iterables.get(shards, 0).creationTime, "creationTime");
+ ImmutableMap.Builder revokes = new ImmutableMap.Builder<>();
+ for (SignedMarkRevocationList shard : shards) {
+ revokes.putAll(shard.revokes);
+ checkState(
+ creationTime.equals(shard.creationTime),
+ "Inconsistent creation times: %s vs. %s",
+ creationTime,
+ shard.creationTime);
+ }
+ return create(creationTime, revokes.build());
+ });
+ }
+
+ /** Save this list to Datastore in sharded form. */
+ private SignedMarkRevocationList saveToDatastore() {
+ tm().transact(
() -> {
ofy()
.deleteWithoutBackup()
@@ -165,8 +208,7 @@ public class SignedMarkRevocationList extends ImmutableObject {
ofy()
.saveWithoutBackup()
.entities(
- CollectionUtils.partitionMap(revokes, SHARD_SIZE)
- .stream()
+ CollectionUtils.partitionMap(revokes, SHARD_SIZE).stream()
.map(
shardRevokes -> {
SignedMarkRevocationList shard = create(creationTime, shardRevokes);
@@ -180,6 +222,38 @@ public class SignedMarkRevocationList extends ImmutableObject {
return this;
}
+ private static void loadAndCompareCloudSqlList(SignedMarkRevocationList datastoreList) {
+ // Lifted with some modifications from ClaimsListShard
+ Optional maybeCloudSqlList =
+ SignedMarkRevocationListDao.getLatestRevision();
+ if (maybeCloudSqlList.isPresent()) {
+ SignedMarkRevocationList cloudSqlList = maybeCloudSqlList.get();
+ MapDifference diff =
+ Maps.difference(datastoreList.revokes, cloudSqlList.revokes);
+ if (!diff.areEqual()) {
+ if (diff.entriesDiffering().size() > 10) {
+ logger.atWarning().log(
+ String.format(
+ "Unequal SM revocation lists detected, Cloud SQL list with revision id %d has %d"
+ + " different records than the current Datastore list.",
+ cloudSqlList.revisionId, diff.entriesDiffering().size()));
+ } else {
+ StringBuilder diffMessage = new StringBuilder("Unequal SM revocation lists detected:\n");
+ diff.entriesDiffering()
+ .forEach(
+ (label, valueDiff) ->
+ diffMessage.append(
+ String.format(
+ "SMD %s has key %s in Datastore and key %s in Cloud SQL.\n",
+ label, valueDiff.leftValue(), valueDiff.rightValue())));
+ logger.atWarning().log(diffMessage.toString());
+ }
+ }
+ } else {
+ logger.atWarning().log("Signed mark revocation list in Cloud SQL is empty.");
+ }
+ }
+
/** As a safety mechanism, fail if someone tries to save this class directly. */
@OnSave
void disallowUnshardedSaves() {
@@ -188,6 +262,16 @@ public class SignedMarkRevocationList extends ImmutableObject {
}
}
+ @Override
+ public ImmutableList toSqlEntities() {
+ return ImmutableList.of(); // Dually-written every day
+ }
+
+ @Override
+ public ImmutableList toDatastoreEntities() {
+ return ImmutableList.of(); // Dually-written every day
+ }
+
/** Exception when trying to directly save a {@link SignedMarkRevocationList} without sharding. */
public static class UnshardedSaveException extends RuntimeException {}
}
diff --git a/core/src/main/java/google/registry/model/smd/SignedMarkRevocationListDao.java b/core/src/main/java/google/registry/model/smd/SignedMarkRevocationListDao.java
new file mode 100644
index 000000000..2bc7e2e30
--- /dev/null
+++ b/core/src/main/java/google/registry/model/smd/SignedMarkRevocationListDao.java
@@ -0,0 +1,76 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.smd;
+
+import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.base.Supplier;
+import com.google.common.flogger.FluentLogger;
+import java.util.Optional;
+import javax.persistence.EntityManager;
+
+public class SignedMarkRevocationListDao {
+
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ private static final Supplier> CACHE =
+ memoizeWithShortExpiration(SignedMarkRevocationListDao::getLatestRevision);
+
+ /** Returns the most recent revision of the {@link SignedMarkRevocationList}, from cache. */
+ public static Optional getLatestRevisionCached() {
+ return CACHE.get();
+ }
+
+ public static Optional getLatestRevision() {
+ return jpaTm()
+ .transact(
+ () -> {
+ EntityManager em = jpaTm().getEntityManager();
+ Long revisionId =
+ em.createQuery("SELECT MAX(revisionId) FROM SignedMarkRevocationList", Long.class)
+ .getSingleResult();
+ return em.createQuery(
+ "FROM SignedMarkRevocationList smrl LEFT JOIN FETCH smrl.revokes "
+ + "WHERE smrl.revisionId = :revisionId",
+ SignedMarkRevocationList.class)
+ .setParameter("revisionId", revisionId)
+ .getResultStream()
+ .findFirst();
+ });
+ }
+
+ /**
+ * Try to save the given {@link SignedMarkRevocationList} into Cloud SQL. If the save fails, the
+ * error will be logged but no exception will be thrown.
+ *
+ *
This method is used during the dual-write phase of database migration as Datastore is still
+ * the authoritative database.
+ */
+ static void trySave(SignedMarkRevocationList signedMarkRevocationList) {
+ try {
+ SignedMarkRevocationListDao.save(signedMarkRevocationList);
+ logger.atInfo().log(
+ "Inserted %,d signed mark revocations into Cloud SQL",
+ signedMarkRevocationList.revokes.size());
+ } catch (Throwable e) {
+ logger.atSevere().withCause(e).log("Error inserting signed mark revocations into Cloud SQL");
+ }
+ }
+
+ private static void save(SignedMarkRevocationList signedMarkRevocationList) {
+ jpaTm().transact(() -> jpaTm().getEntityManager().persist(signedMarkRevocationList));
+ }
+}
diff --git a/core/src/main/resources/META-INF/persistence.xml b/core/src/main/resources/META-INF/persistence.xml
index d614862b4..da7007ac4 100644
--- a/core/src/main/resources/META-INF/persistence.xml
+++ b/core/src/main/resources/META-INF/persistence.xml
@@ -62,6 +62,7 @@
google.registry.model.registry.Registrygoogle.registry.model.reporting.DomainTransactionRecordgoogle.registry.model.reporting.Spec11ThreatMatch
+ google.registry.model.smd.SignedMarkRevocationListgoogle.registry.model.tmch.ClaimsListShardgoogle.registry.persistence.transaction.TransactionEntitygoogle.registry.schema.cursor.Cursor
diff --git a/core/src/test/java/google/registry/model/smd/SignedMarkRevocationListDaoTest.java b/core/src/test/java/google/registry/model/smd/SignedMarkRevocationListDaoTest.java
new file mode 100644
index 000000000..3870137d1
--- /dev/null
+++ b/core/src/test/java/google/registry/model/smd/SignedMarkRevocationListDaoTest.java
@@ -0,0 +1,104 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.smd;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
+
+import com.google.common.collect.ImmutableMap;
+import google.registry.persistence.transaction.JpaTestRules;
+import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension;
+import google.registry.testing.DatastoreEntityExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.FakeClock;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@DualDatabaseTest
+public class SignedMarkRevocationListDaoTest {
+
+ private final FakeClock fakeClock = new FakeClock();
+
+ @RegisterExtension
+ final JpaIntegrationWithCoverageExtension jpa =
+ new JpaTestRules.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension();
+
+ @RegisterExtension
+ @Order(value = 1)
+ final DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension();
+
+ @Test
+ void testSave_success() {
+ SignedMarkRevocationList list =
+ SignedMarkRevocationList.create(
+ fakeClock.nowUtc(), ImmutableMap.of("mark", fakeClock.nowUtc().minusHours(1)));
+ SignedMarkRevocationListDao.trySave(list);
+ SignedMarkRevocationList fromDb = SignedMarkRevocationListDao.getLatestRevision().get();
+ assertAboutImmutableObjects().that(fromDb).isEqualExceptFields(list);
+ }
+
+ @Test
+ void trySave_failureIsSwallowed() {
+ SignedMarkRevocationList list =
+ SignedMarkRevocationList.create(
+ fakeClock.nowUtc(), ImmutableMap.of("mark", fakeClock.nowUtc().minusHours(1)));
+ SignedMarkRevocationListDao.trySave(list);
+ SignedMarkRevocationList fromDb = SignedMarkRevocationListDao.getLatestRevision().get();
+ assertAboutImmutableObjects().that(fromDb).isEqualExceptFields(list);
+
+ // This should throw an exception, which is swallowed and nothing changed
+ SignedMarkRevocationListDao.trySave(list);
+ SignedMarkRevocationList secondFromDb = SignedMarkRevocationListDao.getLatestRevision().get();
+ assertAboutImmutableObjects().that(secondFromDb).isEqualExceptFields(fromDb);
+ }
+
+ @Test
+ void testRetrieval_notPresent() {
+ assertThat(SignedMarkRevocationListDao.getLatestRevision().isPresent()).isFalse();
+ }
+
+ @Test
+ void testSaveAndRetrieval_emptyList() {
+ SignedMarkRevocationList list =
+ SignedMarkRevocationList.create(fakeClock.nowUtc(), ImmutableMap.of());
+ SignedMarkRevocationListDao.trySave(list);
+ SignedMarkRevocationList fromDb = SignedMarkRevocationListDao.getLatestRevision().get();
+ assertAboutImmutableObjects().that(fromDb).isEqualExceptFields(list);
+ }
+
+ @Test
+ void testSave_multipleVersions() {
+ SignedMarkRevocationList list =
+ SignedMarkRevocationList.create(
+ fakeClock.nowUtc(), ImmutableMap.of("mark", fakeClock.nowUtc().minusHours(1)));
+ SignedMarkRevocationListDao.trySave(list);
+ assertThat(
+ SignedMarkRevocationListDao.getLatestRevision()
+ .get()
+ .isSmdRevoked("mark", fakeClock.nowUtc()))
+ .isTrue();
+
+ // Now remove the revocation
+ SignedMarkRevocationList secondList =
+ SignedMarkRevocationList.create(fakeClock.nowUtc(), ImmutableMap.of());
+ SignedMarkRevocationListDao.trySave(secondList);
+ assertThat(
+ SignedMarkRevocationListDao.getLatestRevision()
+ .get()
+ .isSmdRevoked("mark", fakeClock.nowUtc()))
+ .isFalse();
+ }
+}
diff --git a/core/src/test/java/google/registry/model/smd/SignedMarkRevocationListTest.java b/core/src/test/java/google/registry/model/smd/SignedMarkRevocationListTest.java
index b9b4a0b35..e4fdde32f 100644
--- a/core/src/test/java/google/registry/model/smd/SignedMarkRevocationListTest.java
+++ b/core/src/test/java/google/registry/model/smd/SignedMarkRevocationListTest.java
@@ -15,6 +15,7 @@
package google.registry.model.smd;
import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.smd.SignedMarkRevocationList.SHARD_SIZE;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -72,10 +73,11 @@ public class SignedMarkRevocationListTest {
revokes.put(Integer.toString(i), clock.nowUtc());
}
// Save it with sharding, and make sure that reloading it works.
- SignedMarkRevocationList unsharded = SignedMarkRevocationList
- .create(clock.nowUtc(), revokes.build())
- .save();
- assertThat(SignedMarkRevocationList.get()).isEqualTo(unsharded);
+ SignedMarkRevocationList unsharded =
+ SignedMarkRevocationList.create(clock.nowUtc(), revokes.build()).save();
+ assertAboutImmutableObjects()
+ .that(SignedMarkRevocationList.get())
+ .isEqualExceptFields(unsharded, "revisionId");
assertThat(ofy().load().type(SignedMarkRevocationList.class).count()).isEqualTo(2);
}
@@ -91,7 +93,9 @@ public class SignedMarkRevocationListTest {
SignedMarkRevocationList unsharded = SignedMarkRevocationList
.create(clock.nowUtc(), revokes.build())
.save();
- assertThat(SignedMarkRevocationList.get()).isEqualTo(unsharded);
+ assertAboutImmutableObjects()
+ .that(SignedMarkRevocationList.get())
+ .isEqualExceptFields(unsharded, "revisionId");
assertThat(ofy().load().type(SignedMarkRevocationList.class).count()).isEqualTo(4);
}
diff --git a/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java b/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java
index 22757d034..24d1e6ca5 100644
--- a/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java
+++ b/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java
@@ -29,6 +29,7 @@ import google.registry.model.registry.RegistryLockDaoTest;
import google.registry.model.registry.RegistryTest;
import google.registry.model.registry.label.ReservedListSqlDaoTest;
import google.registry.model.reporting.Spec11ThreatMatchTest;
+import google.registry.model.smd.SignedMarkRevocationListDaoTest;
import google.registry.model.tmch.ClaimsListDaoTest;
import google.registry.persistence.transaction.JpaEntityCoverageExtension;
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension;
@@ -92,6 +93,7 @@ import org.junit.runner.RunWith;
RegistryTest.class,
ReservedListSqlDaoTest.class,
RegistryLockDaoTest.class,
+ SignedMarkRevocationListDaoTest.class,
Spec11ThreatMatchTest.class,
// AfterSuiteTest must be the last entry. See class javadoc for details.
AfterSuiteTest.class
diff --git a/db/src/main/resources/sql/er_diagram/brief_er_diagram.html b/db/src/main/resources/sql/er_diagram/brief_er_diagram.html
index 4766be2b7..9ab1fcb7c 100644
--- a/db/src/main/resources/sql/er_diagram/brief_er_diagram.html
+++ b/db/src/main/resources/sql/er_diagram/brief_er_diagram.html
@@ -261,2339 +261,2405 @@ td.section {