Add an extension to verify transaction replay (#857)

* Add an extension to verify transaction replay

Add ReplayExtension, which can be applied to test suites to verify that
transactions committed to datastore can be replayed to SQL.

This introduces a ReplayQueue class, which serves as a stand-in for the
current lack of replay-from-commit-logs.  It also includes replay logic in
TransactionInfo which introduces the concept of "entity class weights."
Entity weighting allows us store and delete objects in an order that is
consistent with the direction of foreign key and deferred foreign key
relationships.  As a general rule, lower weight classes must have no direct or
indirect non-deferred foreign key relationships on higher weight classes.

It is expected that much of this code will change when the final replay
mechanism is implemented.

* Minor fixes:

- Initialize "requestedByRegistrar" to false (it's non-nullable). [reverted
  during rebase: non-nullable was removed in another PR]
- Store test entities (registrar, hosts and contacts) in JPA.

* Make testbed save replay

This changes the replay system to make datastore saves initiated from the
testbed (as opposed to just the tested code) replay when the ReplayExtension
is enabled.  This requires modifications to DatastoreHelper and the
AppEngineExtension that the ReplayExtension can plug into.

This changes also has some necessary fixes to objects that are persisted by
the testbed (such as PremiumList).

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -57,6 +57,7 @@ import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
 import javax.persistence.Table;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /** An entity representing an allocation token. */
@@ -105,7 +106,8 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
   @javax.persistence.Id @Id String token;
 
   /** The key of the history entry for which the token was used. Null if not yet used. */
-  @Nullable @Index VKey<HistoryEntry> redemptionHistoryEntry;
+  // TODO(b/172848495): Remove the "Transient" when we can finally persist and restore this.
+  @Transient @Nullable @Index VKey<HistoryEntry> redemptionHistoryEntry;
 
   /** The fully-qualified domain name that this token is limited to, if any. */
   @Nullable @Index String domainName;

core/src/main/java/google/registry/model/ofy/CommitLoggedWork.java

@@ -161,6 +161,7 @@ class CommitLoggedWork<R> implements Runnable {
           .addAll(untouchedRootsWithTouchedChildren)
           .build())
       .now();
+    ReplayQueue.addInTests(info);
   }
 
   /** Check that the timestamp of each BackupGroupRoot is in the past. */

core/src/main/java/google/registry/model/ofy/ReplayQueue.java

@@ -0,0 +1,46 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.ofy;
+
+import google.registry.config.RegistryEnvironment;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+/**
+ * Implements simplified datastore to SQL transaction replay.
+ *
+ * <p>This code is to be removed when the actual replay cron job is implemented.
+ */
+public class ReplayQueue {
+
+  static ConcurrentLinkedQueue<TransactionInfo> queue =
+      new ConcurrentLinkedQueue<TransactionInfo>();
+
+  static void addInTests(TransactionInfo info) {
+    if (RegistryEnvironment.get() == RegistryEnvironment.UNITTEST) {
+      queue.add(info);
+    }
+  }
+
+  public static void replay() {
+    TransactionInfo info;
+    while ((info = queue.poll()) != null) {
+      info.saveToJpa();
+    }
+  }
+
+  public static void clear() {
+    queue.clear();
+  }
+}

core/src/main/java/google/registry/model/ofy/TransactionInfo.java

@@ -21,17 +21,25 @@ import static com.google.common.collect.Maps.filterValues;
 import static com.google.common.collect.Maps.toMap;
 import static google.registry.model.ofy.CommitLogBucket.getArbitraryBucketId;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
+import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.util.Map;
 import org.joda.time.DateTime;
 
 /** Metadata for an {@link Ofy} transaction that saves commit logs. */
 class TransactionInfo {
 
-  private enum Delete { SENTINEL }
+  @VisibleForTesting
+  enum Delete {
+    SENTINEL
+  }
 
   /** Logical "now" of the transaction. */
   DateTime transactionTime;
@@ -92,4 +100,49 @@ class TransactionInfo {
         .filter(not(Delete.SENTINEL::equals))
         .collect(toImmutableSet());
   }
+
+  // Mapping from class name to "weight" (which in this case is the order in which the class must
+  // be "put" in a transaction with respect to instances of other classes).  Lower weight classes
+  // are put first, by default all classes have a weight of zero.
+  static final ImmutableMap<String, Integer> CLASS_WEIGHTS =
+      ImmutableMap.of(
+          "HistoryEntry", -1,
+          "DomainBase", 1);
+
+  // The beginning of the range of weights reserved for delete.  This must be greater than any of
+  // the values in CLASS_WEIGHTS by enough overhead to accomodate any negative values in it.
+  @VisibleForTesting static final int DELETE_RANGE = Integer.MAX_VALUE / 2;
+
+  /** Returns the weight of the entity type in the map entry. */
+  @VisibleForTesting
+  static int getWeight(ImmutableMap.Entry<Key<?>, Object> entry) {
+    int weight = CLASS_WEIGHTS.getOrDefault(entry.getKey().getKind(), 0);
+    return entry.getValue().equals(Delete.SENTINEL) ? DELETE_RANGE - weight : weight;
+  }
+
+  private static int compareByWeight(
+      ImmutableMap.Entry<Key<?>, Object> a, ImmutableMap.Entry<Key<?>, Object> b) {
+    return getWeight(a) - getWeight(b);
+  }
+
+  void saveToJpa() {
+    // Sort the changes into an order that will work for insertion into the database.
+    jpaTm()
+        .transact(
+            () -> {
+              changesBuilder.build().entrySet().stream()
+                  .sorted(TransactionInfo::compareByWeight)
+                  .forEach(
+                      entry -> {
+                        if (entry.getValue().equals(Delete.SENTINEL)) {
+                          jpaTm().delete(VKey.from(entry.getKey()));
+                        } else {
+                          for (SqlEntity entity :
+                              ((DatastoreEntity) entry.getValue()).toSqlEntities()) {
+                            jpaTm().put(entity);
+                          }
+                        }
+                      });
+            });
+  }
 }

core/src/main/java/google/registry/model/registry/label/BaseDomainLabelList.java

@@ -79,7 +79,7 @@ public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends Dom
   // set to the timestamp when the list is created. In Datastore, we have two fields and the
   // lastUpdateTime is set to the current timestamp when creating and updating a list. So, we use
   // lastUpdateTime as the creation_timestamp column during the dual-write phase for compatibility.
-  @Column(name = "creation_timestamp", nullable = false)
+  @Column(name = "creation_timestamp")
   DateTime lastUpdateTime;
 
   /** Returns the ID of this revision, or throws if null. */

core/src/test/java/google/registry/flows/FlowTestCase.java

@@ -59,6 +59,7 @@ import java.util.Map;
 import javax.annotation.Nullable;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 /**
@@ -80,12 +81,6 @@ public abstract class FlowTestCase<F extends Flow> {
     SUPERUSER
   }
 
-  @RegisterExtension
-  final AppEngineExtension appEngine =
-      AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
-
-  @RegisterExtension final InjectExtension inject = new InjectExtension();
-
   protected EppLoader eppLoader;
   protected SessionMetadata sessionMetadata;
   protected FakeClock clock = new FakeClock(DateTime.now(UTC));
@@ -95,14 +90,24 @@ public abstract class FlowTestCase<F extends Flow> {
 
   private EppMetric.Builder eppMetricBuilder;
 
+  // Set the clock for transactional flows.  We have to order this before the AppEngineExtension
+  // which populates data (and may do so with clock-dependent commit logs if mixed with
+  // ReplayExtension).
+  @Order(value = Order.DEFAULT - 1)
+  @RegisterExtension
+  final InjectExtension inject =
+      new InjectExtension().withStaticFieldOverride(Ofy.class, "clock", clock);
+
+  @RegisterExtension
+  final AppEngineExtension appEngine =
+      AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
+
   @BeforeEach
   public void beforeEachFlowTestCase() {
     sessionMetadata = new HttpSessionMetadata(new FakeHttpSession());
     sessionMetadata.setClientId("TheRegistrar");
     sessionMetadata.setServiceExtensionUris(ProtocolDefinition.getVisibleServiceExtensionUris());
     ofy().saveWithoutBackup().entity(new ClaimsListSingleton()).now();
-    // For transactional flows
-    inject.setStaticField(Ofy.class, "clock", clock);
  }
 
   protected void removeServiceExtensionUri(String uri) {

core/src/test/java/google/registry/flows/domain/DomainCreateFlowTest.java

@@ -163,6 +163,7 @@ import google.registry.model.reporting.DomainTransactionRecord.TransactionReport
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.persistence.VKey;
+import google.registry.testing.ReplayExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import java.math.BigDecimal;
 import java.util.Map;
@@ -171,7 +172,9 @@ import org.joda.money.Money;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link DomainCreateFlow}. */
 class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, DomainBase> {
@@ -180,6 +183,10 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
 
   private AllocationToken allocationToken;
 
+  @Order(value = Order.DEFAULT - 2)
+  @RegisterExtension
+  final ReplayExtension replayExtension = new ReplayExtension(clock);
+
   DomainCreateFlowTest() {
     setEppInput("domain_create.xml", ImmutableMap.of("DOMAIN", "example.tld"));
     clock.setTo(DateTime.parse("1999-04-03T22:00:00.0Z").minus(Duration.millis(1)));
@@ -222,7 +229,6 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     }
     persistActiveContact("jd1234");
     persistActiveContact("sh8013");
-    clock.advanceOneMilli();
   }
 
   private void persistContactsAndHosts() {
@@ -350,7 +356,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
         .hasLaunchNotice(null);
     String expectedPayload =
         String.format(
-            "%s,%s,0000001761376042759136-65535,1,2014-09-09T09:09:09.001Z",
+            "%s,%s,0000001761376042759136-65535,1,2014-09-09T09:09:09.016Z",
             reloadResourceByForeignKey().getRepoId(), domainName);
     assertTasksEnqueued(QUEUE_SUNRISE, new TaskMatcher().payload(expectedPayload));
   }
@@ -371,7 +377,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
             .payload(
                 reloadResourceByForeignKey().getRepoId()
                     + ",example-one.tld,370d0b7c9223372036854775807,1,"
-                    + "2009-08-16T09:00:00.001Z,2009-08-16T09:00:00.000Z");
+                    + "2009-08-16T09:00:00.016Z,2009-08-16T09:00:00.000Z");
     assertTasksEnqueued(QUEUE_CLAIMS, task);
   }
 

core/src/test/java/google/registry/model/ofy/TransactionInfoTest.java

@@ -0,0 +1,49 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.ofy;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.collect.ImmutableMap;
+import com.googlecode.objectify.Key;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.testing.AppEngineExtension;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+class TransactionInfoTest {
+
+  @RegisterExtension
+  AppEngineExtension appEngine = new AppEngineExtension.Builder().withDatastore().build();
+
+  @Test
+  void testGetWeight() {
+    // just verify that the lowest is what we expect for both save and delete and verify that the
+    // Registrar class is zero.
+    ImmutableMap<Key<?>, Object> actions =
+        ImmutableMap.of(
+            Key.create(HistoryEntry.class, 100), TransactionInfo.Delete.SENTINEL,
+            Key.create(HistoryEntry.class, 200), "fake history entry",
+            Key.create(Registrar.class, 300), "fake registrar");
+    ImmutableMap<Long, Integer> expectedValues =
+        ImmutableMap.of(100L, TransactionInfo.DELETE_RANGE + 1, 200L, -1, 300L, 0);
+
+    for (ImmutableMap.Entry<Key<?>, Object> entry : actions.entrySet()) {
+      assertThat(TransactionInfo.getWeight(entry))
+          .isEqualTo(expectedValues.get(entry.getKey().getId()));
+    }
+  }
+}

core/src/test/java/google/registry/testing/AppEngineExtension.java

@@ -477,6 +477,24 @@ public final class AppEngineExtension implements BeforeEachCallback, AfterEachCa
   public void afterEach(ExtensionContext context) throws Exception {
     checkArgumentNotNull(context, "The ExtensionContext must not be null");
     try {
+      // If there is a replay extension, we'll want to call its replayToSql() method.
+      //
+      // We have to provide this hook here for ReplayExtension instead of relying on
+      // ReplayExtension's afterEach() method because of ordering and the conflation of environment
+      // initialization and basic entity initialization.
+      //
+      // ReplayExtension's beforeEach() has to be called before this so that the entities that we
+      // initialize (e.g. "TheRegistrar") also get replayed.  But that means that ReplayExtension's
+      // afterEach() won't be called until after ours.  Since we tear down the datastore and SQL
+      // database in our own afterEach(), ReplayExtension's afterEach() would fail if we let the
+      // replay happen there.
+      ReplayExtension replayer =
+          (ReplayExtension)
+              context.getStore(ExtensionContext.Namespace.GLOBAL).get(ReplayExtension.class);
+      if (replayer != null) {
+        replayer.replayToSql();
+      }
+
       if (withCloudSql) {
         if (enableJpaEntityCoverageCheck) {
           jpaIntegrationWithCoverageExtension.afterEach(context);

core/src/test/java/google/registry/testing/DatastoreHelper.java

@@ -122,6 +122,14 @@ import org.joda.time.DateTimeZone;
 /** Static utils for setting up test resources. */
 public class DatastoreHelper {
 
+  // The following two fields are injected by ReplayExtension.
+
+  // If this is true, all of the methods that save to the datastore do so with backup.
+  private static boolean alwaysSaveWithBackup;
+
+  // If the clock is defined, it will always be advanced by one millsecond after a transaction.
+  private static FakeClock clock;
+
   private static final Supplier<String[]> DEFAULT_PREMIUM_LIST_CONTENTS =
       memoize(
           () ->
@@ -132,6 +140,20 @@ public class DatastoreHelper {
                               DatastoreHelper.class, "default_premium_list_testdata.csv")),
                   String.class));
 
+  public static void setAlwaysSaveWithBackup(boolean enable) {
+    alwaysSaveWithBackup = enable;
+  }
+
+  public static void setClock(FakeClock fakeClock) {
+    clock = fakeClock;
+  }
+
+  private static void maybeAdvanceClock() {
+    if (clock != null) {
+      clock.advanceOneMilli();
+    }
+  }
+
   public static HostResource newHostResource(String hostName) {
     return newHostResourceWithRoid(hostName, generateNewContactHostRoid());
   }
@@ -312,6 +334,7 @@ public class DatastoreHelper {
     // the
     // transaction time is set correctly.
     tm().transactNew(() -> LordnTaskUtils.enqueueDomainBaseTask(persistedDomain));
+    maybeAdvanceClock();
     return persistedDomain;
   }
 
@@ -365,14 +388,25 @@ public class DatastoreHelper {
     PremiumListRevision revision = PremiumListRevision.create(premiumList, entries.keySet());
 
     if (tm().isOfy()) {
-      tm().putAllWithoutBackup(
+      ImmutableList<Object> premiumLists =
-              ImmutableList.of(
+          ImmutableList.of(
-                  premiumList.asBuilder().setRevision(Key.create(revision)).build(), revision));
+              premiumList.asBuilder().setRevision(Key.create(revision)).build(), revision);
-      tm().putAllWithoutBackup(
+      ImmutableSet<PremiumListEntry> entriesOnRevision =
-              parentPremiumListEntriesOnRevision(entries.values(), Key.create(revision)));
+          parentPremiumListEntriesOnRevision(entries.values(), Key.create(revision));
+      if (alwaysSaveWithBackup) {
+        tm().transact(
+                () -> {
+                  tm().putAll(premiumLists);
+                  tm().putAll(entriesOnRevision);
+                });
+      } else {
+        tm().putAllWithoutBackup(premiumLists);
+        tm().putAllWithoutBackup(entriesOnRevision);
+      }
     } else {
       tm().transact(() -> tm().insert(premiumList));
     }
+    maybeAdvanceClock();
     // The above premiumList is in the session cache and it is different from the corresponding
     // entity stored in Datastore because it has some @Ignore fields set dedicated for SQL. This
     // breaks the assumption we have in our application code, see
@@ -934,7 +968,7 @@ public class DatastoreHelper {
 
   private static <R> void saveResource(R resource, boolean wantBackup) {
     if (tm().isOfy()) {
-      Saver saver = wantBackup ? ofy().save() : ofy().saveWithoutBackup();
+      Saver saver = wantBackup || alwaysSaveWithBackup ? ofy().save() : ofy().saveWithoutBackup();
       saver.entity(resource);
       if (resource instanceof EppResource) {
         EppResource eppResource = (EppResource) resource;
@@ -962,6 +996,7 @@ public class DatastoreHelper {
         .that(resource)
         .isNotInstanceOf(Buildable.Builder.class);
     tm().transact(() -> saveResource(resource, wantBackup));
+    maybeAdvanceClock();
     // Force the session cache to be cleared so that when we read the resource back, we read from
     // Datastore and not from the session cache. This is needed to trigger Objectify's load process
     // (unmarshalling entity protos to POJOs, nulling out empty collections, calling @OnLoad
@@ -984,6 +1019,7 @@ public class DatastoreHelper {
                 tm().put(resource);
               }
             });
+    maybeAdvanceClock();
     tm().clearSessionCache();
     return transactIfJpaTm(() -> tm().load(resource));
   }
@@ -1001,6 +1037,7 @@ public class DatastoreHelper {
     // Persist domains ten at a time, to avoid exceeding the entity group limit.
     for (final List<R> chunk : Iterables.partition(resources, 10)) {
       tm().transact(() -> chunk.forEach(resource -> saveResource(resource, wantBackup)));
+      maybeAdvanceClock();
     }
     // Force the session to be cleared so that when we read it back, we read from Datastore
     // and not from the transaction's session cache.
@@ -1035,6 +1072,7 @@ public class DatastoreHelper {
               ofyTmOrDoNothing(
                   () -> tm().put(ForeignKeyIndex.create(resource, resource.getDeletionTime())));
             });
+    maybeAdvanceClock();
     tm().clearSessionCache();
     return transactIfJpaTm(() -> tm().load(resource));
   }
@@ -1128,7 +1166,15 @@ public class DatastoreHelper {
    * ForeignKeyedEppResources.
    */
   public static <R> ImmutableList<R> persistSimpleResources(final Iterable<R> resources) {
-    tm().transact(() -> tm().putAllWithoutBackup(ImmutableList.copyOf(resources)));
+    tm().transact(
+            () -> {
+              if (alwaysSaveWithBackup) {
+                tm().putAll(ImmutableList.copyOf(resources));
+              } else {
+                tm().putAllWithoutBackup(ImmutableList.copyOf(resources));
+              }
+            });
+    maybeAdvanceClock();
     // Force the session to be cleared so that when we read it back, we read from Datastore
     // and not from the transaction's session cache.
     tm().clearSessionCache();
@@ -1136,7 +1182,11 @@ public class DatastoreHelper {
   }
 
   public static void deleteResource(final Object resource) {
-    transactIfJpaTm(() -> tm().deleteWithoutBackup(resource));
+    if (alwaysSaveWithBackup) {
+      tm().transact(() -> tm().delete(resource));
+    } else {
+      transactIfJpaTm(() -> tm().deleteWithoutBackup(resource));
+    }
     // Force the session to be cleared so that when we read it back, we read from Datastore and
     // not from the transaction's session cache.
     tm().clearSessionCache();
@@ -1144,15 +1194,18 @@ public class DatastoreHelper {
 
   /** Force the create and update timestamps to get written into the resource. **/
   public static <R> R cloneAndSetAutoTimestamps(final R resource) {
+    R result;
     if (tm().isOfy()) {
-      return tm().transact(() -> ofy().load().fromEntity(ofy().save().toEntity(resource)));
+      result = tm().transact(() -> ofy().load().fromEntity(ofy().save().toEntity(resource)));
     } else {
       // We have to separate the read and write operation into different transactions
       // otherwise JPA would just return the input entity instead of actually creating a
       // clone.
       tm().transact(() -> tm().put(resource));
-      return tm().transact(() -> tm().load(resource));
+      result = tm().transact(() -> tm().load(resource));
     }
+    maybeAdvanceClock();
+    return result;
   }
 
   /** Returns the entire map of {@link PremiumListEntry}s for the given {@link PremiumList}. */

core/src/test/java/google/registry/testing/ReplayExtension.java

@@ -0,0 +1,58 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.testing;
+
+import google.registry.model.ofy.ReplayQueue;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+/**
+ * A JUnit extension that replays datastore transactions against postgresql.
+ *
+ * <p>This extension must be ordered before AppEngineExtension so that the test entities saved in
+ * that extension are also replayed. If AppEngineExtension is not used,
+ * JpaTransactionManagerExtension must be, and this extension should be ordered _after_
+ * JpaTransactionManagerExtension so that writes to SQL work.
+ */
+public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
+
+  FakeClock clock;
+
+  public ReplayExtension(FakeClock clock) {
+    this.clock = clock;
+  }
+
+  @Override
+  public void beforeEach(ExtensionContext context) {
+    DatastoreHelper.setClock(clock);
+    DatastoreHelper.setAlwaysSaveWithBackup(true);
+    ReplayQueue.clear();
+    context.getStore(ExtensionContext.Namespace.GLOBAL).put(ReplayExtension.class, this);
+  }
+
+  @Override
+  public void afterEach(ExtensionContext context) {
+    // This ensures that we do the replay even if we're not called from AppEngineExtension.  It
+    // should be safe to call replayToSql() twice, as the replay queue should be empty the second
+    // time.
+    replayToSql();
+  }
+
+  public void replayToSql() {
+    DatastoreHelper.setAlwaysSaveWithBackup(false);
+    ReplayQueue.replay();
+  }
+}

core/src/test/resources/google/registry/flows/domain/domain_create_response_eap_fee.xml

@@ -15,7 +15,7 @@
       <fee:creData xmlns:fee="urn:ietf:params:xml:ns:fee-%FEE_VERSION%">
         <fee:currency>USD</fee:currency>
         <fee:fee description="create">26.00</fee:fee>
-        <fee:fee description="Early Access Period, fee expires: 1999-04-04T22:00:00.000Z">100.00</fee:fee>
+        <fee:fee description="Early Access Period, fee expires: 1999-04-04T22:00:00.024Z">100.00</fee:fee>
       </fee:creData>
     </extension>
     <trID>

core/src/test/resources/google/registry/flows/domain/domain_create_response_premium_eap.xml

@@ -15,7 +15,7 @@
       <fee:creData xmlns:fee="urn:ietf:params:xml:ns:fee-0.6">
         <fee:currency>USD</fee:currency>
         <fee:fee description="create">200.00</fee:fee>
-        <fee:fee description="Early Access Period, fee expires: 1999-04-04T22:00:00.000Z">100.00
+        <fee:fee description="Early Access Period, fee expires: 1999-04-04T22:00:00.028Z">100.00
         </fee:fee>
       </fee:creData>
     </extension>

db/src/main/resources/sql/er_diagram/brief_er_diagram.html

@@ -261,11 +261,11 @@ td.section {
     </tr> 
     <tr> 
      <td class="property_name">generated on</td> 
-     <td class="property_value">2020-11-13 19:34:54.398919</td> 
+     <td class="property_value">2020-11-16 16:45:08.581361</td> 
     </tr> 
     <tr>
      <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V76__change_history_nullability.sql</td>
+     <td id="lastFlywayFile" class="property_value">V77__fixes_for_replay.sql</td>
     </tr>
    </tbody>
   </table> 
@@ -284,7 +284,7 @@ td.section {
      generated on
     </text> 
     <text text-anchor="start" x="5830.94" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-     2020-11-13 19:34:54.398919
+     2020-11-16 16:45:08.581361
     </text> 
     <polygon fill="none" stroke="#888888" points="5743.44,-4 5743.44,-44 6008.44,-44 6008.44,-4 5743.44,-4" /> <!-- allocationtoken_a08ccbef --> 
     <g id="node1" class="node"> 

db/src/main/resources/sql/er_diagram/full_er_diagram.html

@@ -261,11 +261,11 @@ td.section {
     </tr> 
     <tr> 
      <td class="property_name">generated on</td> 
-     <td class="property_value">2020-11-13 19:34:52.404634</td> 
+     <td class="property_value">2020-11-16 16:45:06.707088</td> 
     </tr> 
     <tr>
      <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V76__change_history_nullability.sql</td>
+     <td id="lastFlywayFile" class="property_value">V77__fixes_for_replay.sql</td>
     </tr>
    </tbody>
   </table> 
@@ -284,7 +284,7 @@ td.section {
      generated on
     </text> 
     <text text-anchor="start" x="6501.68" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-     2020-11-13 19:34:52.404634
+     2020-11-16 16:45:06.707088
     </text> 
     <polygon fill="none" stroke="#888888" points="6414.18,-4 6414.18,-44 6679.18,-44 6679.18,-4 6414.18,-4" /> <!-- allocationtoken_a08ccbef --> 
     <g id="node1" class="node"> 
@@ -5690,70 +5690,70 @@ td.section {
     </g> <!-- premiumlist_7c3ea68b --> 
     <g id="node26" class="node"> 
      <title>premiumlist_7c3ea68b</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="5717.5,-3349.5 5717.5,-3368.5 5849.5,-3368.5 5849.5,-3349.5 5717.5,-3349.5" /> 
+     <polygon fill="#ebcef2" stroke="transparent" points="5725.5,-3349.5 5725.5,-3368.5 5857.5,-3368.5 5857.5,-3349.5 5725.5,-3349.5" /> 
-     <text text-anchor="start" x="5719.5" y="-3356.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="5727.5" y="-3356.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.PremiumList
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="5849.5,-3349.5 5849.5,-3368.5 5975.5,-3368.5 5975.5,-3349.5 5849.5,-3349.5" /> 
+     <polygon fill="#ebcef2" stroke="transparent" points="5857.5,-3349.5 5857.5,-3368.5 5967.5,-3368.5 5967.5,-3349.5 5857.5,-3349.5" /> 
-     <text text-anchor="start" x="5936.5" y="-3355.3" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="5928.5" y="-3355.3" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="5719.5" y="-3337.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="5727.5" y="-3337.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       revision_id
      </text> 
-     <text text-anchor="start" x="5843.5" y="-3336.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="5851.5" y="-3336.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="5851.5" y="-3336.3" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="5859.5" y="-3336.3" font-family="Helvetica,sans-Serif" font-size="14.00">
       bigserial not null
      </text> 
-     <text text-anchor="start" x="5843.5" y="-3317.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="5851.5" y="-3317.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="5851.5" y="-3317.3" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="5859.5" y="-3317.3" font-family="Helvetica,sans-Serif" font-size="14.00">
       auto-incremented
      </text> 
-     <text text-anchor="start" x="5719.5" y="-3298.3" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="5727.5" y="-3298.3" font-family="Helvetica,sans-Serif" font-size="14.00">
       creation_timestamp
      </text> 
-     <text text-anchor="start" x="5843.5" y="-3298.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="5851.5" y="-3298.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="5851.5" y="-3298.3" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="5859.5" y="-3298.3" font-family="Helvetica,sans-Serif" font-size="14.00">
-      timestamptz not null
+      timestamptz
      </text> 
-     <text text-anchor="start" x="5719.5" y="-3279.3" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="5727.5" y="-3279.3" font-family="Helvetica,sans-Serif" font-size="14.00">
       name
      </text> 
-     <text text-anchor="start" x="5843.5" y="-3279.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="5851.5" y="-3279.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="5851.5" y="-3279.3" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="5859.5" y="-3279.3" font-family="Helvetica,sans-Serif" font-size="14.00">
       text not null
      </text> 
-     <text text-anchor="start" x="5719.5" y="-3260.3" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="5727.5" y="-3260.3" font-family="Helvetica,sans-Serif" font-size="14.00">
       bloom_filter
      </text> 
-     <text text-anchor="start" x="5843.5" y="-3260.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="5851.5" y="-3260.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="5851.5" y="-3260.3" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="5859.5" y="-3260.3" font-family="Helvetica,sans-Serif" font-size="14.00">
       bytea not null
      </text> 
-     <text text-anchor="start" x="5719.5" y="-3241.3" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="5727.5" y="-3241.3" font-family="Helvetica,sans-Serif" font-size="14.00">
       currency
      </text> 
-     <text text-anchor="start" x="5843.5" y="-3241.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="5851.5" y="-3241.3" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="5851.5" y="-3241.3" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="5859.5" y="-3241.3" font-family="Helvetica,sans-Serif" font-size="14.00">
       text not null
      </text> 
-     <polygon fill="none" stroke="#888888" points="5716.5,-3235 5716.5,-3370 5976.5,-3370 5976.5,-3235 5716.5,-3235" /> 
+     <polygon fill="none" stroke="#888888" points="5724.5,-3235 5724.5,-3370 5968.5,-3370 5968.5,-3235 5724.5,-3235" /> 
     </g> <!-- premiumentry_b0060b91&#45;&gt;premiumlist_7c3ea68b --> 
     <g id="edge52" class="edge"> 
      <title>premiumentry_b0060b91:w-&gt;premiumlist_7c3ea68b:e</title> 
-     <path fill="none" stroke="black" d="M6320.18,-3340.5C6177.56,-3340.5 6133.1,-3340.5 5986.75,-3340.5" /> 
+     <path fill="none" stroke="black" d="M6320.23,-3340.5C6173.96,-3340.5 6128.62,-3340.5 5978.54,-3340.5" /> 
      <polygon fill="black" stroke="black" points="6328.5,-3340.5 6338.5,-3345 6333.5,-3340.5 6338.5,-3340.5 6338.5,-3340.5 6338.5,-3340.5 6333.5,-3340.5 6338.5,-3336 6328.5,-3340.5 6328.5,-3340.5" /> 
      <ellipse fill="none" stroke="black" cx="6324.5" cy="-3340.5" rx="4" ry="4" /> 
-     <polygon fill="black" stroke="black" points="5977.5,-3345.5 5977.5,-3335.5 5979.5,-3335.5 5979.5,-3345.5 5977.5,-3345.5" /> 
+     <polygon fill="black" stroke="black" points="5969.5,-3345.5 5969.5,-3335.5 5971.5,-3335.5 5971.5,-3345.5 5969.5,-3345.5" /> 
-     <polyline fill="none" stroke="black" points="5976.5,-3340.5 5981.5,-3340.5 " /> 
+     <polyline fill="none" stroke="black" points="5968.5,-3340.5 5973.5,-3340.5 " /> 
-     <polygon fill="black" stroke="black" points="5982.5,-3345.5 5982.5,-3335.5 5984.5,-3335.5 5984.5,-3345.5 5982.5,-3345.5" /> 
+     <polygon fill="black" stroke="black" points="5974.5,-3345.5 5974.5,-3335.5 5976.5,-3335.5 5976.5,-3345.5 5974.5,-3345.5" /> 
-     <polyline fill="none" stroke="black" points="5981.5,-3340.5 5986.5,-3340.5 " /> 
+     <polyline fill="none" stroke="black" points="5973.5,-3340.5 5978.5,-3340.5 " /> 
      <text text-anchor="start" x="6081.5" y="-3344.3" font-family="Helvetica,sans-Serif" font-size="14.00">
       fko0gw90lpo1tuee56l0nb6y6g5
      </text> 
@@ -11600,7 +11600,7 @@ td.section {
     <tr> 
      <td class="spacer"></td> 
      <td class="minwidth">creation_timestamp</td> 
-     <td class="minwidth">timestamptz not null</td> 
+     <td class="minwidth">timestamptz</td> 
     </tr> 
     <tr> 
      <td class="spacer"></td> 

db/src/main/resources/sql/flyway.txt

@@ -74,3 +74,4 @@ V73__singleton_entities.sql
 V74__sql_replay_checkpoint.sql
 V75__add_grace_period_history.sql
 V76__change_history_nullability.sql
+V77__fixes_for_replay.sql

db/src/main/resources/sql/flyway/V77__fixes_for_replay.sql

@@ -0,0 +1,15 @@
+-- Copyright 2020 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+ALTER TABLE "PremiumList" ALTER COLUMN creation_timestamp DROP NOT NULL;

db/src/main/resources/sql/schema/db-schema.sql.generated

@@ -22,7 +22,6 @@
         discount_premiums boolean not null,
         discount_years int4 not null,
         domain_name text,
-        redemption_history_entry text,
         token_status_transitions hstore,
         token_type text,
         primary key (token)
@@ -524,7 +523,7 @@
 
     create table "PremiumList" (
        revision_id  bigserial not null,
-        creation_timestamp timestamptz not null,
+        creation_timestamp timestamptz,
         name text not null,
         bloom_filter bytea not null,
         currency text not null,
@@ -635,7 +634,7 @@
 
     create table "ReservedList" (
        revision_id  bigserial not null,
-        creation_timestamp timestamptz not null,
+        creation_timestamp timestamptz,
         name text not null,
         should_publish boolean not null,
         primary key (revision_id)

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -686,7 +686,7 @@ CREATE TABLE public."PremiumEntry" (
 
 CREATE TABLE public."PremiumList" (
     revision_id bigint NOT NULL,
-    creation_timestamp timestamp with time zone NOT NULL,
+    creation_timestamp timestamp with time zone,
     name text NOT NULL,
     bloom_filter bytea NOT NULL,
     currency text NOT NULL