Write ClaimsList to Cloud SQL (#223)

* Rewrite ClaimsListShard with new API

* Write ClaimsList to Cloud SQL

* Add creationTimestamp

core/src/main/java/google/registry/model/tmch/ClaimsListDao.java

@@ -0,0 +1,72 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.tmch;
+
+import static google.registry.model.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.flogger.FluentLogger;
+import google.registry.schema.tmch.ClaimsList;
+import javax.persistence.EntityManager;
+
+/** Data access object for {@link ClaimsList}. */
+public class ClaimsListDao {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private static void save(ClaimsList claimsList) {
+    jpaTm().transact(() -> jpaTm().getEntityManager().persist(claimsList));
+  }
+
+  /**
+   * Try to save the given {@link ClaimsList} into Cloud SQL. If the save fails, the error will be
+   * logged but no exception will be thrown.
+   *
+   * <p>This method is used during the dual-write phase of database migration as Datastore is still
+   * the authoritative database.
+   */
+  public static void trySave(ClaimsList claimsList) {
+    try {
+      ClaimsListDao.save(claimsList);
+      logger.atInfo().log(
+          "Inserted %,d claims into Cloud SQL, created at %s",
+          claimsList.getLabelsToKeys().size(), claimsList.getTmdbGenerationTime());
+    } catch (Throwable e) {
+      logger.atSevere().withCause(e).log("Error inserting claims into Cloud SQL");
+    }
+  }
+
+  /**
+   * Returns the current revision of the {@link ClaimsList} in Cloud SQL. Throws exception if there
+   * is no claims in the table.
+   */
+  public static ClaimsList getCurrent() {
+    return jpaTm()
+        .transact(
+            () -> {
+              EntityManager em = jpaTm().getEntityManager();
+              Long revisionId =
+                  em.createQuery("SELECT MAX(revisionId) FROM ClaimsList", Long.class)
+                      .getSingleResult();
+              return em.createQuery(
+                      "FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId ="
+                          + " :revisionId",
+                      ClaimsList.class)
+                  .setParameter("revisionId", revisionId)
+                  .getSingleResult();
+            });
+  }
+
+  private ClaimsListDao() {}
+}

core/src/main/java/google/registry/model/tmch/ClaimsListShard.java

@@ -217,8 +217,7 @@ public class ClaimsListShard extends ImmutableObject {
             });
   }
 
-  public static ClaimsListShard create(
+  public static ClaimsListShard create(DateTime creationTime, Map<String, String> labelsToKeys) {
-      DateTime creationTime, ImmutableMap<String, String> labelsToKeys) {
     ClaimsListShard instance = new ClaimsListShard();
     instance.id = allocateId();
     instance.creationTime = checkNotNull(creationTime);

core/src/main/java/google/registry/schema/tmch/ClaimsList.java

@@ -15,7 +15,11 @@
 package google.registry.schema.tmch;
 
 import static com.google.common.base.Preconditions.checkState;
+import static google.registry.util.DateTimeUtils.toJodaDateTime;
+import static google.registry.util.DateTimeUtils.toZonedDateTime;
 
+import google.registry.model.CreateAutoTimestamp;
+import google.registry.model.ImmutableObject;
 import java.time.ZonedDateTime;
 import java.util.Map;
 import java.util.Optional;
@@ -29,6 +33,7 @@ import javax.persistence.Id;
 import javax.persistence.JoinColumn;
 import javax.persistence.MapKeyColumn;
 import javax.persistence.Table;
+import org.joda.time.DateTime;
 
 /**
  * A list of TMCH claims labels and their associated claims keys.
@@ -40,26 +45,29 @@ import javax.persistence.Table;
  * highest {@link #revisionId}.
  */
 @Entity
-@Table(name = "ClaimsList")
+@Table
-public class ClaimsList {
+public class ClaimsList extends ImmutableObject {
   @Id
   @GeneratedValue(strategy = GenerationType.IDENTITY)
-  @Column(name = "revision_id")
+  @Column
   private Long revisionId;
 
-  @Column(name = "creation_timestamp", nullable = false)
+  @Column(nullable = false)
-  private ZonedDateTime creationTimestamp;
+  private CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
+
+  @Column(nullable = false)
+  private ZonedDateTime tmdbGenerationTime;
 
   @ElementCollection
   @CollectionTable(
       name = "ClaimsEntry",
-      joinColumns = @JoinColumn(name = "revision_id", referencedColumnName = "revision_id"))
+      joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
-  @MapKeyColumn(name = "domain_label", nullable = false)
+  @MapKeyColumn(name = "domainLabel", nullable = false)
-  @Column(name = "claim_key", nullable = false)
+  @Column(name = "claimKey", nullable = false)
   private Map<String, String> labelsToKeys;
 
-  private ClaimsList(ZonedDateTime creationTimestamp, Map<String, String> labelsToKeys) {
+  private ClaimsList(ZonedDateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
-    this.creationTimestamp = creationTimestamp;
+    this.tmdbGenerationTime = tmdbGenerationTime;
     this.labelsToKeys = labelsToKeys;
   }
 
@@ -67,9 +75,8 @@ public class ClaimsList {
   private ClaimsList() {}
 
   /** Constructs a {@link ClaimsList} object. */
-  public static ClaimsList create(
+  public static ClaimsList create(DateTime creationTimestamp, Map<String, String> labelsToKeys) {
-      ZonedDateTime creationTimestamp, Map<String, String> labelsToKeys) {
+    return new ClaimsList(toZonedDateTime(creationTimestamp), labelsToKeys);
-    return new ClaimsList(creationTimestamp, labelsToKeys);
   }
 
   /** Returns the revision id of this claims list, or throws exception if it is null. */
@@ -79,9 +86,14 @@ public class ClaimsList {
     return revisionId;
   }
 
+  /** Returns the TMDB generation time of this claims list. */
+  public DateTime getTmdbGenerationTime() {
+    return toJodaDateTime(tmdbGenerationTime);
+  }
+
   /** Returns the creation time of this claims list. */
-  public ZonedDateTime getCreationTimestamp() {
+  public DateTime getCreationTimestamp() {
-    return creationTimestamp;
+    return creationTimestamp.getTimestamp();
   }
 
   /** Returns an {@link Map} mapping domain label to its lookup key. */

core/src/main/java/google/registry/tmch/ClaimsListParser.java

@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableMap;
-import google.registry.model.tmch.ClaimsListShard;
+import google.registry.schema.tmch.ClaimsList;
 import java.util.List;
 import org.joda.time.DateTime;
 
@@ -34,11 +34,11 @@ import org.joda.time.DateTime;
 public class ClaimsListParser {
 
   /**
-   * Converts the lines from the DNL CSV file into a {@link ClaimsListShard} object.
+   * Converts the lines from the DNL CSV file into a {@link ClaimsList} object.
    *
    * <p>Please note that this does <b>not</b> insert the object into Datastore.
    */
-  public static ClaimsListShard parse(List<String> lines) {
+  public static ClaimsList parse(List<String> lines) {
     ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
 
     // First line: <version>,<DNL List creation datetime>
@@ -74,6 +74,6 @@ public class ClaimsListParser {
       builder.put(label, lookupKey);
     }
 
-    return ClaimsListShard.create(creationTime, builder.build());
+    return ClaimsList.create(creationTime, builder.build());
   }
 }

core/src/main/java/google/registry/tmch/TmchDnlAction.java

@@ -18,9 +18,11 @@ import static google.registry.request.Action.Method.POST;
 
 import com.google.common.flogger.FluentLogger;
 import google.registry.keyring.api.KeyModule.Key;
+import google.registry.model.tmch.ClaimsListDao;
 import google.registry.model.tmch.ClaimsListShard;
 import google.registry.request.Action;
 import google.registry.request.auth.Auth;
+import google.registry.schema.tmch.ClaimsList;
 import java.io.IOException;
 import java.security.SignatureException;
 import java.util.List;
@@ -54,10 +56,14 @@ public final class TmchDnlAction implements Runnable {
     } catch (SignatureException | IOException | PGPException e) {
       throw new RuntimeException(e);
     }
-    ClaimsListShard claims = ClaimsListParser.parse(lines);
+    ClaimsList claims = ClaimsListParser.parse(lines);
-    claims.save();
+    ClaimsListShard claimsListShard =
+        ClaimsListShard.create(claims.getTmdbGenerationTime(), claims.getLabelsToKeys());
+    claimsListShard.save();
     logger.atInfo().log(
         "Inserted %,d claims into Datastore, created at %s",
-        claims.size(), claims.getCreationTime());
+        claimsListShard.size(), claimsListShard.getCreationTime());
+
+    ClaimsListDao.trySave(claims);
   }
 }

core/src/main/java/google/registry/tools/UploadClaimsListCommand.java

@@ -21,7 +21,9 @@ import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.common.base.Joiner;
 import com.google.common.io.Files;
+import google.registry.model.tmch.ClaimsListDao;
 import google.registry.model.tmch.ClaimsListShard;
+import google.registry.schema.tmch.ClaimsList;
 import google.registry.tmch.ClaimsListParser;
 import java.io.File;
 import java.io.IOException;
@@ -35,9 +37,14 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
   @Parameter(description = "Claims list filename")
   private List<String> mainParameters = new ArrayList<>();
 
+  @Parameter(
+      names = {"--also_cloud_sql"},
+      description = "Persist claims list to Cloud SQL in addition to Datastore; defaults to false.")
+  boolean alsoCloudSql;
+
   private String claimsListFilename;
 
-  private ClaimsListShard claimsList;
+  private ClaimsList claimsList;
 
   @Override
   protected void init() throws IOException {
@@ -56,7 +63,10 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
 
   @Override
   public String execute() {
-    claimsList.save();
+    ClaimsListShard.create(claimsList.getTmdbGenerationTime(), claimsList.getLabelsToKeys()).save();
+    if (alsoCloudSql) {
+      ClaimsListDao.trySave(claimsList);
+    }
     return String.format("Successfully uploaded claims list %s", claimsListFilename);
   }
 }

core/src/test/java/google/registry/model/tmch/ClaimsListDaoTest.java

@@ -0,0 +1,95 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.tmch;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.JUnitBackports.assertThrows;
+
+import com.google.common.collect.ImmutableMap;
+import google.registry.model.transaction.JpaTransactionManagerRule;
+import google.registry.schema.tmch.ClaimsList;
+import google.registry.testing.FakeClock;
+import javax.persistence.NoResultException;
+import javax.persistence.PersistenceException;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link ClaimsListDao}. */
+@RunWith(JUnit4.class)
+public class ClaimsListDaoTest {
+
+  private FakeClock fakeClock = new FakeClock();
+
+  @Rule
+  public final JpaTransactionManagerRule jpaTmRule =
+      new JpaTransactionManagerRule.Builder().build();
+
+  @Test
+  public void trySave_insertsClaimsListSuccessfully() {
+    ClaimsList claimsList =
+        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
+    ClaimsListDao.trySave(claimsList);
+    ClaimsList insertedClaimsList = ClaimsListDao.getCurrent();
+    assertClaimsListEquals(claimsList, insertedClaimsList);
+    assertThat(insertedClaimsList.getCreationTimestamp())
+        .isEqualTo(jpaTmRule.getTxnClock().nowUtc());
+  }
+
+  @Test
+  public void trySave_noExceptionThrownWhenSaveFail() {
+    ClaimsList claimsList =
+        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
+    ClaimsListDao.trySave(claimsList);
+    ClaimsList insertedClaimsList = ClaimsListDao.getCurrent();
+    assertClaimsListEquals(claimsList, insertedClaimsList);
+    // Save ClaimsList with existing revisionId should fail because revisionId is the primary key.
+    ClaimsListDao.trySave(insertedClaimsList);
+  }
+
+  @Test
+  public void trySave_claimsListWithNoEntries() {
+    ClaimsList claimsList = ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of());
+    ClaimsListDao.trySave(claimsList);
+    ClaimsList insertedClaimsList = ClaimsListDao.getCurrent();
+    assertClaimsListEquals(claimsList, insertedClaimsList);
+    assertThat(insertedClaimsList.getLabelsToKeys()).isEmpty();
+  }
+
+  @Test
+  public void getCurrent_throwsNoResultExceptionIfTableIsEmpty() {
+    PersistenceException thrown =
+        assertThrows(PersistenceException.class, () -> ClaimsListDao.getCurrent());
+    assertThat(thrown).hasCauseThat().isInstanceOf(NoResultException.class);
+  }
+
+  @Test
+  public void getCurrent_returnsLatestClaims() {
+    ClaimsList oldClaimsList =
+        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
+    ClaimsList newClaimsList =
+        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4"));
+    ClaimsListDao.trySave(oldClaimsList);
+    ClaimsListDao.trySave(newClaimsList);
+    assertClaimsListEquals(newClaimsList, ClaimsListDao.getCurrent());
+  }
+
+  private void assertClaimsListEquals(ClaimsList left, ClaimsList right) {
+    assertThat(left.getRevisionId()).isEqualTo(right.getRevisionId());
+    assertThat(left.getTmdbGenerationTime()).isEqualTo(right.getTmdbGenerationTime());
+    assertThat(left.getLabelsToKeys()).isEqualTo(right.getLabelsToKeys());
+  }
+}

db/src/main/resources/sql/flyway/V7__update_claims_list.sql

@@ -0,0 +1,15 @@
+-- Copyright 2019 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+alter table "ClaimsList" add column if not exists tmdb_generation_time timestamp with time zone not null;

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -50,7 +50,8 @@ CREATE TABLE public."ClaimsEntry" (
 
 CREATE TABLE public."ClaimsList" (
     revision_id bigint NOT NULL,
-    creation_timestamp timestamp with time zone NOT NULL
+    creation_timestamp timestamp with time zone NOT NULL,
+    tmdb_generation_time timestamp with time zone NOT NULL
 );