diff --git a/core/src/main/java/google/registry/model/tmch/ClaimsListDao.java b/core/src/main/java/google/registry/model/tmch/ClaimsListDao.java new file mode 100644 index 000000000..efafd2fcb --- /dev/null +++ b/core/src/main/java/google/registry/model/tmch/ClaimsListDao.java @@ -0,0 +1,72 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model.tmch; + +import static google.registry.model.transaction.TransactionManagerFactory.jpaTm; + +import com.google.common.flogger.FluentLogger; +import google.registry.schema.tmch.ClaimsList; +import javax.persistence.EntityManager; + +/** Data access object for {@link ClaimsList}. */ +public class ClaimsListDao { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private static void save(ClaimsList claimsList) { + jpaTm().transact(() -> jpaTm().getEntityManager().persist(claimsList)); + } + + /** + * Try to save the given {@link ClaimsList} into Cloud SQL. If the save fails, the error will be + * logged but no exception will be thrown. + * + *

This method is used during the dual-write phase of database migration as Datastore is still + * the authoritative database. + */ + public static void trySave(ClaimsList claimsList) { + try { + ClaimsListDao.save(claimsList); + logger.atInfo().log( + "Inserted %,d claims into Cloud SQL, created at %s", + claimsList.getLabelsToKeys().size(), claimsList.getTmdbGenerationTime()); + } catch (Throwable e) { + logger.atSevere().withCause(e).log("Error inserting claims into Cloud SQL"); + } + } + + /** + * Returns the current revision of the {@link ClaimsList} in Cloud SQL. Throws exception if there + * is no claims in the table. + */ + public static ClaimsList getCurrent() { + return jpaTm() + .transact( + () -> { + EntityManager em = jpaTm().getEntityManager(); + Long revisionId = + em.createQuery("SELECT MAX(revisionId) FROM ClaimsList", Long.class) + .getSingleResult(); + return em.createQuery( + "FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId =" + + " :revisionId", + ClaimsList.class) + .setParameter("revisionId", revisionId) + .getSingleResult(); + }); + } + + private ClaimsListDao() {} +} diff --git a/core/src/main/java/google/registry/model/tmch/ClaimsListShard.java b/core/src/main/java/google/registry/model/tmch/ClaimsListShard.java index b3dd8cbd0..46cdcaf32 100644 --- a/core/src/main/java/google/registry/model/tmch/ClaimsListShard.java +++ b/core/src/main/java/google/registry/model/tmch/ClaimsListShard.java @@ -217,8 +217,7 @@ public class ClaimsListShard extends ImmutableObject { }); } - public static ClaimsListShard create( - DateTime creationTime, ImmutableMap labelsToKeys) { + public static ClaimsListShard create(DateTime creationTime, Map labelsToKeys) { ClaimsListShard instance = new ClaimsListShard(); instance.id = allocateId(); instance.creationTime = checkNotNull(creationTime); diff --git a/core/src/main/java/google/registry/schema/tmch/ClaimsList.java b/core/src/main/java/google/registry/schema/tmch/ClaimsList.java index 3ef3135dc..a09767dd4 100644 --- a/core/src/main/java/google/registry/schema/tmch/ClaimsList.java +++ b/core/src/main/java/google/registry/schema/tmch/ClaimsList.java @@ -15,7 +15,11 @@ package google.registry.schema.tmch; import static com.google.common.base.Preconditions.checkState; +import static google.registry.util.DateTimeUtils.toJodaDateTime; +import static google.registry.util.DateTimeUtils.toZonedDateTime; +import google.registry.model.CreateAutoTimestamp; +import google.registry.model.ImmutableObject; import java.time.ZonedDateTime; import java.util.Map; import java.util.Optional; @@ -29,6 +33,7 @@ import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.MapKeyColumn; import javax.persistence.Table; +import org.joda.time.DateTime; /** * A list of TMCH claims labels and their associated claims keys. @@ -40,26 +45,29 @@ import javax.persistence.Table; * highest {@link #revisionId}. */ @Entity -@Table(name = "ClaimsList") -public class ClaimsList { +@Table +public class ClaimsList extends ImmutableObject { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(name = "revision_id") + @Column private Long revisionId; - @Column(name = "creation_timestamp", nullable = false) - private ZonedDateTime creationTimestamp; + @Column(nullable = false) + private CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null); + + @Column(nullable = false) + private ZonedDateTime tmdbGenerationTime; @ElementCollection @CollectionTable( name = "ClaimsEntry", - joinColumns = @JoinColumn(name = "revision_id", referencedColumnName = "revision_id")) - @MapKeyColumn(name = "domain_label", nullable = false) - @Column(name = "claim_key", nullable = false) + joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId")) + @MapKeyColumn(name = "domainLabel", nullable = false) + @Column(name = "claimKey", nullable = false) private Map labelsToKeys; - private ClaimsList(ZonedDateTime creationTimestamp, Map labelsToKeys) { - this.creationTimestamp = creationTimestamp; + private ClaimsList(ZonedDateTime tmdbGenerationTime, Map labelsToKeys) { + this.tmdbGenerationTime = tmdbGenerationTime; this.labelsToKeys = labelsToKeys; } @@ -67,9 +75,8 @@ public class ClaimsList { private ClaimsList() {} /** Constructs a {@link ClaimsList} object. */ - public static ClaimsList create( - ZonedDateTime creationTimestamp, Map labelsToKeys) { - return new ClaimsList(creationTimestamp, labelsToKeys); + public static ClaimsList create(DateTime creationTimestamp, Map labelsToKeys) { + return new ClaimsList(toZonedDateTime(creationTimestamp), labelsToKeys); } /** Returns the revision id of this claims list, or throws exception if it is null. */ @@ -79,9 +86,14 @@ public class ClaimsList { return revisionId; } + /** Returns the TMDB generation time of this claims list. */ + public DateTime getTmdbGenerationTime() { + return toJodaDateTime(tmdbGenerationTime); + } + /** Returns the creation time of this claims list. */ - public ZonedDateTime getCreationTimestamp() { - return creationTimestamp; + public DateTime getCreationTimestamp() { + return creationTimestamp.getTimestamp(); } /** Returns an {@link Map} mapping domain label to its lookup key. */ diff --git a/core/src/main/java/google/registry/tmch/ClaimsListParser.java b/core/src/main/java/google/registry/tmch/ClaimsListParser.java index b2b83dcc9..25813e98c 100644 --- a/core/src/main/java/google/registry/tmch/ClaimsListParser.java +++ b/core/src/main/java/google/registry/tmch/ClaimsListParser.java @@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; -import google.registry.model.tmch.ClaimsListShard; +import google.registry.schema.tmch.ClaimsList; import java.util.List; import org.joda.time.DateTime; @@ -34,11 +34,11 @@ import org.joda.time.DateTime; public class ClaimsListParser { /** - * Converts the lines from the DNL CSV file into a {@link ClaimsListShard} object. + * Converts the lines from the DNL CSV file into a {@link ClaimsList} object. * *

Please note that this does not insert the object into Datastore. */ - public static ClaimsListShard parse(List lines) { + public static ClaimsList parse(List lines) { ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); // First line: , @@ -74,6 +74,6 @@ public class ClaimsListParser { builder.put(label, lookupKey); } - return ClaimsListShard.create(creationTime, builder.build()); + return ClaimsList.create(creationTime, builder.build()); } } diff --git a/core/src/main/java/google/registry/tmch/TmchDnlAction.java b/core/src/main/java/google/registry/tmch/TmchDnlAction.java index 787b7a4c8..7b866d899 100644 --- a/core/src/main/java/google/registry/tmch/TmchDnlAction.java +++ b/core/src/main/java/google/registry/tmch/TmchDnlAction.java @@ -18,9 +18,11 @@ import static google.registry.request.Action.Method.POST; import com.google.common.flogger.FluentLogger; import google.registry.keyring.api.KeyModule.Key; +import google.registry.model.tmch.ClaimsListDao; import google.registry.model.tmch.ClaimsListShard; import google.registry.request.Action; import google.registry.request.auth.Auth; +import google.registry.schema.tmch.ClaimsList; import java.io.IOException; import java.security.SignatureException; import java.util.List; @@ -54,10 +56,14 @@ public final class TmchDnlAction implements Runnable { } catch (SignatureException | IOException | PGPException e) { throw new RuntimeException(e); } - ClaimsListShard claims = ClaimsListParser.parse(lines); - claims.save(); + ClaimsList claims = ClaimsListParser.parse(lines); + ClaimsListShard claimsListShard = + ClaimsListShard.create(claims.getTmdbGenerationTime(), claims.getLabelsToKeys()); + claimsListShard.save(); logger.atInfo().log( "Inserted %,d claims into Datastore, created at %s", - claims.size(), claims.getCreationTime()); + claimsListShard.size(), claimsListShard.getCreationTime()); + + ClaimsListDao.trySave(claims); } } diff --git a/core/src/main/java/google/registry/tools/UploadClaimsListCommand.java b/core/src/main/java/google/registry/tools/UploadClaimsListCommand.java index 14a797748..c70ad9d38 100644 --- a/core/src/main/java/google/registry/tools/UploadClaimsListCommand.java +++ b/core/src/main/java/google/registry/tools/UploadClaimsListCommand.java @@ -21,7 +21,9 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.google.common.base.Joiner; import com.google.common.io.Files; +import google.registry.model.tmch.ClaimsListDao; import google.registry.model.tmch.ClaimsListShard; +import google.registry.schema.tmch.ClaimsList; import google.registry.tmch.ClaimsListParser; import java.io.File; import java.io.IOException; @@ -35,9 +37,14 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command @Parameter(description = "Claims list filename") private List mainParameters = new ArrayList<>(); + @Parameter( + names = {"--also_cloud_sql"}, + description = "Persist claims list to Cloud SQL in addition to Datastore; defaults to false.") + boolean alsoCloudSql; + private String claimsListFilename; - private ClaimsListShard claimsList; + private ClaimsList claimsList; @Override protected void init() throws IOException { @@ -56,7 +63,10 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command @Override public String execute() { - claimsList.save(); + ClaimsListShard.create(claimsList.getTmdbGenerationTime(), claimsList.getLabelsToKeys()).save(); + if (alsoCloudSql) { + ClaimsListDao.trySave(claimsList); + } return String.format("Successfully uploaded claims list %s", claimsListFilename); } } diff --git a/core/src/test/java/google/registry/model/tmch/ClaimsListDaoTest.java b/core/src/test/java/google/registry/model/tmch/ClaimsListDaoTest.java new file mode 100644 index 000000000..c238f8efe --- /dev/null +++ b/core/src/test/java/google/registry/model/tmch/ClaimsListDaoTest.java @@ -0,0 +1,95 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model.tmch; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.testing.JUnitBackports.assertThrows; + +import com.google.common.collect.ImmutableMap; +import google.registry.model.transaction.JpaTransactionManagerRule; +import google.registry.schema.tmch.ClaimsList; +import google.registry.testing.FakeClock; +import javax.persistence.NoResultException; +import javax.persistence.PersistenceException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link ClaimsListDao}. */ +@RunWith(JUnit4.class) +public class ClaimsListDaoTest { + + private FakeClock fakeClock = new FakeClock(); + + @Rule + public final JpaTransactionManagerRule jpaTmRule = + new JpaTransactionManagerRule.Builder().build(); + + @Test + public void trySave_insertsClaimsListSuccessfully() { + ClaimsList claimsList = + ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2")); + ClaimsListDao.trySave(claimsList); + ClaimsList insertedClaimsList = ClaimsListDao.getCurrent(); + assertClaimsListEquals(claimsList, insertedClaimsList); + assertThat(insertedClaimsList.getCreationTimestamp()) + .isEqualTo(jpaTmRule.getTxnClock().nowUtc()); + } + + @Test + public void trySave_noExceptionThrownWhenSaveFail() { + ClaimsList claimsList = + ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2")); + ClaimsListDao.trySave(claimsList); + ClaimsList insertedClaimsList = ClaimsListDao.getCurrent(); + assertClaimsListEquals(claimsList, insertedClaimsList); + // Save ClaimsList with existing revisionId should fail because revisionId is the primary key. + ClaimsListDao.trySave(insertedClaimsList); + } + + @Test + public void trySave_claimsListWithNoEntries() { + ClaimsList claimsList = ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of()); + ClaimsListDao.trySave(claimsList); + ClaimsList insertedClaimsList = ClaimsListDao.getCurrent(); + assertClaimsListEquals(claimsList, insertedClaimsList); + assertThat(insertedClaimsList.getLabelsToKeys()).isEmpty(); + } + + @Test + public void getCurrent_throwsNoResultExceptionIfTableIsEmpty() { + PersistenceException thrown = + assertThrows(PersistenceException.class, () -> ClaimsListDao.getCurrent()); + assertThat(thrown).hasCauseThat().isInstanceOf(NoResultException.class); + } + + @Test + public void getCurrent_returnsLatestClaims() { + ClaimsList oldClaimsList = + ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2")); + ClaimsList newClaimsList = + ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4")); + ClaimsListDao.trySave(oldClaimsList); + ClaimsListDao.trySave(newClaimsList); + assertClaimsListEquals(newClaimsList, ClaimsListDao.getCurrent()); + } + + private void assertClaimsListEquals(ClaimsList left, ClaimsList right) { + assertThat(left.getRevisionId()).isEqualTo(right.getRevisionId()); + assertThat(left.getTmdbGenerationTime()).isEqualTo(right.getTmdbGenerationTime()); + assertThat(left.getLabelsToKeys()).isEqualTo(right.getLabelsToKeys()); + } +} diff --git a/db/src/main/resources/sql/flyway/V7__update_claims_list.sql b/db/src/main/resources/sql/flyway/V7__update_claims_list.sql new file mode 100644 index 000000000..bde0d9361 --- /dev/null +++ b/db/src/main/resources/sql/flyway/V7__update_claims_list.sql @@ -0,0 +1,15 @@ +-- Copyright 2019 The Nomulus Authors. All Rights Reserved. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +alter table "ClaimsList" add column if not exists tmdb_generation_time timestamp with time zone not null; diff --git a/db/src/main/resources/sql/schema/nomulus.golden.sql b/db/src/main/resources/sql/schema/nomulus.golden.sql index ba78f3b34..e6d84ca07 100644 --- a/db/src/main/resources/sql/schema/nomulus.golden.sql +++ b/db/src/main/resources/sql/schema/nomulus.golden.sql @@ -50,7 +50,8 @@ CREATE TABLE public."ClaimsEntry" ( CREATE TABLE public."ClaimsList" ( revision_id bigint NOT NULL, - creation_timestamp timestamp with time zone NOT NULL + creation_timestamp timestamp with time zone NOT NULL, + tmdb_generation_time timestamp with time zone NOT NULL );