From f1eb1a8fe7c67260e625585d51dea24bb90371c9 Mon Sep 17 00:00:00 2001 From: gbrodman Date: Tue, 10 Nov 2020 17:09:18 -0500 Subject: [PATCH] Add SQL replay checkpoint object to SQL (#868) * Add SQL replay checkpoint object to Datastore This will be part of the asynchronous commit-log replay to SQL. Whenever we successfully export commits up to a particular time, we should persist that time so we don't replay the same commits again (it is not idempotent) * Move SqlReplayCheckpoint from DS to SQL * Responses to CR --- .../schema/replay/SqlReplayCheckpoint.java | 58 ++++++ .../main/resources/META-INF/persistence.xml | 1 + .../model/server/ServerSecretTest.java | 11 -- .../integration/SqlIntegrationTestSuite.java | 2 + .../replay/SqlReplayCheckpointTest.java | 61 ++++++ ...peRecurringBillingEventIdsCommandTest.java | 6 - .../sql/er_diagram/brief_er_diagram.html | 124 ++++++++---- .../sql/er_diagram/full_er_diagram.html | 187 +++++++++++++----- db/src/main/resources/sql/flyway.txt | 1 + .../sql/flyway/V74__sql_replay_checkpoint.sql | 19 ++ .../sql/schema/db-schema.sql.generated | 6 + .../resources/sql/schema/nomulus.golden.sql | 18 ++ 12 files changed, 387 insertions(+), 107 deletions(-) create mode 100644 core/src/main/java/google/registry/schema/replay/SqlReplayCheckpoint.java create mode 100644 core/src/test/java/google/registry/schema/replay/SqlReplayCheckpointTest.java create mode 100644 db/src/main/resources/sql/flyway/V74__sql_replay_checkpoint.sql diff --git a/core/src/main/java/google/registry/schema/replay/SqlReplayCheckpoint.java b/core/src/main/java/google/registry/schema/replay/SqlReplayCheckpoint.java new file mode 100644 index 000000000..6826d537d --- /dev/null +++ b/core/src/main/java/google/registry/schema/replay/SqlReplayCheckpoint.java @@ -0,0 +1,58 @@ +// Copyright 2020 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.schema.replay; + +import static google.registry.model.common.CrossTldSingleton.SINGLETON_ID; +import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; +import static google.registry.util.DateTimeUtils.START_OF_TIME; + +import com.google.common.collect.ImmutableList; +import javax.persistence.Entity; +import javax.persistence.Id; +import org.joda.time.DateTime; + +@Entity +public class SqlReplayCheckpoint implements SqlEntity { + + // Hibernate doesn't allow us to have a converted DateTime as our primary key so we need this + @Id private long revisionId = SINGLETON_ID; + + private DateTime lastReplayTime; + + @Override + public ImmutableList toDatastoreEntities() { + return ImmutableList.of(); // not necessary to persist in Datastore + } + + public static DateTime get() { + jpaTm().assertInTransaction(); + return jpaTm() + .getEntityManager() + .createQuery("FROM SqlReplayCheckpoint", SqlReplayCheckpoint.class) + .setMaxResults(1) + .getResultStream() + .findFirst() + .map(checkpoint -> checkpoint.lastReplayTime) + .orElse(START_OF_TIME); + } + + public static void set(DateTime lastReplayTime) { + jpaTm().assertInTransaction(); + SqlReplayCheckpoint checkpoint = new SqlReplayCheckpoint(); + checkpoint.lastReplayTime = lastReplayTime; + // this will overwrite the existing object due to the constant revisionId + jpaTm().put(checkpoint); + } +} diff --git a/core/src/main/resources/META-INF/persistence.xml b/core/src/main/resources/META-INF/persistence.xml index 4edaed2de..08ad1e9b1 100644 --- a/core/src/main/resources/META-INF/persistence.xml +++ b/core/src/main/resources/META-INF/persistence.xml @@ -70,6 +70,7 @@ google.registry.persistence.transaction.TransactionEntity google.registry.schema.cursor.Cursor google.registry.schema.domain.RegistryLock + google.registry.schema.replay.SqlReplayCheckpoint google.registry.schema.server.Lock google.registry.schema.tld.PremiumEntry diff --git a/core/src/test/java/google/registry/model/server/ServerSecretTest.java b/core/src/test/java/google/registry/model/server/ServerSecretTest.java index 7a104d427..fdeb3e07a 100644 --- a/core/src/test/java/google/registry/model/server/ServerSecretTest.java +++ b/core/src/test/java/google/registry/model/server/ServerSecretTest.java @@ -15,15 +15,11 @@ package google.registry.model.server; import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.common.CrossTldSingleton.SINGLETON_ID; -import static google.registry.model.common.EntityGroupRoot.getCrossTldKey; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import com.googlecode.objectify.Key; import google.registry.model.EntityTestCase; import google.registry.model.ofy.RequestCapturingAsyncDatastoreService; -import google.registry.persistence.VKey; import java.util.UUID; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -85,11 +81,4 @@ public class ServerSecretTest extends EntityTestCase { .findFirst() .get()); } - - private static VKey createKey() { - return VKey.create( - ServerSecret.class, - SINGLETON_ID, - Key.create(getCrossTldKey(), ServerSecret.class, SINGLETON_ID)); - } } diff --git a/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java b/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java index 59ae333cb..216db2753 100644 --- a/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java +++ b/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java @@ -40,6 +40,7 @@ import google.registry.schema.cursor.CursorDaoTest; import google.registry.schema.integration.SqlIntegrationTestSuite.AfterSuiteTest; import google.registry.schema.integration.SqlIntegrationTestSuite.BeforeSuiteTest; import google.registry.schema.registrar.RegistrarDaoTest; +import google.registry.schema.replay.SqlReplayCheckpointTest; import google.registry.schema.server.LockDaoTest; import google.registry.schema.tld.PremiumListDaoTest; import google.registry.testing.AppEngineExtension; @@ -100,6 +101,7 @@ import org.junit.runner.RunWith; ServerSecretTest.class, SignedMarkRevocationListDaoTest.class, Spec11ThreatMatchTest.class, + SqlReplayCheckpointTest.class, TmchCrlTest.class, // AfterSuiteTest must be the last entry. See class javadoc for details. AfterSuiteTest.class diff --git a/core/src/test/java/google/registry/schema/replay/SqlReplayCheckpointTest.java b/core/src/test/java/google/registry/schema/replay/SqlReplayCheckpointTest.java new file mode 100644 index 000000000..4b6bfe0c7 --- /dev/null +++ b/core/src/test/java/google/registry/schema/replay/SqlReplayCheckpointTest.java @@ -0,0 +1,61 @@ +// Copyright 2020 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.schema.replay; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; +import static google.registry.util.DateTimeUtils.START_OF_TIME; + +import google.registry.model.EntityTestCase; +import org.joda.time.DateTime; +import org.junit.jupiter.api.Test; + +/** Tests for {@link SqlReplayCheckpoint}. */ +public class SqlReplayCheckpointTest extends EntityTestCase { + + SqlReplayCheckpointTest() { + super(JpaEntityCoverageCheck.ENABLED); + } + + @Test + void testEmpty_startOfTime() { + assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(START_OF_TIME); + } + + @Test + void testSuccess_writes() { + DateTime dateTime = DateTime.parse("2012-02-29T00:00:00Z"); + jpaTm().transact(() -> SqlReplayCheckpoint.set(dateTime)); + assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(dateTime); + } + + @Test + void testSuccess_multipleWrites() { + DateTime firstTime = DateTime.parse("2012-02-29T00:00:00Z"); + jpaTm().transact(() -> SqlReplayCheckpoint.set(firstTime)); + DateTime secondTime = DateTime.parse("2013-02-28T00:00:00Z"); + jpaTm().transact(() -> SqlReplayCheckpoint.set(secondTime)); + assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(secondTime); + jpaTm() + .transact( + () -> + assertThat( + jpaTm() + .getEntityManager() + .createQuery("SELECT COUNT(*) FROM SqlReplayCheckpoint", Long.class) + .getSingleResult()) + .isEqualTo(1L)); + } +} diff --git a/core/src/test/java/google/registry/tools/DedupeRecurringBillingEventIdsCommandTest.java b/core/src/test/java/google/registry/tools/DedupeRecurringBillingEventIdsCommandTest.java index b8a03b170..2770c9e7c 100644 --- a/core/src/test/java/google/registry/tools/DedupeRecurringBillingEventIdsCommandTest.java +++ b/core/src/test/java/google/registry/tools/DedupeRecurringBillingEventIdsCommandTest.java @@ -221,12 +221,6 @@ class DedupeRecurringBillingEventIdsCommandTest } } - private static void assertNotChangeInDatastore(ImmutableObject... entities) { - for (ImmutableObject entity : entities) { - assertThat(ofy().load().entity(entity).now()).isEqualTo(entity); - } - } - private static void assertNotChangeExceptUpdateTime(ImmutableObject... entities) { for (ImmutableObject entity : entities) { assertAboutImmutableObjects() diff --git a/db/src/main/resources/sql/er_diagram/brief_er_diagram.html b/db/src/main/resources/sql/er_diagram/brief_er_diagram.html index 4e2791b44..f2f266712 100644 --- a/db/src/main/resources/sql/er_diagram/brief_er_diagram.html +++ b/db/src/main/resources/sql/er_diagram/brief_er_diagram.html @@ -261,32 +261,32 @@ td.section { generated on - 2020-11-09 17:11:19.905881 + 2020-11-10 19:44:25.39289 last flyway file - V73__singleton_entities.sql + V74__sql_replay_checkpoint.sql

 

 

- + SchemaCrawler_Diagram - - + + generated by - + SchemaCrawler 16.10.1 - + generated on - - 2020-11-09 17:11:19.905881 + + 2020-11-10 19:44:25.39289 - + allocationtoken_a08ccbef @@ -1352,7 +1352,7 @@ td.section { fk_domain_transfer_losing_registrar_id - + tld_f1fa57e2 @@ -3042,67 +3042,87 @@ td.section { fk5ivlhvs3121yx2li5tqh54u4 + + + sqlreplaycheckpoint_342081b3 + + + public.SqlReplayCheckpoint + + + + [table] + + + revision_id + + + + + int8 not null + + - + tmchcrl_d282355 - - + + public.TmchCrl - - + + [table] - + certificate_revocations - + - + text not null - + update_timestamp - + - + timestamptz not null - + url - + - + text not null - + - + transaction_d50389d4 - - + + public.Transaction - - + + [table] - + id - + - + bigserial not null - + - + auto-incremented - + @@ -6755,6 +6775,36 @@ td.section {

 

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
public.SqlReplayCheckpoint [table] +
revision_idint8 not null
Primary Key
SqlReplayCheckpoint_pkey[primary key]
revision_id
+

 

diff --git a/db/src/main/resources/sql/er_diagram/full_er_diagram.html b/db/src/main/resources/sql/er_diagram/full_er_diagram.html index b7c709b60..ca9f0da2b 100644 --- a/db/src/main/resources/sql/er_diagram/full_er_diagram.html +++ b/db/src/main/resources/sql/er_diagram/full_er_diagram.html @@ -261,32 +261,32 @@ td.section { - + - +
public.Tld [table]
generated on2020-11-09 17:11:18.253282020-11-10 19:44:23.457057
last flyway fileV73__singleton_entities.sqlV74__sql_replay_checkpoint.sql

 

 

- + SchemaCrawler_Diagram - - + + generated by - + SchemaCrawler 16.10.1 - + generated on - - 2020-11-09 17:11:18.25328 + + 2020-11-10 19:44:23.457057 - + allocationtoken_a08ccbef @@ -3048,7 +3048,7 @@ td.section { fk_domain_transfer_losing_registrar_id - + tld_f1fa57e2 @@ -6170,75 +6170,103 @@ td.section { fk5ivlhvs3121yx2li5tqh54u4 - - - tmchcrl_d282355 - - - public.TmchCrl + + + sqlreplaycheckpoint_342081b3 + + + public.SqlReplayCheckpoint - - + + [table] - - certificate_revocations + + revision_id - + - - text not null + + int8 not null - - update_timestamp + + last_replay_time - + - + timestamptz not null - - url - - - - - text not null - - - + + - transaction_d50389d4 - - - public.Transaction + tmchcrl_d282355 + + + public.TmchCrl - - + + [table] - + + certificate_revocations + + + + + text not null + + + update_timestamp + + + + + timestamptz not null + + + url + + + + + text not null + + + + + transaction_d50389d4 + + + public.Transaction + + + + [table] + + id - + - + bigserial not null - + - + auto-incremented - + contents - + - + bytea - + @@ -12958,6 +12986,59 @@ td.section {

 

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
public.SqlReplayCheckpoint [table] +
revision_idint8 not null
last_replay_timetimestamptz not null
Primary Key
SqlReplayCheckpoint_pkey[primary key]
revision_id
Indexes
SqlReplayCheckpoint_pkey[unique index]
revision_idascending
+

 

diff --git a/db/src/main/resources/sql/flyway.txt b/db/src/main/resources/sql/flyway.txt index cd6151565..e20235b3d 100644 --- a/db/src/main/resources/sql/flyway.txt +++ b/db/src/main/resources/sql/flyway.txt @@ -71,3 +71,4 @@ V70__signed_mark_revocation_list.sql V71__create_kms_secret.sql V72__add_missing_foreign_keys.sql V73__singleton_entities.sql +V74__sql_replay_checkpoint.sql diff --git a/db/src/main/resources/sql/flyway/V74__sql_replay_checkpoint.sql b/db/src/main/resources/sql/flyway/V74__sql_replay_checkpoint.sql new file mode 100644 index 000000000..17f2c179f --- /dev/null +++ b/db/src/main/resources/sql/flyway/V74__sql_replay_checkpoint.sql @@ -0,0 +1,19 @@ +-- Copyright 2020 The Nomulus Authors. All Rights Reserved. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE TABLE "SqlReplayCheckpoint" ( + revision_id int8 not null, + last_replay_time timestamptz not null, + PRIMARY KEY(revision_id) +); diff --git a/db/src/main/resources/sql/schema/db-schema.sql.generated b/db/src/main/resources/sql/schema/db-schema.sql.generated index 25f6a3c0e..6fa05a019 100644 --- a/db/src/main/resources/sql/schema/db-schema.sql.generated +++ b/db/src/main/resources/sql/schema/db-schema.sql.generated @@ -655,6 +655,12 @@ primary key (id) ); + create table "SqlReplayCheckpoint" ( + revision_id int8 not null, + last_replay_time timestamptz, + primary key (revision_id) + ); + create table "Tld" ( tld_name text not null, add_grace_period_length interval not null, diff --git a/db/src/main/resources/sql/schema/nomulus.golden.sql b/db/src/main/resources/sql/schema/nomulus.golden.sql index ae0637542..0ed172a3f 100644 --- a/db/src/main/resources/sql/schema/nomulus.golden.sql +++ b/db/src/main/resources/sql/schema/nomulus.golden.sql @@ -948,6 +948,16 @@ CREATE SEQUENCE public."SignedMarkRevocationList_revision_id_seq" ALTER SEQUENCE public."SignedMarkRevocationList_revision_id_seq" OWNED BY public."SignedMarkRevocationList".revision_id; +-- +-- Name: SqlReplayCheckpoint; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."SqlReplayCheckpoint" ( + revision_id bigint NOT NULL, + last_replay_time timestamp with time zone NOT NULL +); + + -- -- Name: Tld; Type: TABLE; Schema: public; Owner: - -- @@ -1346,6 +1356,14 @@ ALTER TABLE ONLY public."SignedMarkRevocationList" ADD CONSTRAINT "SignedMarkRevocationList_pkey" PRIMARY KEY (revision_id); +-- +-- Name: SqlReplayCheckpoint SqlReplayCheckpoint_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SqlReplayCheckpoint" + ADD CONSTRAINT "SqlReplayCheckpoint_pkey" PRIMARY KEY (revision_id); + + -- -- Name: Tld Tld_pkey; Type: CONSTRAINT; Schema: public; Owner: - --
public.Tld [table]