Add non-SQL removal code for Transaction and SqlReplayCheckpoint (#1700)

This commit is contained in:
gbrodman 2022-07-07 14:36:01 -04:00 committed by GitHub
parent 7071bb91f2
commit d5c65bf8d7
19 changed files with 10 additions and 438 deletions

View file

@ -32,8 +32,6 @@ import google.registry.model.index.ForeignKeyIndex;
import google.registry.model.poll.PollMessage;
import google.registry.model.rde.RdeRevision;
import google.registry.model.registrar.Registrar;
import google.registry.model.replay.LastSqlTransaction;
import google.registry.model.replay.ReplayGap;
import google.registry.model.reporting.HistoryEntry;
import google.registry.model.server.Lock;
import google.registry.model.server.ServerSecret;
@ -64,14 +62,12 @@ public final class EntityClasses {
HistoryEntry.class,
HostHistory.class,
HostResource.class,
LastSqlTransaction.class,
Lock.class,
PollMessage.class,
PollMessage.Autorenew.class,
PollMessage.OneTime.class,
RdeRevision.class,
Registrar.class,
ReplayGap.class,
ServerSecret.class);
private EntityClasses() {}

View file

@ -1,73 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.replay;
import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import com.google.common.annotations.VisibleForTesting;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.annotation.Entity;
import com.googlecode.objectify.annotation.Id;
import google.registry.model.ImmutableObject;
import google.registry.model.annotations.DeleteAfterMigration;
/** Datastore entity to keep track of the last SQL transaction imported into the datastore. */
@Entity
@DeleteAfterMigration
public class LastSqlTransaction extends ImmutableObject {
/** The key for this singleton. */
public static final Key<LastSqlTransaction> KEY = Key.create(LastSqlTransaction.class, 1);
@SuppressWarnings("unused")
@Id
private long id = 1;
private long transactionId;
LastSqlTransaction() {}
@VisibleForTesting
LastSqlTransaction(long newTransactionId) {
transactionId = newTransactionId;
}
LastSqlTransaction cloneWithNewTransactionId(long transactionId) {
checkArgument(
transactionId > this.transactionId,
"New transaction id (%s) must be greater than the current transaction id (%s)",
transactionId,
this.transactionId);
return new LastSqlTransaction(transactionId);
}
long getTransactionId() {
return transactionId;
}
/**
* Loads the instance.
*
* <p>Must be called within an Ofy transaction.
*
* <p>Creates a new instance of the singleton if it is not already present in Cloud Datastore,
*/
static LastSqlTransaction load() {
auditedOfy().assertInTransaction();
LastSqlTransaction result = auditedOfy().load().key(KEY).now();
return result == null ? new LastSqlTransaction() : result;
}
}

View file

@ -1,60 +0,0 @@
// Copyright 2022 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.replay;
import static google.registry.model.annotations.NotBackedUp.Reason.TRANSIENT;
import com.googlecode.objectify.annotation.Entity;
import com.googlecode.objectify.annotation.Id;
import google.registry.model.ImmutableObject;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.model.annotations.NotBackedUp;
import org.joda.time.DateTime;
/**
* Tracks gaps in transaction ids when replicating from SQL to datastore.
*
* <p>SQL -&gt; DS replication uses a Transaction table indexed by a SEQUENCE column, which normally
* increments monotonically for each committed transaction. Gaps in this sequence can occur when a
* transaction is rolled back or when a transaction has been initiated but not committed to the
* table at the time of a query. To protect us from the latter scenario, we need to keep track of
* these gaps and replay any of them that have been filled in since we processed their batch.
*/
@DeleteAfterMigration
@NotBackedUp(reason = TRANSIENT)
@Entity
public class ReplayGap extends ImmutableObject {
@Id long transactionId;
// We can't use a CreateAutoTimestamp here because this ends up getting persisted in an ofy
// transaction that happens in JPA mode, so we don't end up getting an active transaction manager
// when the timestamp needs to be set.
DateTime timestamp;
ReplayGap() {}
ReplayGap(DateTime timestamp, long transactionId) {
this.timestamp = timestamp;
this.transactionId = transactionId;
}
long getTransactionId() {
return transactionId;
}
DateTime getTimestamp() {
return timestamp;
}
}

View file

@ -1,48 +0,0 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.replay;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.model.common.CrossTldSingleton;
import javax.persistence.Column;
import javax.persistence.Entity;
import org.joda.time.DateTime;
@Entity
@DeleteAfterMigration
public class SqlReplayCheckpoint extends CrossTldSingleton {
@Column(nullable = false)
private DateTime lastReplayTime;
public static DateTime get() {
jpaTm().assertInTransaction();
return jpaTm()
.loadSingleton(SqlReplayCheckpoint.class)
.map(checkpoint -> checkpoint.lastReplayTime)
.orElse(START_OF_TIME);
}
public static void set(DateTime lastReplayTime) {
jpaTm().assertInTransaction();
SqlReplayCheckpoint checkpoint = new SqlReplayCheckpoint();
checkpoint.lastReplayTime = lastReplayTime;
// this will overwrite the existing object due to the constant revisionId
jpaTm().put(checkpoint);
}
}

View file

@ -1,56 +0,0 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.persistence.transaction;
import com.google.common.annotations.VisibleForTesting;
import google.registry.model.ImmutableObject;
import google.registry.model.annotations.DeleteAfterMigration;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
/**
* Object to be stored in the transaction table.
*
* <p>This consists of a sequential identifier and a serialized {@code Tranaction} object.
*/
@Entity
@Table(name = "Transaction")
@DeleteAfterMigration
public class TransactionEntity extends ImmutableObject {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private long id;
private byte[] contents;
TransactionEntity() {}
@VisibleForTesting
public TransactionEntity(byte[] contents) {
this.contents = contents;
}
public long getId() {
return id;
}
public byte[] getContents() {
return contents;
}
}

View file

@ -97,7 +97,6 @@ public final class RegistryTool {
.put("save_sql_credential", SaveSqlCredentialCommand.class)
.put("send_escrow_report_to_icann", SendEscrowReportToIcannCommand.class)
.put("set_num_instances", SetNumInstancesCommand.class)
.put("set_sql_replay_checkpoint", SetSqlReplayCheckpointCommand.class)
.put("setup_ote", SetupOteCommand.class)
.put("uniform_rapid_suspension", UniformRapidSuspensionCommand.class)
.put("unlock_domain", UnlockDomainCommand.class)

View file

@ -142,8 +142,6 @@ interface RegistryToolComponent {
void inject(SetNumInstancesCommand command);
void inject(SetSqlReplayCheckpointCommand command);
void inject(SetupOteCommand command);
void inject(UnlockDomainCommand command);

View file

@ -1,48 +0,0 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.collect.Iterables;
import google.registry.model.replay.SqlReplayCheckpoint;
import java.util.List;
import org.joda.time.DateTime;
/** Command to set {@link SqlReplayCheckpoint} to a particular, post-initial-population time. */
@Parameters(separators = " =", commandDescription = "Set SqlReplayCheckpoint to a particular time")
public class SetSqlReplayCheckpointCommand extends ConfirmingCommand
implements CommandWithRemoteApi {
@Parameter(description = "Time to which SqlReplayCheckpoint will be set", required = true)
List<DateTime> mainParameters;
@Override
protected String prompt() {
checkArgument(mainParameters.size() == 1, "Must provide exactly one DateTime to set");
return String.format(
"Set SqlReplayCheckpoint to %s?", Iterables.getOnlyElement(mainParameters));
}
@Override
protected String execute() {
DateTime dateTime = Iterables.getOnlyElement(mainParameters);
jpaTm().transact(() -> SqlReplayCheckpoint.set(dateTime));
return String.format("Set SqlReplayCheckpoint time to %s", dateTime);
}
}

View file

@ -73,9 +73,7 @@
<class>google.registry.model.tmch.ClaimsList</class>
<class>google.registry.model.tmch.ClaimsEntry</class>
<class>google.registry.model.tmch.TmchCrl</class>
<class>google.registry.persistence.transaction.TransactionEntity</class>
<class>google.registry.model.domain.RegistryLock</class>
<class>google.registry.model.replay.SqlReplayCheckpoint</class>
<!-- Customized type converters -->
<class>google.registry.persistence.converter.AllocationTokenStatusTransitionConverter</class>

View file

@ -34,8 +34,6 @@ import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
import google.registry.model.poll.PollMessage;
import google.registry.model.rde.RdeRevision;
import google.registry.model.registrar.Registrar;
import google.registry.model.replay.LastSqlTransaction;
import google.registry.model.replay.ReplayGap;
import google.registry.model.reporting.HistoryEntry;
import google.registry.model.server.Lock;
import google.registry.model.server.ServerSecret;
@ -64,10 +62,8 @@ public class ClassPathManagerTest {
assertThat(ClassPathManager.getClass("HostResource")).isEqualTo(HostResource.class);
assertThat(ClassPathManager.getClass("Recurring")).isEqualTo(Recurring.class);
assertThat(ClassPathManager.getClass("Registrar")).isEqualTo(Registrar.class);
assertThat(ClassPathManager.getClass("ReplayGap")).isEqualTo(ReplayGap.class);
assertThat(ClassPathManager.getClass("ContactResource")).isEqualTo(ContactResource.class);
assertThat(ClassPathManager.getClass("Cancellation")).isEqualTo(Cancellation.class);
assertThat(ClassPathManager.getClass("LastSqlTransaction")).isEqualTo(LastSqlTransaction.class);
assertThat(ClassPathManager.getClass("GaeUserIdConverter")).isEqualTo(GaeUserIdConverter.class);
assertThat(ClassPathManager.getClass("EppResourceIndexBucket"))
.isEqualTo(EppResourceIndexBucket.class);
@ -121,11 +117,8 @@ public class ClassPathManagerTest {
assertThat(ClassPathManager.getClassName(HostResource.class)).isEqualTo("HostResource");
assertThat(ClassPathManager.getClassName(Recurring.class)).isEqualTo("Recurring");
assertThat(ClassPathManager.getClassName(Registrar.class)).isEqualTo("Registrar");
assertThat(ClassPathManager.getClassName(ReplayGap.class)).isEqualTo("ReplayGap");
assertThat(ClassPathManager.getClassName(ContactResource.class)).isEqualTo("ContactResource");
assertThat(ClassPathManager.getClassName(Cancellation.class)).isEqualTo("Cancellation");
assertThat(ClassPathManager.getClassName(LastSqlTransaction.class))
.isEqualTo("LastSqlTransaction");
assertThat(ClassPathManager.getClassName(GaeUserIdConverter.class))
.isEqualTo("GaeUserIdConverter");
assertThat(ClassPathManager.getClassName(EppResourceIndexBucket.class))

View file

@ -1,60 +0,0 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.replay;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import google.registry.model.EntityTestCase;
import org.joda.time.DateTime;
import org.junit.jupiter.api.Test;
/** Tests for {@link SqlReplayCheckpoint}. */
public class SqlReplayCheckpointTest extends EntityTestCase {
SqlReplayCheckpointTest() {
super(JpaEntityCoverageCheck.ENABLED);
}
@Test
void testEmpty_startOfTime() {
assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(START_OF_TIME);
}
@Test
void testSuccess_writes() {
DateTime dateTime = DateTime.parse("2012-02-29T00:00:00Z");
jpaTm().transact(() -> SqlReplayCheckpoint.set(dateTime));
assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(dateTime);
}
@Test
void testSuccess_multipleWrites() {
DateTime firstTime = DateTime.parse("2012-02-29T00:00:00Z");
jpaTm().transact(() -> SqlReplayCheckpoint.set(firstTime));
DateTime secondTime = DateTime.parse("2013-02-28T00:00:00Z");
jpaTm().transact(() -> SqlReplayCheckpoint.set(secondTime));
assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(secondTime);
jpaTm()
.transact(
() ->
assertThat(
jpaTm()
.query("SELECT COUNT(*) FROM SqlReplayCheckpoint", Long.class)
.getSingleResult())
.isEqualTo(1L));
}
}

View file

@ -45,9 +45,7 @@ public class JpaEntityCoverageExtension implements BeforeEachCallback, AfterEach
// needs to remove it in order to avoid affecting any other tests running in the same JVM.
// TODO(gbrodman): remove this when we implement proper read-only modes for the
// transaction managers.
"DatabaseMigrationStateSchedule",
// TransactionEntity is trivial; its persistence is tested in TransactionTest.
"TransactionEntity");
"DatabaseMigrationStateSchedule");
public static final ImmutableSet<Class<?>> ALL_JPA_ENTITIES =
PersistenceXmlUtility.getManagedClasses().stream()

View file

@ -26,7 +26,6 @@ import google.registry.model.history.DomainHistoryTest;
import google.registry.model.history.HostHistoryTest;
import google.registry.model.poll.PollMessageTest;
import google.registry.model.rde.RdeRevisionTest;
import google.registry.model.replay.SqlReplayCheckpointTest;
import google.registry.model.reporting.Spec11ThreatMatchTest;
import google.registry.model.server.LockTest;
import google.registry.model.server.ServerSecretTest;
@ -99,7 +98,6 @@ import org.junit.runner.RunWith;
ServerSecretTest.class,
SignedMarkRevocationListDaoTest.class,
Spec11ThreatMatchTest.class,
SqlReplayCheckpointTest.class,
TmchCrlTest.class,
// AfterSuiteTest must be the last entry. See class javadoc for details.
AfterSuiteTest.class

View file

@ -1,42 +0,0 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static org.junit.Assert.assertThrows;
import google.registry.model.replay.SqlReplayCheckpoint;
import org.joda.time.DateTime;
import org.junit.jupiter.api.Test;
class SetSqlReplayCheckpointCommandTest extends CommandTestCase<SetSqlReplayCheckpointCommand> {
@Test
void testSuccess() throws Exception {
assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(START_OF_TIME);
DateTime timeToSet = DateTime.parse("2000-06-06T22:00:00.0Z");
runCommandForced(timeToSet.toString());
assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(timeToSet);
}
@Test
void testFailure_multipleParams() {
DateTime one = DateTime.parse("2000-06-06T22:00:00.0Z");
DateTime two = DateTime.parse("2001-06-06T22:00:00.0Z");
assertThrows(IllegalArgumentException.class, () -> runCommand(one.toString(), two.toString()));
}
}

View file

@ -9,7 +9,6 @@ ForeignKeyDomainIndex
ForeignKeyHostIndex
HistoryEntry
HostResource
LastSqlTransaction
Modification
OneTime
PollMessage

View file

@ -553,14 +553,6 @@ class google.registry.model.registrar.RegistrarAddress {
java.lang.String zip;
java.util.List<java.lang.String> street;
}
class google.registry.model.replay.LastSqlTransaction {
@Id long id;
long transactionId;
}
class google.registry.model.replay.ReplayGap {
@Id long transactionId;
org.joda.time.DateTime timestamp;
}
class google.registry.model.reporting.DomainTransactionRecord {
google.registry.model.reporting.DomainTransactionRecord$TransactionReportField reportField;
java.lang.Integer reportAmount;

View file

@ -261,7 +261,7 @@ td.section {
</tr>
<tr>
<td class="property_name">generated on</td>
<td class="property_value">2022-07-01 21:39:06.50205</td>
<td class="property_value">2022-07-06 18:25:44.018541</td>
</tr>
<tr>
<td class="property_name">last flyway file</td>
@ -274,19 +274,19 @@ td.section {
<svg viewbox="0.00 0.00 4249.00 2959.50" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="erDiagram" style="overflow: hidden; width: 100%; height: 800px"><g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 2955.5)">
<title>SchemaCrawler_Diagram</title>
<polygon fill="white" stroke="transparent" points="-4,4 -4,-2955.5 4245,-2955.5 4245,4 -4,4" />
<text text-anchor="start" x="3980.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
<text text-anchor="start" x="3972.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
generated by
</text>
<text text-anchor="start" x="4063.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
<text text-anchor="start" x="4055.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
SchemaCrawler 16.10.1
</text>
<text text-anchor="start" x="3979.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
<text text-anchor="start" x="3971.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
generated on
</text>
<text text-anchor="start" x="4063.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
2022-07-01 21:39:06.50205
<text text-anchor="start" x="4055.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
2022-07-06 18:25:44.018541
</text>
<polygon fill="none" stroke="#888888" points="3976,-4 3976,-44 4233,-44 4233,-4 3976,-4" /> <!-- allocationtoken_a08ccbef -->
<polygon fill="none" stroke="#888888" points="3968,-4 3968,-44 4233,-44 4233,-4 3968,-4" /> <!-- allocationtoken_a08ccbef -->
<g id="node1" class="node">
<title>allocationtoken_a08ccbef</title>
<polygon fill="#ebcef2" stroke="transparent" points="2521.5,-271 2521.5,-290 2707.5,-290 2707.5,-271 2521.5,-271" />

View file

@ -261,7 +261,7 @@ td.section {
</tr>
<tr>
<td class="property_name">generated on</td>
<td class="property_value">2022-07-01 21:39:02.483121</td>
<td class="property_value">2022-07-06 18:25:39.145212</td>
</tr>
<tr>
<td class="property_name">last flyway file</td>
@ -284,7 +284,7 @@ td.section {
generated on
</text>
<text text-anchor="start" x="4755.52" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
2022-07-01 21:39:02.483121
2022-07-06 18:25:39.145212
</text>
<polygon fill="none" stroke="#888888" points="4668.02,-4 4668.02,-44 4933.02,-44 4933.02,-4 4668.02,-4" /> <!-- allocationtoken_a08ccbef -->
<g id="node1" class="node">

View file

@ -682,12 +682,6 @@
primary key (id)
);
create table "SqlReplayCheckpoint" (
id int8 not null,
last_replay_time timestamptz not null,
primary key (id)
);
create table "Tld" (
tld_name text not null,
add_grace_period_length interval not null,
@ -737,12 +731,6 @@
url text not null,
primary key (id)
);
create table "Transaction" (
id bigserial not null,
contents bytea,
primary key (id)
);
create index allocation_token_domain_name_idx on "AllocationToken" (domain_name);
create index IDX9g3s7mjv1yn4t06nqid39whss on "AllocationToken" (token_type);
create index IDXtmlqd31dpvvd2g1h9i7erw6aj on "AllocationToken" (redemption_domain_repo_id);