Add dual write for Cursors (#414)

* Add dual write for Cursors

* Fix UpdateCursorCommand to dual write multiple cursors

* Small fixes

* Make UpdateCursorsCommand implement CommandWithCloudSql
This commit is contained in:
sarahcaseybot 2020-01-21 15:04:14 -05:00 committed by GitHub
parent ad2cf933c2
commit 0b717d40ff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 319 additions and 62 deletions

View file

@ -17,7 +17,6 @@ package google.registry.backup;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.util.DateTimeUtils.END_OF_TIME;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
@ -31,6 +30,7 @@ import google.registry.model.ofy.DatastoreTransactionManager;
import google.registry.model.ofy.Ofy;
import google.registry.model.registry.Registry;
import google.registry.model.transaction.TransactionManager;
import google.registry.schema.cursor.CursorDao;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock;
import google.registry.testing.InjectRule;
@ -54,6 +54,7 @@ public class CommitLogCheckpointStrategyTest {
@Rule
public final InjectRule inject = new InjectRule();
final FakeClock clock = new FakeClock(DateTime.parse("2000-01-01TZ"));
final Ofy ofy = new Ofy(clock);
final TransactionManager tm = new DatastoreTransactionManager(ofy);
@ -293,11 +294,10 @@ public class CommitLogCheckpointStrategyTest {
private void writeCommitLogToBucket(final int bucketId) {
fakeBucketIdSupplier.value = bucketId;
tm.transact(
() -> {
Cursor cursor =
Cursor.create(RDE_REPORT, tm.getTransactionTime(), Registry.get("tld" + bucketId));
ofy().save().entity(cursor);
});
() ->
CursorDao.saveCursor(
Cursor.create(RDE_REPORT, tm.getTransactionTime(), Registry.get("tld" + bucketId)),
"tld" + bucketId));
fakeBucketIdSupplier.value = null;
}

View file

@ -19,7 +19,6 @@ import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING;
import static google.registry.model.domain.Period.Unit.YEARS;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_AUTORENEW;
import static google.registry.model.transaction.TransactionManagerFactory.tm;
import static google.registry.testing.DatastoreHelper.assertBillingEvents;
import static google.registry.testing.DatastoreHelper.assertBillingEventsForResource;
import static google.registry.testing.DatastoreHelper.createTld;
@ -50,6 +49,7 @@ import google.registry.model.registry.Registry;
import google.registry.model.reporting.DomainTransactionRecord;
import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
import google.registry.model.reporting.HistoryEntry;
import google.registry.schema.cursor.CursorDao;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeResponse;
import google.registry.testing.InjectRule;
@ -103,7 +103,9 @@ public class ExpandRecurringBillingEventsActionTest
}
private void saveCursor(final DateTime cursorTime) {
tm().transact(() -> ofy().save().entity(Cursor.createGlobal(RECURRING_BILLING, cursorTime)));
CursorDao.saveCursor(
Cursor.createGlobal(RECURRING_BILLING, cursorTime),
google.registry.schema.cursor.Cursor.GLOBAL);
}
private void runMapreduce() throws Exception {

View file

@ -62,6 +62,7 @@ public class SyncRegistrarsSheetTest {
@Rule public final MockitoRule mocks = MockitoJUnit.rule();
@Rule public final InjectRule inject = new InjectRule();
@Captor private ArgumentCaptor<ImmutableList<ImmutableMap<String, String>>> rowsCaptor;
@Mock private SheetSynchronizer sheetSynchronizer;

View file

@ -19,7 +19,6 @@ import static google.registry.model.common.Cursor.CursorType.BRDA;
import static google.registry.model.common.Cursor.CursorType.RDE_UPLOAD;
import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.transaction.TransactionManagerFactory.tm;
import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.testing.DatastoreHelper.persistActiveDomain;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
@ -28,6 +27,7 @@ import static org.junit.Assert.assertThrows;
import google.registry.model.EntityTestCase;
import google.registry.model.domain.DomainBase;
import google.registry.model.registry.Registry;
import google.registry.schema.cursor.CursorDao;
import org.joda.time.DateTime;
import org.junit.Test;
@ -39,7 +39,7 @@ public class CursorTest extends EntityTestCase {
createTld("tld");
clock.advanceOneMilli();
final DateTime time = DateTime.parse("2012-07-12T03:30:00.000Z");
tm().transact(() -> ofy().save().entity(Cursor.create(RDE_UPLOAD, time, Registry.get("tld"))));
CursorDao.saveCursor(Cursor.create(RDE_UPLOAD, time, Registry.get("tld")), "tld");
assertThat(ofy().load().key(Cursor.createKey(BRDA, Registry.get("tld"))).now()).isNull();
assertThat(
ofy()
@ -53,7 +53,8 @@ public class CursorTest extends EntityTestCase {
@Test
public void testSuccess_persistGlobalCursor() {
final DateTime time = DateTime.parse("2012-07-12T03:30:00.000Z");
tm().transact(() -> ofy().save().entity(Cursor.createGlobal(RECURRING_BILLING, time)));
CursorDao.saveCursor(
Cursor.createGlobal(RECURRING_BILLING, time), google.registry.schema.cursor.Cursor.GLOBAL);
assertThat(ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now().getCursorTime())
.isEqualTo(time);
}
@ -61,7 +62,8 @@ public class CursorTest extends EntityTestCase {
@Test
public void testIndexing() throws Exception {
final DateTime time = DateTime.parse("2012-07-12T03:30:00.000Z");
tm().transact(() -> ofy().save().entity(Cursor.createGlobal(RECURRING_BILLING, time)));
CursorDao.saveCursor(
Cursor.createGlobal(RECURRING_BILLING, time), google.registry.schema.cursor.Cursor.GLOBAL);
Cursor cursor = ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
verifyIndexing(cursor);
}
@ -75,8 +77,7 @@ public class CursorTest extends EntityTestCase {
IllegalArgumentException thrown =
assertThrows(
IllegalArgumentException.class,
() ->
tm().transact(() -> ofy().save().entity(Cursor.create(RDE_UPLOAD, time, domain))));
() -> CursorDao.saveCursor(Cursor.create(RDE_UPLOAD, time, domain), domain.getTld()));
assertThat(thrown)
.hasMessageThat()
.contains("Class required for cursor does not match scope class");

View file

@ -59,7 +59,6 @@ public class EscrowTaskRunnerTest {
private EscrowTaskRunner runner;
private Registry registry;
@Before
public void before() {
createTld("lol");

View file

@ -20,7 +20,6 @@ import static google.registry.model.common.Cursor.CursorType.RDE_STAGING;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.rde.RdeMode.FULL;
import static google.registry.model.rde.RdeMode.THIN;
import static google.registry.model.transaction.TransactionManagerFactory.tm;
import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.testing.DatastoreHelper.persistResource;
import static org.joda.time.DateTimeConstants.TUESDAY;
@ -31,6 +30,7 @@ import google.registry.model.common.Cursor;
import google.registry.model.common.Cursor.CursorType;
import google.registry.model.ofy.Ofy;
import google.registry.model.registry.Registry;
import google.registry.schema.cursor.CursorDao;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock;
import google.registry.testing.InjectRule;
@ -164,7 +164,7 @@ public class PendingDepositCheckerTest {
private static void setCursor(
final Registry registry, final CursorType cursorType, final DateTime value) {
tm().transact(() -> ofy().save().entity(Cursor.create(cursorType, value, registry)));
CursorDao.saveCursor(Cursor.create(cursorType, value, registry), registry.getTldStr());
}
private static void createTldWithEscrowEnabled(final String tld) {

View file

@ -19,7 +19,6 @@ import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.common.Cursor.CursorType.BRDA;
import static google.registry.model.common.Cursor.CursorType.RDE_STAGING;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.transaction.TransactionManagerFactory.tm;
import static google.registry.rde.RdeFixtures.makeContactResource;
import static google.registry.rde.RdeFixtures.makeDomainBase;
import static google.registry.rde.RdeFixtures.makeHostResource;
@ -55,6 +54,7 @@ import google.registry.model.ofy.Ofy;
import google.registry.model.registry.Registry;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.RequestParameters;
import google.registry.schema.cursor.CursorDao;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeKeyringModule;
import google.registry.testing.FakeLockHandler;
@ -848,7 +848,7 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
private void setCursor(
final Registry registry, final CursorType cursorType, final DateTime value) {
clock.advanceOneMilli();
tm().transact(() -> ofy().save().entity(Cursor.create(cursorType, value, registry)).now());
CursorDao.saveCursor(Cursor.create(cursorType, value, registry), registry.getTldStr());
}
public static <T> T unmarshal(Class<T> clazz, byte[] xml) throws XmlException {

View file

@ -133,6 +133,7 @@ public class RdeUploadActionTest {
.withTaskQueue()
.build();
private final FakeResponse response = new FakeResponse();
private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class);
private final FakeClock clock = new FakeClock(DateTime.parse("2010-10-17TZ"));

View file

@ -15,12 +15,23 @@
package google.registry.schema.cursor;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.testing.DatastoreHelper.createTlds;
import static google.registry.testing.LogsSubject.assertAboutLogs;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.testing.TestLogHandler;
import google.registry.model.common.Cursor.CursorType;
import google.registry.model.registry.Registry;
import google.registry.model.transaction.JpaTestRules;
import google.registry.model.transaction.JpaTestRules.JpaIntegrationTestRule;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -32,10 +43,15 @@ public class CursorDaoTest {
private FakeClock fakeClock = new FakeClock();
private final TestLogHandler logHandler = new TestLogHandler();
private final Logger loggerToIntercept = Logger.getLogger(CursorDao.class.getCanonicalName());
@Rule
public final JpaIntegrationTestRule jpaRule =
new JpaTestRules.Builder().buildIntegrationTestRule();
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build();
@Test
public void save_worksSuccessfullyOnNewCursor() {
Cursor cursor = Cursor.create(CursorType.BRDA, "tld", fakeClock.nowUtc());
@ -73,16 +89,30 @@ public class CursorDaoTest {
assertThat(returnedCursor.getCursorTime()).isEqualTo(cursor2.getCursorTime());
}
@Test
public void saveAll_worksSuccessfully() {
Cursor cursor = Cursor.createGlobal(CursorType.RECURRING_BILLING, fakeClock.nowUtc());
Cursor cursor2 = Cursor.create(CursorType.RDE_REPORT, "tld", fakeClock.nowUtc());
ImmutableSet<Cursor> cursors = ImmutableSet.<Cursor>builder().add(cursor, cursor2).build();
CursorDao.saveAll(cursors);
assertThat(CursorDao.loadAll()).hasSize(2);
assertThat(CursorDao.load(CursorType.RECURRING_BILLING).getCursorTime())
.isEqualTo(cursor.getCursorTime());
}
@Test
public void saveAll_worksSuccessfullyEmptySet() {
CursorDao.saveAll(ImmutableSet.of());
assertThat(CursorDao.loadAll()).isEmpty();
}
@Test
public void load_worksSuccessfully() {
Cursor cursor = Cursor.createGlobal(CursorType.RECURRING_BILLING, fakeClock.nowUtc());
Cursor cursor2 = Cursor.create(CursorType.RDE_REPORT, "tld", fakeClock.nowUtc());
Cursor cursor3 = Cursor.create(CursorType.RDE_REPORT, "foo", fakeClock.nowUtc());
Cursor cursor4 = Cursor.create(CursorType.BRDA, "foo", fakeClock.nowUtc());
CursorDao.save(cursor);
CursorDao.save(cursor2);
CursorDao.save(cursor3);
CursorDao.save(cursor4);
CursorDao.saveAll(ImmutableSet.of(cursor, cursor2, cursor3, cursor4));
Cursor returnedCursor = CursorDao.load(CursorType.RDE_REPORT, "tld");
assertThat(returnedCursor.getCursorTime()).isEqualTo(cursor2.getCursorTime());
returnedCursor = CursorDao.load(CursorType.BRDA, "foo");
@ -97,10 +127,7 @@ public class CursorDaoTest {
Cursor cursor2 = Cursor.create(CursorType.RDE_REPORT, "tld", fakeClock.nowUtc());
Cursor cursor3 = Cursor.create(CursorType.RDE_REPORT, "foo", fakeClock.nowUtc());
Cursor cursor4 = Cursor.create(CursorType.BRDA, "foo", fakeClock.nowUtc());
CursorDao.save(cursor);
CursorDao.save(cursor2);
CursorDao.save(cursor3);
CursorDao.save(cursor4);
CursorDao.saveAll(ImmutableSet.of(cursor, cursor2, cursor3, cursor4));
List<Cursor> returnedCursors = CursorDao.loadAll();
assertThat(returnedCursors.size()).isEqualTo(4);
}
@ -117,10 +144,7 @@ public class CursorDaoTest {
Cursor cursor2 = Cursor.create(CursorType.RDE_REPORT, "tld", fakeClock.nowUtc());
Cursor cursor3 = Cursor.create(CursorType.RDE_REPORT, "foo", fakeClock.nowUtc());
Cursor cursor4 = Cursor.create(CursorType.BRDA, "foo", fakeClock.nowUtc());
CursorDao.save(cursor);
CursorDao.save(cursor2);
CursorDao.save(cursor3);
CursorDao.save(cursor4);
CursorDao.saveAll(ImmutableSet.of(cursor, cursor2, cursor3, cursor4));
List<Cursor> returnedCursors = CursorDao.loadByType(CursorType.RDE_REPORT);
assertThat(returnedCursors.size()).isEqualTo(2);
}
@ -130,4 +154,109 @@ public class CursorDaoTest {
List<Cursor> returnedCursors = CursorDao.loadByType(CursorType.RDE_REPORT);
assertThat(returnedCursors.size()).isEqualTo(0);
}
@Test
public void saveCursor_worksSuccessfully() {
createTld("tld");
google.registry.model.common.Cursor cursor =
google.registry.model.common.Cursor.create(
CursorType.ICANN_UPLOAD_ACTIVITY, fakeClock.nowUtc(), Registry.get("tld"));
CursorDao.saveCursor(cursor, "tld");
Cursor createdCursor = CursorDao.load(CursorType.ICANN_UPLOAD_ACTIVITY, "tld");
google.registry.model.common.Cursor dataStoreCursor =
ofy()
.load()
.key(
google.registry.model.common.Cursor.createKey(
CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("tld")))
.now();
assertThat(createdCursor.getCursorTime()).isEqualTo(cursor.getCursorTime());
assertThat(cursor).isEqualTo(dataStoreCursor);
}
@Test
public void saveCursor_worksSuccessfullyOnGlobalCursor() {
google.registry.model.common.Cursor cursor =
google.registry.model.common.Cursor.createGlobal(
CursorType.RECURRING_BILLING, fakeClock.nowUtc());
CursorDao.saveCursor(cursor, Cursor.GLOBAL);
Cursor createdCursor = CursorDao.load(CursorType.RECURRING_BILLING);
google.registry.model.common.Cursor dataStoreCursor =
ofy()
.load()
.key(google.registry.model.common.Cursor.createGlobalKey(CursorType.RECURRING_BILLING))
.now();
assertThat(createdCursor.getCursorTime()).isEqualTo(cursor.getCursorTime());
assertThat(cursor).isEqualTo(dataStoreCursor);
}
@Test
public void saveCursor_logsErrorWhenSaveToCloudSqlFails() {
loggerToIntercept.addHandler(logHandler);
createTld("tld");
google.registry.model.common.Cursor cursor =
google.registry.model.common.Cursor.create(
CursorType.ICANN_UPLOAD_ACTIVITY, fakeClock.nowUtc(), Registry.get("tld"));
CursorDao.saveCursor(cursor, null);
assertAboutLogs()
.that(logHandler)
.hasLogAtLevelWithMessage(Level.SEVERE, "Error saving cursor to Cloud SQL.");
google.registry.model.common.Cursor dataStoreCursor =
ofy()
.load()
.key(
google.registry.model.common.Cursor.createKey(
CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("tld")))
.now();
assertThat(cursor).isEqualTo(dataStoreCursor);
}
@Test
public void saveCursors_worksSuccessfully() {
createTlds("tld", "foo");
google.registry.model.common.Cursor cursor1 =
google.registry.model.common.Cursor.create(
CursorType.ICANN_UPLOAD_ACTIVITY, fakeClock.nowUtc(), Registry.get("tld"));
google.registry.model.common.Cursor cursor2 =
google.registry.model.common.Cursor.create(
CursorType.ICANN_UPLOAD_ACTIVITY, fakeClock.nowUtc(), Registry.get("foo"));
google.registry.model.common.Cursor cursor3 =
google.registry.model.common.Cursor.createGlobal(
CursorType.RECURRING_BILLING, fakeClock.nowUtc());
ImmutableMap<google.registry.model.common.Cursor, String> cursors =
ImmutableMap.<google.registry.model.common.Cursor, String>builder()
.put(cursor1, "tld")
.put(cursor2, "foo")
.put(cursor3, Cursor.GLOBAL)
.build();
CursorDao.saveCursors(cursors);
Cursor createdCursor1 = CursorDao.load(CursorType.ICANN_UPLOAD_ACTIVITY, "tld");
google.registry.model.common.Cursor dataStoreCursor1 =
ofy()
.load()
.key(
google.registry.model.common.Cursor.createKey(
CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("tld")))
.now();
assertThat(createdCursor1.getCursorTime()).isEqualTo(cursor1.getCursorTime());
assertThat(cursor1).isEqualTo(dataStoreCursor1);
Cursor createdCursor2 = CursorDao.load(CursorType.ICANN_UPLOAD_ACTIVITY, "foo");
google.registry.model.common.Cursor dataStoreCursor2 =
ofy()
.load()
.key(
google.registry.model.common.Cursor.createKey(
CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("foo")))
.now();
assertThat(createdCursor2.getCursorTime()).isEqualTo(cursor2.getCursorTime());
assertThat(cursor2).isEqualTo(dataStoreCursor2);
Cursor createdCursor3 = CursorDao.load(CursorType.RECURRING_BILLING);
google.registry.model.common.Cursor dataStoreCursor3 =
ofy()
.load()
.key(google.registry.model.common.Cursor.createGlobalKey(CursorType.RECURRING_BILLING))
.now();
assertThat(createdCursor3.getCursorTime()).isEqualTo(cursor3.getCursorTime());
assertThat(cursor3).isEqualTo(dataStoreCursor3);
}
}

View file

@ -44,6 +44,9 @@ public class UpdateCursorsCommandTest extends CommandTestCase<UpdateCursorsComma
runCommandForced("--type=brda", "--timestamp=1984-12-18T00:00:00Z", "foo");
assertThat(ofy().load().key(Cursor.createKey(CursorType.BRDA, registry)).now().getCursorTime())
.isEqualTo(DateTime.parse("1984-12-18TZ"));
String changes = command.prompt();
assertThat(changes)
.isEqualTo("Change cursorTime of BRDA for Scope:foo to 1984-12-18T00:00:00.000Z\n");
}
void doGlobalUpdateTest() throws Exception {
@ -55,6 +58,11 @@ public class UpdateCursorsCommandTest extends CommandTestCase<UpdateCursorsComma
.now()
.getCursorTime())
.isEqualTo(DateTime.parse("1984-12-18TZ"));
String changes = command.prompt();
assertThat(changes)
.isEqualTo(
"Change cursorTime of RECURRING_BILLING for Scope:GLOBAL to"
+ " 1984-12-18T00:00:00.000Z\n");
}
@Test
@ -94,6 +102,11 @@ public class UpdateCursorsCommandTest extends CommandTestCase<UpdateCursorsComma
.isEqualTo(DateTime.parse("1984-12-18TZ"));
assertThat(ofy().load().key(Cursor.createKey(CursorType.BRDA, registry2)).now().getCursorTime())
.isEqualTo(DateTime.parse("1984-12-18TZ"));
String changes = command.prompt();
assertThat(changes)
.isEqualTo(
"Change cursorTime of BRDA for Scope:foo to 1984-12-18T00:00:00.000Z\n"
+ "Change cursorTime of BRDA for Scope:bar to 1984-12-18T00:00:00.000Z\n");
}
@Test
@ -107,6 +120,11 @@ public class UpdateCursorsCommandTest extends CommandTestCase<UpdateCursorsComma
.isEqualTo(DateTime.parse("1984-12-18TZ"));
assertThat(ofy().load().key(Cursor.createKey(CursorType.BRDA, registry2)).now().getCursorTime())
.isEqualTo(DateTime.parse("1984-12-18TZ"));
String changes = command.prompt();
assertThat(changes)
.isEqualTo(
"Change cursorTime of BRDA for Scope:foo to 1984-12-18T00:00:00.000Z\n"
+ "Change cursorTime of BRDA for Scope:bar to 1984-12-18T00:00:00.000Z\n");
}
@Test