mirror of
https://github.com/google/nomulus.git
synced 2025-08-03 08:22:13 +02:00
mv com/google/domain/registry google/registry
This change renames directories in preparation for the great package rename. The repository is now in a broken state because the code itself hasn't been updated. However this should ensure that git correctly preserves history for each file.
This commit is contained in:
Justine Tunney
parent
a41677aea1
commit
5012893c1d
2396 changed files with 0 additions and 0 deletions
39
javatests/google/registry/backup/BUILD
Normal file
39
javatests/google/registry/backup/BUILD
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
package(
|
||||
default_visibility = ["//java/com/google/domain/registry:registry_project"],
|
||||
)
|
||||
|
||||
load("//java/com/google/testing/builddefs:GenTestRules.bzl", "GenTestRules")
|
||||
|
||||
|
||||
java_library(
|
||||
name = "backup",
|
||||
srcs = glob(["*.java"]),
|
||||
resources = glob(["testdata/*"]),
|
||||
deps = [
|
||||
"//java/com/google/common/base",
|
||||
"//java/com/google/common/collect",
|
||||
"//java/com/google/common/net",
|
||||
"//java/com/google/common/primitives",
|
||||
"//java/com/google/common/util/concurrent",
|
||||
"//java/com/google/domain/registry/backup",
|
||||
"//java/com/google/domain/registry/config",
|
||||
"//java/com/google/domain/registry/model",
|
||||
"//java/com/google/domain/registry/util",
|
||||
"//javatests/com/google/domain/registry/testing",
|
||||
"//third_party/java/appengine:appengine-api-testonly",
|
||||
"//third_party/java/appengine_gcs_client",
|
||||
"//third_party/java/joda_time",
|
||||
"//third_party/java/jsr305_annotations",
|
||||
"//third_party/java/junit",
|
||||
"//third_party/java/mockito",
|
||||
"//third_party/java/objectify:objectify-v4_1",
|
||||
"//third_party/java/servlet/servlet_api",
|
||||
"//third_party/java/truth",
|
||||
],
|
||||
)
|
||||
|
||||
GenTestRules(
|
||||
name = "GeneratedTestRules",
|
||||
test_files = glob(["*Test.java"]),
|
||||
deps = [":backup"],
|
||||
)
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.backup;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.domain.registry.model.ofy.CommitLogCheckpointRoot.loadRoot;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistResource;
|
||||
import static com.google.domain.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||
import static com.google.domain.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.domain.registry.model.ofy.CommitLogCheckpoint;
|
||||
import com.google.domain.registry.model.ofy.CommitLogCheckpointRoot;
|
||||
import com.google.domain.registry.testing.AppEngineRule;
|
||||
import com.google.domain.registry.testing.FakeClock;
|
||||
import com.google.domain.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||
import com.google.domain.registry.util.Retrier;
|
||||
import com.google.domain.registry.util.TaskEnqueuer;
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
/** Unit tests for {@link CommitLogCheckpointAction}. */
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class CommitLogCheckpointActionTest {
|
||||
|
||||
private static final String QUEUE_NAME = "export-commits";
|
||||
|
||||
@Rule
|
||||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
||||
.withDatastore()
|
||||
.withTaskQueue()
|
||||
.build();
|
||||
|
||||
@Mock
|
||||
CommitLogCheckpointStrategy strategy;
|
||||
|
||||
DateTime now = DateTime.now(UTC);
|
||||
CommitLogCheckpointAction task = new CommitLogCheckpointAction();
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
task.clock = new FakeClock(now);
|
||||
task.strategy = strategy;
|
||||
task.taskEnqueuer = new TaskEnqueuer(new Retrier(null, 1));
|
||||
when(strategy.computeCheckpoint()).thenReturn(
|
||||
CommitLogCheckpoint.create(now, ImmutableMap.of(1, START_OF_TIME)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_noCheckpointEverWritten_writesCheckpointAndEnqueuesTask() throws Exception {
|
||||
task.run();
|
||||
assertTasksEnqueued(
|
||||
QUEUE_NAME,
|
||||
new TaskMatcher()
|
||||
.url(ExportCommitLogDiffAction.PATH)
|
||||
.param(ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM, START_OF_TIME.toString())
|
||||
.param(ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM, now.toString()));
|
||||
assertThat(loadRoot().getLastWrittenTime()).isEqualTo(now);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_checkpointWrittenBeforeNow_writesCheckpointAndEnqueuesTask()
|
||||
throws Exception {
|
||||
DateTime oneMinuteAgo = now.minusMinutes(1);
|
||||
persistResource(CommitLogCheckpointRoot.create(oneMinuteAgo));
|
||||
task.run();
|
||||
assertTasksEnqueued(
|
||||
QUEUE_NAME,
|
||||
new TaskMatcher()
|
||||
.url(ExportCommitLogDiffAction.PATH)
|
||||
.param(ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM, oneMinuteAgo.toString())
|
||||
.param(ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM, now.toString()));
|
||||
assertThat(loadRoot().getLastWrittenTime()).isEqualTo(now);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_checkpointWrittenAfterNow_doesntOverwrite_orEnqueueTask() throws Exception {
|
||||
DateTime oneMinuteFromNow = now.plusMinutes(1);
|
||||
persistResource(CommitLogCheckpointRoot.create(oneMinuteFromNow));
|
||||
task.run();
|
||||
assertNoTasksEnqueued(QUEUE_NAME);
|
||||
assertThat(loadRoot().getLastWrittenTime()).isEqualTo(oneMinuteFromNow);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,327 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.backup;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.domain.registry.model.ofy.CommitLogBucket.getBucketKey;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.createTld;
|
||||
import static com.google.domain.registry.util.DateTimeUtils.END_OF_TIME;
|
||||
import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.google.common.base.Supplier;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.domain.registry.config.TestRegistryConfig;
|
||||
import com.google.domain.registry.model.ofy.CommitLogBucket;
|
||||
import com.google.domain.registry.model.ofy.CommitLogCheckpoint;
|
||||
import com.google.domain.registry.model.ofy.Ofy;
|
||||
import com.google.domain.registry.model.registry.Registry;
|
||||
import com.google.domain.registry.model.registry.RegistryCursor;
|
||||
import com.google.domain.registry.model.registry.RegistryCursor.CursorType;
|
||||
import com.google.domain.registry.testing.AppEngineRule;
|
||||
import com.google.domain.registry.testing.FakeClock;
|
||||
import com.google.domain.registry.testing.InjectRule;
|
||||
import com.google.domain.registry.testing.RegistryConfigRule;
|
||||
|
||||
import com.googlecode.objectify.VoidWork;
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
/** Unit tests for {@link CommitLogCheckpointStrategy}. */
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class CommitLogCheckpointStrategyTest {
|
||||
|
||||
@Rule
|
||||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
||||
.withDatastore()
|
||||
.build();
|
||||
|
||||
@Rule
|
||||
public final InjectRule inject = new InjectRule();
|
||||
|
||||
@Rule
|
||||
public final RegistryConfigRule configRule = new RegistryConfigRule();
|
||||
|
||||
final FakeClock clock = new FakeClock(DateTime.parse("2000-01-01TZ"));
|
||||
final Ofy ofy = new Ofy(clock);
|
||||
final CommitLogCheckpointStrategy strategy = new CommitLogCheckpointStrategy();
|
||||
|
||||
/**
|
||||
* Supplier to inject into CommitLogBucket for doling out predictable bucket IDs.
|
||||
*
|
||||
* <p>If not overridden, the supplier returns 1 so that other saves won't hit an NPE (since even
|
||||
* if they use saveWithoutBackup() the transaction still selects a bucket key early).
|
||||
*/
|
||||
final FakeSupplier<Integer> fakeBucketIdSupplier = new FakeSupplier<>(1);
|
||||
|
||||
/** Gross but necessary supplier that can be modified to return the desired value. */
|
||||
private static class FakeSupplier<T> implements Supplier<T> {
|
||||
/** Default value to return if 'value' is not set. */
|
||||
final T defaultValue;
|
||||
|
||||
/** Set this value field to make the supplier return this value. */
|
||||
T value = null;
|
||||
|
||||
public FakeSupplier(T defaultValue) {
|
||||
this.defaultValue = defaultValue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public T get() {
|
||||
return value == null ? defaultValue : value;
|
||||
}
|
||||
}
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
strategy.clock = clock;
|
||||
strategy.ofy = ofy;
|
||||
|
||||
// Use three commit log buckets for easier but sufficiently complex testing.
|
||||
configRule.override(new TestRegistryConfig() {
|
||||
@Override
|
||||
public int getCommitLogBucketCount() {
|
||||
return 3;
|
||||
}});
|
||||
|
||||
// Need to inject clock into Ofy so that createTld() below will get the right time.
|
||||
inject.setStaticField(Ofy.class, "clock", clock);
|
||||
// Inject a fake bucket ID supplier so we can dole out specific bucket IDs to commit logs.
|
||||
inject.setStaticField(CommitLogBucket.class, "bucketIdSupplier", fakeBucketIdSupplier);
|
||||
|
||||
// Create some fake TLDs to parent RegistryCursor test objects under.
|
||||
createTld("tld1");
|
||||
createTld("tld2");
|
||||
createTld("tld3");
|
||||
clock.advanceOneMilli();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_readBucketTimestamps_noCommitLogs() throws Exception {
|
||||
assertThat(strategy.readBucketTimestamps())
|
||||
.containsExactly(1, START_OF_TIME, 2, START_OF_TIME, 3, START_OF_TIME);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_readBucketTimestamps_withSomeCommitLogs() throws Exception {
|
||||
DateTime startTime = clock.nowUtc();
|
||||
writeCommitLogToBucket(1);
|
||||
clock.advanceOneMilli();
|
||||
writeCommitLogToBucket(2);
|
||||
assertThat(strategy.readBucketTimestamps())
|
||||
.containsExactly(1, startTime, 2, startTime.plusMillis(1), 3, START_OF_TIME);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_readBucketTimestamps_againAfterUpdate_reflectsUpdate() throws Exception {
|
||||
DateTime firstTime = clock.nowUtc();
|
||||
writeCommitLogToBucket(1);
|
||||
writeCommitLogToBucket(2);
|
||||
writeCommitLogToBucket(3);
|
||||
assertThat(strategy.readBucketTimestamps().values())
|
||||
.containsExactly(firstTime, firstTime, firstTime);
|
||||
clock.advanceOneMilli();
|
||||
writeCommitLogToBucket(1);
|
||||
DateTime secondTime = clock.nowUtc();
|
||||
assertThat(strategy.readBucketTimestamps())
|
||||
.containsExactly(1, secondTime, 2, firstTime, 3, firstTime);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_readNewCommitLogsAndFindThreshold_noCommitsAtAll_returnsEndOfTime() {
|
||||
ImmutableMap<Integer, DateTime> bucketTimes =
|
||||
ImmutableMap.of(1, START_OF_TIME, 2, START_OF_TIME, 3, START_OF_TIME);
|
||||
assertThat(strategy.readNewCommitLogsAndFindThreshold(bucketTimes)).isEqualTo(END_OF_TIME);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_readNewCommitLogsAndFindThreshold_noNewCommits_returnsEndOfTime() {
|
||||
DateTime now = clock.nowUtc();
|
||||
writeCommitLogToBucket(1);
|
||||
clock.advanceOneMilli();
|
||||
writeCommitLogToBucket(2);
|
||||
clock.advanceOneMilli();
|
||||
writeCommitLogToBucket(3);
|
||||
ImmutableMap<Integer, DateTime> bucketTimes =
|
||||
ImmutableMap.of(1, now, 2, now.plusMillis(1), 3, now.plusMillis(2));
|
||||
assertThat(strategy.readNewCommitLogsAndFindThreshold(bucketTimes)).isEqualTo(END_OF_TIME);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_readNewCommitLogsAndFindThreshold_tiedNewCommits_returnsCommitTimeMinusOne() {
|
||||
DateTime now = clock.nowUtc();
|
||||
writeCommitLogToBucket(1);
|
||||
writeCommitLogToBucket(2);
|
||||
writeCommitLogToBucket(3);
|
||||
assertThat(strategy.readNewCommitLogsAndFindThreshold(
|
||||
ImmutableMap.of(1, START_OF_TIME, 2, START_OF_TIME, 3, START_OF_TIME)))
|
||||
.isEqualTo(now.minusMillis(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_readNewCommitLogsAndFindThreshold_someNewCommits_returnsEarliestTimeMinusOne() {
|
||||
DateTime now = clock.nowUtc();
|
||||
writeCommitLogToBucket(1); // 1A
|
||||
writeCommitLogToBucket(2); // 2A
|
||||
writeCommitLogToBucket(3); // 3A
|
||||
clock.advanceBy(Duration.millis(5));
|
||||
writeCommitLogToBucket(1); // 1B
|
||||
writeCommitLogToBucket(2); // 2B
|
||||
writeCommitLogToBucket(3); // 3B
|
||||
clock.advanceBy(Duration.millis(5));
|
||||
writeCommitLogToBucket(1); // 1C
|
||||
writeCommitLogToBucket(2); // 2C
|
||||
writeCommitLogToBucket(3); // 3C
|
||||
// First pass times: 1 at T0, 2 at T+5, 3 at T+10.
|
||||
// Commits 1A, 2B, 3C are the commits seen in the first pass.
|
||||
// Commits 2A, 3A, 3B are all old prior commits that should be ignored.
|
||||
// Commit 1B is the first new commit for bucket 1, at T+5.
|
||||
// Commit 1C is the second new commit for bucket 1, at T+10, and should be ignored.
|
||||
// Commit 2C is the first new commit for bucket 2, at T+10.
|
||||
// Since 1B as a new commit is older than 1C, T+5 is the oldest new commit time.
|
||||
// Therefore, expect T+4 as the threshold time.
|
||||
assertThat(strategy.readNewCommitLogsAndFindThreshold(
|
||||
ImmutableMap.of(1, now, 2, now.plusMillis(5), 3, now.plusMillis(10))))
|
||||
.isEqualTo(now.plusMillis(4));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_readNewCommitLogsAndFindThreshold_commitsAtBucketTimes() {
|
||||
DateTime now = clock.nowUtc();
|
||||
ImmutableMap<Integer, DateTime> bucketTimes =
|
||||
ImmutableMap.of(1, now.minusMillis(1), 2, now, 3, now.plusMillis(1));
|
||||
assertThat(strategy.readNewCommitLogsAndFindThreshold(bucketTimes)).isEqualTo(END_OF_TIME);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_computeBucketCheckpointTimes_earlyThreshold_setsEverythingToThreshold() {
|
||||
DateTime now = clock.nowUtc();
|
||||
ImmutableMap<Integer, DateTime> bucketTimes =
|
||||
ImmutableMap.of(1, now.minusMillis(1), 2, now, 3, now.plusMillis(1));
|
||||
assertThat(strategy.computeBucketCheckpointTimes(bucketTimes, now.minusMillis(2)).values())
|
||||
.containsExactly(now.minusMillis(2), now.minusMillis(2), now.minusMillis(2));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_computeBucketCheckpointTimes_middleThreshold_clampsToThreshold() {
|
||||
DateTime now = clock.nowUtc();
|
||||
ImmutableMap<Integer, DateTime> bucketTimes =
|
||||
ImmutableMap.of(1, now.minusMillis(1), 2, now, 3, now.plusMillis(1));
|
||||
assertThat(strategy.computeBucketCheckpointTimes(bucketTimes, now))
|
||||
.containsExactly(1, now.minusMillis(1), 2, now, 3, now);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_computeBucketCheckpointTimes_lateThreshold_leavesBucketTimesAsIs() {
|
||||
DateTime now = clock.nowUtc();
|
||||
ImmutableMap<Integer, DateTime> bucketTimes =
|
||||
ImmutableMap.of(1, now.minusMillis(1), 2, now, 3, now.plusMillis(1));
|
||||
assertThat(strategy.computeBucketCheckpointTimes(bucketTimes, now.plusMillis(2)))
|
||||
.isEqualTo(bucketTimes);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_computeCheckpoint_noCommitsAtAll_bucketCheckpointTimesAreStartOfTime() {
|
||||
assertThat(strategy.computeCheckpoint())
|
||||
.isEqualTo(CommitLogCheckpoint.create(
|
||||
clock.nowUtc(),
|
||||
ImmutableMap.of(1, START_OF_TIME, 2, START_OF_TIME, 3, START_OF_TIME)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_computeCheckpoint_noNewCommitLogs_bucketCheckpointTimesAreBucketTimes() {
|
||||
DateTime now = clock.nowUtc();
|
||||
writeCommitLogToBucket(1);
|
||||
clock.advanceOneMilli();
|
||||
writeCommitLogToBucket(2);
|
||||
clock.advanceOneMilli();
|
||||
writeCommitLogToBucket(3);
|
||||
clock.advanceOneMilli();
|
||||
DateTime checkpointTime = clock.nowUtc();
|
||||
assertThat(strategy.computeCheckpoint())
|
||||
.isEqualTo(CommitLogCheckpoint.create(
|
||||
checkpointTime,
|
||||
ImmutableMap.of(1, now, 2, now.plusMillis(1), 3, now.plusMillis(2))));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_computeCheckpoint_someNewCommits_bucketCheckpointTimesAreClampedToThreshold() {
|
||||
DateTime now = clock.nowUtc();
|
||||
writeCommitLogToBucket(1); // 1A
|
||||
writeCommitLogToBucket(2); // 2A
|
||||
writeCommitLogToBucket(3); // 3A
|
||||
clock.advanceBy(Duration.millis(5));
|
||||
writeCommitLogToBucket(1); // 1B
|
||||
writeCommitLogToBucket(2); // 2B
|
||||
writeCommitLogToBucket(3); // 3B
|
||||
clock.advanceBy(Duration.millis(5));
|
||||
writeCommitLogToBucket(1); // 1C
|
||||
writeCommitLogToBucket(2); // 2C
|
||||
writeCommitLogToBucket(3); // 3C
|
||||
|
||||
// Set first pass times: 1 at T0, 2 at T+5, 3 at T+10.
|
||||
saveBucketWithLastWrittenTime(1, now);
|
||||
saveBucketWithLastWrittenTime(2, now.plusMillis(5));
|
||||
saveBucketWithLastWrittenTime(3, now.plusMillis(10));
|
||||
|
||||
// Commits 1A, 2B, 3C are the commits seen in the first pass.
|
||||
// Commits 2A, 3A, 3B are all old prior commits that should be ignored.
|
||||
// Commit 1B is the first new commit for bucket 1, at T+5.
|
||||
// Commit 1C is the second new commit for bucket 1, at T+10, and should be ignored.
|
||||
// Commit 2C is the first new commit for bucket 2, at T+10.
|
||||
// Since 1B as a new commit is older than 1C, T+5 is the oldest new commit time.
|
||||
// Therefore, expect T+4 as the threshold time.
|
||||
DateTime threshold = now.plusMillis(4);
|
||||
|
||||
// Advance clock before taking checkpoint.
|
||||
clock.advanceBy(Duration.millis(10));
|
||||
DateTime checkpointTime = clock.nowUtc();
|
||||
|
||||
// Bucket checkpoint times should be clamped as expected.
|
||||
assertThat(strategy.computeCheckpoint())
|
||||
.isEqualTo(CommitLogCheckpoint.create(
|
||||
checkpointTime,
|
||||
ImmutableMap.of(1, now, 2, threshold, 3, threshold)));
|
||||
}
|
||||
|
||||
private void writeCommitLogToBucket(final int bucketId) {
|
||||
fakeBucketIdSupplier.value = bucketId;
|
||||
ofy.transact(
|
||||
new VoidWork() {
|
||||
@Override
|
||||
public void vrun() {
|
||||
String tld = "tld" + bucketId;
|
||||
RegistryCursor.save(Registry.get(tld), CursorType.RDE_REPORT, ofy.getTransactionTime());
|
||||
}
|
||||
});
|
||||
fakeBucketIdSupplier.value = null;
|
||||
}
|
||||
|
||||
private void saveBucketWithLastWrittenTime(final int bucketId, final DateTime lastWrittenTime) {
|
||||
ofy.transact(new VoidWork() {
|
||||
@Override
|
||||
public void vrun() {
|
||||
ofy.saveWithoutBackup().entity(
|
||||
CommitLogBucket.loadBucket(getBucketKey(bucketId)).asBuilder()
|
||||
.setLastWrittenTime(lastWrittenTime)
|
||||
.build());
|
||||
}});
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.backup;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static org.joda.time.Duration.millis;
|
||||
|
||||
import com.google.domain.registry.config.TestRegistryConfig;
|
||||
import com.google.domain.registry.model.ofy.CommitLogManifest;
|
||||
import com.google.domain.registry.model.ofy.CommitLogMutation;
|
||||
import com.google.domain.registry.model.ofy.Ofy;
|
||||
import com.google.domain.registry.model.registrar.Registrar;
|
||||
import com.google.domain.registry.testing.AppEngineRule;
|
||||
import com.google.domain.registry.testing.ExceptionRule;
|
||||
import com.google.domain.registry.testing.FakeClock;
|
||||
import com.google.domain.registry.testing.RegistryConfigRule;
|
||||
|
||||
import com.googlecode.objectify.VoidWork;
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
|
||||
/** Unit tests for {@link DeleteOldCommitLogsAction}. */
|
||||
@RunWith(JUnit4.class)
|
||||
public class DeleteOldCommitLogsActionTest {
|
||||
|
||||
@Rule
|
||||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
||||
.withDatastore()
|
||||
.build();
|
||||
|
||||
@Rule
|
||||
public final RegistryConfigRule configRule = new RegistryConfigRule();
|
||||
|
||||
@Rule
|
||||
public final ExceptionRule thrown = new ExceptionRule();
|
||||
|
||||
private final FakeClock clock = new FakeClock(DateTime.parse("2000-01-01TZ"));
|
||||
private final Ofy ofy = new Ofy(clock);
|
||||
private final DeleteOldCommitLogsAction task = new DeleteOldCommitLogsAction();
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
task.bucketNum = 1;
|
||||
task.clock = clock;
|
||||
task.maxAge = Duration.millis(2);
|
||||
task.maxDeletes = 4;
|
||||
task.ofy = ofy;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_noCommitLogs_doesNothing() throws Exception {
|
||||
assertManifestAndMutationCounts(0, 0);
|
||||
task.run();
|
||||
assertManifestAndMutationCounts(0, 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_commitLogNewerThanThreshold_doesntGetDeleted() throws Exception {
|
||||
createCommitLog();
|
||||
clock.advanceOneMilli();
|
||||
assertManifestAndMutationCounts(1, 2);
|
||||
task.run();
|
||||
assertManifestAndMutationCounts(1, 2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_commitLogEqualToThreshold_doesntGetDeleted() throws Exception {
|
||||
createCommitLog();
|
||||
clock.advanceBy(millis(2));
|
||||
task.run();
|
||||
assertManifestAndMutationCounts(1, 2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_commitLogOlderThanThreshold_getsDeleted() throws Exception {
|
||||
createCommitLog();
|
||||
clock.advanceBy(millis(3));
|
||||
task.run();
|
||||
assertManifestAndMutationCounts(0, 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_oneOlderThanThresholdAndOneNewer_onlyOldOneIsDeleted() throws Exception {
|
||||
createCommitLog();
|
||||
clock.advanceBy(millis(3));
|
||||
createCommitLog();
|
||||
assertManifestAndMutationCounts(2, 4);
|
||||
task.run();
|
||||
assertManifestAndMutationCounts(1, 2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_twoOlderThanThreshold_bothGetDeletedInSameTransaction() throws Exception {
|
||||
task.maxDeletes = 2;
|
||||
createCommitLog();
|
||||
clock.advanceOneMilli();
|
||||
createCommitLog();
|
||||
clock.advanceBy(millis(3));
|
||||
assertManifestAndMutationCounts(2, 4);
|
||||
task.run();
|
||||
assertManifestAndMutationCounts(0, 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_twoOlderThanThreshold_bothGetDeletedInTwoTransactions() throws Exception {
|
||||
task.maxDeletes = 1;
|
||||
createCommitLog();
|
||||
clock.advanceOneMilli();
|
||||
createCommitLog();
|
||||
clock.advanceBy(millis(3));
|
||||
createCommitLog();
|
||||
assertManifestAndMutationCounts(3, 6);
|
||||
task.run();
|
||||
assertManifestAndMutationCounts(2, 4);
|
||||
task.run();
|
||||
assertManifestAndMutationCounts(1, 2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_commitLogOlderButInADifferentBucket_doesntGetDeleted() throws Exception {
|
||||
createCommitLog();
|
||||
clock.advanceBy(millis(31337));
|
||||
configRule.override(new TestRegistryConfig() {
|
||||
@Override public int getCommitLogBucketCount() { return 2; }
|
||||
});
|
||||
task.bucketNum = 2;
|
||||
task.run();
|
||||
assertManifestAndMutationCounts(1, 2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_lessThanATenthOfOldData_doesntGetDeleted() throws Exception {
|
||||
task.maxDeletes = 20;
|
||||
createCommitLog();
|
||||
clock.advanceBy(millis(2));
|
||||
task.run();
|
||||
assertManifestAndMutationCounts(1, 2);
|
||||
}
|
||||
|
||||
private void assertManifestAndMutationCounts(int manifestCount, int mutationCount) {
|
||||
assertThat(ofy.load().type(CommitLogManifest.class).count()).isEqualTo(manifestCount);
|
||||
assertThat(ofy.load().type(CommitLogMutation.class).count()).isEqualTo(mutationCount);
|
||||
}
|
||||
|
||||
private void createCommitLog() {
|
||||
ofy.transact(new VoidWork() {
|
||||
@Override
|
||||
public void vrun() {
|
||||
ofy.save().entity(
|
||||
Registrar.loadByClientId("NewRegistrar").asBuilder()
|
||||
.setEmailAddress("pumpkin@cat.test")
|
||||
.build());
|
||||
ofy.save().entity(
|
||||
Registrar.loadByClientId("TheRegistrar").asBuilder()
|
||||
.setReferralUrl("http://justine.test")
|
||||
.build());
|
||||
}});
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,397 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.backup;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth.assertWithMessage;
|
||||
import static com.google.domain.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static com.google.domain.registry.backup.BackupUtils.GcsMetadataKeys.NUM_TRANSACTIONS;
|
||||
import static com.google.domain.registry.backup.BackupUtils.GcsMetadataKeys.UPPER_BOUND_CHECKPOINT;
|
||||
import static com.google.domain.registry.backup.BackupUtils.deserializeEntities;
|
||||
import static com.google.domain.registry.testing.DatastoreHelper.persistResource;
|
||||
import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.domain.registry.config.TestRegistryConfig;
|
||||
import com.google.domain.registry.model.ImmutableObject;
|
||||
import com.google.domain.registry.model.ofy.CommitLogBucket;
|
||||
import com.google.domain.registry.model.ofy.CommitLogCheckpoint;
|
||||
import com.google.domain.registry.model.ofy.CommitLogManifest;
|
||||
import com.google.domain.registry.model.ofy.CommitLogMutation;
|
||||
import com.google.domain.registry.testing.AppEngineRule;
|
||||
import com.google.domain.registry.testing.GcsTestingUtils;
|
||||
import com.google.domain.registry.testing.RegistryConfigRule;
|
||||
import com.google.domain.registry.testing.TestObject;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.ObjectifyService;
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Unit tests for {@link ExportCommitLogDiffAction}. */
|
||||
@RunWith(JUnit4.class)
|
||||
public class ExportCommitLogDiffActionTest {
|
||||
|
||||
private static final int NUM_BUCKETS = 3;
|
||||
|
||||
@Rule
|
||||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
||||
.withDatastore()
|
||||
.build();
|
||||
|
||||
@Rule
|
||||
public final RegistryConfigRule configRule = new RegistryConfigRule(
|
||||
new TestRegistryConfig() {
|
||||
@Override public int getCommitLogBucketCount() {
|
||||
return NUM_BUCKETS;
|
||||
}});
|
||||
|
||||
/** Local GCS service available for testing. */
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
|
||||
private final DateTime now = DateTime.now(UTC);
|
||||
private final DateTime oneMinuteAgo = now.minusMinutes(1);
|
||||
|
||||
private final ExportCommitLogDiffAction task = new ExportCommitLogDiffAction();
|
||||
|
||||
@Before
|
||||
public void before() {
|
||||
ObjectifyService.register(TestObject.class);
|
||||
task.gcsService = gcsService;
|
||||
task.gcsBucket = "gcs bucket";
|
||||
task.batchSize = 5;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_noCommitHistory_onlyUpperCheckpointExported() throws Exception {
|
||||
task.lowerCheckpointTime = oneMinuteAgo;
|
||||
task.upperCheckpointTime = now;
|
||||
|
||||
persistResource(CommitLogCheckpoint.create(
|
||||
oneMinuteAgo,
|
||||
ImmutableMap.of(1, oneMinuteAgo, 2, oneMinuteAgo, 3, oneMinuteAgo)));
|
||||
CommitLogCheckpoint upperCheckpoint = persistResource(CommitLogCheckpoint.create(
|
||||
now,
|
||||
ImmutableMap.of(1, now, 2, now, 3, now)));
|
||||
|
||||
// Don't persist any manifests or mutations.
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
UPPER_BOUND_CHECKPOINT,
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"0");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
assertThat(exported).containsExactly(upperCheckpoint);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_regularCommitHistory_exportsCorrectCheckpointDiff() throws Exception {
|
||||
task.lowerCheckpointTime = oneMinuteAgo;
|
||||
task.upperCheckpointTime = now;
|
||||
|
||||
// Persist the lower and upper checkpoints, with 3 buckets each and staggered times. We respect
|
||||
// the real invariant that the time for bucket n in the lower checkpoint is <= the time for
|
||||
// that bucket in the upper.
|
||||
persistResource(CommitLogCheckpoint.create(
|
||||
oneMinuteAgo,
|
||||
ImmutableMap.of(
|
||||
1, oneMinuteAgo,
|
||||
2, oneMinuteAgo.minusDays(1),
|
||||
3, oneMinuteAgo.minusDays(2))));
|
||||
CommitLogCheckpoint upperCheckpoint = persistResource(CommitLogCheckpoint.create(
|
||||
now,
|
||||
ImmutableMap.of(
|
||||
1, now,
|
||||
2, now.minusDays(1),
|
||||
3, oneMinuteAgo.minusDays(2)))); // Note that this matches the lower bound.
|
||||
|
||||
// Persist some fake commit log manifests.
|
||||
// These shouldn't be in the diff because the lower bound is exclusive.
|
||||
persistManifestAndMutation(1, oneMinuteAgo);
|
||||
persistManifestAndMutation(2, oneMinuteAgo.minusDays(1));
|
||||
persistManifestAndMutation(3, oneMinuteAgo.minusDays(2)); // Even though it's == upper bound.
|
||||
// These shouldn't be in the diff because they are above the upper bound.
|
||||
persistManifestAndMutation(1, now.plusMillis(1));
|
||||
persistManifestAndMutation(2, now.minusDays(1).plusMillis(1));
|
||||
persistManifestAndMutation(3, oneMinuteAgo.minusDays(2).plusMillis(1));
|
||||
// These should be in the diff because they are between the bounds. (Not possible for bucket 3.)
|
||||
persistManifestAndMutation(1, now.minusMillis(1));
|
||||
persistManifestAndMutation(2, now.minusDays(1).minusMillis(1));
|
||||
// These should be in the diff because they are at the upper bound. (Not possible for bucket 3.)
|
||||
persistManifestAndMutation(1, now);
|
||||
persistManifestAndMutation(2, now.minusDays(1));
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
UPPER_BOUND_CHECKPOINT,
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"4");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in time order, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(2, now.minusDays(1).minusMillis(1));
|
||||
CommitLogManifest manifest2 = createManifest(2, now.minusDays(1));
|
||||
CommitLogManifest manifest3 = createManifest(1, now.minusMillis(1));
|
||||
CommitLogManifest manifest4 = createManifest(1, now);
|
||||
assertThat(exported).containsExactly(
|
||||
upperCheckpoint,
|
||||
manifest1,
|
||||
createMutation(manifest1),
|
||||
manifest2,
|
||||
createMutation(manifest2),
|
||||
manifest3,
|
||||
createMutation(manifest3),
|
||||
manifest4,
|
||||
createMutation(manifest4))
|
||||
.inOrder();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_simultaneousTransactions_bothExported() throws Exception {
|
||||
task.lowerCheckpointTime = oneMinuteAgo;
|
||||
task.upperCheckpointTime = now;
|
||||
|
||||
persistResource(CommitLogCheckpoint.create(
|
||||
oneMinuteAgo,
|
||||
ImmutableMap.of(1, START_OF_TIME, 2, START_OF_TIME, 3, START_OF_TIME)));
|
||||
CommitLogCheckpoint upperCheckpoint = persistResource(CommitLogCheckpoint.create(
|
||||
now,
|
||||
ImmutableMap.of(1, now, 2, now, 3, now)));
|
||||
|
||||
// Persist some fake commit log manifests that are at the same time but in different buckets.
|
||||
persistManifestAndMutation(1, oneMinuteAgo);
|
||||
persistManifestAndMutation(2, oneMinuteAgo);
|
||||
persistManifestAndMutation(1, now);
|
||||
persistManifestAndMutation(2, now);
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
UPPER_BOUND_CHECKPOINT,
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"4");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in the order below, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(1, oneMinuteAgo);
|
||||
CommitLogManifest manifest2 = createManifest(2, oneMinuteAgo);
|
||||
CommitLogManifest manifest3 = createManifest(1, now);
|
||||
CommitLogManifest manifest4 = createManifest(2, now);
|
||||
assertThat(exported).containsExactly(
|
||||
upperCheckpoint,
|
||||
manifest1,
|
||||
createMutation(manifest1),
|
||||
manifest2,
|
||||
createMutation(manifest2),
|
||||
manifest3,
|
||||
createMutation(manifest3),
|
||||
manifest4,
|
||||
createMutation(manifest4))
|
||||
.inOrder();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_exportsAcrossMultipleBatches() throws Exception {
|
||||
task.batchSize = 2;
|
||||
task.lowerCheckpointTime = oneMinuteAgo;
|
||||
task.upperCheckpointTime = now;
|
||||
|
||||
persistResource(CommitLogCheckpoint.create(
|
||||
oneMinuteAgo,
|
||||
ImmutableMap.of(1, START_OF_TIME, 2, START_OF_TIME, 3, START_OF_TIME)));
|
||||
CommitLogCheckpoint upperCheckpoint = persistResource(CommitLogCheckpoint.create(
|
||||
now,
|
||||
ImmutableMap.of(1, now, 2, now, 3, now)));
|
||||
|
||||
// Persist some fake commit log manifests.
|
||||
persistManifestAndMutation(1, oneMinuteAgo);
|
||||
persistManifestAndMutation(2, oneMinuteAgo);
|
||||
persistManifestAndMutation(3, oneMinuteAgo);
|
||||
persistManifestAndMutation(1, now);
|
||||
persistManifestAndMutation(2, now);
|
||||
persistManifestAndMutation(3, now);
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
UPPER_BOUND_CHECKPOINT,
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"6");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in the order below, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(1, oneMinuteAgo);
|
||||
CommitLogManifest manifest2 = createManifest(2, oneMinuteAgo);
|
||||
CommitLogManifest manifest3 = createManifest(3, oneMinuteAgo);
|
||||
CommitLogManifest manifest4 = createManifest(1, now);
|
||||
CommitLogManifest manifest5 = createManifest(2, now);
|
||||
CommitLogManifest manifest6 = createManifest(3, now);
|
||||
assertThat(exported).containsExactly(
|
||||
upperCheckpoint,
|
||||
manifest1,
|
||||
createMutation(manifest1),
|
||||
manifest2,
|
||||
createMutation(manifest2),
|
||||
manifest3,
|
||||
createMutation(manifest3),
|
||||
manifest4,
|
||||
createMutation(manifest4),
|
||||
manifest5,
|
||||
createMutation(manifest5),
|
||||
manifest6,
|
||||
createMutation(manifest6))
|
||||
.inOrder();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_checkpointDiffWithNeverTouchedBuckets_exportsCorrectly() throws Exception {
|
||||
task.lowerCheckpointTime = oneMinuteAgo;
|
||||
task.upperCheckpointTime = now;
|
||||
|
||||
persistResource(CommitLogCheckpoint.create(
|
||||
oneMinuteAgo,
|
||||
ImmutableMap.of(1, START_OF_TIME, 2, START_OF_TIME, 3, START_OF_TIME)));
|
||||
CommitLogCheckpoint upperCheckpoint = persistResource(CommitLogCheckpoint.create(
|
||||
now,
|
||||
ImmutableMap.of(1, START_OF_TIME, 2, START_OF_TIME, 3, START_OF_TIME)));
|
||||
|
||||
// Don't persist any commit log manifests; we're just checking that the task runs correctly
|
||||
// even if the upper timestamp contains START_OF_TIME values.
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
UPPER_BOUND_CHECKPOINT,
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"0");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
// We expect no manifests or mutations, only the upper checkpoint.
|
||||
assertThat(exported).containsExactly(upperCheckpoint);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_exportingFromStartOfTime_exportsAllCommits() throws Exception {
|
||||
task.lowerCheckpointTime = START_OF_TIME;
|
||||
task.upperCheckpointTime = now;
|
||||
|
||||
CommitLogCheckpoint upperCheckpoint = persistResource(CommitLogCheckpoint.create(
|
||||
now,
|
||||
ImmutableMap.of(1, now, 2, now, 3, now)));
|
||||
|
||||
// Persist some fake commit log manifests.
|
||||
persistManifestAndMutation(1, START_OF_TIME.plusMillis(1)); // Oldest possible manifest time.
|
||||
persistManifestAndMutation(2, oneMinuteAgo);
|
||||
persistManifestAndMutation(3, now);
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
START_OF_TIME.toString(),
|
||||
UPPER_BOUND_CHECKPOINT,
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"3");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in the order below, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(1, START_OF_TIME.plusMillis(1));
|
||||
CommitLogManifest manifest2 = createManifest(2, oneMinuteAgo);
|
||||
CommitLogManifest manifest3 = createManifest(3, now);
|
||||
assertThat(exported).containsExactly(
|
||||
upperCheckpoint,
|
||||
manifest1,
|
||||
createMutation(manifest1),
|
||||
manifest2,
|
||||
createMutation(manifest2),
|
||||
manifest3,
|
||||
createMutation(manifest3))
|
||||
.inOrder();
|
||||
}
|
||||
|
||||
private CommitLogManifest createManifest(int bucketNum, DateTime commitTime) {
|
||||
return CommitLogManifest.create(CommitLogBucket.getBucketKey(bucketNum), commitTime, null);
|
||||
}
|
||||
|
||||
private CommitLogMutation createMutation(CommitLogManifest manifest) {
|
||||
return CommitLogMutation.create(
|
||||
Key.create(manifest),
|
||||
TestObject.create(manifest.getCommitTime().toString()));
|
||||
}
|
||||
|
||||
private void persistManifestAndMutation(int bucketNum, DateTime commitTime) {
|
||||
persistResource(
|
||||
createMutation(persistResource(createManifest(bucketNum, commitTime))));
|
||||
}
|
||||
}
|
||||
153
javatests/google/registry/backup/GcsDiffFileListerTest.java
Normal file
153
javatests/google/registry/backup/GcsDiffFileListerTest.java
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.backup;
|
||||
|
||||
import static com.google.common.collect.Iterables.transform;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
|
||||
import static com.google.domain.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static com.google.domain.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
import static java.lang.reflect.Proxy.newProxyInstance;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.ListItem;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.Iterators;
|
||||
import com.google.domain.registry.testing.AppEngineRule;
|
||||
import com.google.domain.registry.testing.TestObject;
|
||||
|
||||
import com.googlecode.objectify.ObjectifyService;
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
|
||||
import java.lang.reflect.InvocationHandler;
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
/** Unit tests for {@link GcsDiffFileLister}. */
|
||||
@RunWith(JUnit4.class)
|
||||
public class GcsDiffFileListerTest {
|
||||
|
||||
static final String GCS_BUCKET = "gcs bucket";
|
||||
|
||||
final DateTime now = DateTime.now(UTC);
|
||||
final GcsDiffFileLister diffLister = new GcsDiffFileLister();
|
||||
final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
|
||||
@Rule
|
||||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
||||
.withDatastore()
|
||||
.build();
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
ObjectifyService.register(TestObject.class);
|
||||
diffLister.gcsService = gcsService;
|
||||
diffLister.gcsBucket = GCS_BUCKET;
|
||||
diffLister.executor = newDirectExecutorService();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
gcsService.createOrReplace(
|
||||
new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(i)),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(i + 1).toString())
|
||||
.build(),
|
||||
ByteBuffer.wrap(new byte[]{1, 2, 3}));
|
||||
}
|
||||
}
|
||||
|
||||
private Iterable<DateTime> extractTimesFromDiffFiles(List<GcsFileMetadata> diffFiles) {
|
||||
return transform(
|
||||
diffFiles,
|
||||
new Function<GcsFileMetadata, DateTime>() {
|
||||
@Override
|
||||
public DateTime apply(GcsFileMetadata metadata) {
|
||||
return DateTime.parse(
|
||||
metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()));
|
||||
}});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testList_noFilesFound() {
|
||||
DateTime fromTime = now.plusMillis(1);
|
||||
assertThat(extractTimesFromDiffFiles(diffLister.listDiffFiles(fromTime))).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testList_findsFiles_backToFromTimeExclusive() {
|
||||
DateTime fromTime = now.minusMinutes(2);
|
||||
assertThat(extractTimesFromDiffFiles(diffLister.listDiffFiles(fromTime)))
|
||||
.containsExactly(now.minusMinutes(1), now)
|
||||
.inOrder();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testList_patchesHoles() throws Exception {
|
||||
// Fake out the GCS list() method to return only the first and last file.
|
||||
// We can't use Mockito.spy() because GcsService's impl is final.
|
||||
diffLister.gcsService = (GcsService) newProxyInstance(
|
||||
GcsService.class.getClassLoader(),
|
||||
new Class<?>[] {GcsService.class},
|
||||
new InvocationHandler() {
|
||||
@Override
|
||||
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
|
||||
if (method.getName().equals("list")) {
|
||||
// ListResult is an incredibly annoying thing to construct. It needs to be fed from a
|
||||
// Callable that returns Iterators, each representing a batch of results.
|
||||
return new ListResult(new Callable<Iterator<ListItem>>() {
|
||||
boolean called = false;
|
||||
|
||||
@Override
|
||||
public Iterator<ListItem> call() throws Exception {
|
||||
try {
|
||||
return called ? null : Iterators.forArray(
|
||||
new ListItem.Builder()
|
||||
.setName(DIFF_FILE_PREFIX + now)
|
||||
.build(),
|
||||
new ListItem.Builder()
|
||||
.setName(DIFF_FILE_PREFIX + now.minusMinutes(4))
|
||||
.build());
|
||||
} finally {
|
||||
called = true;
|
||||
}
|
||||
}});
|
||||
}
|
||||
return method.invoke(gcsService, args);
|
||||
}});
|
||||
DateTime fromTime = now.minusMinutes(4).minusSeconds(1);
|
||||
// Request all files with checkpoint > fromTime.
|
||||
assertThat(extractTimesFromDiffFiles(diffLister.listDiffFiles(fromTime)))
|
||||
.containsExactly(
|
||||
now.minusMinutes(4),
|
||||
now.minusMinutes(3),
|
||||
now.minusMinutes(2),
|
||||
now.minusMinutes(1),
|
||||
now)
|
||||
.inOrder();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,319 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.backup;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.base.Functions.constant;
|
||||
import static com.google.common.collect.Iterables.transform;
|
||||
import static com.google.common.collect.Maps.toMap;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
|
||||
import static com.google.domain.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static com.google.domain.registry.backup.BackupUtils.serializeEntity;
|
||||
import static com.google.domain.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
import static com.google.domain.registry.model.ofy.CommitLogBucket.getBucketIds;
|
||||
import static com.google.domain.registry.model.ofy.CommitLogBucket.getBucketKey;
|
||||
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static java.util.Arrays.asList;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.api.datastore.DatastoreServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.primitives.Longs;
|
||||
import com.google.domain.registry.config.TestRegistryConfig;
|
||||
import com.google.domain.registry.model.ImmutableObject;
|
||||
import com.google.domain.registry.model.ofy.CommitLogBucket;
|
||||
import com.google.domain.registry.model.ofy.CommitLogCheckpoint;
|
||||
import com.google.domain.registry.model.ofy.CommitLogCheckpointRoot;
|
||||
import com.google.domain.registry.model.ofy.CommitLogManifest;
|
||||
import com.google.domain.registry.model.ofy.CommitLogMutation;
|
||||
import com.google.domain.registry.testing.AppEngineRule;
|
||||
import com.google.domain.registry.testing.FakeClock;
|
||||
import com.google.domain.registry.testing.FakeSleeper;
|
||||
import com.google.domain.registry.testing.RegistryConfigRule;
|
||||
import com.google.domain.registry.testing.TestObject;
|
||||
import com.google.domain.registry.util.Retrier;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.ObjectifyService;
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
/** Unit tests for {@link RestoreCommitLogsAction}. */
|
||||
@RunWith(JUnit4.class)
|
||||
public class RestoreCommitLogsActionTest {
|
||||
|
||||
static final String GCS_BUCKET = "gcs bucket";
|
||||
|
||||
final DateTime now = DateTime.now(UTC);
|
||||
final RestoreCommitLogsAction action = new RestoreCommitLogsAction();
|
||||
final GcsService gcsService = createGcsService();
|
||||
|
||||
@Rule
|
||||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
||||
.withDatastore()
|
||||
.build();
|
||||
|
||||
@Rule
|
||||
public final RegistryConfigRule configRule = new RegistryConfigRule();
|
||||
|
||||
@Before
|
||||
public void init() {
|
||||
ObjectifyService.register(TestObject.class);
|
||||
action.gcsService = gcsService;
|
||||
action.dryRun = false;
|
||||
action.datastoreService = DatastoreServiceFactory.getDatastoreService();
|
||||
action.fromTime = now.minusMillis(1);
|
||||
action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 1);
|
||||
action.diffLister = new GcsDiffFileLister();
|
||||
action.diffLister.gcsService = gcsService;
|
||||
action.diffLister.gcsBucket = GCS_BUCKET;
|
||||
action.diffLister.executor = newDirectExecutorService();
|
||||
configRule.override(new TestRegistryConfig() {
|
||||
@Override
|
||||
public int getCommitLogBucketCount() {
|
||||
return 3;
|
||||
}});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRestore_multipleDiffFiles() throws Exception {
|
||||
ofy().saveWithoutBackup().entities(
|
||||
TestObject.create("previous to keep"),
|
||||
TestObject.create("previous to delete")).now();
|
||||
// Create 3 transactions, across two diff files.
|
||||
// Before: {"previous to keep", "previous to delete"}
|
||||
// 1a: Add {"a", "b"}, Delete {"previous to delete"}
|
||||
// 1b: Add {"c", "d"}, Delete {"a"}
|
||||
// 2: Add {"e", "f"}, Delete {"c"}
|
||||
// After: {"previous to keep", "b", "d", "e", "f"}
|
||||
Key<CommitLogManifest> manifest1aKey =
|
||||
CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(3));
|
||||
Key<CommitLogManifest> manifest1bKey =
|
||||
CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2));
|
||||
Key<CommitLogManifest> manifest2Key =
|
||||
CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(now.minusMinutes(2));
|
||||
Iterable<ImmutableObject> file1CommitLogs = saveDiffFile(
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
now.minusMinutes(3),
|
||||
ImmutableSet.<Key<?>>of(Key.create(TestObject.create("previous to delete")))),
|
||||
CommitLogMutation.create(manifest1aKey, TestObject.create("a")),
|
||||
CommitLogMutation.create(manifest1aKey, TestObject.create("b")),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(2),
|
||||
now.minusMinutes(2),
|
||||
ImmutableSet.<Key<?>>of(Key.create(TestObject.create("a")))),
|
||||
CommitLogMutation.create(manifest1bKey, TestObject.create("c")),
|
||||
CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
|
||||
Iterable<ImmutableObject> file2CommitLogs = saveDiffFile(
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
now.minusMinutes(1),
|
||||
ImmutableSet.<Key<?>>of(Key.create(TestObject.create("c")))),
|
||||
CommitLogMutation.create(manifest2Key, TestObject.create("e")),
|
||||
CommitLogMutation.create(manifest2Key, TestObject.create("f")));
|
||||
action.fromTime = now.minusMinutes(1).minusMillis(1);
|
||||
action.run();
|
||||
ofy().clearSessionCache();
|
||||
assertExpectedIds("previous to keep", "b", "d", "e", "f");
|
||||
assertInDatastore(file1CommitLogs);
|
||||
assertInDatastore(file2CommitLogs);
|
||||
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
||||
assertCommitLogBuckets(ImmutableMap.of(1, now.minusMinutes(1), 2, now.minusMinutes(2)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRestore_noManifests() throws Exception {
|
||||
ofy().saveWithoutBackup().entity(
|
||||
TestObject.create("previous to keep")).now();
|
||||
saveDiffFileNotToRestore(now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs = saveDiffFile(createCheckpoint(now));
|
||||
action.run();
|
||||
ofy().clearSessionCache();
|
||||
assertExpectedIds("previous to keep");
|
||||
assertInDatastore(commitLogs);
|
||||
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
||||
assertCommitLogBuckets(ImmutableMap.<Integer, DateTime>of());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRestore_manifestWithNoDeletions() throws Exception {
|
||||
ofy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now();
|
||||
Key<CommitLogBucket> bucketKey = getBucketKey(1);
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
|
||||
saveDiffFileNotToRestore(now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs = saveDiffFile(
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(bucketKey, now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("a")),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("b")));
|
||||
action.run();
|
||||
ofy().clearSessionCache();
|
||||
assertExpectedIds("previous to keep", "a", "b");
|
||||
assertInDatastore(commitLogs);
|
||||
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
||||
assertCommitLogBuckets(ImmutableMap.of(1, now));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRestore_manifestWithNoMutations() throws Exception {
|
||||
ofy().saveWithoutBackup().entities(
|
||||
TestObject.create("previous to keep"),
|
||||
TestObject.create("previous to delete")).now();
|
||||
saveDiffFileNotToRestore(now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs = saveDiffFile(
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
now,
|
||||
ImmutableSet.<Key<?>>of(Key.create(TestObject.create("previous to delete")))));
|
||||
action.run();
|
||||
ofy().clearSessionCache();
|
||||
assertExpectedIds("previous to keep");
|
||||
assertInDatastore(commitLogs);
|
||||
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
||||
assertCommitLogBuckets(ImmutableMap.of(1, now));
|
||||
}
|
||||
|
||||
// This is a pathological case that shouldn't be possible, but we should be robust to it.
|
||||
@Test
|
||||
public void testRestore_manifestWithNoMutationsOrDeletions() throws Exception {
|
||||
ofy().saveWithoutBackup().entities(
|
||||
TestObject.create("previous to keep")).now();
|
||||
saveDiffFileNotToRestore(now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs = saveDiffFile(
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(getBucketKey(1), now, null));
|
||||
action.run();
|
||||
ofy().clearSessionCache();
|
||||
assertExpectedIds("previous to keep");
|
||||
assertInDatastore(commitLogs);
|
||||
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
||||
assertCommitLogBuckets(ImmutableMap.of(1, now));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRestore_mutateExistingEntity() throws Exception {
|
||||
ofy().saveWithoutBackup().entity(TestObject.create("existing", "a")).now();
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(getBucketKey(1), now);
|
||||
saveDiffFileNotToRestore(now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs = saveDiffFile(
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(getBucketKey(1), now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
|
||||
action.run();
|
||||
ofy().clearSessionCache();
|
||||
assertThat(ofy().load().entity(TestObject.create("existing")).now().getField()).isEqualTo("b");
|
||||
assertInDatastore(commitLogs);
|
||||
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
||||
assertCommitLogBuckets(ImmutableMap.of(1, now));
|
||||
}
|
||||
|
||||
// This should be harmless; deletes are idempotent.
|
||||
@Test
|
||||
public void testRestore_deleteMissingEntity() throws Exception {
|
||||
ofy().saveWithoutBackup().entity(TestObject.create("previous to keep", "a")).now();
|
||||
saveDiffFileNotToRestore(now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs = saveDiffFile(
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
now,
|
||||
ImmutableSet.<Key<?>>of(Key.create(TestObject.create("previous to delete")))));
|
||||
action.run();
|
||||
ofy().clearSessionCache();
|
||||
assertExpectedIds("previous to keep");
|
||||
assertInDatastore(commitLogs);
|
||||
assertCommitLogBuckets(ImmutableMap.of(1, now));
|
||||
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
||||
}
|
||||
|
||||
private CommitLogCheckpoint createCheckpoint(DateTime now) {
|
||||
return CommitLogCheckpoint.create(now, toMap(getBucketIds(), constant(now)));
|
||||
}
|
||||
|
||||
private Iterable<ImmutableObject> saveDiffFile(
|
||||
CommitLogCheckpoint checkpoint, ImmutableObject... entities) throws IOException {
|
||||
DateTime now = checkpoint.getCheckpointTime();
|
||||
List<ImmutableObject> allEntities = Lists.<ImmutableObject>asList(checkpoint, entities);
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
for (ImmutableObject entity : allEntities) {
|
||||
serializeEntity(entity, output);
|
||||
}
|
||||
gcsService.createOrReplace(
|
||||
new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(1).toString())
|
||||
.build(),
|
||||
ByteBuffer.wrap(output.toByteArray()));
|
||||
return allEntities;
|
||||
}
|
||||
|
||||
private void saveDiffFileNotToRestore(DateTime now) throws Exception {
|
||||
saveDiffFile(
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(getBucketKey(1), now, null),
|
||||
CommitLogMutation.create(
|
||||
CommitLogManifest.createKey(getBucketKey(1), now),
|
||||
TestObject.create("should not be restored")));
|
||||
}
|
||||
|
||||
private void assertExpectedIds(String... ids) {
|
||||
assertThat(transform(
|
||||
ofy().load().type(TestObject.class),
|
||||
new Function<TestObject, String>() {
|
||||
@Override
|
||||
public String apply(TestObject test) {
|
||||
return test.getId();
|
||||
}})).containsExactly((Object[]) ids);
|
||||
}
|
||||
|
||||
private void assertInDatastore(Iterable<? extends ImmutableObject> entities) {
|
||||
assertThat(ofy().load().entities(entities).values()).containsExactlyElementsIn(entities);
|
||||
}
|
||||
|
||||
private void assertCommitLogBuckets(Map<Integer, DateTime> bucketIdsAndTimestamps) {
|
||||
Map<Long, CommitLogBucket> buckets = ofy().load()
|
||||
.type(CommitLogBucket.class)
|
||||
.ids(Longs.asList(Longs.toArray(CommitLogBucket.getBucketIds())));
|
||||
assertThat(buckets).hasSize(bucketIdsAndTimestamps.size());
|
||||
for (Entry<Integer, DateTime> bucketIdAndTimestamp : bucketIdsAndTimestamps.entrySet()) {
|
||||
assertThat(buckets.get((long) bucketIdAndTimestamp.getKey()).getLastWrittenTime())
|
||||
.isEqualTo(bucketIdAndTimestamp.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue