mirror of
https://github.com/google/nomulus.git
synced 2025-04-30 12:07:51 +02:00
c458c05801
The dark lord Gosling designed the Java package naming system so that ownership flows from the DNS system. Since we own the domain name registry.google, it seems only appropriate that we should use google.registry as our package name.
320 lines
13 KiB
Java
320 lines
13 KiB
Java
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package google.registry.backup;
|
|
|
|
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
|
import static com.google.common.base.Functions.constant;
|
|
import static com.google.common.collect.Iterables.transform;
|
|
import static com.google.common.collect.Maps.toMap;
|
|
import static com.google.common.truth.Truth.assertThat;
|
|
import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
|
|
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
|
import static google.registry.backup.BackupUtils.serializeEntity;
|
|
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
|
import static google.registry.model.ofy.CommitLogBucket.getBucketIds;
|
|
import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
|
|
import static google.registry.model.ofy.ObjectifyService.ofy;
|
|
import static java.util.Arrays.asList;
|
|
import static org.joda.time.DateTimeZone.UTC;
|
|
|
|
import com.google.appengine.api.datastore.DatastoreServiceFactory;
|
|
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
|
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
|
import com.google.appengine.tools.cloudstorage.GcsService;
|
|
import com.google.common.base.Function;
|
|
import com.google.common.collect.ImmutableMap;
|
|
import com.google.common.collect.ImmutableSet;
|
|
import com.google.common.collect.Lists;
|
|
import com.google.common.primitives.Longs;
|
|
|
|
import com.googlecode.objectify.Key;
|
|
import com.googlecode.objectify.ObjectifyService;
|
|
|
|
import google.registry.config.TestRegistryConfig;
|
|
import google.registry.model.ImmutableObject;
|
|
import google.registry.model.ofy.CommitLogBucket;
|
|
import google.registry.model.ofy.CommitLogCheckpoint;
|
|
import google.registry.model.ofy.CommitLogCheckpointRoot;
|
|
import google.registry.model.ofy.CommitLogManifest;
|
|
import google.registry.model.ofy.CommitLogMutation;
|
|
import google.registry.testing.AppEngineRule;
|
|
import google.registry.testing.FakeClock;
|
|
import google.registry.testing.FakeSleeper;
|
|
import google.registry.testing.RegistryConfigRule;
|
|
import google.registry.testing.TestObject;
|
|
import google.registry.util.Retrier;
|
|
|
|
import org.joda.time.DateTime;
|
|
import org.junit.Before;
|
|
import org.junit.Rule;
|
|
import org.junit.Test;
|
|
import org.junit.runner.RunWith;
|
|
import org.junit.runners.JUnit4;
|
|
|
|
import java.io.ByteArrayOutputStream;
|
|
import java.io.IOException;
|
|
import java.nio.ByteBuffer;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Map.Entry;
|
|
|
|
/** Unit tests for {@link RestoreCommitLogsAction}. */
|
|
@RunWith(JUnit4.class)
|
|
public class RestoreCommitLogsActionTest {
|
|
|
|
static final String GCS_BUCKET = "gcs bucket";
|
|
|
|
final DateTime now = DateTime.now(UTC);
|
|
final RestoreCommitLogsAction action = new RestoreCommitLogsAction();
|
|
final GcsService gcsService = createGcsService();
|
|
|
|
@Rule
|
|
public final AppEngineRule appEngine = AppEngineRule.builder()
|
|
.withDatastore()
|
|
.build();
|
|
|
|
@Rule
|
|
public final RegistryConfigRule configRule = new RegistryConfigRule();
|
|
|
|
@Before
|
|
public void init() {
|
|
ObjectifyService.register(TestObject.class);
|
|
action.gcsService = gcsService;
|
|
action.dryRun = false;
|
|
action.datastoreService = DatastoreServiceFactory.getDatastoreService();
|
|
action.fromTime = now.minusMillis(1);
|
|
action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 1);
|
|
action.diffLister = new GcsDiffFileLister();
|
|
action.diffLister.gcsService = gcsService;
|
|
action.diffLister.gcsBucket = GCS_BUCKET;
|
|
action.diffLister.executor = newDirectExecutorService();
|
|
configRule.override(new TestRegistryConfig() {
|
|
@Override
|
|
public int getCommitLogBucketCount() {
|
|
return 3;
|
|
}});
|
|
}
|
|
|
|
@Test
|
|
public void testRestore_multipleDiffFiles() throws Exception {
|
|
ofy().saveWithoutBackup().entities(
|
|
TestObject.create("previous to keep"),
|
|
TestObject.create("previous to delete")).now();
|
|
// Create 3 transactions, across two diff files.
|
|
// Before: {"previous to keep", "previous to delete"}
|
|
// 1a: Add {"a", "b"}, Delete {"previous to delete"}
|
|
// 1b: Add {"c", "d"}, Delete {"a"}
|
|
// 2: Add {"e", "f"}, Delete {"c"}
|
|
// After: {"previous to keep", "b", "d", "e", "f"}
|
|
Key<CommitLogManifest> manifest1aKey =
|
|
CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(3));
|
|
Key<CommitLogManifest> manifest1bKey =
|
|
CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2));
|
|
Key<CommitLogManifest> manifest2Key =
|
|
CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
|
|
saveDiffFileNotToRestore(now.minusMinutes(2));
|
|
Iterable<ImmutableObject> file1CommitLogs = saveDiffFile(
|
|
createCheckpoint(now.minusMinutes(1)),
|
|
CommitLogManifest.create(
|
|
getBucketKey(1),
|
|
now.minusMinutes(3),
|
|
ImmutableSet.<Key<?>>of(Key.create(TestObject.create("previous to delete")))),
|
|
CommitLogMutation.create(manifest1aKey, TestObject.create("a")),
|
|
CommitLogMutation.create(manifest1aKey, TestObject.create("b")),
|
|
CommitLogManifest.create(
|
|
getBucketKey(2),
|
|
now.minusMinutes(2),
|
|
ImmutableSet.<Key<?>>of(Key.create(TestObject.create("a")))),
|
|
CommitLogMutation.create(manifest1bKey, TestObject.create("c")),
|
|
CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
|
|
Iterable<ImmutableObject> file2CommitLogs = saveDiffFile(
|
|
createCheckpoint(now),
|
|
CommitLogManifest.create(
|
|
getBucketKey(1),
|
|
now.minusMinutes(1),
|
|
ImmutableSet.<Key<?>>of(Key.create(TestObject.create("c")))),
|
|
CommitLogMutation.create(manifest2Key, TestObject.create("e")),
|
|
CommitLogMutation.create(manifest2Key, TestObject.create("f")));
|
|
action.fromTime = now.minusMinutes(1).minusMillis(1);
|
|
action.run();
|
|
ofy().clearSessionCache();
|
|
assertExpectedIds("previous to keep", "b", "d", "e", "f");
|
|
assertInDatastore(file1CommitLogs);
|
|
assertInDatastore(file2CommitLogs);
|
|
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
|
assertCommitLogBuckets(ImmutableMap.of(1, now.minusMinutes(1), 2, now.minusMinutes(2)));
|
|
}
|
|
|
|
@Test
|
|
public void testRestore_noManifests() throws Exception {
|
|
ofy().saveWithoutBackup().entity(
|
|
TestObject.create("previous to keep")).now();
|
|
saveDiffFileNotToRestore(now.minusMinutes(1));
|
|
Iterable<ImmutableObject> commitLogs = saveDiffFile(createCheckpoint(now));
|
|
action.run();
|
|
ofy().clearSessionCache();
|
|
assertExpectedIds("previous to keep");
|
|
assertInDatastore(commitLogs);
|
|
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
|
assertCommitLogBuckets(ImmutableMap.<Integer, DateTime>of());
|
|
}
|
|
|
|
@Test
|
|
public void testRestore_manifestWithNoDeletions() throws Exception {
|
|
ofy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now();
|
|
Key<CommitLogBucket> bucketKey = getBucketKey(1);
|
|
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
|
|
saveDiffFileNotToRestore(now.minusMinutes(1));
|
|
Iterable<ImmutableObject> commitLogs = saveDiffFile(
|
|
createCheckpoint(now),
|
|
CommitLogManifest.create(bucketKey, now, null),
|
|
CommitLogMutation.create(manifestKey, TestObject.create("a")),
|
|
CommitLogMutation.create(manifestKey, TestObject.create("b")));
|
|
action.run();
|
|
ofy().clearSessionCache();
|
|
assertExpectedIds("previous to keep", "a", "b");
|
|
assertInDatastore(commitLogs);
|
|
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
|
assertCommitLogBuckets(ImmutableMap.of(1, now));
|
|
}
|
|
|
|
@Test
|
|
public void testRestore_manifestWithNoMutations() throws Exception {
|
|
ofy().saveWithoutBackup().entities(
|
|
TestObject.create("previous to keep"),
|
|
TestObject.create("previous to delete")).now();
|
|
saveDiffFileNotToRestore(now.minusMinutes(1));
|
|
Iterable<ImmutableObject> commitLogs = saveDiffFile(
|
|
createCheckpoint(now),
|
|
CommitLogManifest.create(
|
|
getBucketKey(1),
|
|
now,
|
|
ImmutableSet.<Key<?>>of(Key.create(TestObject.create("previous to delete")))));
|
|
action.run();
|
|
ofy().clearSessionCache();
|
|
assertExpectedIds("previous to keep");
|
|
assertInDatastore(commitLogs);
|
|
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
|
assertCommitLogBuckets(ImmutableMap.of(1, now));
|
|
}
|
|
|
|
// This is a pathological case that shouldn't be possible, but we should be robust to it.
|
|
@Test
|
|
public void testRestore_manifestWithNoMutationsOrDeletions() throws Exception {
|
|
ofy().saveWithoutBackup().entities(
|
|
TestObject.create("previous to keep")).now();
|
|
saveDiffFileNotToRestore(now.minusMinutes(1));
|
|
Iterable<ImmutableObject> commitLogs = saveDiffFile(
|
|
createCheckpoint(now),
|
|
CommitLogManifest.create(getBucketKey(1), now, null));
|
|
action.run();
|
|
ofy().clearSessionCache();
|
|
assertExpectedIds("previous to keep");
|
|
assertInDatastore(commitLogs);
|
|
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
|
assertCommitLogBuckets(ImmutableMap.of(1, now));
|
|
}
|
|
|
|
@Test
|
|
public void testRestore_mutateExistingEntity() throws Exception {
|
|
ofy().saveWithoutBackup().entity(TestObject.create("existing", "a")).now();
|
|
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(getBucketKey(1), now);
|
|
saveDiffFileNotToRestore(now.minusMinutes(1));
|
|
Iterable<ImmutableObject> commitLogs = saveDiffFile(
|
|
createCheckpoint(now),
|
|
CommitLogManifest.create(getBucketKey(1), now, null),
|
|
CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
|
|
action.run();
|
|
ofy().clearSessionCache();
|
|
assertThat(ofy().load().entity(TestObject.create("existing")).now().getField()).isEqualTo("b");
|
|
assertInDatastore(commitLogs);
|
|
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
|
assertCommitLogBuckets(ImmutableMap.of(1, now));
|
|
}
|
|
|
|
// This should be harmless; deletes are idempotent.
|
|
@Test
|
|
public void testRestore_deleteMissingEntity() throws Exception {
|
|
ofy().saveWithoutBackup().entity(TestObject.create("previous to keep", "a")).now();
|
|
saveDiffFileNotToRestore(now.minusMinutes(1));
|
|
Iterable<ImmutableObject> commitLogs = saveDiffFile(
|
|
createCheckpoint(now),
|
|
CommitLogManifest.create(
|
|
getBucketKey(1),
|
|
now,
|
|
ImmutableSet.<Key<?>>of(Key.create(TestObject.create("previous to delete")))));
|
|
action.run();
|
|
ofy().clearSessionCache();
|
|
assertExpectedIds("previous to keep");
|
|
assertInDatastore(commitLogs);
|
|
assertCommitLogBuckets(ImmutableMap.of(1, now));
|
|
assertInDatastore(asList(CommitLogCheckpointRoot.create(now)));
|
|
}
|
|
|
|
private CommitLogCheckpoint createCheckpoint(DateTime now) {
|
|
return CommitLogCheckpoint.create(now, toMap(getBucketIds(), constant(now)));
|
|
}
|
|
|
|
private Iterable<ImmutableObject> saveDiffFile(
|
|
CommitLogCheckpoint checkpoint, ImmutableObject... entities) throws IOException {
|
|
DateTime now = checkpoint.getCheckpointTime();
|
|
List<ImmutableObject> allEntities = Lists.<ImmutableObject>asList(checkpoint, entities);
|
|
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
|
for (ImmutableObject entity : allEntities) {
|
|
serializeEntity(entity, output);
|
|
}
|
|
gcsService.createOrReplace(
|
|
new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now),
|
|
new GcsFileOptions.Builder()
|
|
.addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(1).toString())
|
|
.build(),
|
|
ByteBuffer.wrap(output.toByteArray()));
|
|
return allEntities;
|
|
}
|
|
|
|
private void saveDiffFileNotToRestore(DateTime now) throws Exception {
|
|
saveDiffFile(
|
|
createCheckpoint(now),
|
|
CommitLogManifest.create(getBucketKey(1), now, null),
|
|
CommitLogMutation.create(
|
|
CommitLogManifest.createKey(getBucketKey(1), now),
|
|
TestObject.create("should not be restored")));
|
|
}
|
|
|
|
private void assertExpectedIds(String... ids) {
|
|
assertThat(transform(
|
|
ofy().load().type(TestObject.class),
|
|
new Function<TestObject, String>() {
|
|
@Override
|
|
public String apply(TestObject test) {
|
|
return test.getId();
|
|
}})).containsExactly((Object[]) ids);
|
|
}
|
|
|
|
private void assertInDatastore(Iterable<? extends ImmutableObject> entities) {
|
|
assertThat(ofy().load().entities(entities).values()).containsExactlyElementsIn(entities);
|
|
}
|
|
|
|
private void assertCommitLogBuckets(Map<Integer, DateTime> bucketIdsAndTimestamps) {
|
|
Map<Long, CommitLogBucket> buckets = ofy().load()
|
|
.type(CommitLogBucket.class)
|
|
.ids(Longs.asList(Longs.toArray(CommitLogBucket.getBucketIds())));
|
|
assertThat(buckets).hasSize(bucketIdsAndTimestamps.size());
|
|
for (Entry<Integer, DateTime> bucketIdAndTimestamp : bucketIdsAndTimestamps.entrySet()) {
|
|
assertThat(buckets.get((long) bucketIdAndTimestamp.getKey()).getLastWrittenTime())
|
|
.isEqualTo(bucketIdAndTimestamp.getValue());
|
|
}
|
|
}
|
|
}
|