diff --git a/core/src/main/java/google/registry/tools/CompareDbBackups.java b/core/src/main/java/google/registry/tools/CompareDbBackups.java index 445a9c6aa..c0595d794 100644 --- a/core/src/main/java/google/registry/tools/CompareDbBackups.java +++ b/core/src/main/java/google/registry/tools/CompareDbBackups.java @@ -18,9 +18,20 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.common.collect.Sets.SetView; import java.io.File; +import java.util.function.Predicate; -/** Compare two database backups. */ +/** + * Compares two Datastore backups in V3 format on local file system. This is for use in tests and + * experiments with small data sizes. + * + *

This utility only supports the current Datastore backup format (version 3). A backup is a + * two-level directory hierarchy with data files in level-db format (output-*) and Datastore + * metadata files (*.export_metadata). + */ class CompareDbBackups { + private static final String DS_V3_BACKUP_FILE_PREFIX = "output-"; + private static final Predicate DATA_FILE_MATCHER = + file -> file.isFile() && file.getName().startsWith(DS_V3_BACKUP_FILE_PREFIX); public static void main(String[] args) { if (args.length != 2) { @@ -29,9 +40,13 @@ class CompareDbBackups { } ImmutableSet entities1 = - new RecordAccumulator().readDirectory(new File(args[0])).getComparableEntitySet(); + new RecordAccumulator() + .readDirectory(new File(args[0]), DATA_FILE_MATCHER) + .getComparableEntitySet(); ImmutableSet entities2 = - new RecordAccumulator().readDirectory(new File(args[1])).getComparableEntitySet(); + new RecordAccumulator() + .readDirectory(new File(args[1]), DATA_FILE_MATCHER) + .getComparableEntitySet(); // Calculate the entities added and removed. SetView added = Sets.difference(entities2, entities1); @@ -54,6 +69,10 @@ class CompareDbBackups { System.out.println(entity); } } + + if (added.isEmpty() && removed.isEmpty()) { + System.out.printf("\nBoth sets have the same %d entities.\n", entities1.size()); + } } /** Print out multi-line text in a pretty ASCII header frame. */ diff --git a/core/src/main/java/google/registry/tools/RecordAccumulator.java b/core/src/main/java/google/registry/tools/RecordAccumulator.java index 2a0791b86..676c88f9a 100644 --- a/core/src/main/java/google/registry/tools/RecordAccumulator.java +++ b/core/src/main/java/google/registry/tools/RecordAccumulator.java @@ -20,17 +20,18 @@ import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.util.function.Predicate; -/** Utility class that accumulates Entity records from level db files. */ +/** Accumulates Entity records from level db files under a directory hierarchy. */ class RecordAccumulator { private final LevelDbLogReader reader = new LevelDbLogReader(); /** Recursively reads all records in the directory. */ - public final RecordAccumulator readDirectory(File dir) { + public final RecordAccumulator readDirectory(File dir, Predicate fileMatcher) { for (File child : dir.listFiles()) { if (child.isDirectory()) { - readDirectory(child); - } else if (child.isFile()) { + readDirectory(child, fileMatcher); + } else if (fileMatcher.test(child)) { try { reader.readFrom(new FileInputStream(child)); } catch (IOException e) { diff --git a/core/src/test/java/google/registry/persistence/VKeyTest.java b/core/src/test/java/google/registry/persistence/VKeyTest.java index 0e798ac9b..2cfd9fdca 100644 --- a/core/src/test/java/google/registry/persistence/VKeyTest.java +++ b/core/src/test/java/google/registry/persistence/VKeyTest.java @@ -17,11 +17,11 @@ import static com.google.common.truth.Truth.assertThat; import com.googlecode.objectify.Key; import google.registry.testing.AppEngineRule; +import google.registry.testing.TestObject; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import google.registry.testing.TestObject; @RunWith(JUnit4.class) public class VKeyTest { diff --git a/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java b/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java index be8022830..6598c6859 100644 --- a/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java +++ b/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java @@ -17,11 +17,15 @@ package google.registry.tools; import static com.google.common.truth.Truth.assertThat; import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.io.Resources; import google.registry.testing.AppEngineRule; import google.registry.tools.LevelDbFileBuilder.Property; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; +import java.net.URL; +import org.junit.After; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -35,18 +39,38 @@ public class CompareDbBackupsTest { // Capture standard output. private final ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + private PrintStream orgStdout; @Rule public final TemporaryFolder tempFs = new TemporaryFolder(); @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastoreAndCloudSql().build(); + @Before + public void before() { + orgStdout = System.out; + System.setOut(new PrintStream(stdout)); + } + + @After + public void after() { + System.setOut(orgStdout); + } + @Test - public void testCommand() throws Exception { + public void testLoadBackup() { + URL backupRootFolder = Resources.getResource("google/registry/tools/datastore-export"); + CompareDbBackups.main(new String[] {backupRootFolder.getPath(), backupRootFolder.getPath()}); + String output = new String(stdout.toByteArray(), UTF_8); + assertThat(output).containsMatch("Both sets have the same 41 entities"); + } + + @Test + public void testCompareBackups() throws Exception { // Create two directories corresponding to data dumps. File dump1 = tempFs.newFolder("dump1"); - LevelDbFileBuilder builder = new LevelDbFileBuilder(new File(dump1, "data1")); + LevelDbFileBuilder builder = new LevelDbFileBuilder(new File(dump1, "output-data1")); builder.addEntityProto( BASE_ID, Property.create("eeny", 100L), @@ -60,7 +84,7 @@ public class CompareDbBackupsTest { builder.build(); File dump2 = tempFs.newFolder("dump2"); - builder = new LevelDbFileBuilder(new File(dump2, "data2")); + builder = new LevelDbFileBuilder(new File(dump2, "output-data2")); builder.addEntityProto( BASE_ID + 1, Property.create("moxey", 100L), @@ -73,7 +97,6 @@ public class CompareDbBackupsTest { Property.create("strutz", 300L)); builder.build(); - System.setOut(new PrintStream(stdout)); CompareDbBackups.main(new String[] {dump1.getCanonicalPath(), dump2.getCanonicalPath()}); String output = new String(stdout.toByteArray(), UTF_8); assertThat(output) diff --git a/core/src/test/java/google/registry/tools/GhostrydeCommandTest.java b/core/src/test/java/google/registry/tools/GhostrydeCommandTest.java index 346e2f044..49c3af328 100644 --- a/core/src/test/java/google/registry/tools/GhostrydeCommandTest.java +++ b/core/src/test/java/google/registry/tools/GhostrydeCommandTest.java @@ -23,11 +23,8 @@ import google.registry.rde.Ghostryde; import google.registry.testing.BouncyCastleProviderRule; import google.registry.testing.FakeKeyringModule; import google.registry.testing.InjectRule; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; -import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -61,19 +58,12 @@ public class GhostrydeCommandTest extends CommandTestCase { public final BouncyCastleProviderRule bouncy = new BouncyCastleProviderRule(); private Keyring keyring; - private PrintStream orgStdout; @Before public void before() { keyring = new FakeKeyringModule().get(); command.rdeStagingDecryptionKey = keyring::getRdeStagingDecryptionKey; command.rdeStagingEncryptionKey = keyring::getRdeStagingEncryptionKey; - orgStdout = System.out; - } - - @After - public void after() { - System.setOut(orgStdout); } @Test @@ -153,9 +143,7 @@ public class GhostrydeCommandTest extends CommandTestCase { Path inFile = tmpDir.newFolder().toPath().resolve("atrain.ghostryde"); Files.write( inFile, Ghostryde.encode(SONG_BY_CHRISTINA_ROSSETTI, keyring.getRdeStagingEncryptionKey())); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - System.setOut(new PrintStream(out)); runCommand("--decrypt", "--input=" + inFile); - assertThat(out.toByteArray()).isEqualTo(SONG_BY_CHRISTINA_ROSSETTI); + assertThat(getStdoutAsString().getBytes(UTF_8)).isEqualTo(SONG_BY_CHRISTINA_ROSSETTI); } } diff --git a/core/src/test/java/google/registry/tools/RecordAccumulatorTest.java b/core/src/test/java/google/registry/tools/RecordAccumulatorTest.java index 1c198fb51..9073176eb 100644 --- a/core/src/test/java/google/registry/tools/RecordAccumulatorTest.java +++ b/core/src/test/java/google/registry/tools/RecordAccumulatorTest.java @@ -76,7 +76,7 @@ public class RecordAccumulatorTest { builder.build(); ImmutableSet entities = - new RecordAccumulator().readDirectory(subdir).getComparableEntitySet(); + new RecordAccumulator().readDirectory(subdir, any -> true).getComparableEntitySet(); assertThat(entities).containsExactly(e1, e2, e3); } } diff --git a/core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/all_namespaces_kind_AllocationToken.export_metadata b/core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/all_namespaces_kind_AllocationToken.export_metadata new file mode 100644 index 000000000..9d8db2bac Binary files /dev/null and b/core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/all_namespaces_kind_AllocationToken.export_metadata differ diff --git a/core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/output-0 b/core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/output-0 new file mode 100644 index 000000000..74665d8d6 Binary files /dev/null and b/core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/output-0 differ diff --git a/core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/output-1 b/core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/output-1 new file mode 100644 index 000000000..e69de29bb