From 3a7d71e4119f7577e2646d2626d30ff394c90e64 Mon Sep 17 00:00:00 2001 From: Weimin Yu Date: Mon, 6 Apr 2020 10:50:38 -0400 Subject: [PATCH] Upgrade CompareDbBackup for Datastore V3 (#543) * Upgrade CompareDbBackup for Datastore V3 Upgrade the CompareDbBackup class to work with latest Datastore backup directory structure. Also fixed a few unrelated minor issues: - Remaining cases of improper use of System.setOut - Wrong import order in one class --- .../registry/tools/CompareDbBackups.java | 25 ++++++++++++-- .../registry/tools/RecordAccumulator.java | 9 ++--- .../google/registry/persistence/VKeyTest.java | 2 +- .../registry/tools/CompareDbBackupsTest.java | 31 +++++++++++++++--- .../registry/tools/GhostrydeCommandTest.java | 14 +------- .../registry/tools/RecordAccumulatorTest.java | 2 +- ...paces_kind_AllocationToken.export_metadata | Bin 0 -> 320 bytes .../kind_AllocationToken/output-0 | Bin 0 -> 32768 bytes .../kind_AllocationToken/output-1 | 0 9 files changed, 57 insertions(+), 26 deletions(-) create mode 100644 core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/all_namespaces_kind_AllocationToken.export_metadata create mode 100644 core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/output-0 create mode 100644 core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/output-1 diff --git a/core/src/main/java/google/registry/tools/CompareDbBackups.java b/core/src/main/java/google/registry/tools/CompareDbBackups.java index 445a9c6aa..c0595d794 100644 --- a/core/src/main/java/google/registry/tools/CompareDbBackups.java +++ b/core/src/main/java/google/registry/tools/CompareDbBackups.java @@ -18,9 +18,20 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.common.collect.Sets.SetView; import java.io.File; +import java.util.function.Predicate; -/** Compare two database backups. */ +/** + * Compares two Datastore backups in V3 format on local file system. This is for use in tests and + * experiments with small data sizes. + * + *

This utility only supports the current Datastore backup format (version 3). A backup is a + * two-level directory hierarchy with data files in level-db format (output-*) and Datastore + * metadata files (*.export_metadata). + */ class CompareDbBackups { + private static final String DS_V3_BACKUP_FILE_PREFIX = "output-"; + private static final Predicate DATA_FILE_MATCHER = + file -> file.isFile() && file.getName().startsWith(DS_V3_BACKUP_FILE_PREFIX); public static void main(String[] args) { if (args.length != 2) { @@ -29,9 +40,13 @@ class CompareDbBackups { } ImmutableSet entities1 = - new RecordAccumulator().readDirectory(new File(args[0])).getComparableEntitySet(); + new RecordAccumulator() + .readDirectory(new File(args[0]), DATA_FILE_MATCHER) + .getComparableEntitySet(); ImmutableSet entities2 = - new RecordAccumulator().readDirectory(new File(args[1])).getComparableEntitySet(); + new RecordAccumulator() + .readDirectory(new File(args[1]), DATA_FILE_MATCHER) + .getComparableEntitySet(); // Calculate the entities added and removed. SetView added = Sets.difference(entities2, entities1); @@ -54,6 +69,10 @@ class CompareDbBackups { System.out.println(entity); } } + + if (added.isEmpty() && removed.isEmpty()) { + System.out.printf("\nBoth sets have the same %d entities.\n", entities1.size()); + } } /** Print out multi-line text in a pretty ASCII header frame. */ diff --git a/core/src/main/java/google/registry/tools/RecordAccumulator.java b/core/src/main/java/google/registry/tools/RecordAccumulator.java index 2a0791b86..676c88f9a 100644 --- a/core/src/main/java/google/registry/tools/RecordAccumulator.java +++ b/core/src/main/java/google/registry/tools/RecordAccumulator.java @@ -20,17 +20,18 @@ import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.util.function.Predicate; -/** Utility class that accumulates Entity records from level db files. */ +/** Accumulates Entity records from level db files under a directory hierarchy. */ class RecordAccumulator { private final LevelDbLogReader reader = new LevelDbLogReader(); /** Recursively reads all records in the directory. */ - public final RecordAccumulator readDirectory(File dir) { + public final RecordAccumulator readDirectory(File dir, Predicate fileMatcher) { for (File child : dir.listFiles()) { if (child.isDirectory()) { - readDirectory(child); - } else if (child.isFile()) { + readDirectory(child, fileMatcher); + } else if (fileMatcher.test(child)) { try { reader.readFrom(new FileInputStream(child)); } catch (IOException e) { diff --git a/core/src/test/java/google/registry/persistence/VKeyTest.java b/core/src/test/java/google/registry/persistence/VKeyTest.java index 0e798ac9b..2cfd9fdca 100644 --- a/core/src/test/java/google/registry/persistence/VKeyTest.java +++ b/core/src/test/java/google/registry/persistence/VKeyTest.java @@ -17,11 +17,11 @@ import static com.google.common.truth.Truth.assertThat; import com.googlecode.objectify.Key; import google.registry.testing.AppEngineRule; +import google.registry.testing.TestObject; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import google.registry.testing.TestObject; @RunWith(JUnit4.class) public class VKeyTest { diff --git a/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java b/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java index be8022830..6598c6859 100644 --- a/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java +++ b/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java @@ -17,11 +17,15 @@ package google.registry.tools; import static com.google.common.truth.Truth.assertThat; import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.io.Resources; import google.registry.testing.AppEngineRule; import google.registry.tools.LevelDbFileBuilder.Property; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; +import java.net.URL; +import org.junit.After; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -35,18 +39,38 @@ public class CompareDbBackupsTest { // Capture standard output. private final ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + private PrintStream orgStdout; @Rule public final TemporaryFolder tempFs = new TemporaryFolder(); @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastoreAndCloudSql().build(); + @Before + public void before() { + orgStdout = System.out; + System.setOut(new PrintStream(stdout)); + } + + @After + public void after() { + System.setOut(orgStdout); + } + @Test - public void testCommand() throws Exception { + public void testLoadBackup() { + URL backupRootFolder = Resources.getResource("google/registry/tools/datastore-export"); + CompareDbBackups.main(new String[] {backupRootFolder.getPath(), backupRootFolder.getPath()}); + String output = new String(stdout.toByteArray(), UTF_8); + assertThat(output).containsMatch("Both sets have the same 41 entities"); + } + + @Test + public void testCompareBackups() throws Exception { // Create two directories corresponding to data dumps. File dump1 = tempFs.newFolder("dump1"); - LevelDbFileBuilder builder = new LevelDbFileBuilder(new File(dump1, "data1")); + LevelDbFileBuilder builder = new LevelDbFileBuilder(new File(dump1, "output-data1")); builder.addEntityProto( BASE_ID, Property.create("eeny", 100L), @@ -60,7 +84,7 @@ public class CompareDbBackupsTest { builder.build(); File dump2 = tempFs.newFolder("dump2"); - builder = new LevelDbFileBuilder(new File(dump2, "data2")); + builder = new LevelDbFileBuilder(new File(dump2, "output-data2")); builder.addEntityProto( BASE_ID + 1, Property.create("moxey", 100L), @@ -73,7 +97,6 @@ public class CompareDbBackupsTest { Property.create("strutz", 300L)); builder.build(); - System.setOut(new PrintStream(stdout)); CompareDbBackups.main(new String[] {dump1.getCanonicalPath(), dump2.getCanonicalPath()}); String output = new String(stdout.toByteArray(), UTF_8); assertThat(output) diff --git a/core/src/test/java/google/registry/tools/GhostrydeCommandTest.java b/core/src/test/java/google/registry/tools/GhostrydeCommandTest.java index 346e2f044..49c3af328 100644 --- a/core/src/test/java/google/registry/tools/GhostrydeCommandTest.java +++ b/core/src/test/java/google/registry/tools/GhostrydeCommandTest.java @@ -23,11 +23,8 @@ import google.registry.rde.Ghostryde; import google.registry.testing.BouncyCastleProviderRule; import google.registry.testing.FakeKeyringModule; import google.registry.testing.InjectRule; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; -import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -61,19 +58,12 @@ public class GhostrydeCommandTest extends CommandTestCase { public final BouncyCastleProviderRule bouncy = new BouncyCastleProviderRule(); private Keyring keyring; - private PrintStream orgStdout; @Before public void before() { keyring = new FakeKeyringModule().get(); command.rdeStagingDecryptionKey = keyring::getRdeStagingDecryptionKey; command.rdeStagingEncryptionKey = keyring::getRdeStagingEncryptionKey; - orgStdout = System.out; - } - - @After - public void after() { - System.setOut(orgStdout); } @Test @@ -153,9 +143,7 @@ public class GhostrydeCommandTest extends CommandTestCase { Path inFile = tmpDir.newFolder().toPath().resolve("atrain.ghostryde"); Files.write( inFile, Ghostryde.encode(SONG_BY_CHRISTINA_ROSSETTI, keyring.getRdeStagingEncryptionKey())); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - System.setOut(new PrintStream(out)); runCommand("--decrypt", "--input=" + inFile); - assertThat(out.toByteArray()).isEqualTo(SONG_BY_CHRISTINA_ROSSETTI); + assertThat(getStdoutAsString().getBytes(UTF_8)).isEqualTo(SONG_BY_CHRISTINA_ROSSETTI); } } diff --git a/core/src/test/java/google/registry/tools/RecordAccumulatorTest.java b/core/src/test/java/google/registry/tools/RecordAccumulatorTest.java index 1c198fb51..9073176eb 100644 --- a/core/src/test/java/google/registry/tools/RecordAccumulatorTest.java +++ b/core/src/test/java/google/registry/tools/RecordAccumulatorTest.java @@ -76,7 +76,7 @@ public class RecordAccumulatorTest { builder.build(); ImmutableSet entities = - new RecordAccumulator().readDirectory(subdir).getComparableEntitySet(); + new RecordAccumulator().readDirectory(subdir, any -> true).getComparableEntitySet(); assertThat(entities).containsExactly(e1, e2, e3); } } diff --git a/core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/all_namespaces_kind_AllocationToken.export_metadata b/core/src/test/resources/google/registry/tools/datastore-export/kind_AllocationToken/all_namespaces_kind_AllocationToken.export_metadata new file mode 100644 index 0000000000000000000000000000000000000000..9d8db2baca5f29b900dbbc4b074b6c30709b24f4 GIT binary patch literal 320 zcmZ{eJxT;Y5QW{@3D!cGroctSXvIWW(IXfrMkY=!%}fQ&?sPTX)!^JfPoS4D_6(*X z8krk<1iNQo6ES;KeeVYjp`ts-UT(>X5ko&5ahr zw&oRCoN*KvWqG<$XZt^szgz{|k_~n52n#hu1<-P*jlI_0c$tH0O=vh;x%9fp9f{mR zgq_CI*l7l~#3dXJaW|qRhT3Zz(rUKhL^it4a5e~d4@{@Z9v>DuesE-Jb9j3)(EV9a666kYAfF+HyOyk2h7Szz zX;RAa9Kka*x9(OlpEopG0hh_$tRHM>V~Q>SlcxFqhO|WbA8-jdr|qU zrKTmJt;Dt{fkz0hM_FZRnRy!_=c}J+G_^)Ihq{8^ZLz3VSyS1kWyL&2N_2HkMTund zGrM(U>W9Ox?wDM#(=boJ7thHqeI=}vHCK#EPy(@p>UUA07waqGi%=djHwYyVOQ?SG z=(g|mmFOn$6eclH0jiButtl{=GA0fiT z7>ri419#bdSO*>sOQK@@`+RtZ;g`|R_AZhrm=w=-QK9B|fu4R?-wHGZG( z$95x^aA)EvTTqnJr630-Y9A9CzwA#(FCmxkS$%TU!T8u!C_ar`f|VVXWVs{=TZkBxsJ(>7@BI&6M_+gn@Vp?hvZqYO0#Ks%5*oiZ4hHXe zv~ol7fjMvZ3G4r5S$uhxpIsHl+#z4T*-^I;J6?JD*!m z3eW+qgUa{u*!>Ub>d+qzbTQsEbU^E%@;x$k=9_Un=V`{ygv!?b8@BFeaXWF_Ci5KY zexDxL``yC-a^0`NPy1?(FJkW)bPX?NFv(@GAPg_G;id9T8NYg5R|nqO9qYlmp#xe6 zmG8>XC;N1DDDpYG#nA~J&^oAm7an}=oW2fn#!?FOK?lSRvwSPwCGJJ-fJK~EGsQXs zmPDd%9Y&r$@ML3No@@B9yOYOJJ2>6RNX{83+x&qLbf|eeX!uUPy>c~b2X|O