diff --git a/java/google/registry/tools/BUILD b/java/google/registry/tools/BUILD index b8c343411..469713d49 100644 --- a/java/google/registry/tools/BUILD +++ b/java/google/registry/tools/BUILD @@ -13,6 +13,7 @@ package_group( "//java/google/registry/eclipse", "//java/google/registry/testing", "//java/google/registry/tools", + "//javatests/google/registry/testing", "//javatests/google/registry/tools", ], ) @@ -100,3 +101,18 @@ java_binary( "@com_google_appengine_remote_api//:link", ], ) + +java_binary( + name = "compare_db_backups", + srcs = [ + "CompareDbBackups.java", + ], + create_executable = 1, + main_class = "google.registry.tools.CompareDbBackups", + deps = [ + ":tools", + "@com_google_appengine_api_1_0_sdk", + "@com_google_guava", + "@com_google_protobuf_java", + ], +) diff --git a/java/google/registry/tools/CompareDbBackups.java b/java/google/registry/tools/CompareDbBackups.java new file mode 100644 index 000000000..ed5e5318b --- /dev/null +++ b/java/google/registry/tools/CompareDbBackups.java @@ -0,0 +1,67 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.collect.Sets.SetView; +import java.io.File; + +/** Compare two database backups. */ +class CompareDbBackups { + + public static void main(String[] args) throws Exception { + if (args.length != 2) { + System.err.println("Usage: compare_db_backups "); + return; + } + + ImmutableSet entities1 = + new RecordAccumulator().readDirectory(new File(args[0])).getComparableEntitySet(); + ImmutableSet entities2 = + new RecordAccumulator().readDirectory(new File(args[1])).getComparableEntitySet(); + + // Calculate the entities added and removed. + SetView added = Sets.difference(entities2, entities1); + SetView removed = Sets.difference(entities1, entities2); + + printHeader( + String.format("First backup: %d records", entities1.size()), + String.format("Second backup: %d records", entities2.size())); + + if (!removed.isEmpty()) { + printHeader(removed.size() + " records were removed:"); + for (ComparableEntity entity : removed) { + System.out.println(entity); + } + } + + if (!added.isEmpty()) { + printHeader(added.size() + " records were added:"); + for (ComparableEntity entity : added) { + System.out.println(entity); + } + } + } + + /** Print out multi-line text in a pretty ASCII header frame. */ + private static void printHeader(String... headerLines) { + System.out.println("========================================================================"); + for (String line : headerLines) { + System.out.println("| " + line); + } + System.out.println("========================================================================"); + } +} diff --git a/javatests/google/registry/tools/CompareDbBackupsTest.java b/javatests/google/registry/tools/CompareDbBackupsTest.java new file mode 100644 index 000000000..37d9313e8 --- /dev/null +++ b/javatests/google/registry/tools/CompareDbBackupsTest.java @@ -0,0 +1,80 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import google.registry.testing.AppEngineRule; +import google.registry.tools.LevelDbFileBuilder.Property; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintStream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class CompareDbBackupsTest { + + private static final int BASE_ID = 1001; + + // Capture standard output. + private final ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + + @Rule public final TemporaryFolder tempFs = new TemporaryFolder(); + @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build(); + + @Test + public void testCommand() throws Exception { + + // Create two directories corresponding to data dumps. + File dump1 = tempFs.newFolder("dump1"); + LevelDbFileBuilder builder = new LevelDbFileBuilder(new File(dump1, "data1")); + builder.addEntityProto( + BASE_ID, + Property.create("eeny", 100L), + Property.create("meeny", 200L), + Property.create("miney", 300L)); + builder.addEntityProto( + BASE_ID + 1, + Property.create("moxey", 100L), + Property.create("minney", 200L), + Property.create("motz", 300L)); + builder.build(); + + File dump2 = tempFs.newFolder("dump2"); + builder = new LevelDbFileBuilder(new File(dump2, "data2")); + builder.addEntityProto( + BASE_ID + 1, + Property.create("moxey", 100L), + Property.create("minney", 200L), + Property.create("motz", 300L)); + builder.addEntityProto( + BASE_ID + 2, + Property.create("blutzy", 100L), + Property.create("fishey", 200L), + Property.create("strutz", 300L)); + builder.build(); + + System.setOut(new PrintStream(stdout)); + CompareDbBackups.main(new String[] {dump1.getCanonicalPath(), dump2.getCanonicalPath()}); + String output = new String(stdout.toByteArray(), UTF_8); + assertThat(output) + .containsMatch("(?s)1 records were removed.*eeny.*1 records were added.*blutzy"); + } +} diff --git a/javatests/google/registry/tools/LevelDbFileBuilder.java b/javatests/google/registry/tools/LevelDbFileBuilder.java new file mode 100644 index 000000000..e7837e9dc --- /dev/null +++ b/javatests/google/registry/tools/LevelDbFileBuilder.java @@ -0,0 +1,83 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import static google.registry.tools.LevelDbLogReader.BLOCK_SIZE; +import static google.registry.tools.LevelDbLogReader.HEADER_SIZE; + +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.EntityTranslator; +import com.google.auto.value.AutoValue; +import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; +import google.registry.tools.LevelDbLogReader.ChunkType; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; + +/** Utility class for building a leveldb logfile. */ +final class LevelDbFileBuilder { + private static final String TEST_ENTITY_KIND = "TestEntity"; + + private final FileOutputStream out; + private byte[] currentBlock = new byte[BLOCK_SIZE]; + + // Write position in the current block. + private int currentPos = 0; + + LevelDbFileBuilder(File file) throws FileNotFoundException { + out = new FileOutputStream(file); + } + + /** + * Adds a record containing a new entity protobuf to the file. + * + *

Returns the ComparableEntity object rather than "this" so that we can check for the presence + * of the entity in the result set. + */ + ComparableEntity addEntityProto(int id, Property... properties) throws IOException { + Entity entity = new Entity(TEST_ENTITY_KIND, id); + for (Property prop : properties) { + entity.setProperty(prop.name(), prop.value()); + } + EntityProto proto = EntityTranslator.convertToPb(entity); + byte[] protoBytes = proto.toByteArray(); + if (protoBytes.length > BLOCK_SIZE - (currentPos + HEADER_SIZE)) { + out.write(currentBlock); + currentBlock = new byte[BLOCK_SIZE]; + currentPos = 0; + } + + currentPos = LevelDbUtil.addRecord(currentBlock, currentPos, ChunkType.FULL, protoBytes); + return new ComparableEntity(entity); + } + + /** Writes all remaining data and closes the block. */ + void build() throws IOException { + out.write(currentBlock); + out.close(); + } + + @AutoValue + abstract static class Property { + static Property create(String name, Object value) { + return new AutoValue_LevelDbFileBuilder_Property(name, value); + } + + abstract String name(); + + abstract Object value(); + } +} diff --git a/javatests/google/registry/tools/LevelDbFileBuilderTest.java b/javatests/google/registry/tools/LevelDbFileBuilderTest.java new file mode 100644 index 000000000..afd5ce151 --- /dev/null +++ b/javatests/google/registry/tools/LevelDbFileBuilderTest.java @@ -0,0 +1,98 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.EntityTranslator; +import com.google.common.collect.ImmutableList; +import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; +import google.registry.testing.AppEngineRule; +import google.registry.tools.LevelDbFileBuilder.Property; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class LevelDbFileBuilderTest { + + public static final int BASE_ID = 1001; + + @Rule public final TemporaryFolder tempFs = new TemporaryFolder(); + @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build(); + + @Test + public void testSingleRecordWrites() throws FileNotFoundException, IOException { + File subdir = tempFs.newFolder("folder"); + File logFile = new File(subdir, "testfile"); + LevelDbFileBuilder builder = new LevelDbFileBuilder(logFile); + ComparableEntity entity = + builder.addEntityProto( + BASE_ID, Property.create("first", 100L), Property.create("second", 200L)); + builder.build(); + + LevelDbLogReader reader = new LevelDbLogReader(); + reader.readFrom(new FileInputStream(logFile)); + + ImmutableList records = reader.getRecords(); + assertThat(records).hasSize(1); + + // Reconstitute an entity, make sure that what we've got is the same as what we started with. + EntityProto proto = new EntityProto(); + proto.parseFrom(records.get(0)); + Entity materializedEntity = EntityTranslator.createFromPb(proto); + assertThat(new ComparableEntity(materializedEntity)).isEqualTo(entity); + } + + @Test + public void testMultipleRecordWrites() throws FileNotFoundException, IOException { + File subdir = tempFs.newFolder("folder"); + File logFile = new File(subdir, "testfile"); + LevelDbFileBuilder builder = new LevelDbFileBuilder(logFile); + + // Generate enough records to cross a block boundary. These records end up being around 80 + // bytes, so 1000 works. + ImmutableList.Builder originalEntitiesBuilder = new ImmutableList.Builder<>(); + for (int i = 0; i < 1000; ++i) { + ComparableEntity entity = + builder.addEntityProto( + BASE_ID + i, Property.create("first", 100L), Property.create("second", 200L)); + originalEntitiesBuilder.add(entity); + } + builder.build(); + ImmutableList originalEntities = originalEntitiesBuilder.build(); + + LevelDbLogReader reader = new LevelDbLogReader(); + reader.readFrom(new FileInputStream(logFile)); + + ImmutableList records = reader.getRecords(); + assertThat(records).hasSize(1000); + int index = 0; + for (byte[] record : records) { + EntityProto proto = new EntityProto(); + proto.parseFrom(record); + Entity materializedEntity = EntityTranslator.createFromPb(proto); + assertThat(new ComparableEntity(materializedEntity)).isEqualTo(originalEntities.get(index)); + ++index; + } + } +} diff --git a/javatests/google/registry/tools/RecordAccumulatorTest.java b/javatests/google/registry/tools/RecordAccumulatorTest.java index 69c2172fe..ea15e3742 100644 --- a/javatests/google/registry/tools/RecordAccumulatorTest.java +++ b/javatests/google/registry/tools/RecordAccumulatorTest.java @@ -15,18 +15,11 @@ package google.registry.tools; import static com.google.common.truth.Truth.assertThat; -import static google.registry.tools.LevelDbLogReader.BLOCK_SIZE; -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityTranslator; -import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableSet; -import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; import google.registry.testing.AppEngineRule; -import google.registry.tools.LevelDbLogReader.ChunkType; +import google.registry.tools.LevelDbFileBuilder.Property; import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import org.junit.Rule; import org.junit.Test; @@ -38,7 +31,6 @@ import org.junit.runners.JUnit4; public class RecordAccumulatorTest { private static final int BASE_ID = 1001; - private static final String TEST_ENTITY_KIND = "TestEntity"; @Rule public final TemporaryFolder tempFs = new TemporaryFolder(); @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build(); @@ -85,56 +77,4 @@ public class RecordAccumulatorTest { new RecordAccumulator().readDirectory(subdir).getComparableEntitySet(); assertThat(entities).containsExactly(e1, e2, e3); } - - /** Utility class for building a leveldb logfile. */ - private static final class LevelDbFileBuilder { - private final FileOutputStream out; - private byte[] currentBlock = new byte[BLOCK_SIZE]; - - // Write position in the current block. - private int currentPos = 0; - - LevelDbFileBuilder(File file) throws FileNotFoundException { - out = new FileOutputStream(file); - } - - /** - * Adds a record containing a new entity protobuf to the file. - * - *

Returns the ComparableEntity object rather than "this" so that we can check for the - * presence of the entity in the result set. - */ - private ComparableEntity addEntityProto(int id, Property... properties) throws IOException { - Entity entity = new Entity(TEST_ENTITY_KIND, id); - for (Property prop : properties) { - entity.setProperty(prop.name(), prop.value()); - } - EntityProto proto = EntityTranslator.convertToPb(entity); - byte[] protoBytes = proto.toByteArray(); - if (protoBytes.length > BLOCK_SIZE - currentPos) { - out.write(currentBlock); - currentBlock = new byte[BLOCK_SIZE]; - } - - currentPos = LevelDbUtil.addRecord(currentBlock, currentPos, ChunkType.FULL, protoBytes); - return new ComparableEntity(entity); - } - - /** Writes all remaining data and closes the block. */ - void build() throws IOException { - out.write(currentBlock); - out.close(); - } - } - - @AutoValue - abstract static class Property { - static Property create(String name, Object value) { - return new AutoValue_RecordAccumulatorTest_Property(name, value); - } - - abstract String name(); - - abstract Object value(); - } }