From ebe55146c3ccb8699b494b4f3f55b49f5468a6cb Mon Sep 17 00:00:00 2001 From: Lai Jiang Date: Tue, 11 Jan 2022 11:47:58 -0500 Subject: [PATCH] Add a command to compare two escrow deposits (#1476) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We already have ValidateEscrowDepositCommand to check for internal reference consistency of two deposits, i. e. making sure that all contacts and hosts referenced by domains exist in the same deposit. Therefore to compare whether two deposits are equal we only need to make sure that they contain the same domains and registrars, assuming they both pass the validation. We don't compare their contents directly because the MapReduce deposit contains all contacts and domains whereas the Beam deposit only contains referenced ones, making a direct comparison impossible. --- This change is [Reviewable](https://reviewable.io/reviews/google/nomulus/1476) --- core/build.gradle | 4 +- .../google/registry/tools/RegistryTool.java | 2 + .../registry/tools/RegistryToolComponent.java | 3 + .../CompareEscrowDepositsCommand.java | 130 ++++++++ .../registry/tools/CommandTestCase.java | 6 +- .../CompareEscrowDepositsCommandTest.java | 63 ++++ .../registry/rde/deposit_full_different.xml | 295 ++++++++++++++++++ .../rde/deposit_full_out_of_order.xml | 261 ++++++++++++++++ 8 files changed, 759 insertions(+), 5 deletions(-) create mode 100644 core/src/main/java/google/registry/tools/javascrap/CompareEscrowDepositsCommand.java create mode 100644 core/src/test/java/google/registry/tools/javascrap/CompareEscrowDepositsCommandTest.java create mode 100644 core/src/test/resources/google/registry/rde/deposit_full_different.xml create mode 100644 core/src/test/resources/google/registry/rde/deposit_full_out_of_order.xml diff --git a/core/build.gradle b/core/build.gradle index 5dd25e36e..2534beae1 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -676,9 +676,9 @@ Optional> getToolArgsList() { // To run the nomulus tools with these command line tokens: // "--foo", "bar baz", "--qux=quz" -// gradle registryTool --args="--foo 'bar baz' --qux=quz" +// gradle core:registryTool --args="--foo 'bar baz' --qux=quz" // or: -// gradle registryTool --PtoolArgs="--foo|bar baz|--qux=quz" +// gradle core:registryTool -PtoolArgs="--foo|bar baz|--qux=quz" // Note that the delimiting pipe can be backslash escaped if it is part of a // parameter. ext.createToolTask = { diff --git a/core/src/main/java/google/registry/tools/RegistryTool.java b/core/src/main/java/google/registry/tools/RegistryTool.java index 50ff73cbb..55802ba0d 100644 --- a/core/src/main/java/google/registry/tools/RegistryTool.java +++ b/core/src/main/java/google/registry/tools/RegistryTool.java @@ -17,6 +17,7 @@ package google.registry.tools; import com.google.common.collect.ImmutableMap; import google.registry.tools.javascrap.BackfillRegistryLocksCommand; import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand; +import google.registry.tools.javascrap.CompareEscrowDepositsCommand; import google.registry.tools.javascrap.DeleteContactByRoidCommand; import google.registry.tools.javascrap.HardDeleteHostCommand; import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand; @@ -40,6 +41,7 @@ public final class RegistryTool { .put("canonicalize_labels", CanonicalizeLabelsCommand.class) .put("check_domain", CheckDomainCommand.class) .put("check_domain_claims", CheckDomainClaimsCommand.class) + .put("compare_escrow_deposits", CompareEscrowDepositsCommand.class) .put("convert_idn", ConvertIdnCommand.class) .put("count_domains", CountDomainsCommand.class) .put("create_anchor_tenant", CreateAnchorTenantCommand.class) diff --git a/core/src/main/java/google/registry/tools/RegistryToolComponent.java b/core/src/main/java/google/registry/tools/RegistryToolComponent.java index 4c8463f3a..c075ca453 100644 --- a/core/src/main/java/google/registry/tools/RegistryToolComponent.java +++ b/core/src/main/java/google/registry/tools/RegistryToolComponent.java @@ -43,6 +43,7 @@ import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UserServiceModule; import google.registry.tools.AuthModule.LocalCredentialModule; import google.registry.tools.javascrap.BackfillRegistryLocksCommand; +import google.registry.tools.javascrap.CompareEscrowDepositsCommand; import google.registry.tools.javascrap.DeleteContactByRoidCommand; import google.registry.tools.javascrap.HardDeleteHostCommand; import google.registry.util.UtilsModule; @@ -95,6 +96,8 @@ interface RegistryToolComponent { void inject(CheckDomainCommand command); + void inject(CompareEscrowDepositsCommand command); + void inject(CountDomainsCommand command); void inject(CreateAnchorTenantCommand command); diff --git a/core/src/main/java/google/registry/tools/javascrap/CompareEscrowDepositsCommand.java b/core/src/main/java/google/registry/tools/javascrap/CompareEscrowDepositsCommand.java new file mode 100644 index 000000000..5fac88bdf --- /dev/null +++ b/core/src/main/java/google/registry/tools/javascrap/CompareEscrowDepositsCommand.java @@ -0,0 +1,130 @@ +// Copyright 2022 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.javascrap; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.Sets.difference; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import google.registry.keyring.api.Keyring; +import google.registry.model.annotations.DeleteAfterMigration; +import google.registry.rde.Ghostryde; +import google.registry.tools.Command; +import google.registry.tools.params.PathParameter; +import google.registry.xjc.XjcXmlTransformer; +import google.registry.xjc.rde.XjcRdeDeposit; +import google.registry.xjc.rdedomain.XjcRdeDomain; +import google.registry.xjc.rderegistrar.XjcRdeRegistrar; +import google.registry.xml.XmlException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.inject.Inject; +import javax.inject.Provider; +import javax.xml.bind.JAXBElement; + +/** + * Command to view and schema validate an XML RDE escrow deposit. + * + *

Note that this command only makes sure that both deposits contain the same registrars and + * domains, regardless of the order. To verify that they are indeed equivalent one still needs to + * verify internal consistency within each deposit (i.e. to check that all hosts and contacts + * referenced by domains are included in the deposit) by calling {@code + * google.registry.tools.ValidateEscrowDepositCommand}. + */ +@DeleteAfterMigration +@Parameters(separators = " =", commandDescription = "Compare two XML escrow deposits.") +public final class CompareEscrowDepositsCommand implements Command { + + @Parameter( + description = + "Two XML escrow deposit files. Each may be a plain XML or an XML GhostRyDE file.", + validateWith = PathParameter.InputFile.class) + private List inputs; + + @Inject Provider keyring; + + private XjcRdeDeposit getDeposit(Path input) throws IOException, XmlException { + InputStream fileStream = Files.newInputStream(input); + InputStream inputStream = fileStream; + if (input.toString().endsWith(".ghostryde")) { + inputStream = Ghostryde.decoder(fileStream, keyring.get().getRdeStagingDecryptionKey()); + } + return XjcXmlTransformer.unmarshal(XjcRdeDeposit.class, inputStream); + } + + @Override + public void run() throws Exception { + checkArgument( + inputs.size() == 2, + "Must supply 2 files to compare, but %s was/were supplied.", + inputs.size()); + XjcRdeDeposit deposit1 = getDeposit(inputs.get(0)); + XjcRdeDeposit deposit2 = getDeposit(inputs.get(1)); + compareXmlDeposits(deposit1, deposit2); + } + + private static void process(XjcRdeDeposit deposit, Set domains, Set registrars) { + for (JAXBElement item : deposit.getContents().getContents()) { + if (XjcRdeDomain.class.isAssignableFrom(item.getDeclaredType())) { + XjcRdeDomain domain = (XjcRdeDomain) item.getValue(); + domains.add(checkNotNull(domain.getName())); + } else if (XjcRdeRegistrar.class.isAssignableFrom(item.getDeclaredType())) { + XjcRdeRegistrar registrar = (XjcRdeRegistrar) item.getValue(); + registrars.add(checkNotNull(registrar.getId())); + } + } + } + + private static boolean printUniqueElements( + Set set1, Set set2, String element, String deposit) { + ImmutableList uniqueElements = ImmutableList.copyOf(difference(set1, set2)); + if (!uniqueElements.isEmpty()) { + System.out.printf( + "%s only in %s:\n%s\n", element, deposit, Joiner.on("\n").join(uniqueElements)); + return false; + } + return true; + } + + private static void compareXmlDeposits(XjcRdeDeposit deposit1, XjcRdeDeposit deposit2) { + Set domains1 = new HashSet<>(); + Set domains2 = new HashSet<>(); + Set registrars1 = new HashSet<>(); + Set registrars2 = new HashSet<>(); + process(deposit1, domains1, registrars1); + process(deposit2, domains2, registrars2); + boolean good = true; + good &= printUniqueElements(domains1, domains2, "domains", "deposit1"); + good &= printUniqueElements(domains2, domains1, "domains", "deposit2"); + good &= printUniqueElements(registrars1, registrars2, "registrars", "deposit1"); + good &= printUniqueElements(registrars2, registrars1, "registrars", "deposit2"); + if (good) { + System.out.println( + "The two deposits contain the same domains and registrars. " + + "You still need to run validate_escrow_deposit to check reference consistency."); + } else { + System.out.println("The two deposits differ."); + } + } +} diff --git a/core/src/test/java/google/registry/tools/CommandTestCase.java b/core/src/test/java/google/registry/tools/CommandTestCase.java index 5f404119f..958d252a4 100644 --- a/core/src/test/java/google/registry/tools/CommandTestCase.java +++ b/core/src/test/java/google/registry/tools/CommandTestCase.java @@ -134,7 +134,7 @@ public abstract class CommandTestCase { } /** Writes the data to a named temporary file and then returns a path to the file. */ - private String writeToNamedTmpFile(String filename, byte[] data) throws IOException { + protected String writeToNamedTmpFile(String filename, byte[] data) throws IOException { Path tmpFile = tmpDir.resolve(filename); Files.write(data, tmpFile.toFile()); return tmpFile.toString(); @@ -151,7 +151,7 @@ public abstract class CommandTestCase { } /** Writes the data to a temporary file and then returns a path to the file. */ - String writeToTmpFile(byte[] data) throws IOException { + public String writeToTmpFile(byte[] data) throws IOException { return writeToNamedTmpFile("tmp_file", data); } @@ -220,7 +220,7 @@ public abstract class CommandTestCase { assertThat(getStderrAsString()).doesNotContain(expected); } - String getStdoutAsString() { + protected String getStdoutAsString() { return new String(stdout.toByteArray(), UTF_8); } diff --git a/core/src/test/java/google/registry/tools/javascrap/CompareEscrowDepositsCommandTest.java b/core/src/test/java/google/registry/tools/javascrap/CompareEscrowDepositsCommandTest.java new file mode 100644 index 000000000..b1706453a --- /dev/null +++ b/core/src/test/java/google/registry/tools/javascrap/CompareEscrowDepositsCommandTest.java @@ -0,0 +1,63 @@ +// Copyright 2021 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.javascrap; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import google.registry.rde.RdeTestData; +import google.registry.tools.CommandTestCase; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; + +/** Unit tests for {@link CompareEscrowDepositsCommand}. */ +class CompareEscrowDepositsCommandTest extends CommandTestCase { + + @Test + void testFailure_wrongNumberOfFiles() throws Exception { + String file1 = writeToNamedTmpFile("file1", "foo".getBytes(StandardCharsets.UTF_8)); + String file2 = writeToNamedTmpFile("file2", "bar".getBytes(StandardCharsets.UTF_8)); + String file3 = writeToNamedTmpFile("file3", "baz".getBytes(StandardCharsets.UTF_8)); + assertThrows(IllegalArgumentException.class, () -> runCommand(file1)); + assertThrows(IllegalArgumentException.class, () -> runCommand(file1, file2, file3)); + } + + @Test + void testSuccess_sameContentDifferentOrder() throws Exception { + String file1 = writeToNamedTmpFile("file1", RdeTestData.loadBytes("deposit_full.xml").read()); + String file2 = + writeToNamedTmpFile("file2", RdeTestData.loadBytes("deposit_full_out_of_order.xml").read()); + runCommand(file1, file2); + assertThat(getStdoutAsString()) + .contains("The two deposits contain the same domains and registrars."); + } + + @Test + void testSuccess_differentContent() throws Exception { + String file1 = writeToNamedTmpFile("file1", RdeTestData.loadBytes("deposit_full.xml").read()); + String file2 = + writeToNamedTmpFile("file2", RdeTestData.loadBytes("deposit_full_different.xml").read()); + runCommand(file1, file2); + assertThat(getStdoutAsString()) + .isEqualTo( + "domains only in deposit1:\n" + + "example2.test\n" + + "domains only in deposit2:\n" + + "example3.test\n" + + "registrars only in deposit2:\n" + + "RegistrarY\n" + + "The two deposits differ.\n"); + } +} diff --git a/core/src/test/resources/google/registry/rde/deposit_full_different.xml b/core/src/test/resources/google/registry/rde/deposit_full_different.xml new file mode 100644 index 000000000..c426f5410 --- /dev/null +++ b/core/src/test/resources/google/registry/rde/deposit_full_different.xml @@ -0,0 +1,295 @@ + + + + 2010-10-17T00:00:00Z + + 1.0 + urn:ietf:params:xml:ns:rdeHeader-1.0 + urn:ietf:params:xml:ns:rdeContact-1.0 + urn:ietf:params:xml:ns:rdeHost-1.0 + urn:ietf:params:xml:ns:rdeDomain-1.0 + urn:ietf:params:xml:ns:rdeRegistrar-1.0 + urn:ietf:params:xml:ns:rdeIDN-1.0 + urn:ietf:params:xml:ns:rdeNNDN-1.0 + urn:ietf:params:xml:ns:rdeEppParams-1.0 + + + + + + + test + 2 + + 1 + + 1 + + 1 + + 1 + + 1 + + 1 + + + + + + example1.test + Dexample1-TEST + + jd1234 + sh8013 + sh8013 + + ns1.example.com + ns1.example1.test + + RegistrarX + RegistrarX + 1999-04-03T22:00:00.0Z + 2015-04-03T22:00:00.0Z + + + + + example3.test + Dexample3-TEST + + + jd1234 + sh8013 + sh8013 + RegistrarY + RegistrarY + 1999-04-03T22:00:00.0Z + 2015-04-03T22:00:00.0Z + + + + + ns1.example.com + Hns1_example_com-TEST + + + 192.0.2.2 + 192.0.2.29 + 1080:0:0:0:8:800:200C:417A + + RegistrarX + RegistrarX + 1999-05-08T12:10:00.0Z + RegistrarX + 2009-10-03T09:34:00.0Z + + + + + ns1.example1.test + Hns1_example1_test-TEST + + + 192.0.2.2 + 192.0.2.29 + 1080:0:0:0:8:800:200C:417A + + RegistrarX + RegistrarX + 1999-05-08T12:10:00.0Z + RegistrarX + 2009-10-03T09:34:00.0Z + + + + + sh8013 + Csh8013-TEST + + + + John Doe + Example Inc. + + 123 Example Dr. + Suite 100 + Dulles + VA + 20166-6503 + US + + + +1.7035555555 + + +1.7035555556 + + jdoe@example.test + + RegistrarX + RegistrarX + + 2009-09-13T08:01:00.0Z + RegistrarX + + 2009-11-26T09:10:00.0Z + 2009-12-03T09:05:00.0Z + + + + + + + + + RegistrarX + Registrar X + 123 + ok + + + 123 Example Dr. + + Suite 100 + + Dulles + VA + 20166-6503 + US + + + +1.7035555555 + + +1.7035555556 + + jdoe@example.test + + http://www.example.test + + + whois.example.test + + http://whois.example.test + + + 2005-04-23T11:49:00.0Z + 2009-02-17T17:51:00.0Z + + + + + RegistrarY + Registrar Y + 1234 + ok + + + 123 Example Dr. + + Suite 100 + + Dulles + VA + 20166-6503 + US + + + +1.7035555555 + + +1.7035555556 + + jdoe@example.test + + http://www.example.test + + + whois.example.test + + http://whois.example.test + + + 2005-04-23T11:49:00.0Z + 2009-02-17T17:51:00.0Z + + + + + +http://www.iana.org/domains/idn-tables/tables/br_pt-br_1.0.html + + + http://registro.br/dominio/regras.html + + + + + + xn--exampl-gva.test + pt-BR + example1.test + withheld + 2005-04-23T11:49:00.0Z + + + + + 1.0 + en + + urn:ietf:params:xml:ns:domain-1.0 + + + urn:ietf:params:xml:ns:contact-1.0 + + + urn:ietf:params:xml:ns:host-1.0 + + + urn:ietf:params:xml:ns:rgp-1.0 + + urn:ietf:params:xml:ns:secDNS-1.1 + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/test/resources/google/registry/rde/deposit_full_out_of_order.xml b/core/src/test/resources/google/registry/rde/deposit_full_out_of_order.xml new file mode 100644 index 000000000..fad3c36a0 --- /dev/null +++ b/core/src/test/resources/google/registry/rde/deposit_full_out_of_order.xml @@ -0,0 +1,261 @@ + + + + 2010-10-17T00:00:00Z + + 1.0 + urn:ietf:params:xml:ns:rdeHeader-1.0 + urn:ietf:params:xml:ns:rdeContact-1.0 + urn:ietf:params:xml:ns:rdeHost-1.0 + urn:ietf:params:xml:ns:rdeDomain-1.0 + urn:ietf:params:xml:ns:rdeRegistrar-1.0 + urn:ietf:params:xml:ns:rdeIDN-1.0 + urn:ietf:params:xml:ns:rdeNNDN-1.0 + urn:ietf:params:xml:ns:rdeEppParams-1.0 + + + + + + + test + 2 + + 1 + + 1 + + 1 + + 1 + + 1 + + 1 + + + + + + example2.test + Dexample2-TEST + + + jd1234 + sh8013 + sh8013 + RegistrarX + RegistrarX + 1999-04-03T22:00:00.0Z + 2015-04-03T22:00:00.0Z + + + + + example1.test + Dexample1-TEST + + jd1234 + sh8013 + sh8013 + + ns1.example.com + ns1.example1.test + + RegistrarX + RegistrarX + 1999-04-03T22:00:00.0Z + 2015-04-03T22:00:00.0Z + + + + + + + ns1.example.com + Hns1_example_com-TEST + + + 192.0.2.2 + 192.0.2.29 + 1080:0:0:0:8:800:200C:417A + + RegistrarX + RegistrarX + 1999-05-08T12:10:00.0Z + RegistrarX + 2009-10-03T09:34:00.0Z + + + + + ns1.example1.test + Hns1_example1_test-TEST + + + 192.0.2.2 + 192.0.2.29 + 1080:0:0:0:8:800:200C:417A + + RegistrarX + RegistrarX + 1999-05-08T12:10:00.0Z + RegistrarX + 2009-10-03T09:34:00.0Z + + + + + sh8013 + Csh8013-TEST + + + + John Doe + Example Inc. + + 123 Example Dr. + Suite 100 + Dulles + VA + 20166-6503 + US + + + +1.7035555555 + + +1.7035555556 + + jdoe@example.test + + RegistrarX + RegistrarX + + 2009-09-13T08:01:00.0Z + RegistrarX + + 2009-11-26T09:10:00.0Z + 2009-12-03T09:05:00.0Z + + + + + + + + + RegistrarX + Registrar X + 123 + ok + + + 123 Example Dr. + + Suite 100 + + Dulles + VA + 20166-6503 + US + + + +1.7035555555 + + +1.7035555556 + + jdoe@example.test + + http://www.example.test + + + whois.example.test + + http://whois.example.test + + + 2005-04-23T11:49:00.0Z + 2009-02-17T17:51:00.0Z + + + + + +http://www.iana.org/domains/idn-tables/tables/br_pt-br_1.0.html + + + http://registro.br/dominio/regras.html + + + + + + xn--exampl-gva.test + pt-BR + example1.test + withheld + 2005-04-23T11:49:00.0Z + + + + + 1.0 + en + + urn:ietf:params:xml:ns:domain-1.0 + + + urn:ietf:params:xml:ns:contact-1.0 + + + urn:ietf:params:xml:ns:host-1.0 + + + urn:ietf:params:xml:ns:rgp-1.0 + + urn:ietf:params:xml:ns:secDNS-1.1 + + + + + + + + + + + + + + + + + + + + + +