diff --git a/core/build.gradle b/core/build.gradle index 5dd25e36e..2534beae1 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -676,9 +676,9 @@ Optional> getToolArgsList() { // To run the nomulus tools with these command line tokens: // "--foo", "bar baz", "--qux=quz" -// gradle registryTool --args="--foo 'bar baz' --qux=quz" +// gradle core:registryTool --args="--foo 'bar baz' --qux=quz" // or: -// gradle registryTool --PtoolArgs="--foo|bar baz|--qux=quz" +// gradle core:registryTool -PtoolArgs="--foo|bar baz|--qux=quz" // Note that the delimiting pipe can be backslash escaped if it is part of a // parameter. ext.createToolTask = { diff --git a/core/src/main/java/google/registry/tools/RegistryTool.java b/core/src/main/java/google/registry/tools/RegistryTool.java index 50ff73cbb..55802ba0d 100644 --- a/core/src/main/java/google/registry/tools/RegistryTool.java +++ b/core/src/main/java/google/registry/tools/RegistryTool.java @@ -17,6 +17,7 @@ package google.registry.tools; import com.google.common.collect.ImmutableMap; import google.registry.tools.javascrap.BackfillRegistryLocksCommand; import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand; +import google.registry.tools.javascrap.CompareEscrowDepositsCommand; import google.registry.tools.javascrap.DeleteContactByRoidCommand; import google.registry.tools.javascrap.HardDeleteHostCommand; import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand; @@ -40,6 +41,7 @@ public final class RegistryTool { .put("canonicalize_labels", CanonicalizeLabelsCommand.class) .put("check_domain", CheckDomainCommand.class) .put("check_domain_claims", CheckDomainClaimsCommand.class) + .put("compare_escrow_deposits", CompareEscrowDepositsCommand.class) .put("convert_idn", ConvertIdnCommand.class) .put("count_domains", CountDomainsCommand.class) .put("create_anchor_tenant", CreateAnchorTenantCommand.class) diff --git a/core/src/main/java/google/registry/tools/RegistryToolComponent.java b/core/src/main/java/google/registry/tools/RegistryToolComponent.java index 4c8463f3a..c075ca453 100644 --- a/core/src/main/java/google/registry/tools/RegistryToolComponent.java +++ b/core/src/main/java/google/registry/tools/RegistryToolComponent.java @@ -43,6 +43,7 @@ import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UserServiceModule; import google.registry.tools.AuthModule.LocalCredentialModule; import google.registry.tools.javascrap.BackfillRegistryLocksCommand; +import google.registry.tools.javascrap.CompareEscrowDepositsCommand; import google.registry.tools.javascrap.DeleteContactByRoidCommand; import google.registry.tools.javascrap.HardDeleteHostCommand; import google.registry.util.UtilsModule; @@ -95,6 +96,8 @@ interface RegistryToolComponent { void inject(CheckDomainCommand command); + void inject(CompareEscrowDepositsCommand command); + void inject(CountDomainsCommand command); void inject(CreateAnchorTenantCommand command); diff --git a/core/src/main/java/google/registry/tools/javascrap/CompareEscrowDepositsCommand.java b/core/src/main/java/google/registry/tools/javascrap/CompareEscrowDepositsCommand.java new file mode 100644 index 000000000..5fac88bdf --- /dev/null +++ b/core/src/main/java/google/registry/tools/javascrap/CompareEscrowDepositsCommand.java @@ -0,0 +1,130 @@ +// Copyright 2022 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.javascrap; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.Sets.difference; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import google.registry.keyring.api.Keyring; +import google.registry.model.annotations.DeleteAfterMigration; +import google.registry.rde.Ghostryde; +import google.registry.tools.Command; +import google.registry.tools.params.PathParameter; +import google.registry.xjc.XjcXmlTransformer; +import google.registry.xjc.rde.XjcRdeDeposit; +import google.registry.xjc.rdedomain.XjcRdeDomain; +import google.registry.xjc.rderegistrar.XjcRdeRegistrar; +import google.registry.xml.XmlException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.inject.Inject; +import javax.inject.Provider; +import javax.xml.bind.JAXBElement; + +/** + * Command to view and schema validate an XML RDE escrow deposit. + * + *

Note that this command only makes sure that both deposits contain the same registrars and + * domains, regardless of the order. To verify that they are indeed equivalent one still needs to + * verify internal consistency within each deposit (i.e. to check that all hosts and contacts + * referenced by domains are included in the deposit) by calling {@code + * google.registry.tools.ValidateEscrowDepositCommand}. + */ +@DeleteAfterMigration +@Parameters(separators = " =", commandDescription = "Compare two XML escrow deposits.") +public final class CompareEscrowDepositsCommand implements Command { + + @Parameter( + description = + "Two XML escrow deposit files. Each may be a plain XML or an XML GhostRyDE file.", + validateWith = PathParameter.InputFile.class) + private List inputs; + + @Inject Provider keyring; + + private XjcRdeDeposit getDeposit(Path input) throws IOException, XmlException { + InputStream fileStream = Files.newInputStream(input); + InputStream inputStream = fileStream; + if (input.toString().endsWith(".ghostryde")) { + inputStream = Ghostryde.decoder(fileStream, keyring.get().getRdeStagingDecryptionKey()); + } + return XjcXmlTransformer.unmarshal(XjcRdeDeposit.class, inputStream); + } + + @Override + public void run() throws Exception { + checkArgument( + inputs.size() == 2, + "Must supply 2 files to compare, but %s was/were supplied.", + inputs.size()); + XjcRdeDeposit deposit1 = getDeposit(inputs.get(0)); + XjcRdeDeposit deposit2 = getDeposit(inputs.get(1)); + compareXmlDeposits(deposit1, deposit2); + } + + private static void process(XjcRdeDeposit deposit, Set domains, Set registrars) { + for (JAXBElement item : deposit.getContents().getContents()) { + if (XjcRdeDomain.class.isAssignableFrom(item.getDeclaredType())) { + XjcRdeDomain domain = (XjcRdeDomain) item.getValue(); + domains.add(checkNotNull(domain.getName())); + } else if (XjcRdeRegistrar.class.isAssignableFrom(item.getDeclaredType())) { + XjcRdeRegistrar registrar = (XjcRdeRegistrar) item.getValue(); + registrars.add(checkNotNull(registrar.getId())); + } + } + } + + private static boolean printUniqueElements( + Set set1, Set set2, String element, String deposit) { + ImmutableList uniqueElements = ImmutableList.copyOf(difference(set1, set2)); + if (!uniqueElements.isEmpty()) { + System.out.printf( + "%s only in %s:\n%s\n", element, deposit, Joiner.on("\n").join(uniqueElements)); + return false; + } + return true; + } + + private static void compareXmlDeposits(XjcRdeDeposit deposit1, XjcRdeDeposit deposit2) { + Set domains1 = new HashSet<>(); + Set domains2 = new HashSet<>(); + Set registrars1 = new HashSet<>(); + Set registrars2 = new HashSet<>(); + process(deposit1, domains1, registrars1); + process(deposit2, domains2, registrars2); + boolean good = true; + good &= printUniqueElements(domains1, domains2, "domains", "deposit1"); + good &= printUniqueElements(domains2, domains1, "domains", "deposit2"); + good &= printUniqueElements(registrars1, registrars2, "registrars", "deposit1"); + good &= printUniqueElements(registrars2, registrars1, "registrars", "deposit2"); + if (good) { + System.out.println( + "The two deposits contain the same domains and registrars. " + + "You still need to run validate_escrow_deposit to check reference consistency."); + } else { + System.out.println("The two deposits differ."); + } + } +} diff --git a/core/src/test/java/google/registry/tools/CommandTestCase.java b/core/src/test/java/google/registry/tools/CommandTestCase.java index 5f404119f..958d252a4 100644 --- a/core/src/test/java/google/registry/tools/CommandTestCase.java +++ b/core/src/test/java/google/registry/tools/CommandTestCase.java @@ -134,7 +134,7 @@ public abstract class CommandTestCase { } /** Writes the data to a named temporary file and then returns a path to the file. */ - private String writeToNamedTmpFile(String filename, byte[] data) throws IOException { + protected String writeToNamedTmpFile(String filename, byte[] data) throws IOException { Path tmpFile = tmpDir.resolve(filename); Files.write(data, tmpFile.toFile()); return tmpFile.toString(); @@ -151,7 +151,7 @@ public abstract class CommandTestCase { } /** Writes the data to a temporary file and then returns a path to the file. */ - String writeToTmpFile(byte[] data) throws IOException { + public String writeToTmpFile(byte[] data) throws IOException { return writeToNamedTmpFile("tmp_file", data); } @@ -220,7 +220,7 @@ public abstract class CommandTestCase { assertThat(getStderrAsString()).doesNotContain(expected); } - String getStdoutAsString() { + protected String getStdoutAsString() { return new String(stdout.toByteArray(), UTF_8); } diff --git a/core/src/test/java/google/registry/tools/javascrap/CompareEscrowDepositsCommandTest.java b/core/src/test/java/google/registry/tools/javascrap/CompareEscrowDepositsCommandTest.java new file mode 100644 index 000000000..b1706453a --- /dev/null +++ b/core/src/test/java/google/registry/tools/javascrap/CompareEscrowDepositsCommandTest.java @@ -0,0 +1,63 @@ +// Copyright 2021 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.javascrap; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import google.registry.rde.RdeTestData; +import google.registry.tools.CommandTestCase; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; + +/** Unit tests for {@link CompareEscrowDepositsCommand}. */ +class CompareEscrowDepositsCommandTest extends CommandTestCase { + + @Test + void testFailure_wrongNumberOfFiles() throws Exception { + String file1 = writeToNamedTmpFile("file1", "foo".getBytes(StandardCharsets.UTF_8)); + String file2 = writeToNamedTmpFile("file2", "bar".getBytes(StandardCharsets.UTF_8)); + String file3 = writeToNamedTmpFile("file3", "baz".getBytes(StandardCharsets.UTF_8)); + assertThrows(IllegalArgumentException.class, () -> runCommand(file1)); + assertThrows(IllegalArgumentException.class, () -> runCommand(file1, file2, file3)); + } + + @Test + void testSuccess_sameContentDifferentOrder() throws Exception { + String file1 = writeToNamedTmpFile("file1", RdeTestData.loadBytes("deposit_full.xml").read()); + String file2 = + writeToNamedTmpFile("file2", RdeTestData.loadBytes("deposit_full_out_of_order.xml").read()); + runCommand(file1, file2); + assertThat(getStdoutAsString()) + .contains("The two deposits contain the same domains and registrars."); + } + + @Test + void testSuccess_differentContent() throws Exception { + String file1 = writeToNamedTmpFile("file1", RdeTestData.loadBytes("deposit_full.xml").read()); + String file2 = + writeToNamedTmpFile("file2", RdeTestData.loadBytes("deposit_full_different.xml").read()); + runCommand(file1, file2); + assertThat(getStdoutAsString()) + .isEqualTo( + "domains only in deposit1:\n" + + "example2.test\n" + + "domains only in deposit2:\n" + + "example3.test\n" + + "registrars only in deposit2:\n" + + "RegistrarY\n" + + "The two deposits differ.\n"); + } +} diff --git a/core/src/test/resources/google/registry/rde/deposit_full_different.xml b/core/src/test/resources/google/registry/rde/deposit_full_different.xml new file mode 100644 index 000000000..c426f5410 --- /dev/null +++ b/core/src/test/resources/google/registry/rde/deposit_full_different.xml @@ -0,0 +1,295 @@ + + + + 2010-10-17T00:00:00Z + + 1.0 + urn:ietf:params:xml:ns:rdeHeader-1.0 + urn:ietf:params:xml:ns:rdeContact-1.0 + urn:ietf:params:xml:ns:rdeHost-1.0 + urn:ietf:params:xml:ns:rdeDomain-1.0 + urn:ietf:params:xml:ns:rdeRegistrar-1.0 + urn:ietf:params:xml:ns:rdeIDN-1.0 + urn:ietf:params:xml:ns:rdeNNDN-1.0 + urn:ietf:params:xml:ns:rdeEppParams-1.0 + + + + + + + test + 2 + + 1 + + 1 + + 1 + + 1 + + 1 + + 1 + + + + + + example1.test + Dexample1-TEST + + jd1234 + sh8013 + sh8013 + + ns1.example.com + ns1.example1.test + + RegistrarX + RegistrarX + 1999-04-03T22:00:00.0Z + 2015-04-03T22:00:00.0Z + + + + + example3.test + Dexample3-TEST + + + jd1234 + sh8013 + sh8013 + RegistrarY + RegistrarY + 1999-04-03T22:00:00.0Z + 2015-04-03T22:00:00.0Z + + + + + ns1.example.com + Hns1_example_com-TEST + + + 192.0.2.2 + 192.0.2.29 + 1080:0:0:0:8:800:200C:417A + + RegistrarX + RegistrarX + 1999-05-08T12:10:00.0Z + RegistrarX + 2009-10-03T09:34:00.0Z + + + + + ns1.example1.test + Hns1_example1_test-TEST + + + 192.0.2.2 + 192.0.2.29 + 1080:0:0:0:8:800:200C:417A + + RegistrarX + RegistrarX + 1999-05-08T12:10:00.0Z + RegistrarX + 2009-10-03T09:34:00.0Z + + + + + sh8013 + Csh8013-TEST + + + + John Doe + Example Inc. + + 123 Example Dr. + Suite 100 + Dulles + VA + 20166-6503 + US + + + +1.7035555555 + + +1.7035555556 + + jdoe@example.test + + RegistrarX + RegistrarX + + 2009-09-13T08:01:00.0Z + RegistrarX + + 2009-11-26T09:10:00.0Z + 2009-12-03T09:05:00.0Z + + + + + + + + + RegistrarX + Registrar X + 123 + ok + + + 123 Example Dr. + + Suite 100 + + Dulles + VA + 20166-6503 + US + + + +1.7035555555 + + +1.7035555556 + + jdoe@example.test + + http://www.example.test + + + whois.example.test + + http://whois.example.test + + + 2005-04-23T11:49:00.0Z + 2009-02-17T17:51:00.0Z + + + + + RegistrarY + Registrar Y + 1234 + ok + + + 123 Example Dr. + + Suite 100 + + Dulles + VA + 20166-6503 + US + + + +1.7035555555 + + +1.7035555556 + + jdoe@example.test + + http://www.example.test + + + whois.example.test + + http://whois.example.test + + + 2005-04-23T11:49:00.0Z + 2009-02-17T17:51:00.0Z + + + + + +http://www.iana.org/domains/idn-tables/tables/br_pt-br_1.0.html + + + http://registro.br/dominio/regras.html + + + + + + xn--exampl-gva.test + pt-BR + example1.test + withheld + 2005-04-23T11:49:00.0Z + + + + + 1.0 + en + + urn:ietf:params:xml:ns:domain-1.0 + + + urn:ietf:params:xml:ns:contact-1.0 + + + urn:ietf:params:xml:ns:host-1.0 + + + urn:ietf:params:xml:ns:rgp-1.0 + + urn:ietf:params:xml:ns:secDNS-1.1 + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/test/resources/google/registry/rde/deposit_full_out_of_order.xml b/core/src/test/resources/google/registry/rde/deposit_full_out_of_order.xml new file mode 100644 index 000000000..fad3c36a0 --- /dev/null +++ b/core/src/test/resources/google/registry/rde/deposit_full_out_of_order.xml @@ -0,0 +1,261 @@ + + + + 2010-10-17T00:00:00Z + + 1.0 + urn:ietf:params:xml:ns:rdeHeader-1.0 + urn:ietf:params:xml:ns:rdeContact-1.0 + urn:ietf:params:xml:ns:rdeHost-1.0 + urn:ietf:params:xml:ns:rdeDomain-1.0 + urn:ietf:params:xml:ns:rdeRegistrar-1.0 + urn:ietf:params:xml:ns:rdeIDN-1.0 + urn:ietf:params:xml:ns:rdeNNDN-1.0 + urn:ietf:params:xml:ns:rdeEppParams-1.0 + + + + + + + test + 2 + + 1 + + 1 + + 1 + + 1 + + 1 + + 1 + + + + + + example2.test + Dexample2-TEST + + + jd1234 + sh8013 + sh8013 + RegistrarX + RegistrarX + 1999-04-03T22:00:00.0Z + 2015-04-03T22:00:00.0Z + + + + + example1.test + Dexample1-TEST + + jd1234 + sh8013 + sh8013 + + ns1.example.com + ns1.example1.test + + RegistrarX + RegistrarX + 1999-04-03T22:00:00.0Z + 2015-04-03T22:00:00.0Z + + + + + + + ns1.example.com + Hns1_example_com-TEST + + + 192.0.2.2 + 192.0.2.29 + 1080:0:0:0:8:800:200C:417A + + RegistrarX + RegistrarX + 1999-05-08T12:10:00.0Z + RegistrarX + 2009-10-03T09:34:00.0Z + + + + + ns1.example1.test + Hns1_example1_test-TEST + + + 192.0.2.2 + 192.0.2.29 + 1080:0:0:0:8:800:200C:417A + + RegistrarX + RegistrarX + 1999-05-08T12:10:00.0Z + RegistrarX + 2009-10-03T09:34:00.0Z + + + + + sh8013 + Csh8013-TEST + + + + John Doe + Example Inc. + + 123 Example Dr. + Suite 100 + Dulles + VA + 20166-6503 + US + + + +1.7035555555 + + +1.7035555556 + + jdoe@example.test + + RegistrarX + RegistrarX + + 2009-09-13T08:01:00.0Z + RegistrarX + + 2009-11-26T09:10:00.0Z + 2009-12-03T09:05:00.0Z + + + + + + + + + RegistrarX + Registrar X + 123 + ok + + + 123 Example Dr. + + Suite 100 + + Dulles + VA + 20166-6503 + US + + + +1.7035555555 + + +1.7035555556 + + jdoe@example.test + + http://www.example.test + + + whois.example.test + + http://whois.example.test + + + 2005-04-23T11:49:00.0Z + 2009-02-17T17:51:00.0Z + + + + + +http://www.iana.org/domains/idn-tables/tables/br_pt-br_1.0.html + + + http://registro.br/dominio/regras.html + + + + + + xn--exampl-gva.test + pt-BR + example1.test + withheld + 2005-04-23T11:49:00.0Z + + + + + 1.0 + en + + urn:ietf:params:xml:ns:domain-1.0 + + + urn:ietf:params:xml:ns:contact-1.0 + + + urn:ietf:params:xml:ns:host-1.0 + + + urn:ietf:params:xml:ns:rgp-1.0 + + urn:ietf:params:xml:ns:secDNS-1.1 + + + + + + + + + + + + + + + + + + + + + +