diff --git a/java/google/registry/BUILD b/java/google/registry/BUILD index b2bd7352f..1a5ed762e 100644 --- a/java/google/registry/BUILD +++ b/java/google/registry/BUILD @@ -8,6 +8,7 @@ package_group( packages = [ "//java/google/registry/...", "//javatests/google/registry/...", + "//python/...", ], ) diff --git a/javatests/google/registry/flows/testdata/domain_info_response_fakesite_ok.xml b/javatests/google/registry/flows/testdata/domain_info_response_fakesite_ok.xml index b28c93139..6d1a19714 100644 --- a/javatests/google/registry/flows/testdata/domain_info_response_fakesite_ok.xml +++ b/javatests/google/registry/flows/testdata/domain_info_response_fakesite_ok.xml @@ -26,7 +26,7 @@ - + diff --git a/javatests/google/registry/flows/testdata/domain_info_response_fakesite_pending_delete.xml b/javatests/google/registry/flows/testdata/domain_info_response_fakesite_pending_delete.xml index 63d5e621c..b2ac294cd 100644 --- a/javatests/google/registry/flows/testdata/domain_info_response_fakesite_pending_delete.xml +++ b/javatests/google/registry/flows/testdata/domain_info_response_fakesite_pending_delete.xml @@ -26,7 +26,7 @@ - + diff --git a/javatests/google/registry/flows/testdata/domain_info_response_fakesite_pending_transfer_autorenew.xml b/javatests/google/registry/flows/testdata/domain_info_response_fakesite_pending_transfer_autorenew.xml index 85c2a018d..9bf0dc051 100644 --- a/javatests/google/registry/flows/testdata/domain_info_response_fakesite_pending_transfer_autorenew.xml +++ b/javatests/google/registry/flows/testdata/domain_info_response_fakesite_pending_transfer_autorenew.xml @@ -26,7 +26,7 @@ - + diff --git a/javatests/google/registry/flows/testdata/domain_info_response_fakesite_transfer_period.xml b/javatests/google/registry/flows/testdata/domain_info_response_fakesite_transfer_period.xml index 8703fa965..049343940 100644 --- a/javatests/google/registry/flows/testdata/domain_info_response_fakesite_transfer_period.xml +++ b/javatests/google/registry/flows/testdata/domain_info_response_fakesite_transfer_period.xml @@ -27,7 +27,7 @@ - + diff --git a/javatests/google/registry/flows/testdata/domain_info_response_testvalidate_ok.xml b/javatests/google/registry/flows/testdata/domain_info_response_testvalidate_ok.xml index 1df15bd4f..d195316a0 100644 --- a/javatests/google/registry/flows/testdata/domain_info_response_testvalidate_ok.xml +++ b/javatests/google/registry/flows/testdata/domain_info_response_testvalidate_ok.xml @@ -25,7 +25,7 @@ - + diff --git a/javatests/google/registry/flows/testdata/domain_update_add_nameserver_response_fakesite.xml b/javatests/google/registry/flows/testdata/domain_update_add_nameserver_response_fakesite.xml index 601b1a77f..7c0fcd652 100644 --- a/javatests/google/registry/flows/testdata/domain_update_add_nameserver_response_fakesite.xml +++ b/javatests/google/registry/flows/testdata/domain_update_add_nameserver_response_fakesite.xml @@ -1,4 +1,4 @@ - + Command completed successfully diff --git a/javatests/google/registry/testing/AppEngineRule.java b/javatests/google/registry/testing/AppEngineRule.java index 339b8e439..cec30494d 100644 --- a/javatests/google/registry/testing/AppEngineRule.java +++ b/javatests/google/registry/testing/AppEngineRule.java @@ -347,9 +347,12 @@ public final class AppEngineRule extends ExternalResource { ImmutableSet.Builder builder = new ImmutableSet.Builder<>(); try { // To normalize the indexes, we are going to pass them through JSON and then rewrite the xml. - for (JSONObject index : getJsonAsArray(toJSONObject(indexFile) - .getJSONObject("datastore-indexes") - .opt("datastore-index"))) { + JSONObject datastoreIndexes = new JSONObject(); + Object indexes = toJSONObject(indexFile).get("datastore-indexes"); + if (indexes instanceof JSONObject) { + datastoreIndexes = (JSONObject) indexes; + } + for (JSONObject index : getJsonAsArray(datastoreIndexes.opt("datastore-index"))) { builder.add(getIndexXmlString(index)); } } catch (JSONException e) { @@ -384,7 +387,7 @@ public final class AppEngineRule extends ExternalResource { builder.append(String.format( "\n", source.getString("kind"), - source.getString("ancestor"))); + source.get("ancestor").toString())); for (JSONObject property : getJsonAsArray(source.get("property"))) { builder.append(String.format( " \n", diff --git a/javatests/google/registry/xml/BUILD b/javatests/google/registry/xml/BUILD index 670a89d68..7d52b899a 100644 --- a/javatests/google/registry/xml/BUILD +++ b/javatests/google/registry/xml/BUILD @@ -11,6 +11,7 @@ load("//java/com/google/testing/builddefs:GenTestRules.bzl", "GenTestRules") java_library( name = "xml", srcs = glob(["*.java"]), + resources = glob(["testdata/*.xml"]), deps = [ "//java/com/google/common/base", "//java/com/google/common/collect", diff --git a/javatests/google/registry/xml/XmlTestUtils.java b/javatests/google/registry/xml/XmlTestUtils.java index ef10ed09f..0b9aa3b23 100644 --- a/javatests/google/registry/xml/XmlTestUtils.java +++ b/javatests/google/registry/xml/XmlTestUtils.java @@ -14,17 +14,22 @@ package google.registry.xml; -import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.truth.Truth.assert_; import static google.registry.util.DiffUtils.prettyPrintDeepDiff; import static org.joda.time.DateTimeZone.UTC; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.net.InetAddresses; import com.google.common.net.InternetDomainName; +import java.util.AbstractMap; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import javax.annotation.Nullable; @@ -58,91 +63,205 @@ public class XmlTestUtils { } } - /** Deeply explore the object and normalize values so that things we consider equal compare so. */ - private static Object normalize(Object obj, @Nullable String path, Set ignoredPaths) - throws Exception { + /** + * Map an element or attribute name using a namespace map to replace the namespace identifier + * with the complete URI as given in the map. If the name has no namespace identifier, the default + * namespace mapping is used. If the namespace identifier does not exist in the map, the name is + * left unchanged. + */ + private static String mapName( + @Nullable String name, Map nsMap, boolean mapDefaultNamespace) { + if (name == null) { + return null; + } + String ns; + String simpleKey; + List components = Splitter.on(':').splitToList(name); + // Handle names without identifiers, meaning they are in the default namespace. + if (components.size() < 2) { + if (!mapDefaultNamespace) { + return name; + } + ns = ""; + simpleKey = name; + // Handle names with identifiers. + } else { + ns = components.get(0); + simpleKey = components.get(1); + } + // If the map does not contain the specified identifier (or "" for the default), don't do + // anything. + if (nsMap.containsKey(ns)) { + ns = nsMap.get(ns); + } + return ns.isEmpty() ? simpleKey : (ns + ':' + simpleKey); + } + + /** + * Deeply explore the object and normalize values so that things we consider equal compare so. + * The return value consists of two parts: the updated key and the value. The value is + * straightforward enough: it is the rendering of the subtree to be attached at the current point. + * The key is more complicated, because of namespaces. When an XML element specifies namespaces + * using xmlns attributes, those namespaces apply to the element as well as all of its + * descendants. That means that, when prefixing the element name with the full namespace path, + * as required to do proper comparison, the element name depends on its children. When looping + * through a JSONObject map, we can't just recursively generate the value and store it using the + * key. We may have to update the key as well, to get the namespaces correct. A returned key of + * null indicates that we should use the existing key. A non-null key indicates that we should + * replace the existing key. + * + * @param elementName the name under which the current subtree was found, or null if the current + * subtree's name is nonexistent or irrelevant + * @param obj the current subtree + * @param path the (non-namespaced) element path used for ignoredPaths purposes + * @param ignoredPaths the set of paths whose values should be set to IGNORED + * @param nsMap the inherited namespace identifier-to-URI map + * @return the key under which the rendered subtree should be stored (or null), and the rendered + * subtree + */ + private static Map.Entry normalize( + @Nullable String elementName, + Object obj, + @Nullable String path, + Set ignoredPaths, + Map nsMap) throws Exception { if (obj instanceof JSONObject) { JSONObject jsonObject = (JSONObject) obj; Map map = new HashMap<>(); - // getNames helpfully returns null rather than empty when there are no names. - for (String key : firstNonNull(JSONObject.getNames(jsonObject), new String[0])) { - // Recursively transform json maps, remove namespaces and the "xmlns" key. - if (key.startsWith("xmlns")) { - continue; + String[] names = JSONObject.getNames(jsonObject); + if (names != null) { + // Separate all elements and keys into namespace specifications, which we must process + // first, and everything else. + ImmutableList.Builder namespacesBuilder = new ImmutableList.Builder<>(); + ImmutableList.Builder othersBuilder = new ImmutableList.Builder<>(); + for (String key : names) { + (key.startsWith("xmlns") ? namespacesBuilder : othersBuilder).add(key); } - String simpleKey = key.replaceAll(".*:", ""); - String newPath = path == null ? simpleKey : path + "." + simpleKey; - Object value; - if (ignoredPaths.contains(newPath)) { - // Set ignored fields to a value that will compare equal. - value = "IGNORED"; - } else { - value = normalize(jsonObject.get(key), newPath, ignoredPaths); + // First, handle all namespace specifications, updating our ns-to-URI map. Use a HashMap + // rather than an ImmutableMap.Builder so that we can override existing map entries. + HashMap newNsMap = new HashMap<>(); + newNsMap.putAll(nsMap); + for (String key : namespacesBuilder.build()) { + // Parse the attribute name, of the form xmlns:nsid, and extract the namespace identifier. + // If there's no colon, we are setting the default namespace. + List components = Splitter.on(':').splitToList(key); + String ns = (components.size() >= 2) ? components.get(1) : ""; + newNsMap.put(ns, jsonObject.get(key).toString()); + } + nsMap = ImmutableMap.copyOf(newNsMap); + // Now, handle the non-namespace items, recursively transforming the map and mapping all + // namespaces to the full URI for proper comparison. + for (String key : othersBuilder.build()) { + String simpleKey = Iterables.getLast(Splitter.on(':').split(key)); + String newPath = (path == null) ? simpleKey : (path + "." + simpleKey); + String mappedKey; + Object value; + if (ignoredPaths.contains(newPath)) { + mappedKey = null; + // Set ignored fields to a value that will compare equal. + value = "IGNORED"; + } else { + Map.Entry simpleEntry = + normalize(key, jsonObject.get(key), newPath, ignoredPaths, nsMap); + mappedKey = simpleEntry.getKey(); + value = simpleEntry.getValue(); + } + if (mappedKey == null) { + // Note that this does not follow the XML rules exactly. I read somewhere that attribute + // names, unlike element names, never use the default namespace. But after + // JSONification, we cannot distinguish between attributes and child elements, so we + // apply the default namespace to everything. Hopefully that will not cause a problem. + mappedKey = key.equals("content") ? key : mapName(key, nsMap, true); + } + map.put(mappedKey, value); } - map.put(simpleKey, value); } + // Map the namespace of the element name of the map we are normalizing. + elementName = mapName(elementName, nsMap, true); // If a node has both text content and attributes, the text content will end up under a key // called "content". If that's the only thing left (which will only happen if there was an // "xmlns:*" key that we removed), treat the node as just text and recurse. - if (map.size() == 1 && map.get("content") != null) { - return normalize(jsonObject.get("content"), path, ignoredPaths); + if (map.size() == 1 && map.containsKey("content")) { + return new AbstractMap.SimpleEntry<>( + elementName, + normalize(null, jsonObject.get("content"), path, ignoredPaths, nsMap).getValue()); } // The conversion to JSON converts into "" and the semantically equivalent into // an empty map, so normalize that here. - return map.isEmpty() ? "" : map; + return new AbstractMap.SimpleEntry<>(elementName, map.isEmpty() ? "" : map); } if (obj instanceof JSONArray) { + // Another problem resulting from JSONification: If the array contains elements whose names + // are the same before URI expansion, but different after URI expansion, because they use + // xmlns attribute that define the namespaces differently, we will screw up. Again, hopefully + // that doesn't happen much. The reverse is also true: If the array contains names that are + // different before URI expansion, but the same after, we may have a problem, because the + // elements will wind up in different JSONArrays as a result of JSONification. We wave our + // hands and just assume that the URI expansion of the first element holds for all others. Set set = new HashSet<>(); + String mappedKey = null; for (int i = 0; i < ((JSONArray) obj).length(); ++i) { - set.add(normalize(((JSONArray) obj).get(i), path, ignoredPaths)); + Map.Entry simpleEntry = + normalize(null, ((JSONArray) obj).get(i), path, ignoredPaths, nsMap); + if (i == 0) { + mappedKey = simpleEntry.getKey(); + } + set.add(simpleEntry.getValue()); } - return set; + return new AbstractMap.SimpleEntry(mappedKey, set); } if (obj instanceof Number) { - return obj.toString(); + return new AbstractMap.SimpleEntry(null, obj.toString()); } if (obj instanceof Boolean) { - return ((Boolean) obj) ? "1" : "0"; + return new AbstractMap.SimpleEntry(null, ((Boolean) obj) ? "1" : "0"); } if (obj instanceof String) { // Turn stringified booleans into integers. Both are acceptable as xml boolean values, but // we use "true" and "false" whereas the samples use "1" and "0". if (obj.equals("true")) { - return "1"; + return new AbstractMap.SimpleEntry(null, "1"); } if (obj.equals("false")) { - return "0"; + return new AbstractMap.SimpleEntry(null, "0"); } String string = obj.toString(); // We use a slightly different datetime format (both legal) than the samples, so normalize // both into Datetime objects. try { - return ISODateTimeFormat.dateTime().parseDateTime(string).toDateTime(UTC); + return new AbstractMap.SimpleEntry( + null, ISODateTimeFormat.dateTime().parseDateTime(string).toDateTime(UTC)); } catch (IllegalArgumentException e) { // It wasn't a DateTime. } try { - return ISODateTimeFormat.dateTimeNoMillis().parseDateTime(string).toDateTime(UTC); + return new AbstractMap.SimpleEntry( + null, ISODateTimeFormat.dateTimeNoMillis().parseDateTime(string).toDateTime(UTC)); } catch (IllegalArgumentException e) { // It wasn't a DateTime. } try { if (!InternetDomainName.isValid(string)) { // It's not a domain name, but it is an InetAddress. Ergo, it's an ip address. - return InetAddresses.forString(string); + return new AbstractMap.SimpleEntry(null, InetAddresses.forString(string)); } } catch (IllegalArgumentException e) { // Not an ip address. } - return string; + return new AbstractMap.SimpleEntry(null, string); } - return checkNotNull(obj); + return new AbstractMap.SimpleEntry<>(null, checkNotNull(obj)); } @SuppressWarnings("unchecked") private static Map toComparableJson( String xml, String... ignoredPaths) throws Exception { return (Map) normalize( - XML.toJSONObject(xml), null, ImmutableSet.copyOf(ignoredPaths)); + null, + XML.toJSONObject(xml), + null, + ImmutableSet.copyOf(ignoredPaths), + ImmutableMap.of()).getValue(); } } + diff --git a/javatests/google/registry/xml/XmlTestUtilsTest.java b/javatests/google/registry/xml/XmlTestUtilsTest.java new file mode 100644 index 000000000..6dddd5325 --- /dev/null +++ b/javatests/google/registry/xml/XmlTestUtilsTest.java @@ -0,0 +1,60 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.xml; + +import static google.registry.util.ResourceUtils.readResourceUtf8; +import static google.registry.xml.XmlTestUtils.assertXmlEquals; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link XmlTestUtils}. */ +@RunWith(JUnit4.class) +public class XmlTestUtilsTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + void runTest(String file1, String file2) throws Exception { + String s1 = readResourceUtf8(getClass(), "testdata/" + file1); + String s2 = readResourceUtf8(getClass(), "testdata/" + file2); + assertXmlEquals(s1, s2); + } + + @Test + public void testSelfEquality() throws Exception { + runTest("simple.xml", "simple.xml"); + } + + @Test + public void testInequality() throws Exception { + thrown.expect(AssertionError.class); + runTest("simple.xml", "twoextensions_feeThenLaunch.xml"); + } + + @Test + public void testMultipleElementsInDifferentNamespaces() throws Exception { + runTest("twoextensions_feeThenLaunch.xml", "twoextensions_launchThenFee.xml"); + } + + @Test + public void testMultipleElementsInDifferentNamespaces_differentValues() throws Exception { + thrown.expect(AssertionError.class); + runTest("twoextensions_feeThenLaunch.xml", "twoextensions_feeThenLaunch2.xml"); + } +} diff --git a/javatests/google/registry/xml/testdata/simple.xml b/javatests/google/registry/xml/testdata/simple.xml new file mode 100644 index 000000000..1c85491ba --- /dev/null +++ b/javatests/google/registry/xml/testdata/simple.xml @@ -0,0 +1,19 @@ + + + + + Command completed successfully + + + + example.tld + 1999-04-03T22:00:00.0Z + 2001-04-03T22:00:00.0Z + + + + ABC-12345 + server-trid + + + diff --git a/javatests/google/registry/xml/testdata/twoextensions_feeThenLaunch.xml b/javatests/google/registry/xml/testdata/twoextensions_feeThenLaunch.xml new file mode 100644 index 000000000..de8152ac8 --- /dev/null +++ b/javatests/google/registry/xml/testdata/twoextensions_feeThenLaunch.xml @@ -0,0 +1,29 @@ + + + + + Command completed successfully + + + + example.tld + 1999-04-03T22:00:00.0Z + 2001-04-03T22:00:00.0Z + + + + + USD + 26.00 + + + sunrise + 1-TLD + + + + ABC-12345 + server-trid + + + diff --git a/javatests/google/registry/xml/testdata/twoextensions_feeThenLaunch2.xml b/javatests/google/registry/xml/testdata/twoextensions_feeThenLaunch2.xml new file mode 100644 index 000000000..512604120 --- /dev/null +++ b/javatests/google/registry/xml/testdata/twoextensions_feeThenLaunch2.xml @@ -0,0 +1,28 @@ + + + + + Command completed successfully + + + + example.tld + 1999-04-03T22:00:00.0Z + 2001-04-03T22:00:00.0Z + + + + + USD + 26.00 + + + 2-TLD + + + + ABC-12345 + server-trid + + + diff --git a/javatests/google/registry/xml/testdata/twoextensions_launchThenFee.xml b/javatests/google/registry/xml/testdata/twoextensions_launchThenFee.xml new file mode 100644 index 000000000..8d35a9230 --- /dev/null +++ b/javatests/google/registry/xml/testdata/twoextensions_launchThenFee.xml @@ -0,0 +1,29 @@ + + + + + Command completed successfully + + + + example.tld + 1999-04-03T22:00:00.0Z + 2001-04-03T22:00:00.0Z + + + + + sunrise + 1-TLD + + + USD + 26.00 + + + + ABC-12345 + server-trid + + + diff --git a/python/BUILD b/python/BUILD new file mode 100644 index 000000000..96d9b2277 --- /dev/null +++ b/python/BUILD @@ -0,0 +1,27 @@ +# Top-level directory for python code in the domain registry project. +# +# Code within this directory should live under python/google/registry. +# +# For an example of another Google open-source Bazel-based project using a +# top-level python directory in a similar way, see +# https://github.com/google/protobuf/tree/master/python + +package(default_visibility = ["//java/google/registry:registry_project"]) + +licenses(["notice"]) # Apache 2.0 + + +# NOTE: All py_library() rules under this directory should depend on this rule. +# +# This rule serves to add this directory (/python) to the PYTHONPATH of any +# python executable rules that transitively depend on this rule. Adding this +# directory ensures that imports for "google.registry.*" can be correctly +# resolved to files that are under //python/google/registry. +# +# See also: +# - http://www.bazel.io/docs/be/python.html#py_library.imports +# - https://github.com/google/protobuf/blob/v3.0.0-beta-3/BUILD#L568 +py_library( + name = "python_directory_import", + imports = ["."], +)