Add top-level python directory to opensource repo

This adds a home in our opensource repo for python libraries and binaries,
under a top-level "python" directory.  Future CLs will relocate ICANN
reporting bits and pieces to new homes under this directory, and will use
the MOE configuration and python_directory_import rule defined here.

This approach is roughly modeled on the protobuf Bazel opensource project,
which also uses a top-level directory for various languages, and also uses
the "imports" parameter to exclude that directory in python module names:
  https://github.com/google/protobuf/blob/v3.0.0-beta-3/BUILD#L568
-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=127459882

java/google/registry/BUILD

@@ -8,6 +8,7 @@ package_group(
     packages = [
         "//java/google/registry/...",
         "//javatests/google/registry/...",
+        "//python/...",
     ],
 )
 

javatests/google/registry/flows/testdata/domain_info_response_fakesite_ok.xml

@@ -26,7 +26,7 @@
       </domain:infData>
     </resData>
     <extension>
-        <rgp:infData>
+        <rgp:infData xmlns:rgp="urn:ietf:params:xml:ns:rgp-1.0">
             <rgp:rgpStatus s="addPeriod"/>
         </rgp:infData>
     </extension>

javatests/google/registry/flows/testdata/domain_info_response_fakesite_pending_delete.xml

@@ -26,7 +26,7 @@
       </domain:infData>
     </resData>
     <extension>
-        <rgp:infData>
+        <rgp:infData xmlns:rgp="urn:ietf:params:xml:ns:rgp-1.0">
             <rgp:rgpStatus s="redemptionPeriod"/>
         </rgp:infData>
     </extension>

javatests/google/registry/flows/testdata/domain_info_response_fakesite_pending_transfer_autorenew.xml

@@ -26,7 +26,7 @@
       </domain:infData>
     </resData>
     <extension>
-        <rgp:infData>
+        <rgp:infData xmlns:rgp="urn:ietf:params:xml:ns:rgp-1.0">
             <rgp:rgpStatus s="autoRenewPeriod"/>
         </rgp:infData>
     </extension>

javatests/google/registry/flows/testdata/domain_info_response_fakesite_transfer_period.xml

@@ -27,7 +27,7 @@
       </domain:infData>
     </resData>
     <extension>
-        <rgp:infData>
+        <rgp:infData xmlns:rgp="urn:ietf:params:xml:ns:rgp-1.0">
             <rgp:rgpStatus s="transferPeriod"/>
         </rgp:infData>
     </extension>

javatests/google/registry/flows/testdata/domain_info_response_testvalidate_ok.xml

@@ -25,7 +25,7 @@
       </domain:infData>
     </resData>
     <extension>
-        <rgp:infData>
+        <rgp:infData xmlns:rgp="urn:ietf:params:xml:ns:rgp-1.0">
             <rgp:rgpStatus s="addPeriod"/>
         </rgp:infData>
     </extension>

javatests/google/registry/flows/testdata/domain_update_add_nameserver_response_fakesite.xml

@@ -1,4 +1,4 @@
-<epp xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
     <response>
         <result code="1000">
             <msg>Command completed successfully</msg>

javatests/google/registry/testing/AppEngineRule.java

@@ -347,9 +347,12 @@ public final class AppEngineRule extends ExternalResource {
     ImmutableSet.Builder<String> builder = new ImmutableSet.Builder<>();
     try {
       // To normalize the indexes, we are going to pass them through JSON and then rewrite the xml.
-      for (JSONObject index : getJsonAsArray(toJSONObject(indexFile)
+      JSONObject datastoreIndexes = new JSONObject();
-          .getJSONObject("datastore-indexes")
+      Object indexes = toJSONObject(indexFile).get("datastore-indexes");
-          .opt("datastore-index"))) {
+      if (indexes instanceof JSONObject) {
+        datastoreIndexes = (JSONObject) indexes;
+      }
+      for (JSONObject index : getJsonAsArray(datastoreIndexes.opt("datastore-index"))) {
         builder.add(getIndexXmlString(index));
       }
     } catch (JSONException e) {
@@ -384,7 +387,7 @@ public final class AppEngineRule extends ExternalResource {
     builder.append(String.format(
         "<datastore-index kind=\"%s\" ancestor=\"%s\" source=\"manual\">\n",
         source.getString("kind"),
-        source.getString("ancestor")));
+        source.get("ancestor").toString()));
     for (JSONObject property : getJsonAsArray(source.get("property"))) {
       builder.append(String.format(
           "  <property name=\"%s\" direction=\"%s\"/>\n",

javatests/google/registry/xml/BUILD

@@ -11,6 +11,7 @@ load("//java/com/google/testing/builddefs:GenTestRules.bzl", "GenTestRules")
 java_library(
     name = "xml",
     srcs = glob(["*.java"]),
+    resources = glob(["testdata/*.xml"]),
     deps = [
         "//java/com/google/common/base",
         "//java/com/google/common/collect",

javatests/google/registry/xml/XmlTestUtils.java

@@ -14,17 +14,22 @@
 
 package google.registry.xml;
 
-import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.truth.Truth.assert_;
 import static google.registry.util.DiffUtils.prettyPrintDeepDiff;
 import static org.joda.time.DateTimeZone.UTC;
 
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
 import com.google.common.net.InetAddresses;
 import com.google.common.net.InternetDomainName;
+import java.util.AbstractMap;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import javax.annotation.Nullable;
@@ -58,91 +63,205 @@ public class XmlTestUtils {
     }
   }
 
-  /** Deeply explore the object and normalize values so that things we consider equal compare so. */
+  /**
-  private static Object normalize(Object obj, @Nullable String path, Set<String> ignoredPaths)
+   * Map an element or attribute name using a namespace map to replace the namespace identifier
-      throws Exception {
+   * with the complete URI as given in the map. If the name has no namespace identifier, the default
+   * namespace mapping is used. If the namespace identifier does not exist in the map, the name is
+   * left unchanged.
+   */
+  private static String mapName(
+      @Nullable String name, Map<String, String> nsMap, boolean mapDefaultNamespace) {
+    if (name == null) {
+      return null;
+    }
+    String ns;
+    String simpleKey;
+    List<String> components = Splitter.on(':').splitToList(name);
+    // Handle names without identifiers, meaning they are in the default namespace.
+    if (components.size() < 2) {
+      if (!mapDefaultNamespace) {
+        return name;
+      }
+      ns = "";
+      simpleKey = name;
+    // Handle names with identifiers.
+    } else {
+      ns = components.get(0);
+      simpleKey = components.get(1);
+    }
+    // If the map does not contain the specified identifier (or "" for the default), don't do
+    // anything.
+    if (nsMap.containsKey(ns)) {
+      ns = nsMap.get(ns);
+    }
+    return ns.isEmpty() ? simpleKey : (ns + ':' + simpleKey);
+  }
+
+  /**
+   * Deeply explore the object and normalize values so that things we consider equal compare so.
+   * The return value consists of two parts: the updated key and the value. The value is
+   * straightforward enough: it is the rendering of the subtree to be attached at the current point.
+   * The key is more complicated, because of namespaces. When an XML element specifies namespaces
+   * using xmlns attributes, those namespaces apply to the element as well as all of its
+   * descendants. That means that, when prefixing the element name with the full namespace path,
+   * as required to do proper comparison, the element name depends on its children. When looping
+   * through a JSONObject map, we can't just recursively generate the value and store it using the
+   * key. We may have to update the key as well, to get the namespaces correct. A returned key of
+   * null indicates that we should use the existing key. A non-null key indicates that we should
+   * replace the existing key.
+   * 
+   * @param elementName the name under which the current subtree was found, or null if the current
+   *     subtree's name is nonexistent or irrelevant
+   * @param obj the current subtree
+   * @param path the (non-namespaced) element path used for ignoredPaths purposes
+   * @param ignoredPaths the set of paths whose values should be set to IGNORED
+   * @param nsMap the inherited namespace identifier-to-URI map
+   * @return the key under which the rendered subtree should be stored (or null), and the rendered
+   *     subtree
+   */
+  private static Map.Entry<String, Object> normalize(
+      @Nullable String elementName,
+      Object obj,
+      @Nullable String path,
+      Set<String> ignoredPaths,
+      Map<String, String> nsMap) throws Exception {
     if (obj instanceof JSONObject) {
       JSONObject jsonObject = (JSONObject) obj;
       Map<String, Object> map = new HashMap<>();
-      // getNames helpfully returns null rather than empty when there are no names.
+      String[] names = JSONObject.getNames(jsonObject);
-      for (String key : firstNonNull(JSONObject.getNames(jsonObject), new String[0])) {
+      if (names != null) {
-        // Recursively transform json maps, remove namespaces and the "xmlns" key.
+        // Separate all elements and keys into namespace specifications, which we must process
-        if (key.startsWith("xmlns")) {
+        // first, and everything else.
-          continue;
+        ImmutableList.Builder<String> namespacesBuilder = new ImmutableList.Builder<>();
+        ImmutableList.Builder<String> othersBuilder = new ImmutableList.Builder<>();
+        for (String key : names) {
+          (key.startsWith("xmlns") ? namespacesBuilder : othersBuilder).add(key);
         }
-        String simpleKey = key.replaceAll(".*:", "");
+        // First, handle all namespace specifications, updating our ns-to-URI map. Use a HashMap
-        String newPath = path == null ? simpleKey : path + "." + simpleKey;
+        // rather than an ImmutableMap.Builder so that we can override existing map entries.
+        HashMap<String, String> newNsMap = new HashMap<>();
+        newNsMap.putAll(nsMap);
+        for (String key : namespacesBuilder.build()) {
+          // Parse the attribute name, of the form xmlns:nsid, and extract the namespace identifier.
+          // If there's no colon, we are setting the default namespace.
+          List<String> components = Splitter.on(':').splitToList(key);
+          String ns = (components.size() >= 2) ? components.get(1) : "";
+          newNsMap.put(ns, jsonObject.get(key).toString());
+        }
+        nsMap = ImmutableMap.copyOf(newNsMap);
+        // Now, handle the non-namespace items, recursively transforming the map and mapping all
+        // namespaces to the full URI for proper comparison.
+        for (String key : othersBuilder.build()) {
+          String simpleKey = Iterables.getLast(Splitter.on(':').split(key));
+          String newPath = (path == null) ? simpleKey : (path + "." + simpleKey);
+          String mappedKey;
           Object value;
           if (ignoredPaths.contains(newPath)) {
+            mappedKey = null;
             // Set ignored fields to a value that will compare equal.
             value = "IGNORED";
           } else {
-          value = normalize(jsonObject.get(key), newPath, ignoredPaths);
+            Map.Entry<String, Object> simpleEntry =
+                normalize(key, jsonObject.get(key), newPath, ignoredPaths, nsMap);
+            mappedKey = simpleEntry.getKey();
+            value = simpleEntry.getValue();
           }
-        map.put(simpleKey, value);
+          if (mappedKey == null) {
+            // Note that this does not follow the XML rules exactly. I read somewhere that attribute
+            // names, unlike element names, never use the default namespace. But after
+            // JSONification, we cannot distinguish between attributes and child elements, so we
+            // apply the default namespace to everything. Hopefully that will not cause a problem.
+            mappedKey = key.equals("content") ? key : mapName(key, nsMap, true);
           }
+          map.put(mappedKey, value);
+        }
+      }
+      // Map the namespace of the element name of the map we are normalizing.
+      elementName = mapName(elementName, nsMap, true);
       // If a node has both text content and attributes, the text content will end up under a key
       // called "content". If that's the only thing left (which will only happen if there was an
       // "xmlns:*" key that we removed), treat the node as just text and recurse.
-      if (map.size() == 1 && map.get("content") != null) {
+      if (map.size() == 1 && map.containsKey("content")) {
-        return normalize(jsonObject.get("content"), path, ignoredPaths);
+        return new AbstractMap.SimpleEntry<>(
+            elementName,
+            normalize(null, jsonObject.get("content"), path, ignoredPaths, nsMap).getValue());
       }
       // The conversion to JSON converts <a/> into "" and the semantically equivalent <a></a> into
       // an empty map, so normalize that here.
-      return map.isEmpty() ? "" : map;
+      return new AbstractMap.SimpleEntry<>(elementName, map.isEmpty() ? "" : map);
     }
     if (obj instanceof JSONArray) {
+      // Another problem resulting from JSONification: If the array contains elements whose names
+      // are the same before URI expansion, but different after URI expansion, because they use
+      // xmlns attribute that define the namespaces differently, we will screw up. Again, hopefully
+      // that doesn't happen much. The reverse is also true: If the array contains names that are
+      // different before URI expansion, but the same after, we may have a problem, because the
+      // elements will wind up in different JSONArrays as a result of JSONification. We wave our
+      // hands and just assume that the URI expansion of the first element holds for all others.
       Set<Object> set = new HashSet<>();
+      String mappedKey = null;
       for (int i = 0; i < ((JSONArray) obj).length(); ++i) {
-        set.add(normalize(((JSONArray) obj).get(i), path, ignoredPaths));
+        Map.Entry<String, Object> simpleEntry =
+            normalize(null, ((JSONArray) obj).get(i), path, ignoredPaths, nsMap);
+        if (i == 0) {
+          mappedKey = simpleEntry.getKey();
         }
-      return set;
+        set.add(simpleEntry.getValue());
+      }
+      return new AbstractMap.SimpleEntry<String, Object>(mappedKey, set);
     }
     if (obj instanceof Number) {
-      return obj.toString();
+      return new AbstractMap.SimpleEntry<String, Object>(null, obj.toString());
     }
     if (obj instanceof Boolean) {
-      return ((Boolean) obj) ? "1" : "0";
+      return new AbstractMap.SimpleEntry<String, Object>(null, ((Boolean) obj) ? "1" : "0");
     }
     if (obj instanceof String) {
       // Turn stringified booleans into integers. Both are acceptable as xml boolean values, but
       // we use "true" and "false" whereas the samples use "1" and "0".
       if (obj.equals("true")) {
-        return "1";
+        return new AbstractMap.SimpleEntry<String, Object>(null, "1");
       }
       if (obj.equals("false")) {
-        return "0";
+        return new AbstractMap.SimpleEntry<String, Object>(null, "0");
       }
       String string = obj.toString();
       // We use a slightly different datetime format (both legal) than the samples, so normalize
       // both into Datetime objects.
       try {
-        return ISODateTimeFormat.dateTime().parseDateTime(string).toDateTime(UTC);
+        return new AbstractMap.SimpleEntry<String, Object>(
+            null, ISODateTimeFormat.dateTime().parseDateTime(string).toDateTime(UTC));
       } catch (IllegalArgumentException e) {
         // It wasn't a DateTime.
       }
       try {
-        return ISODateTimeFormat.dateTimeNoMillis().parseDateTime(string).toDateTime(UTC);
+        return new AbstractMap.SimpleEntry<String, Object>(
+            null, ISODateTimeFormat.dateTimeNoMillis().parseDateTime(string).toDateTime(UTC));
       } catch (IllegalArgumentException e) {
         // It wasn't a DateTime.
       }
       try {
         if (!InternetDomainName.isValid(string)) {
           // It's not a domain name, but it is an InetAddress. Ergo, it's an ip address.
-          return InetAddresses.forString(string);
+          return new AbstractMap.SimpleEntry<String, Object>(null, InetAddresses.forString(string));
         }
       } catch (IllegalArgumentException e) {
         // Not an ip address.
       }
-      return string;
+      return new AbstractMap.SimpleEntry<String, Object>(null, string);
     }
-    return checkNotNull(obj);
+    return new AbstractMap.SimpleEntry<>(null, checkNotNull(obj));
   }
 
   @SuppressWarnings("unchecked")
   private static Map<String, Object> toComparableJson(
       String xml, String... ignoredPaths) throws Exception {
     return (Map<String, Object>) normalize(
-        XML.toJSONObject(xml), null, ImmutableSet.copyOf(ignoredPaths));
+        null,
+        XML.toJSONObject(xml),
+        null,
+        ImmutableSet.copyOf(ignoredPaths),
+        ImmutableMap.<String, String>of()).getValue();
   }
 }
+

javatests/google/registry/xml/XmlTestUtilsTest.java

@@ -0,0 +1,60 @@
+// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.xml;
+
+import static google.registry.util.ResourceUtils.readResourceUtf8;
+import static google.registry.xml.XmlTestUtils.assertXmlEquals;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link XmlTestUtils}. */
+@RunWith(JUnit4.class)
+public class XmlTestUtilsTest {
+
+  @Rule
+  public final ExpectedException thrown = ExpectedException.none();
+
+  void runTest(String file1, String file2) throws Exception {
+    String s1 = readResourceUtf8(getClass(), "testdata/" + file1);
+    String s2 = readResourceUtf8(getClass(), "testdata/" + file2);
+    assertXmlEquals(s1, s2);
+  }
+
+  @Test
+  public void testSelfEquality() throws Exception {
+    runTest("simple.xml", "simple.xml");
+  }
+
+  @Test
+  public void testInequality() throws Exception {
+    thrown.expect(AssertionError.class);
+    runTest("simple.xml", "twoextensions_feeThenLaunch.xml");
+  }
+  
+  @Test
+  public void testMultipleElementsInDifferentNamespaces() throws Exception {
+    runTest("twoextensions_feeThenLaunch.xml", "twoextensions_launchThenFee.xml");
+  }
+  
+  @Test
+  public void testMultipleElementsInDifferentNamespaces_differentValues() throws Exception {
+    thrown.expect(AssertionError.class);
+    runTest("twoextensions_feeThenLaunch.xml", "twoextensions_feeThenLaunch2.xml");
+  }
+}

javatests/google/registry/xml/testdata/simple.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<test xmlns="urn:ietf:params:xml:ns:fake">
+  <response>
+    <result code="1000">
+      <msg>Command completed successfully</msg>
+    </result>
+    <resData>
+      <green:creData xmlns:green="urn:ietf:params:xml:ns:color">
+        <green:name>example.tld</green:name>
+        <green:crDate>1999-04-03T22:00:00.0Z</green:crDate>
+        <green:exDate>2001-04-03T22:00:00.0Z</green:exDate>
+      </green:creData>
+    </resData>
+    <trID>
+      <clTRID>ABC-12345</clTRID>
+      <svTRID>server-trid</svTRID>
+    </trID>
+  </response>
+</test>

javatests/google/registry/xml/testdata/twoextensions_feeThenLaunch.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<test xmlns="urn:ietf:params:xml:ns:fake">
+  <response>
+    <result code="1000">
+      <msg>Command completed successfully</msg>
+    </result>
+    <resData>
+      <red:creData xmlns:red="urn:ietf:params:xml:ns:color">
+        <red:name>example.tld</red:name>
+        <red:crDate>1999-04-03T22:00:00.0Z</red:crDate>
+        <red:exDate>2001-04-03T22:00:00.0Z</red:exDate>
+      </red:creData>
+    </resData>
+    <extension>
+      <fee:creData xmlns:fee="urn:ietf:params:xml:ns:fee">
+        <fee:currency>USD</fee:currency>
+        <fee:fee description="create">26.00</fee:fee>
+      </fee:creData>
+      <launch:creData xmlns:launch="urn:ietf:params:xml:ns:launch">
+        <launch:phase name="landrush">sunrise</launch:phase>
+        <launch:applicationID>1-TLD</launch:applicationID>
+      </launch:creData>
+    </extension>
+    <trID>
+      <clTRID>ABC-12345</clTRID>
+      <svTRID>server-trid</svTRID>
+    </trID>
+  </response>
+</test>

javatests/google/registry/xml/testdata/twoextensions_feeThenLaunch2.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<test xmlns="urn:ietf:params:xml:ns:fake">
+  <response>
+    <result code="1000">
+      <msg>Command completed successfully</msg>
+    </result>
+    <resData>
+      <blue:creData xmlns:blue="urn:ietf:params:xml:ns:color">
+        <blue:name>example.tld</blue:name>
+        <blue:crDate>1999-04-03T22:00:00.0Z</blue:crDate>
+        <blue:exDate>2001-04-03T22:00:00.0Z</blue:exDate>
+      </blue:creData>
+    </resData>
+    <extension>
+      <fee:creData xmlns:fee="urn:ietf:params:xml:ns:fee">
+        <fee:currency>USD</fee:currency>
+        <fee:fee description="create">26.00</fee:fee>
+      </fee:creData>
+      <launch:creData xmlns:launch="urn:ietf:params:xml:ns:launch">
+        <launch:applicationID>2-TLD</launch:applicationID>
+      </launch:creData>
+    </extension>
+    <trID>
+      <clTRID>ABC-12345</clTRID>
+      <svTRID>server-trid</svTRID>
+    </trID>
+  </response>
+</test>

javatests/google/registry/xml/testdata/twoextensions_launchThenFee.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<test xmlns="urn:ietf:params:xml:ns:fake">
+  <response>
+    <result code="1000">
+      <msg>Command completed successfully</msg>
+    </result>
+    <resData>
+      <orange:creData xmlns:orange="urn:ietf:params:xml:ns:color">
+        <orange:name>example.tld</orange:name>
+        <orange:crDate>1999-04-03T22:00:00.0Z</orange:crDate>
+        <orange:exDate>2001-04-03T22:00:00.0Z</orange:exDate>
+      </orange:creData>
+    </resData>
+    <extension>
+      <launch:creData xmlns:launch="urn:ietf:params:xml:ns:launch">
+        <launch:phase name="landrush">sunrise</launch:phase>
+        <launch:applicationID>1-TLD</launch:applicationID>
+      </launch:creData>
+      <fee:creData xmlns:fee="urn:ietf:params:xml:ns:fee">
+        <fee:currency>USD</fee:currency>
+        <fee:fee description="create">26.00</fee:fee>
+      </fee:creData>
+    </extension>
+    <trID>
+      <clTRID>ABC-12345</clTRID>
+      <svTRID>server-trid</svTRID>
+    </trID>
+  </response>
+</test>

python/BUILD

@@ -0,0 +1,27 @@
+# Top-level directory for python code in the domain registry project.
+#
+# Code within this directory should live under python/google/registry.
+#
+# For an example of another Google open-source Bazel-based project using a
+# top-level python directory in a similar way, see
+#   https://github.com/google/protobuf/tree/master/python
+
+package(default_visibility = ["//java/google/registry:registry_project"])
+
+licenses(["notice"])  # Apache 2.0
+
+
+# NOTE: All py_library() rules under this directory should depend on this rule.
+#
+# This rule serves to add this directory (/python) to the PYTHONPATH of any
+# python executable rules that transitively depend on this rule.  Adding this
+# directory ensures that imports for "google.registry.*" can be correctly
+# resolved to files that are under //python/google/registry.
+#
+# See also:
+#   - http://www.bazel.io/docs/be/python.html#py_library.imports
+#   - https://github.com/google/protobuf/blob/v3.0.0-beta-3/BUILD#L568
+py_library(
+    name = "python_directory_import",
+    imports = ["."],
+)