Sanitize EPP XML requests and responses

Masks user credentials (tags 'pw' and 'newPW') in EPP XML messages.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=207953894

java/google/registry/flows/EppRequestHandler.java

@@ -16,6 +16,7 @@ package google.registry.flows;
 
 import static google.registry.flows.EppXmlTransformer.marshalWithLenientRetry;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_AND_CLOSE;
+import static google.registry.xml.XmlTransformer.prettyPrint;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -53,7 +54,10 @@ public class EppRequestHandler {
           eppController.handleEppCommand(
               sessionMetadata, credentials, eppRequestSource, isDryRun, isSuperuser, inputXmlBytes);
       response.setContentType(APPLICATION_EPP_XML);
-      response.setPayload(new String(marshalWithLenientRetry(eppOutput), UTF_8));
+      byte[] eppResponseXmlBytes = marshalWithLenientRetry(eppOutput);
+      response.setPayload(new String(eppResponseXmlBytes, UTF_8));
+      logger.atInfo().log(
+          "EPP response: %s", prettyPrint(EppXmlSanitizer.sanitizeEppXml(eppResponseXmlBytes)));
       // Note that we always return 200 (OK) even if the EppController returns an error response.
       // This is because returning a non-OK HTTP status code will cause the proxy server to
       // silently close the connection without returning any data. The only time we will ever return

java/google/registry/flows/EppXmlSanitizer.java

@@ -0,0 +1,149 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.package google.registry.flows;
+
+package google.registry.flows;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Locale;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.xml.namespace.QName;
+import javax.xml.stream.XMLEventFactory;
+import javax.xml.stream.XMLEventReader;
+import javax.xml.stream.XMLEventWriter;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.events.Characters;
+import javax.xml.stream.events.XMLEvent;
+
+/**
+ * Sanitizes sensitive data in incoming/outgoing EPP XML messages.
+ *
+ * <p>Current implementation masks user credentials (text following <pw> and <newPW> tags) as
+ * follows:
+ *
+ * <ul>
+ *   <li>A control character (in ranges [0 - 1F] and [7F - 9F]) is replaced with 'C'.
+ *   <li>Everything else is replaced with '*'.
+ * </ul>
+ *
+ * <p>Invalid XML text is not sanitized, and returned as is.
+ */
+public class EppXmlSanitizer {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  /**
+   * Set of EPP XML tags whose data should be sanitized. Tags are converted to lower case for
+   * case-insensitive match.
+   *
+   * <p>Although XML tag names are case sensitive, a tag in wrong case such as {@code newPw} is
+   * likely a user error and may still wrap a real password.
+   */
+  private static final ImmutableSet<String> EPP_TAGS_IN_LOWER_CASE =
+      Stream.of("pw", "newPW").map(String::toLowerCase).collect(ImmutableSet.toImmutableSet());
+
+  // Masks by unicode char categories:
+  // Ctrl chars: [0 - 1F] and [7F - 9F]
+  private static final String CTRL_CHAR_MASK = "C";
+  private static final String DEFAULT_MASK = "*";
+
+  private static final XMLInputFactory XML_INPUT_FACTORY = createXmlInputFactory();
+  private static final XMLOutputFactory XML_OUTPUT_FACTORY = XMLOutputFactory.newFactory();
+  private static final XMLEventFactory XML_EVENT_FACTORY = XMLEventFactory.newFactory();
+
+  /**
+   * Returns sanitized and pretty-printed EPP XML message. For malformed XML messages,
+   * base64-encoded raw bytes will be returned.
+   *
+   * <p>The xml version header {@code <?xml version="1.0" ?>} is added to the result if the input
+   * does not already contain it. Also, an empty element will be formatted as {@code <tag></tag>}
+   * instead of {@code <tag/>}.
+   */
+  public static String sanitizeEppXml(byte[] inputXmlBytes) {
+    try {
+      // Keep exactly one newline at end of sanitized string.
+      return CharMatcher.whitespace()
+              .trimTrailingFrom(new String(sanitize(inputXmlBytes), StandardCharsets.UTF_8))
+          + "\n";
+    } catch (XMLStreamException e) {
+      logger.atWarning().withCause(e).log("Failed to sanitize EPP XML message.");
+      return Base64.getMimeEncoder().encodeToString(inputXmlBytes);
+    }
+  }
+
+  private static byte[] sanitize(byte[] inputXmlBytes) throws XMLStreamException {
+    XMLEventReader xmlEventReader =
+        XML_INPUT_FACTORY.createXMLEventReader(new ByteArrayInputStream(inputXmlBytes));
+
+    ByteArrayOutputStream outputXmlBytes = new ByteArrayOutputStream();
+    XMLEventWriter xmlEventWriter = XML_OUTPUT_FACTORY.createXMLEventWriter(outputXmlBytes);
+
+    while (xmlEventReader.hasNext()) {
+      XMLEvent xmlEvent = xmlEventReader.nextEvent();
+      xmlEventWriter.add(xmlEvent);
+      if (isStartEventForSensitiveData(xmlEvent)) {
+        QName startEventName = xmlEvent.asStartElement().getName();
+        while (xmlEventReader.hasNext()) {
+          XMLEvent event = xmlEventReader.nextEvent();
+          if (event.isCharacters()) {
+            Characters characters = event.asCharacters();
+            event = XML_EVENT_FACTORY.createCharacters(maskSensitiveData(characters.getData()));
+          }
+          xmlEventWriter.add(event);
+          if (isMatchingEndEvent(event, startEventName)) {
+            // The inner while-loop is guaranteed to break here for any valid XML.
+            // If matching event is missing, xmlEventReader will throw XMLStreamException.
+            break;
+          }
+        }
+      }
+    }
+    xmlEventWriter.flush();
+    return outputXmlBytes.toByteArray();
+  }
+
+  private static String maskSensitiveData(String original) {
+    return original
+        .codePoints()
+        .mapToObj(codePoint -> Character.isISOControl(codePoint) ? CTRL_CHAR_MASK : DEFAULT_MASK)
+        .collect(Collectors.joining());
+  }
+
+  private static boolean isStartEventForSensitiveData(XMLEvent xmlEvent) {
+    return xmlEvent.isStartElement()
+        && EPP_TAGS_IN_LOWER_CASE.contains(
+            xmlEvent.asStartElement().getName().getLocalPart().toLowerCase(Locale.ROOT));
+  }
+
+  private static boolean isMatchingEndEvent(XMLEvent xmlEvent, QName startEventName) {
+    return xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().equals(startEventName);
+  }
+
+  private static XMLInputFactory createXmlInputFactory() {
+    XMLInputFactory xmlInputFactory = XMLInputFactory.newFactory();
+    // Coalesce adjacent data, so that all chars in a string will be grouped as one item.
+    xmlInputFactory.setProperty(XMLInputFactory.IS_COALESCING, true);
+    // Preserve Name Space information.
+    xmlInputFactory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, true);
+    return xmlInputFactory;
+  }
+}

java/google/registry/flows/FlowRunner.java

@@ -54,7 +54,7 @@ public class FlowRunner {
 
   /** Runs the EPP flow, and records metrics on the given builder. */
   public EppOutput run(final EppMetric.Builder eppMetricBuilder) throws EppException {
-    String prettyXml = prettyPrint(inputXmlBytes);
+    String prettyXml = prettyPrint(EppXmlSanitizer.sanitizeEppXml(inputXmlBytes));
 
     logger.atInfo().log(
         COMMAND_LOG_FORMAT,

javatests/google/registry/flows/EppXmlSanitizerTest.java

@@ -0,0 +1,125 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.package google.registry.flows;
+
+package google.registry.flows;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.flows.EppXmlSanitizer.sanitizeEppXml;
+import static google.registry.testing.TestDataHelper.loadBytes;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableMap;
+import google.registry.testing.EppLoader;
+import java.util.Base64;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link EppXmlSanitizer}. */
+@RunWith(JUnit4.class)
+public class EppXmlSanitizerTest {
+
+  private static final String XML_HEADER = "<?xml version=\"1.0\" ?>";
+
+  @Test
+  public void testSanitize_noSensitiveData_noop() throws Exception {
+    byte[] inputXmlBytes = loadBytes(getClass(), "host_create.xml").read();
+    String expectedXml = XML_HEADER + new String(inputXmlBytes, UTF_8);
+
+    String sanitizedXml = sanitizeEppXml(inputXmlBytes);
+    assertThat(sanitizedXml).isEqualTo(expectedXml);
+  }
+
+  @Test
+  public void testSanitize_loginPasswords_sanitized() {
+    String inputXml =
+        new EppLoader(
+                this,
+                "login_update_password.xml",
+                ImmutableMap.of("PW", "oldpass", "NEWPW", "newPw"))
+            .getEppXml();
+    String expectedXml =
+        XML_HEADER
+            + new EppLoader(
+                    this,
+                    "login_update_password.xml",
+                    ImmutableMap.of("PW", "*******", "NEWPW", "*****"))
+                .getEppXml();
+
+    String sanitizedXml = sanitizeEppXml(inputXml.getBytes(UTF_8));
+    assertThat(sanitizedXml).isEqualTo(expectedXml);
+  }
+
+  @Test
+  public void testSanitize_loginPasswordTagWrongCase_sanitized() {
+    String inputXml =
+        new EppLoader(
+                this, "login_wrong_case.xml", ImmutableMap.of("PW", "oldpass", "NEWPW", "newPw"))
+            .getEppXml();
+    String expectedXml =
+        XML_HEADER
+            + new EppLoader(
+                    this,
+                    "login_wrong_case.xml",
+                    ImmutableMap.of("PW", "*******", "NEWPW", "*****"))
+                .getEppXml();
+
+    String sanitizedXml = sanitizeEppXml(inputXml.getBytes(UTF_8));
+    assertThat(sanitizedXml).isEqualTo(expectedXml);
+  }
+
+  @Test
+  public void testSanitize_contactAuthInfo_sanitized() throws Exception {
+    byte[] inputXmlBytes = loadBytes(getClass(), "contact_info.xml").read();
+    String expectedXml =
+        XML_HEADER
+            + new EppLoader(this, "contact_info_sanitized.xml", ImmutableMap.of()).getEppXml();
+
+    String sanitizedXml = sanitizeEppXml(inputXmlBytes);
+    assertThat(sanitizedXml).isEqualTo(expectedXml);
+  }
+
+  @Test
+  public void testSanitize_contactCreateResponseAuthInfo_sanitized() throws Exception {
+    byte[] inputXmlBytes = loadBytes(getClass(), "contact_info_from_create_response.xml").read();
+    String expectedXml =
+        XML_HEADER
+            + new EppLoader(
+                    this, "contact_info_from_create_response_sanitized.xml", ImmutableMap.of())
+                .getEppXml();
+
+    String sanitizedXml = sanitizeEppXml(inputXmlBytes);
+    assertThat(sanitizedXml).isEqualTo(expectedXml);
+  }
+
+  @Test
+  public void testSanitize_emptyElement_transformedToLongForm() {
+    byte[] inputXmlBytes = "<pw/>".getBytes(UTF_8);
+    assertThat(sanitizeEppXml(inputXmlBytes)).isEqualTo(XML_HEADER + "<pw></pw>\n");
+  }
+
+  @Test
+  public void testSanitize_invalidXML_throws() {
+    byte[] inputXmlBytes = "<pw>".getBytes(UTF_8);
+    assertThat(sanitizeEppXml(inputXmlBytes))
+        .isEqualTo(Base64.getMimeEncoder().encodeToString(inputXmlBytes));
+  }
+
+  @Test
+  public void testSanitize_unicode_hasCorrectCharCount() {
+    byte[] inputXmlBytes = "<pw>\u007F\u4E43x</pw>".getBytes(UTF_8);
+    String expectedXml = XML_HEADER + "<pw>C**</pw>\n";
+    assertThat(sanitizeEppXml(inputXmlBytes)).isEqualTo(expectedXml);
+  }
+}

javatests/google/registry/flows/FlowRunnerTest.java

@@ -190,12 +190,13 @@ public class FlowRunnerTest extends ShardableTestCase {
   @Test
   public void testRun_loggingStatement_complexEppInput() throws Exception {
     String domainCreateXml = loadFile(getClass(), "domain_create_prettyprinted.xml");
+    String sanitizedDomainCreateXml = domainCreateXml.replace("2fooBAR", "*******");
     flowRunner.inputXmlBytes = domainCreateXml.getBytes(UTF_8);
     flowRunner.run(eppMetricBuilder);
     String logMessage = findFirstLogMessageByPrefix(handler, "EPP Command\n\t");
     List<String> lines = Splitter.on("\n\t").splitToList(logMessage);
     assertThat(lines.size()).named("number of lines in log message").isAtLeast(9);
     String xml = Joiner.on('\n').join(lines.subList(3, lines.size() - 4));
-    assertThat(xml).isEqualTo(domainCreateXml);
+    assertThat(xml).isEqualTo(sanitizedDomainCreateXml);
   }
 }

javatests/google/registry/flows/testdata/contact_info_from_create_response_sanitized.xml

@@ -0,0 +1,43 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <response>
+    <result code="1000">
+      <msg>Command completed successfully</msg>
+    </result>
+    <resData>
+      <contact:infData xmlns:contact="urn:ietf:params:xml:ns:contact-1.0">
+        <contact:id>sh8013</contact:id>
+        <contact:roid>1-Q9JYB4C</contact:roid>
+        <contact:status s="ok"></contact:status>
+        <contact:postalInfo type="int">
+          <contact:name>John Doe</contact:name>
+          <contact:org>Example Inc.</contact:org>
+          <contact:addr>
+            <contact:street>123 Example Dr.</contact:street>
+            <contact:street>Suite 100</contact:street>
+            <contact:city>Dulles</contact:city>
+            <contact:sp>VA</contact:sp>
+            <contact:pc>20166-6503</contact:pc>
+            <contact:cc>US</contact:cc>
+          </contact:addr>
+        </contact:postalInfo>
+        <contact:voice x="1234">+1.7035555555</contact:voice>
+        <contact:fax>+1.7035555556</contact:fax>
+        <contact:email>jdoe@example.com</contact:email>
+        <contact:clID>NewRegistrar</contact:clID>
+        <contact:crID>NewRegistrar</contact:crID>
+        <contact:crDate>2000-06-01T00:00:00.0Z</contact:crDate>
+        <contact:authInfo>
+          <contact:pw>*******</contact:pw>
+        </contact:authInfo>
+        <contact:disclose flag="1">
+          <contact:voice></contact:voice>
+          <contact:email></contact:email>
+        </contact:disclose>
+      </contact:infData>
+    </resData>
+    <trID>
+      <clTRID>ABC-12345</clTRID>
+      <svTRID>server-trid</svTRID>
+    </trID>
+  </response>
+</epp>

javatests/google/registry/flows/testdata/contact_info_sanitized.xml

@@ -0,0 +1,13 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <info>
+      <contact:info xmlns:contact="urn:ietf:params:xml:ns:contact-1.0">
+        <contact:id>sh8013</contact:id>
+        <contact:authInfo>
+          <contact:pw>*******</contact:pw>
+        </contact:authInfo>
+      </contact:info>
+    </info>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>

javatests/google/registry/flows/testdata/login_update_password.xml

@@ -0,0 +1,23 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <login>
+      <clID>%CLID%</clID>
+      <pw>%PW%</pw>
+      <newPW>%NEWPW%</newPW>
+      <options>
+        <version>1.0</version>
+        <lang>en</lang>
+      </options>
+      <svcs>
+        <objURI>urn:ietf:params:xml:ns:host-1.0</objURI>
+        <objURI>urn:ietf:params:xml:ns:domain-1.0</objURI>
+        <objURI>urn:ietf:params:xml:ns:contact-1.0</objURI>
+        <svcExtension>
+          <extURI>urn:ietf:params:xml:ns:launch-1.0</extURI>
+          <extURI>urn:ietf:params:xml:ns:rgp-1.0</extURI>
+        </svcExtension>
+      </svcs>
+    </login>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>

javatests/google/registry/flows/testdata/login_wrong_case.xml

@@ -0,0 +1,23 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <login>
+      <clID>%CLID%</clID>
+      <pw>%PW%</pw>
+      <newPw>%NEWPW%</newPw>
+      <options>
+        <version>1.0</version>
+        <lang>en</lang>
+      </options>
+      <svcs>
+        <objURI>urn:ietf:params:xml:ns:host-1.0</objURI>
+        <objURI>urn:ietf:params:xml:ns:domain-1.0</objURI>
+        <objURI>urn:ietf:params:xml:ns:contact-1.0</objURI>
+        <svcExtension>
+          <extURI>urn:ietf:params:xml:ns:launch-1.0</extURI>
+          <extURI>urn:ietf:params:xml:ns:rgp-1.0</extURI>
+        </svcExtension>
+      </svcs>
+    </login>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>

javatests/google/registry/model/eppinput/EppInputTest.java

@@ -17,8 +17,10 @@ package google.registry.model.eppinput;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.flows.EppXmlTransformer.unmarshal;
+import static google.registry.testing.JUnitBackports.assertThrows;
 import static google.registry.testing.TestDataHelper.loadBytes;
 
+import google.registry.flows.EppException.SyntaxErrorException;
 import google.registry.model.contact.ContactResourceTest;
 import google.registry.model.domain.DomainResourceTest;
 import google.registry.model.eppinput.EppInput.InnerCommand;
@@ -77,4 +79,11 @@ public class EppInputTest {
     assertThat(loginCommand.services.serviceExtensions)
         .containsExactly("urn:ietf:params:xml:ns:launch-1.0", "urn:ietf:params:xml:ns:rgp-1.0");
   }
+
+  @Test
+  public void testUnmarshalling_loginTagInWrongCase_throws() {
+    assertThrows(
+        SyntaxErrorException.class,
+        () -> unmarshal(EppInput.class, loadBytes(getClass(), "login_wrong_case.xml").read()));
+  }
 }

javatests/google/registry/model/eppinput/testdata/login_wrong_case.xml

@@ -0,0 +1,22 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <login>
+      <clID>NewRegistrar</clID>
+      <PW>foo-BAR2</PW>
+      <options>
+        <version>1.0</version>
+        <lang>en</lang>
+      </options>
+      <svcs>
+        <objURI>urn:ietf:params:xml:ns:host-1.0</objURI>
+        <objURI>urn:ietf:params:xml:ns:domain-1.0</objURI>
+        <objURI>urn:ietf:params:xml:ns:contact-1.0</objURI>
+        <svcExtension>
+          <extURI>urn:ietf:params:xml:ns:launch-1.0</extURI>
+          <extURI>urn:ietf:params:xml:ns:rgp-1.0</extURI>
+        </svcExtension>
+      </svcs>
+    </login>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>