diff --git a/java/google/registry/flows/EppRequestHandler.java b/java/google/registry/flows/EppRequestHandler.java index 812c8525f..2d1f568fa 100644 --- a/java/google/registry/flows/EppRequestHandler.java +++ b/java/google/registry/flows/EppRequestHandler.java @@ -16,6 +16,7 @@ package google.registry.flows; import static google.registry.flows.EppXmlTransformer.marshalWithLenientRetry; import static google.registry.model.eppoutput.Result.Code.SUCCESS_AND_CLOSE; +import static google.registry.xml.XmlTransformer.prettyPrint; import static java.nio.charset.StandardCharsets.UTF_8; import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; import static javax.servlet.http.HttpServletResponse.SC_OK; @@ -53,7 +54,10 @@ public class EppRequestHandler { eppController.handleEppCommand( sessionMetadata, credentials, eppRequestSource, isDryRun, isSuperuser, inputXmlBytes); response.setContentType(APPLICATION_EPP_XML); - response.setPayload(new String(marshalWithLenientRetry(eppOutput), UTF_8)); + byte[] eppResponseXmlBytes = marshalWithLenientRetry(eppOutput); + response.setPayload(new String(eppResponseXmlBytes, UTF_8)); + logger.atInfo().log( + "EPP response: %s", prettyPrint(EppXmlSanitizer.sanitizeEppXml(eppResponseXmlBytes))); // Note that we always return 200 (OK) even if the EppController returns an error response. // This is because returning a non-OK HTTP status code will cause the proxy server to // silently close the connection without returning any data. The only time we will ever return diff --git a/java/google/registry/flows/EppXmlSanitizer.java b/java/google/registry/flows/EppXmlSanitizer.java new file mode 100644 index 000000000..2113dc5dc --- /dev/null +++ b/java/google/registry/flows/EppXmlSanitizer.java @@ -0,0 +1,149 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.package google.registry.flows; + +package google.registry.flows; + +import com.google.common.base.CharMatcher; +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.FluentLogger; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Locale; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.xml.namespace.QName; +import javax.xml.stream.XMLEventFactory; +import javax.xml.stream.XMLEventReader; +import javax.xml.stream.XMLEventWriter; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.events.Characters; +import javax.xml.stream.events.XMLEvent; + +/** + * Sanitizes sensitive data in incoming/outgoing EPP XML messages. + * + *

Current implementation masks user credentials (text following and tags) as + * follows: + * + *

+ * + *

Invalid XML text is not sanitized, and returned as is. + */ +public class EppXmlSanitizer { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + /** + * Set of EPP XML tags whose data should be sanitized. Tags are converted to lower case for + * case-insensitive match. + * + *

Although XML tag names are case sensitive, a tag in wrong case such as {@code newPw} is + * likely a user error and may still wrap a real password. + */ + private static final ImmutableSet EPP_TAGS_IN_LOWER_CASE = + Stream.of("pw", "newPW").map(String::toLowerCase).collect(ImmutableSet.toImmutableSet()); + + // Masks by unicode char categories: + // Ctrl chars: [0 - 1F] and [7F - 9F] + private static final String CTRL_CHAR_MASK = "C"; + private static final String DEFAULT_MASK = "*"; + + private static final XMLInputFactory XML_INPUT_FACTORY = createXmlInputFactory(); + private static final XMLOutputFactory XML_OUTPUT_FACTORY = XMLOutputFactory.newFactory(); + private static final XMLEventFactory XML_EVENT_FACTORY = XMLEventFactory.newFactory(); + + /** + * Returns sanitized and pretty-printed EPP XML message. For malformed XML messages, + * base64-encoded raw bytes will be returned. + * + *

The xml version header {@code } is added to the result if the input + * does not already contain it. Also, an empty element will be formatted as {@code } + * instead of {@code }. + */ + public static String sanitizeEppXml(byte[] inputXmlBytes) { + try { + // Keep exactly one newline at end of sanitized string. + return CharMatcher.whitespace() + .trimTrailingFrom(new String(sanitize(inputXmlBytes), StandardCharsets.UTF_8)) + + "\n"; + } catch (XMLStreamException e) { + logger.atWarning().withCause(e).log("Failed to sanitize EPP XML message."); + return Base64.getMimeEncoder().encodeToString(inputXmlBytes); + } + } + + private static byte[] sanitize(byte[] inputXmlBytes) throws XMLStreamException { + XMLEventReader xmlEventReader = + XML_INPUT_FACTORY.createXMLEventReader(new ByteArrayInputStream(inputXmlBytes)); + + ByteArrayOutputStream outputXmlBytes = new ByteArrayOutputStream(); + XMLEventWriter xmlEventWriter = XML_OUTPUT_FACTORY.createXMLEventWriter(outputXmlBytes); + + while (xmlEventReader.hasNext()) { + XMLEvent xmlEvent = xmlEventReader.nextEvent(); + xmlEventWriter.add(xmlEvent); + if (isStartEventForSensitiveData(xmlEvent)) { + QName startEventName = xmlEvent.asStartElement().getName(); + while (xmlEventReader.hasNext()) { + XMLEvent event = xmlEventReader.nextEvent(); + if (event.isCharacters()) { + Characters characters = event.asCharacters(); + event = XML_EVENT_FACTORY.createCharacters(maskSensitiveData(characters.getData())); + } + xmlEventWriter.add(event); + if (isMatchingEndEvent(event, startEventName)) { + // The inner while-loop is guaranteed to break here for any valid XML. + // If matching event is missing, xmlEventReader will throw XMLStreamException. + break; + } + } + } + } + xmlEventWriter.flush(); + return outputXmlBytes.toByteArray(); + } + + private static String maskSensitiveData(String original) { + return original + .codePoints() + .mapToObj(codePoint -> Character.isISOControl(codePoint) ? CTRL_CHAR_MASK : DEFAULT_MASK) + .collect(Collectors.joining()); + } + + private static boolean isStartEventForSensitiveData(XMLEvent xmlEvent) { + return xmlEvent.isStartElement() + && EPP_TAGS_IN_LOWER_CASE.contains( + xmlEvent.asStartElement().getName().getLocalPart().toLowerCase(Locale.ROOT)); + } + + private static boolean isMatchingEndEvent(XMLEvent xmlEvent, QName startEventName) { + return xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().equals(startEventName); + } + + private static XMLInputFactory createXmlInputFactory() { + XMLInputFactory xmlInputFactory = XMLInputFactory.newFactory(); + // Coalesce adjacent data, so that all chars in a string will be grouped as one item. + xmlInputFactory.setProperty(XMLInputFactory.IS_COALESCING, true); + // Preserve Name Space information. + xmlInputFactory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, true); + return xmlInputFactory; + } +} diff --git a/java/google/registry/flows/FlowRunner.java b/java/google/registry/flows/FlowRunner.java index 801f1320d..b830e4e23 100644 --- a/java/google/registry/flows/FlowRunner.java +++ b/java/google/registry/flows/FlowRunner.java @@ -54,7 +54,7 @@ public class FlowRunner { /** Runs the EPP flow, and records metrics on the given builder. */ public EppOutput run(final EppMetric.Builder eppMetricBuilder) throws EppException { - String prettyXml = prettyPrint(inputXmlBytes); + String prettyXml = prettyPrint(EppXmlSanitizer.sanitizeEppXml(inputXmlBytes)); logger.atInfo().log( COMMAND_LOG_FORMAT, diff --git a/javatests/google/registry/flows/EppXmlSanitizerTest.java b/javatests/google/registry/flows/EppXmlSanitizerTest.java new file mode 100644 index 000000000..3cc098608 --- /dev/null +++ b/javatests/google/registry/flows/EppXmlSanitizerTest.java @@ -0,0 +1,125 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.package google.registry.flows; + +package google.registry.flows; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.flows.EppXmlSanitizer.sanitizeEppXml; +import static google.registry.testing.TestDataHelper.loadBytes; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableMap; +import google.registry.testing.EppLoader; +import java.util.Base64; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link EppXmlSanitizer}. */ +@RunWith(JUnit4.class) +public class EppXmlSanitizerTest { + + private static final String XML_HEADER = ""; + + @Test + public void testSanitize_noSensitiveData_noop() throws Exception { + byte[] inputXmlBytes = loadBytes(getClass(), "host_create.xml").read(); + String expectedXml = XML_HEADER + new String(inputXmlBytes, UTF_8); + + String sanitizedXml = sanitizeEppXml(inputXmlBytes); + assertThat(sanitizedXml).isEqualTo(expectedXml); + } + + @Test + public void testSanitize_loginPasswords_sanitized() { + String inputXml = + new EppLoader( + this, + "login_update_password.xml", + ImmutableMap.of("PW", "oldpass", "NEWPW", "newPw")) + .getEppXml(); + String expectedXml = + XML_HEADER + + new EppLoader( + this, + "login_update_password.xml", + ImmutableMap.of("PW", "*******", "NEWPW", "*****")) + .getEppXml(); + + String sanitizedXml = sanitizeEppXml(inputXml.getBytes(UTF_8)); + assertThat(sanitizedXml).isEqualTo(expectedXml); + } + + @Test + public void testSanitize_loginPasswordTagWrongCase_sanitized() { + String inputXml = + new EppLoader( + this, "login_wrong_case.xml", ImmutableMap.of("PW", "oldpass", "NEWPW", "newPw")) + .getEppXml(); + String expectedXml = + XML_HEADER + + new EppLoader( + this, + "login_wrong_case.xml", + ImmutableMap.of("PW", "*******", "NEWPW", "*****")) + .getEppXml(); + + String sanitizedXml = sanitizeEppXml(inputXml.getBytes(UTF_8)); + assertThat(sanitizedXml).isEqualTo(expectedXml); + } + + @Test + public void testSanitize_contactAuthInfo_sanitized() throws Exception { + byte[] inputXmlBytes = loadBytes(getClass(), "contact_info.xml").read(); + String expectedXml = + XML_HEADER + + new EppLoader(this, "contact_info_sanitized.xml", ImmutableMap.of()).getEppXml(); + + String sanitizedXml = sanitizeEppXml(inputXmlBytes); + assertThat(sanitizedXml).isEqualTo(expectedXml); + } + + @Test + public void testSanitize_contactCreateResponseAuthInfo_sanitized() throws Exception { + byte[] inputXmlBytes = loadBytes(getClass(), "contact_info_from_create_response.xml").read(); + String expectedXml = + XML_HEADER + + new EppLoader( + this, "contact_info_from_create_response_sanitized.xml", ImmutableMap.of()) + .getEppXml(); + + String sanitizedXml = sanitizeEppXml(inputXmlBytes); + assertThat(sanitizedXml).isEqualTo(expectedXml); + } + + @Test + public void testSanitize_emptyElement_transformedToLongForm() { + byte[] inputXmlBytes = "".getBytes(UTF_8); + assertThat(sanitizeEppXml(inputXmlBytes)).isEqualTo(XML_HEADER + "\n"); + } + + @Test + public void testSanitize_invalidXML_throws() { + byte[] inputXmlBytes = "".getBytes(UTF_8); + assertThat(sanitizeEppXml(inputXmlBytes)) + .isEqualTo(Base64.getMimeEncoder().encodeToString(inputXmlBytes)); + } + + @Test + public void testSanitize_unicode_hasCorrectCharCount() { + byte[] inputXmlBytes = "\u007F\u4E43x".getBytes(UTF_8); + String expectedXml = XML_HEADER + "C**\n"; + assertThat(sanitizeEppXml(inputXmlBytes)).isEqualTo(expectedXml); + } +} diff --git a/javatests/google/registry/flows/FlowRunnerTest.java b/javatests/google/registry/flows/FlowRunnerTest.java index e9772765e..11bba256d 100644 --- a/javatests/google/registry/flows/FlowRunnerTest.java +++ b/javatests/google/registry/flows/FlowRunnerTest.java @@ -190,12 +190,13 @@ public class FlowRunnerTest extends ShardableTestCase { @Test public void testRun_loggingStatement_complexEppInput() throws Exception { String domainCreateXml = loadFile(getClass(), "domain_create_prettyprinted.xml"); + String sanitizedDomainCreateXml = domainCreateXml.replace("2fooBAR", "*******"); flowRunner.inputXmlBytes = domainCreateXml.getBytes(UTF_8); flowRunner.run(eppMetricBuilder); String logMessage = findFirstLogMessageByPrefix(handler, "EPP Command\n\t"); List lines = Splitter.on("\n\t").splitToList(logMessage); assertThat(lines.size()).named("number of lines in log message").isAtLeast(9); String xml = Joiner.on('\n').join(lines.subList(3, lines.size() - 4)); - assertThat(xml).isEqualTo(domainCreateXml); + assertThat(xml).isEqualTo(sanitizedDomainCreateXml); } } diff --git a/javatests/google/registry/flows/testdata/contact_info_from_create_response_sanitized.xml b/javatests/google/registry/flows/testdata/contact_info_from_create_response_sanitized.xml new file mode 100644 index 000000000..4ac0b567f --- /dev/null +++ b/javatests/google/registry/flows/testdata/contact_info_from_create_response_sanitized.xml @@ -0,0 +1,43 @@ + + + + Command completed successfully + + + + sh8013 + 1-Q9JYB4C + + + John Doe + Example Inc. + + 123 Example Dr. + Suite 100 + Dulles + VA + 20166-6503 + US + + + +1.7035555555 + +1.7035555556 + jdoe@example.com + NewRegistrar + NewRegistrar + 2000-06-01T00:00:00.0Z + + ******* + + + + + + + + + ABC-12345 + server-trid + + + diff --git a/javatests/google/registry/flows/testdata/contact_info_sanitized.xml b/javatests/google/registry/flows/testdata/contact_info_sanitized.xml new file mode 100644 index 000000000..9d34af46f --- /dev/null +++ b/javatests/google/registry/flows/testdata/contact_info_sanitized.xml @@ -0,0 +1,13 @@ + + + + + sh8013 + + ******* + + + + ABC-12345 + + diff --git a/javatests/google/registry/flows/testdata/login_update_password.xml b/javatests/google/registry/flows/testdata/login_update_password.xml new file mode 100644 index 000000000..07011f4e9 --- /dev/null +++ b/javatests/google/registry/flows/testdata/login_update_password.xml @@ -0,0 +1,23 @@ + + + + %CLID% + %PW% + %NEWPW% + + 1.0 + en + + + urn:ietf:params:xml:ns:host-1.0 + urn:ietf:params:xml:ns:domain-1.0 + urn:ietf:params:xml:ns:contact-1.0 + + urn:ietf:params:xml:ns:launch-1.0 + urn:ietf:params:xml:ns:rgp-1.0 + + + + ABC-12345 + + diff --git a/javatests/google/registry/flows/testdata/login_wrong_case.xml b/javatests/google/registry/flows/testdata/login_wrong_case.xml new file mode 100644 index 000000000..c0c8716a1 --- /dev/null +++ b/javatests/google/registry/flows/testdata/login_wrong_case.xml @@ -0,0 +1,23 @@ + + + + %CLID% + %PW% + %NEWPW% + + 1.0 + en + + + urn:ietf:params:xml:ns:host-1.0 + urn:ietf:params:xml:ns:domain-1.0 + urn:ietf:params:xml:ns:contact-1.0 + + urn:ietf:params:xml:ns:launch-1.0 + urn:ietf:params:xml:ns:rgp-1.0 + + + + ABC-12345 + + diff --git a/javatests/google/registry/model/eppinput/EppInputTest.java b/javatests/google/registry/model/eppinput/EppInputTest.java index 6a3084783..e6168dddc 100644 --- a/javatests/google/registry/model/eppinput/EppInputTest.java +++ b/javatests/google/registry/model/eppinput/EppInputTest.java @@ -17,8 +17,10 @@ package google.registry.model.eppinput; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth8.assertThat; import static google.registry.flows.EppXmlTransformer.unmarshal; +import static google.registry.testing.JUnitBackports.assertThrows; import static google.registry.testing.TestDataHelper.loadBytes; +import google.registry.flows.EppException.SyntaxErrorException; import google.registry.model.contact.ContactResourceTest; import google.registry.model.domain.DomainResourceTest; import google.registry.model.eppinput.EppInput.InnerCommand; @@ -77,4 +79,11 @@ public class EppInputTest { assertThat(loginCommand.services.serviceExtensions) .containsExactly("urn:ietf:params:xml:ns:launch-1.0", "urn:ietf:params:xml:ns:rgp-1.0"); } + + @Test + public void testUnmarshalling_loginTagInWrongCase_throws() { + assertThrows( + SyntaxErrorException.class, + () -> unmarshal(EppInput.class, loadBytes(getClass(), "login_wrong_case.xml").read())); + } } diff --git a/javatests/google/registry/model/eppinput/testdata/login_wrong_case.xml b/javatests/google/registry/model/eppinput/testdata/login_wrong_case.xml new file mode 100644 index 000000000..c445ab412 --- /dev/null +++ b/javatests/google/registry/model/eppinput/testdata/login_wrong_case.xml @@ -0,0 +1,22 @@ + + + + NewRegistrar + foo-BAR2 + + 1.0 + en + + + urn:ietf:params:xml:ns:host-1.0 + urn:ietf:params:xml:ns:domain-1.0 + urn:ietf:params:xml:ns:contact-1.0 + + urn:ietf:params:xml:ns:launch-1.0 + urn:ietf:params:xml:ns:rgp-1.0 + + + + ABC-12345 + +