From 74a0defef3b7c395737a8e0cd78dc16ceb5a4a0b Mon Sep 17 00:00:00 2001 From: mountford Date: Mon, 15 May 2017 14:56:02 -0700 Subject: [PATCH] Add the ability to generate RDE deposits in lenient mode We will not want to run this under normal circumstances, but in cases (such as PDT testing in sandbox) where it's desirable to generate an escrow deposit even if it isn't technically valid XML, this gives us that option. Manual-mode RDE generation is also changed so that, if no watermark date is specified, it defaults to the previous midnight, to better support running of RDE in sandbox to catch data problems. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=156108295 --- java/google/registry/rde/RdeMarshaller.java | 27 ++++++++++---- java/google/registry/rde/RdeModule.java | 7 ++++ .../google/registry/rde/RdeStagingAction.java | 11 ++++-- .../google/registry/rde/RdeStagingMapper.java | 7 +++- .../registry/rde/RdeStagingReducer.java | 37 ++++++++++++++----- .../registry/xjc/XjcXmlTransformer.java | 7 ++++ .../registry/xml/XmlFragmentMarshaller.java | 16 ++++++-- .../registry/rde/RdeMarshallerTest.java | 7 +++- .../registry/rde/RdeStagingActionTest.java | 15 +++++--- 9 files changed, 101 insertions(+), 33 deletions(-) diff --git a/java/google/registry/rde/RdeMarshaller.java b/java/google/registry/rde/RdeMarshaller.java index 4a9c20907..db4f243ff 100644 --- a/java/google/registry/rde/RdeMarshaller.java +++ b/java/google/registry/rde/RdeMarshaller.java @@ -35,6 +35,7 @@ import google.registry.xjc.rdeidn.XjcRdeIdn; import google.registry.xjc.rdeidn.XjcRdeIdnElement; import google.registry.xjc.rdepolicy.XjcRdePolicy; import google.registry.xjc.rdepolicy.XjcRdePolicyElement; +import google.registry.xml.ValidationMode; import google.registry.xml.XmlException; import google.registry.xml.XmlFragmentMarshaller; import java.io.ByteArrayOutputStream; @@ -50,11 +51,15 @@ import org.joda.time.DateTime; public final class RdeMarshaller implements Serializable { private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); - private static final long serialVersionUID = 202890386611768455L; + private final ValidationMode validationMode; private transient XmlFragmentMarshaller memoizedMarshaller; + public RdeMarshaller(ValidationMode validationMode) { + this.validationMode = validationMode; + } + /** Returns top-portion of XML document. */ public String makeHeader( String depositId, DateTime watermark, Collection uris, int revision) { @@ -79,7 +84,7 @@ public final class RdeMarshaller implements Serializable { deposit.setContents(contents); ByteArrayOutputStream os = new ByteArrayOutputStream(); try { - XjcXmlTransformer.marshalStrict(deposit, os, UTF_8); + XjcXmlTransformer.marshal(deposit, os, UTF_8, validationMode); } catch (XmlException e) { throw new RuntimeException(e); } @@ -95,10 +100,18 @@ public final class RdeMarshaller implements Serializable { return "\n\n\n"; } - /** Turns XJC element into XML fragment, with schema validation. */ - public String marshalStrictlyOrDie(JAXBElement element) { + /** Turns XJC element into XML fragment, with schema validation unless in lenient mode. */ + public String marshal(JAXBElement element) throws MarshalException { + return getMarshaller().marshal(element, validationMode); + } + + /** + * Turns XJC element into XML fragment, converting {@link MarshalException}s to {@link + * RuntimeException}s. + */ + public String marshalOrDie(JAXBElement element) { try { - return getMarshaller().marshal(element); + return marshal(element); } catch (MarshalException e) { throw new RuntimeException(e); } @@ -141,7 +154,7 @@ public final class RdeMarshaller implements Serializable { bean.setId(idn.getName()); bean.setUrl(idn.getUrl().toString()); bean.setUrlPolicy(idn.getPolicy().toString()); - return marshalStrictlyOrDie(new XjcRdeIdnElement(bean)); + return marshalOrDie(new XjcRdeIdnElement(bean)); } private DepositFragment marshalResource( @@ -149,7 +162,7 @@ public final class RdeMarshaller implements Serializable { String xml = ""; String error = ""; try { - xml = getMarshaller().marshal(element); + xml = marshal(element); } catch (MarshalException e) { error = String.format("RDE XML schema validation failed: %s\n%s%s\n", Key.create(resource), diff --git a/java/google/registry/rde/RdeModule.java b/java/google/registry/rde/RdeModule.java index 82fd84d30..84591e0b5 100644 --- a/java/google/registry/rde/RdeModule.java +++ b/java/google/registry/rde/RdeModule.java @@ -45,6 +45,7 @@ public final class RdeModule { public static final String PARAM_DIRECTORY = "directory"; public static final String PARAM_MODE = "mode"; public static final String PARAM_REVISION = "revision"; + public static final String PARAM_LENIENT = "lenient"; @Provides @Parameter(PARAM_WATERMARK) @@ -82,6 +83,12 @@ public final class RdeModule { return extractOptionalIntParameter(req, PARAM_REVISION); } + @Provides + @Parameter(PARAM_LENIENT) + static boolean provideLenient(HttpServletRequest req) { + return extractBooleanParameter(req, PARAM_REVISION); + } + @Provides @Named("brda") static Queue provideQueueBrda() { diff --git a/java/google/registry/rde/RdeStagingAction.java b/java/google/registry/rde/RdeStagingAction.java index 52481aa36..b55075ea3 100644 --- a/java/google/registry/rde/RdeStagingAction.java +++ b/java/google/registry/rde/RdeStagingAction.java @@ -17,6 +17,8 @@ package google.registry.rde; import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.POST; import static google.registry.util.PipelineUtils.createJobPath; +import static google.registry.xml.ValidationMode.LENIENT; +import static google.registry.xml.ValidationMode.STRICT; import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; import com.google.common.base.Ascii; @@ -212,14 +214,16 @@ public final class RdeStagingAction implements Runnable { @Inject @Parameter(RequestParameters.PARAM_TLD) ImmutableSet tlds; @Inject @Parameter(RdeModule.PARAM_WATERMARK) ImmutableSet watermarks; @Inject @Parameter(RdeModule.PARAM_REVISION) Optional revision; + @Inject @Parameter(RdeModule.PARAM_LENIENT) boolean lenient; @Inject RdeStagingAction() {} + private RdeStagingMapper mapper; + @Override public void run() { ImmutableSetMultimap pendings = manual ? getManualPendingDeposits() : getStandardPendingDeposits(); - if (pendings.isEmpty()) { String message = "Nothing needs to be deposited"; logger.info(message); @@ -227,16 +231,17 @@ public final class RdeStagingAction implements Runnable { response.setPayload(message); return; } - for (PendingDeposit pending : pendings.values()) { logger.infofmt("%s", pending); } + mapper = new RdeStagingMapper(lenient ? LENIENT : STRICT, pendings); + response.sendJavaScriptRedirect(createJobPath(mrRunner .setJobName("Stage escrow deposits for all TLDs") .setModuleName("backend") .setDefaultReduceShards(pendings.size()) .runMapreduce( - new RdeStagingMapper(pendings), + mapper, reducer, ImmutableList.of( // Add an extra shard that maps over a null resource. See the mapper code for why. diff --git a/java/google/registry/rde/RdeStagingMapper.java b/java/google/registry/rde/RdeStagingMapper.java index 6385742ee..a26b75b2e 100644 --- a/java/google/registry/rde/RdeStagingMapper.java +++ b/java/google/registry/rde/RdeStagingMapper.java @@ -34,6 +34,7 @@ import google.registry.model.domain.DomainResource; import google.registry.model.host.HostResource; import google.registry.model.rde.RdeMode; import google.registry.model.registrar.Registrar; +import google.registry.xml.ValidationMode; import java.util.HashMap; import java.util.Map; import org.joda.time.DateTime; @@ -43,10 +44,12 @@ public final class RdeStagingMapper extends Mapper pendings; - private final RdeMarshaller marshaller = new RdeMarshaller(); - RdeStagingMapper(ImmutableSetMultimap pendings) { + RdeStagingMapper( + ValidationMode validationMode, ImmutableSetMultimap pendings) { + this.marshaller = new RdeMarshaller(validationMode); this.pendings = pendings; } diff --git a/java/google/registry/rde/RdeStagingReducer.java b/java/google/registry/rde/RdeStagingReducer.java index 91b0d59ac..d4bc2fbdc 100644 --- a/java/google/registry/rde/RdeStagingReducer.java +++ b/java/google/registry/rde/RdeStagingReducer.java @@ -21,6 +21,8 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime; import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.xml.ValidationMode.LENIENT; +import static google.registry.xml.ValidationMode.STRICT; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; @@ -39,6 +41,7 @@ import google.registry.model.rde.RdeNamingUtils; import google.registry.model.rde.RdeRevision; import google.registry.model.registry.Registry; import google.registry.model.server.Lock; +import google.registry.request.Parameter; import google.registry.request.RequestParameters; import google.registry.tldconfig.idn.IdnTableEnum; import google.registry.util.FormattingLogger; @@ -67,15 +70,31 @@ public final class RdeStagingReducer extends Reducer fragments) { @@ -154,7 +173,7 @@ public final class RdeStagingReducer extends Reducer element) throws MarshalException { - return internalMarshal(element, true); + return marshal(element, STRICT); } /** Turns an individual JAXB element into an XML fragment string. */ public String marshalLenient(JAXBElement element) { try { - return internalMarshal(element, false); + return marshal(element, LENIENT); } catch (MarshalException e) { throw new RuntimeException("MarshalException shouldn't be thrown in lenient mode", e); } } - private String internalMarshal(JAXBElement element, boolean strict) throws MarshalException { + /** + * Turns an individual JAXB element into an XML fragment string using the given validation mode. + * + * @throws MarshalException if schema validation failed + */ + public String marshal(JAXBElement element, ValidationMode validationMode) + throws MarshalException { os.reset(); - marshaller.setSchema(strict ? schema : null); + marshaller.setSchema((validationMode == STRICT) ? schema : null); try { marshaller.marshal(element, os); } catch (JAXBException e) { diff --git a/javatests/google/registry/rde/RdeMarshallerTest.java b/javatests/google/registry/rde/RdeMarshallerTest.java index 1c6f2edec..99ff09f44 100644 --- a/javatests/google/registry/rde/RdeMarshallerTest.java +++ b/javatests/google/registry/rde/RdeMarshallerTest.java @@ -15,6 +15,7 @@ package google.registry.rde; import static com.google.common.truth.Truth.assertThat; +import static google.registry.xml.ValidationMode.STRICT; import google.registry.model.registrar.Registrar; import google.registry.testing.AppEngineRule; @@ -40,7 +41,8 @@ public class RdeMarshallerTest extends ShardableTestCase { @Test public void testMarshalRegistrar_validData_producesXmlFragment() throws Exception { DepositFragment fragment = - new RdeMarshaller().marshalRegistrar(Registrar.loadByClientId("TheRegistrar")); + new RdeMarshaller(STRICT) + .marshalRegistrar(Registrar.loadByClientId("TheRegistrar")); assertThat(fragment.type()).isEqualTo(RdeResourceType.REGISTRAR); assertThat(fragment.error()).isEmpty(); String expected = "" @@ -83,7 +85,8 @@ public class RdeMarshallerTest extends ShardableTestCase { @Test public void testMarshalRegistrar_unicodeCharacters_dontGetMangled() throws Exception { DepositFragment fragment = - new RdeMarshaller().marshalRegistrar(Registrar.loadByClientId("TheRegistrar")); + new RdeMarshaller(STRICT) + .marshalRegistrar(Registrar.loadByClientId("TheRegistrar")); assertThat(fragment.xml()).contains("123 Example Bőulevard"); } } diff --git a/javatests/google/registry/rde/RdeStagingActionTest.java b/javatests/google/registry/rde/RdeStagingActionTest.java index 1ccf85f9a..492d3e4be 100644 --- a/javatests/google/registry/rde/RdeStagingActionTest.java +++ b/javatests/google/registry/rde/RdeStagingActionTest.java @@ -135,12 +135,15 @@ public class RdeStagingActionTest extends MapreduceTestCase { action = new RdeStagingAction(); action.clock = clock; action.mrRunner = makeDefaultRunner(); - action.reducer = new RdeStagingReducer(); - action.reducer.ghostrydeBufferSize = 31337; - action.reducer.lockTimeout = Duration.standardHours(1); - action.reducer.bucket = "rde-bucket"; - action.reducer.taskEnqueuer = new TaskEnqueuer(new Retrier(new SystemSleeper(), 1)); - action.reducer.stagingKeyBytes = PgpHelper.convertPublicKeyToBytes(encryptKey); + action.lenient = false; + action.reducer = new RdeStagingReducer( + new TaskEnqueuer(new Retrier(new SystemSleeper(), 1)), // taskEnqueuer + 0, // gcsBufferSize + "rde-bucket", // bucket + 31337, // ghostrydeBufferSize + Duration.standardHours(1), // lockTimeout + PgpHelper.convertPublicKeyToBytes(encryptKey), // stagingKeyBytes + false); // lenient action.pendingDepositChecker = new PendingDepositChecker(); action.pendingDepositChecker.brdaDayOfWeek = DateTimeConstants.TUESDAY; action.pendingDepositChecker.brdaInterval = Duration.standardDays(7);