diff --git a/java/google/registry/rde/RdeMarshaller.java b/java/google/registry/rde/RdeMarshaller.java index 4a9c20907..db4f243ff 100644 --- a/java/google/registry/rde/RdeMarshaller.java +++ b/java/google/registry/rde/RdeMarshaller.java @@ -35,6 +35,7 @@ import google.registry.xjc.rdeidn.XjcRdeIdn; import google.registry.xjc.rdeidn.XjcRdeIdnElement; import google.registry.xjc.rdepolicy.XjcRdePolicy; import google.registry.xjc.rdepolicy.XjcRdePolicyElement; +import google.registry.xml.ValidationMode; import google.registry.xml.XmlException; import google.registry.xml.XmlFragmentMarshaller; import java.io.ByteArrayOutputStream; @@ -50,11 +51,15 @@ import org.joda.time.DateTime; public final class RdeMarshaller implements Serializable { private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); - private static final long serialVersionUID = 202890386611768455L; + private final ValidationMode validationMode; private transient XmlFragmentMarshaller memoizedMarshaller; + public RdeMarshaller(ValidationMode validationMode) { + this.validationMode = validationMode; + } + /** Returns top-portion of XML document. */ public String makeHeader( String depositId, DateTime watermark, Collection uris, int revision) { @@ -79,7 +84,7 @@ public final class RdeMarshaller implements Serializable { deposit.setContents(contents); ByteArrayOutputStream os = new ByteArrayOutputStream(); try { - XjcXmlTransformer.marshalStrict(deposit, os, UTF_8); + XjcXmlTransformer.marshal(deposit, os, UTF_8, validationMode); } catch (XmlException e) { throw new RuntimeException(e); } @@ -95,10 +100,18 @@ public final class RdeMarshaller implements Serializable { return "\n\n\n"; } - /** Turns XJC element into XML fragment, with schema validation. */ - public String marshalStrictlyOrDie(JAXBElement element) { + /** Turns XJC element into XML fragment, with schema validation unless in lenient mode. */ + public String marshal(JAXBElement element) throws MarshalException { + return getMarshaller().marshal(element, validationMode); + } + + /** + * Turns XJC element into XML fragment, converting {@link MarshalException}s to {@link + * RuntimeException}s. + */ + public String marshalOrDie(JAXBElement element) { try { - return getMarshaller().marshal(element); + return marshal(element); } catch (MarshalException e) { throw new RuntimeException(e); } @@ -141,7 +154,7 @@ public final class RdeMarshaller implements Serializable { bean.setId(idn.getName()); bean.setUrl(idn.getUrl().toString()); bean.setUrlPolicy(idn.getPolicy().toString()); - return marshalStrictlyOrDie(new XjcRdeIdnElement(bean)); + return marshalOrDie(new XjcRdeIdnElement(bean)); } private DepositFragment marshalResource( @@ -149,7 +162,7 @@ public final class RdeMarshaller implements Serializable { String xml = ""; String error = ""; try { - xml = getMarshaller().marshal(element); + xml = marshal(element); } catch (MarshalException e) { error = String.format("RDE XML schema validation failed: %s\n%s%s\n", Key.create(resource), diff --git a/java/google/registry/rde/RdeModule.java b/java/google/registry/rde/RdeModule.java index 82fd84d30..84591e0b5 100644 --- a/java/google/registry/rde/RdeModule.java +++ b/java/google/registry/rde/RdeModule.java @@ -45,6 +45,7 @@ public final class RdeModule { public static final String PARAM_DIRECTORY = "directory"; public static final String PARAM_MODE = "mode"; public static final String PARAM_REVISION = "revision"; + public static final String PARAM_LENIENT = "lenient"; @Provides @Parameter(PARAM_WATERMARK) @@ -82,6 +83,12 @@ public final class RdeModule { return extractOptionalIntParameter(req, PARAM_REVISION); } + @Provides + @Parameter(PARAM_LENIENT) + static boolean provideLenient(HttpServletRequest req) { + return extractBooleanParameter(req, PARAM_REVISION); + } + @Provides @Named("brda") static Queue provideQueueBrda() { diff --git a/java/google/registry/rde/RdeStagingAction.java b/java/google/registry/rde/RdeStagingAction.java index 52481aa36..b55075ea3 100644 --- a/java/google/registry/rde/RdeStagingAction.java +++ b/java/google/registry/rde/RdeStagingAction.java @@ -17,6 +17,8 @@ package google.registry.rde; import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.POST; import static google.registry.util.PipelineUtils.createJobPath; +import static google.registry.xml.ValidationMode.LENIENT; +import static google.registry.xml.ValidationMode.STRICT; import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; import com.google.common.base.Ascii; @@ -212,14 +214,16 @@ public final class RdeStagingAction implements Runnable { @Inject @Parameter(RequestParameters.PARAM_TLD) ImmutableSet tlds; @Inject @Parameter(RdeModule.PARAM_WATERMARK) ImmutableSet watermarks; @Inject @Parameter(RdeModule.PARAM_REVISION) Optional revision; + @Inject @Parameter(RdeModule.PARAM_LENIENT) boolean lenient; @Inject RdeStagingAction() {} + private RdeStagingMapper mapper; + @Override public void run() { ImmutableSetMultimap pendings = manual ? getManualPendingDeposits() : getStandardPendingDeposits(); - if (pendings.isEmpty()) { String message = "Nothing needs to be deposited"; logger.info(message); @@ -227,16 +231,17 @@ public final class RdeStagingAction implements Runnable { response.setPayload(message); return; } - for (PendingDeposit pending : pendings.values()) { logger.infofmt("%s", pending); } + mapper = new RdeStagingMapper(lenient ? LENIENT : STRICT, pendings); + response.sendJavaScriptRedirect(createJobPath(mrRunner .setJobName("Stage escrow deposits for all TLDs") .setModuleName("backend") .setDefaultReduceShards(pendings.size()) .runMapreduce( - new RdeStagingMapper(pendings), + mapper, reducer, ImmutableList.of( // Add an extra shard that maps over a null resource. See the mapper code for why. diff --git a/java/google/registry/rde/RdeStagingMapper.java b/java/google/registry/rde/RdeStagingMapper.java index 6385742ee..a26b75b2e 100644 --- a/java/google/registry/rde/RdeStagingMapper.java +++ b/java/google/registry/rde/RdeStagingMapper.java @@ -34,6 +34,7 @@ import google.registry.model.domain.DomainResource; import google.registry.model.host.HostResource; import google.registry.model.rde.RdeMode; import google.registry.model.registrar.Registrar; +import google.registry.xml.ValidationMode; import java.util.HashMap; import java.util.Map; import org.joda.time.DateTime; @@ -43,10 +44,12 @@ public final class RdeStagingMapper extends Mapper pendings; - private final RdeMarshaller marshaller = new RdeMarshaller(); - RdeStagingMapper(ImmutableSetMultimap pendings) { + RdeStagingMapper( + ValidationMode validationMode, ImmutableSetMultimap pendings) { + this.marshaller = new RdeMarshaller(validationMode); this.pendings = pendings; } diff --git a/java/google/registry/rde/RdeStagingReducer.java b/java/google/registry/rde/RdeStagingReducer.java index 91b0d59ac..d4bc2fbdc 100644 --- a/java/google/registry/rde/RdeStagingReducer.java +++ b/java/google/registry/rde/RdeStagingReducer.java @@ -21,6 +21,8 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime; import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.xml.ValidationMode.LENIENT; +import static google.registry.xml.ValidationMode.STRICT; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; @@ -39,6 +41,7 @@ import google.registry.model.rde.RdeNamingUtils; import google.registry.model.rde.RdeRevision; import google.registry.model.registry.Registry; import google.registry.model.server.Lock; +import google.registry.request.Parameter; import google.registry.request.RequestParameters; import google.registry.tldconfig.idn.IdnTableEnum; import google.registry.util.FormattingLogger; @@ -67,15 +70,31 @@ public final class RdeStagingReducer extends Reducer fragments) { @@ -154,7 +173,7 @@ public final class RdeStagingReducer extends Reducer element) throws MarshalException { - return internalMarshal(element, true); + return marshal(element, STRICT); } /** Turns an individual JAXB element into an XML fragment string. */ public String marshalLenient(JAXBElement element) { try { - return internalMarshal(element, false); + return marshal(element, LENIENT); } catch (MarshalException e) { throw new RuntimeException("MarshalException shouldn't be thrown in lenient mode", e); } } - private String internalMarshal(JAXBElement element, boolean strict) throws MarshalException { + /** + * Turns an individual JAXB element into an XML fragment string using the given validation mode. + * + * @throws MarshalException if schema validation failed + */ + public String marshal(JAXBElement element, ValidationMode validationMode) + throws MarshalException { os.reset(); - marshaller.setSchema(strict ? schema : null); + marshaller.setSchema((validationMode == STRICT) ? schema : null); try { marshaller.marshal(element, os); } catch (JAXBException e) { diff --git a/javatests/google/registry/rde/RdeMarshallerTest.java b/javatests/google/registry/rde/RdeMarshallerTest.java index 1c6f2edec..99ff09f44 100644 --- a/javatests/google/registry/rde/RdeMarshallerTest.java +++ b/javatests/google/registry/rde/RdeMarshallerTest.java @@ -15,6 +15,7 @@ package google.registry.rde; import static com.google.common.truth.Truth.assertThat; +import static google.registry.xml.ValidationMode.STRICT; import google.registry.model.registrar.Registrar; import google.registry.testing.AppEngineRule; @@ -40,7 +41,8 @@ public class RdeMarshallerTest extends ShardableTestCase { @Test public void testMarshalRegistrar_validData_producesXmlFragment() throws Exception { DepositFragment fragment = - new RdeMarshaller().marshalRegistrar(Registrar.loadByClientId("TheRegistrar")); + new RdeMarshaller(STRICT) + .marshalRegistrar(Registrar.loadByClientId("TheRegistrar")); assertThat(fragment.type()).isEqualTo(RdeResourceType.REGISTRAR); assertThat(fragment.error()).isEmpty(); String expected = "" @@ -83,7 +85,8 @@ public class RdeMarshallerTest extends ShardableTestCase { @Test public void testMarshalRegistrar_unicodeCharacters_dontGetMangled() throws Exception { DepositFragment fragment = - new RdeMarshaller().marshalRegistrar(Registrar.loadByClientId("TheRegistrar")); + new RdeMarshaller(STRICT) + .marshalRegistrar(Registrar.loadByClientId("TheRegistrar")); assertThat(fragment.xml()).contains("123 Example Bőulevard"); } } diff --git a/javatests/google/registry/rde/RdeStagingActionTest.java b/javatests/google/registry/rde/RdeStagingActionTest.java index 1ccf85f9a..492d3e4be 100644 --- a/javatests/google/registry/rde/RdeStagingActionTest.java +++ b/javatests/google/registry/rde/RdeStagingActionTest.java @@ -135,12 +135,15 @@ public class RdeStagingActionTest extends MapreduceTestCase { action = new RdeStagingAction(); action.clock = clock; action.mrRunner = makeDefaultRunner(); - action.reducer = new RdeStagingReducer(); - action.reducer.ghostrydeBufferSize = 31337; - action.reducer.lockTimeout = Duration.standardHours(1); - action.reducer.bucket = "rde-bucket"; - action.reducer.taskEnqueuer = new TaskEnqueuer(new Retrier(new SystemSleeper(), 1)); - action.reducer.stagingKeyBytes = PgpHelper.convertPublicKeyToBytes(encryptKey); + action.lenient = false; + action.reducer = new RdeStagingReducer( + new TaskEnqueuer(new Retrier(new SystemSleeper(), 1)), // taskEnqueuer + 0, // gcsBufferSize + "rde-bucket", // bucket + 31337, // ghostrydeBufferSize + Duration.standardHours(1), // lockTimeout + PgpHelper.convertPublicKeyToBytes(encryptKey), // stagingKeyBytes + false); // lenient action.pendingDepositChecker = new PendingDepositChecker(); action.pendingDepositChecker.brdaDayOfWeek = DateTimeConstants.TUESDAY; action.pendingDepositChecker.brdaInterval = Duration.standardDays(7);