Add the ability to generate RDE deposits in lenient mode

We will not want to run this under normal circumstances, but in cases (such as PDT testing in sandbox) where it's desirable to generate an escrow deposit even if it isn't technically valid XML, this gives us that option. Manual-mode RDE generation is also changed so that, if no watermark date is specified, it defaults to the previous midnight, to better support running of RDE in sandbox to catch data problems.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=156108295

java/google/registry/rde/RdeMarshaller.java

@@ -35,6 +35,7 @@ import google.registry.xjc.rdeidn.XjcRdeIdn;
 import google.registry.xjc.rdeidn.XjcRdeIdnElement;
 import google.registry.xjc.rdepolicy.XjcRdePolicy;
 import google.registry.xjc.rdepolicy.XjcRdePolicyElement;
+import google.registry.xml.ValidationMode;
 import google.registry.xml.XmlException;
 import google.registry.xml.XmlFragmentMarshaller;
 import java.io.ByteArrayOutputStream;
@@ -50,11 +51,15 @@ import org.joda.time.DateTime;
 public final class RdeMarshaller implements Serializable {
 
   private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
-
   private static final long serialVersionUID = 202890386611768455L;
 
+  private final ValidationMode validationMode;
   private transient XmlFragmentMarshaller memoizedMarshaller;
 
+  public RdeMarshaller(ValidationMode validationMode) {
+    this.validationMode = validationMode;
+  }
+
   /** Returns top-portion of XML document. */
   public String makeHeader(
       String depositId, DateTime watermark, Collection<String> uris, int revision) {
@@ -79,7 +84,7 @@ public final class RdeMarshaller implements Serializable {
     deposit.setContents(contents);
     ByteArrayOutputStream os = new ByteArrayOutputStream();
     try {
-      XjcXmlTransformer.marshalStrict(deposit, os, UTF_8);
+      XjcXmlTransformer.marshal(deposit, os, UTF_8, validationMode);
     } catch (XmlException e) {
       throw new RuntimeException(e);
     }
@@ -95,10 +100,18 @@ public final class RdeMarshaller implements Serializable {
     return "\n</rde:contents>\n</rde:deposit>\n";
   }
 
-  /** Turns XJC element into XML fragment, with schema validation. */
+  /** Turns XJC element into XML fragment, with schema validation unless in lenient mode. */
-  public String marshalStrictlyOrDie(JAXBElement<?> element) {
+  public String marshal(JAXBElement<?> element) throws MarshalException {
+    return getMarshaller().marshal(element, validationMode);
+  }
+
+  /**
+   * Turns XJC element into XML fragment, converting {@link MarshalException}s to {@link
+   * RuntimeException}s.
+   */
+  public String marshalOrDie(JAXBElement<?> element) {
     try {
-      return getMarshaller().marshal(element);
+      return marshal(element);
     } catch (MarshalException e) {
       throw new RuntimeException(e);
     }
@@ -141,7 +154,7 @@ public final class RdeMarshaller implements Serializable {
     bean.setId(idn.getName());
     bean.setUrl(idn.getUrl().toString());
     bean.setUrlPolicy(idn.getPolicy().toString());
-    return marshalStrictlyOrDie(new XjcRdeIdnElement(bean));
+    return marshalOrDie(new XjcRdeIdnElement(bean));
   }
 
   private DepositFragment marshalResource(
@@ -149,7 +162,7 @@ public final class RdeMarshaller implements Serializable {
     String xml = "";
     String error = "";
     try {
-      xml = getMarshaller().marshal(element);
+      xml = marshal(element);
     } catch (MarshalException e) {
       error = String.format("RDE XML schema validation failed: %s\n%s%s\n",
           Key.create(resource),

java/google/registry/rde/RdeModule.java

@@ -45,6 +45,7 @@ public final class RdeModule {
   public static final String PARAM_DIRECTORY = "directory";
   public static final String PARAM_MODE = "mode";
   public static final String PARAM_REVISION = "revision";
+  public static final String PARAM_LENIENT = "lenient";
 
   @Provides
   @Parameter(PARAM_WATERMARK)
@@ -82,6 +83,12 @@ public final class RdeModule {
     return extractOptionalIntParameter(req, PARAM_REVISION);
   }
 
+  @Provides
+  @Parameter(PARAM_LENIENT)
+  static boolean provideLenient(HttpServletRequest req) {
+    return extractBooleanParameter(req, PARAM_REVISION);
+  }
+
   @Provides
   @Named("brda")
   static Queue provideQueueBrda() {

java/google/registry/rde/RdeStagingAction.java

@@ -17,6 +17,8 @@ package google.registry.rde;
 import static google.registry.request.Action.Method.GET;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.util.PipelineUtils.createJobPath;
+import static google.registry.xml.ValidationMode.LENIENT;
+import static google.registry.xml.ValidationMode.STRICT;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 
 import com.google.common.base.Ascii;
@@ -212,14 +214,16 @@ public final class RdeStagingAction implements Runnable {
   @Inject @Parameter(RequestParameters.PARAM_TLD) ImmutableSet<String> tlds;
   @Inject @Parameter(RdeModule.PARAM_WATERMARK) ImmutableSet<DateTime> watermarks;
   @Inject @Parameter(RdeModule.PARAM_REVISION) Optional<Integer> revision;
+  @Inject @Parameter(RdeModule.PARAM_LENIENT) boolean lenient;
 
   @Inject RdeStagingAction() {}
 
+  private RdeStagingMapper mapper;
+
   @Override
   public void run() {
     ImmutableSetMultimap<String, PendingDeposit> pendings =
         manual ? getManualPendingDeposits() : getStandardPendingDeposits();
-
     if (pendings.isEmpty()) {
       String message = "Nothing needs to be deposited";
       logger.info(message);
@@ -227,16 +231,17 @@ public final class RdeStagingAction implements Runnable {
       response.setPayload(message);
       return;
     }
-
     for (PendingDeposit pending : pendings.values()) {
       logger.infofmt("%s", pending);
     }
+    mapper = new RdeStagingMapper(lenient ? LENIENT : STRICT, pendings);
+
     response.sendJavaScriptRedirect(createJobPath(mrRunner
         .setJobName("Stage escrow deposits for all TLDs")
         .setModuleName("backend")
         .setDefaultReduceShards(pendings.size())
         .runMapreduce(
-            new RdeStagingMapper(pendings),
+            mapper,
             reducer,
             ImmutableList.of(
                 // Add an extra shard that maps over a null resource. See the mapper code for why.

java/google/registry/rde/RdeStagingMapper.java

@@ -34,6 +34,7 @@ import google.registry.model.domain.DomainResource;
 import google.registry.model.host.HostResource;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.registrar.Registrar;
+import google.registry.xml.ValidationMode;
 import java.util.HashMap;
 import java.util.Map;
 import org.joda.time.DateTime;
@@ -43,10 +44,12 @@ public final class RdeStagingMapper extends Mapper<EppResource, PendingDeposit,
 
   private static final long serialVersionUID = -1518185703789372524L;
 
+  private final RdeMarshaller marshaller;
   private final ImmutableSetMultimap<String, PendingDeposit> pendings;
-  private final RdeMarshaller marshaller = new RdeMarshaller();
 
-  RdeStagingMapper(ImmutableSetMultimap<String, PendingDeposit> pendings) {
+  RdeStagingMapper(
+      ValidationMode validationMode, ImmutableSetMultimap<String, PendingDeposit> pendings) {
+    this.marshaller = new RdeMarshaller(validationMode);
     this.pendings = pendings;
   }
 

java/google/registry/rde/RdeStagingReducer.java

@@ -21,6 +21,8 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Verify.verify;
 import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.xml.ValidationMode.LENIENT;
+import static google.registry.xml.ValidationMode.STRICT;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
@@ -39,6 +41,7 @@ import google.registry.model.rde.RdeNamingUtils;
 import google.registry.model.rde.RdeRevision;
 import google.registry.model.registry.Registry;
 import google.registry.model.server.Lock;
+import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
 import google.registry.tldconfig.idn.IdnTableEnum;
 import google.registry.util.FormattingLogger;
@@ -67,15 +70,31 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
 
   private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
 
-  private final RdeMarshaller marshaller = new RdeMarshaller();
+  private final TaskEnqueuer taskEnqueuer;
+  private final int gcsBufferSize;
+  private final String bucket;
+  private final int ghostrydeBufferSize;
+  private final Duration lockTimeout;
+  private final byte[] stagingKeyBytes;
+  private final RdeMarshaller marshaller;
 
-  @Inject TaskEnqueuer taskEnqueuer;
+  @Inject
-  @Inject @Config("gcsBufferSize") int gcsBufferSize;
+  RdeStagingReducer(
-  @Inject @Config("rdeBucket") String bucket;
+      TaskEnqueuer taskEnqueuer,
-  @Inject @Config("rdeGhostrydeBufferSize") int ghostrydeBufferSize;
+      @Config("gcsBufferSize") int gcsBufferSize,
-  @Inject @Config("rdeStagingLockTimeout") Duration lockTimeout;
+      @Config("rdeBucket") String bucket,
-  @Inject @KeyModule.Key("rdeStagingEncryptionKey") byte[] stagingKeyBytes;
+      @Config("rdeGhostrydeBufferSize") int ghostrydeBufferSize,
-  @Inject RdeStagingReducer() {}
+      @Config("rdeStagingLockTimeout") Duration lockTimeout,
+      @KeyModule.Key("rdeStagingEncryptionKey") byte[] stagingKeyBytes,
+      @Parameter(RdeModule.PARAM_LENIENT) boolean lenient) {
+    this.taskEnqueuer = taskEnqueuer;
+    this.gcsBufferSize = gcsBufferSize;
+    this.bucket = bucket;
+    this.ghostrydeBufferSize = ghostrydeBufferSize;
+    this.lockTimeout = lockTimeout;
+    this.stagingKeyBytes = stagingKeyBytes;
+    this.marshaller = new RdeMarshaller(lenient ? LENIENT : STRICT);
+  }
 
   @Override
   public void reduce(final PendingDeposit key, final ReducerInput<DepositFragment> fragments) {
@@ -154,7 +173,7 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
 
       // Output XML that says how many resources were emitted.
       header = counter.makeHeader(tld, mode);
-      output.write(marshaller.marshalStrictlyOrDie(new XjcRdeHeaderElement(header)));
+      output.write(marshaller.marshalOrDie(new XjcRdeHeaderElement(header)));
 
       // Output the bottom of the XML document.
       output.write(marshaller.makeFooter());

java/google/registry/xjc/XjcXmlTransformer.java

@@ -18,6 +18,7 @@ import static google.registry.xml.ValidationMode.LENIENT;
 import static google.registry.xml.ValidationMode.STRICT;
 
 import com.google.common.collect.ImmutableMap;
+import google.registry.xml.ValidationMode;
 import google.registry.xml.XmlException;
 import google.registry.xml.XmlTransformer;
 import java.io.InputStream;
@@ -82,4 +83,10 @@ public class XjcXmlTransformer {
       throws XmlException {
     INSTANCE.marshal(root, out, charset, STRICT);
   }
+
+  public static void marshal(
+      Object root, OutputStream out, Charset charset, ValidationMode validationMode)
+      throws XmlException {
+    INSTANCE.marshal(root, out, charset, validationMode);
+  }
 }

java/google/registry/xml/XmlFragmentMarshaller.java

@@ -16,6 +16,8 @@ package google.registry.xml;
 
 import static com.google.common.base.Throwables.throwIfInstanceOf;
 import static com.google.common.base.Verify.verify;
+import static google.registry.xml.ValidationMode.LENIENT;
+import static google.registry.xml.ValidationMode.STRICT;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.re2j.Pattern;
@@ -56,21 +58,27 @@ public final class XmlFragmentMarshaller {
    * @throws MarshalException if schema validation failed
    */
   public String marshal(JAXBElement<?> element) throws MarshalException {
-    return internalMarshal(element, true);
+    return marshal(element, STRICT);
   }
 
   /** Turns an individual JAXB element into an XML fragment string. */
   public String marshalLenient(JAXBElement<?> element) {
     try {
-      return internalMarshal(element, false);
+      return marshal(element, LENIENT);
     } catch (MarshalException e) {
       throw new RuntimeException("MarshalException shouldn't be thrown in lenient mode", e);
     }
   }
 
-  private String internalMarshal(JAXBElement<?> element, boolean strict) throws MarshalException {
+  /**
+   * Turns an individual JAXB element into an XML fragment string using the given validation mode.
+   *
+   * @throws MarshalException if schema validation failed
+   */
+  public String marshal(JAXBElement<?> element, ValidationMode validationMode)
+      throws MarshalException {
     os.reset();
-    marshaller.setSchema(strict ? schema : null);
+    marshaller.setSchema((validationMode == STRICT) ? schema : null);
     try {
       marshaller.marshal(element, os);
     } catch (JAXBException e) {

javatests/google/registry/rde/RdeMarshallerTest.java

@@ -15,6 +15,7 @@
 package google.registry.rde;
 
 import static com.google.common.truth.Truth.assertThat;
+import static google.registry.xml.ValidationMode.STRICT;
 
 import google.registry.model.registrar.Registrar;
 import google.registry.testing.AppEngineRule;
@@ -40,7 +41,8 @@ public class RdeMarshallerTest extends ShardableTestCase {
   @Test
   public void testMarshalRegistrar_validData_producesXmlFragment() throws Exception {
     DepositFragment fragment =
-        new RdeMarshaller().marshalRegistrar(Registrar.loadByClientId("TheRegistrar"));
+        new RdeMarshaller(STRICT)
+            .marshalRegistrar(Registrar.loadByClientId("TheRegistrar"));
     assertThat(fragment.type()).isEqualTo(RdeResourceType.REGISTRAR);
     assertThat(fragment.error()).isEmpty();
     String expected = ""
@@ -83,7 +85,8 @@ public class RdeMarshallerTest extends ShardableTestCase {
   @Test
   public void testMarshalRegistrar_unicodeCharacters_dontGetMangled() throws Exception {
     DepositFragment fragment =
-        new RdeMarshaller().marshalRegistrar(Registrar.loadByClientId("TheRegistrar"));
+        new RdeMarshaller(STRICT)
+            .marshalRegistrar(Registrar.loadByClientId("TheRegistrar"));
     assertThat(fragment.xml()).contains("123 Example Bőulevard");
   }
 }

javatests/google/registry/rde/RdeStagingActionTest.java

@@ -135,12 +135,15 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
     action = new RdeStagingAction();
     action.clock = clock;
     action.mrRunner = makeDefaultRunner();
-    action.reducer = new RdeStagingReducer();
+    action.lenient = false;
-    action.reducer.ghostrydeBufferSize = 31337;
+    action.reducer = new RdeStagingReducer(
-    action.reducer.lockTimeout = Duration.standardHours(1);
+        new TaskEnqueuer(new Retrier(new SystemSleeper(), 1)), // taskEnqueuer
-    action.reducer.bucket = "rde-bucket";
+        0, // gcsBufferSize
-    action.reducer.taskEnqueuer = new TaskEnqueuer(new Retrier(new SystemSleeper(), 1));
+        "rde-bucket", // bucket
-    action.reducer.stagingKeyBytes = PgpHelper.convertPublicKeyToBytes(encryptKey);
+        31337, // ghostrydeBufferSize
+        Duration.standardHours(1), // lockTimeout
+        PgpHelper.convertPublicKeyToBytes(encryptKey), // stagingKeyBytes
+        false); // lenient
     action.pendingDepositChecker = new PendingDepositChecker();
     action.pendingDepositChecker.brdaDayOfWeek = DateTimeConstants.TUESDAY;
     action.pendingDepositChecker.brdaInterval = Duration.standardDays(7);