Add mapper to import domains from RDE deposits

With some additional cleanup by Ben McIlwain.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=146695665

java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -275,6 +275,12 @@
     <url-pattern>/_dr/task/importRdeHosts</url-pattern>
   </servlet-mapping>
 
+  <!-- Mapreduce to import domains from escrow file -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/importRdeDomains</url-pattern>
+  </servlet-mapping>
+
   <!-- Security config -->
   <security-constraint>
     <web-resource-collection>

java/google/registry/module/backend/BackendRequestComponent.java

@@ -59,6 +59,7 @@ import google.registry.rde.RdeReporter;
 import google.registry.rde.RdeStagingAction;
 import google.registry.rde.RdeUploadAction;
 import google.registry.rde.imports.RdeContactImportAction;
+import google.registry.rde.imports.RdeDomainImportAction;
 import google.registry.rde.imports.RdeHostImportAction;
 import google.registry.rde.imports.RdeImportsModule;
 import google.registry.request.RequestComponentBuilder;
@@ -114,6 +115,7 @@ interface BackendRequestComponent {
   PublishDnsUpdatesAction publishDnsUpdatesAction();
   ReadDnsQueueAction readDnsQueueAction();
   RdeContactImportAction rdeContactImportAction();
+  RdeDomainImportAction rdeDomainImportAction();
   RdeHostImportAction rdeHostImportAction();
   RdeReportAction rdeReportAction();
   RdeStagingAction rdeStagingAction();

java/google/registry/rde/imports/BUILD

@@ -15,6 +15,7 @@ java_library(
         "//java/google/registry/request",
         "//java/google/registry/util",
         "//java/google/registry/xjc",
+        "//java/google/registry/xml",
         "//third_party/java/objectify:objectify-v4_1",
         "@com_google_appengine_api_1_0_sdk",
         "@com_google_appengine_tools_appengine_gcs_client",

java/google/registry/rde/imports/RdeContactImportAction.java

@@ -24,6 +24,7 @@ import com.google.appengine.tools.cloudstorage.RetryParams;
 import com.google.appengine.tools.mapreduce.Mapper;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.VoidWork;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
@@ -32,7 +33,11 @@ import google.registry.model.contact.ContactResource;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
+import google.registry.util.FormattingLogger;
 import google.registry.util.SystemClock;
+import google.registry.xjc.JaxbFragment;
+import google.registry.xjc.rdecontact.XjcRdeContact;
+import google.registry.xjc.rdecontact.XjcRdeContactElement;
 import javax.inject.Inject;
 
 /**
@@ -43,6 +48,7 @@ import javax.inject.Inject;
 @Action(path = "/_dr/task/importRdeContacts")
 public class RdeContactImportAction implements Runnable {
 
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
   private static final GcsService GCS_SERVICE =
       GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
 
@@ -91,9 +97,11 @@ public class RdeContactImportAction implements Runnable {
   }
 
   /** Mapper to import contacts from an escrow file. */
-  public static class RdeContactImportMapper extends Mapper<ContactResource, Void, Void> {
+  public static class RdeContactImportMapper
+      extends Mapper<JaxbFragment<XjcRdeContactElement>, Void, Void> {
 
     private static final long serialVersionUID = -7645091075256589374L;
+
     private final String importBucketName;
     private transient RdeImportUtils importUtils;
 
@@ -120,8 +128,39 @@ public class RdeContactImportAction implements Runnable {
     }
 
     @Override
-    public void map(ContactResource contact) {
-      getImportUtils().importContact(contact);
+    public void map(JaxbFragment<XjcRdeContactElement> fragment) {
+      final XjcRdeContact xjcContact = fragment.getInstance().getValue();
+      try {
+        logger.infofmt("Converting xml for contact %s", xjcContact.getId());
+        // Record number of attempted map operations
+        getContext().incrementCounter("contact imports attempted");
+        logger.infofmt("Saving contact %s", xjcContact.getId());
+        ofy().transact(new VoidWork() {
+          @Override
+          public void vrun() {
+            ContactResource contact =
+                XjcToContactResourceConverter.convertContact(xjcContact);
+            getImportUtils().importContact(contact);
+          }
+        });
+        // Record number of contacts imported
+        getContext().incrementCounter("contacts saved");
+        logger.infofmt("Contact %s was imported successfully", xjcContact.getId());
+      } catch (ResourceExistsException e) {
+        // Record the number of contacts already in the registry
+        getContext().incrementCounter("contacts skipped");
+        logger.infofmt("Contact %s already exists", xjcContact.getId());
+      } catch (Exception e) {
+        // Record the number of contacts with unexpected errors
+        getContext().incrementCounter("contact import errors");
+        throw new ContactImportException(xjcContact.getId(), xjcContact.toString(), e);
+      }
+    }
+  }
+
+  private static class ContactImportException extends RuntimeException {
+    ContactImportException(String contactId, String xml, Throwable cause) {
+      super(String.format("Error importing contact %s; xml=%s", contactId, xml), cause);
     }
   }
 }

java/google/registry/rde/imports/RdeContactInput.java

@@ -30,6 +30,8 @@ import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.contact.ContactResource;
 import google.registry.rde.imports.RdeParser.RdeHeader;
+import google.registry.xjc.JaxbFragment;
+import google.registry.xjc.rdecontact.XjcRdeContactElement;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
@@ -41,7 +43,7 @@ import java.util.List;
  * so that each map shard has one reader. If a mapShards parameter has not been specified, a
  * default number of readers will be created.
  */
-public class RdeContactInput extends Input<ContactResource> {
+public class RdeContactInput extends Input<JaxbFragment<XjcRdeContactElement>> {
 
   private static final long serialVersionUID = -366966393494008712L;
   private static final GcsService GCS_SERVICE =
@@ -64,13 +66,6 @@ public class RdeContactInput extends Input<ContactResource> {
   private final String importBucketName;
   private final String importFileName;
 
-  /**
-   * Creates a new {@link RdeContactInput}
-   *
-   * @param mapShards Number of readers that should be created
-   * @param importBucketName Name of GCS bucket for escrow file imports
-   * @param importFileName Name of escrow file in GCS
-   */
   public RdeContactInput(Optional<Integer> mapShards, String importBucketName,
       String importFileName) {
     this.numReaders = mapShards.or(DEFAULT_READERS);
@@ -79,7 +74,8 @@ public class RdeContactInput extends Input<ContactResource> {
   }
 
   @Override
-  public List<? extends InputReader<ContactResource>> createReaders() throws IOException {
+  public List<? extends InputReader<JaxbFragment<XjcRdeContactElement>>> createReaders()
+      throws IOException {
     int numReaders = this.numReaders;
     RdeHeader header = newParser().getHeader();
     int numberOfContacts = header.getContactCount().intValue();
@@ -99,16 +95,10 @@ public class RdeContactInput extends Input<ContactResource> {
     return builder.build();
   }
 
-  /**
-   * Creates a new instance of {@link RdeContactReader}
-   */
   private RdeContactReader newReader(int offset, int maxResults) {
     return new RdeContactReader(importBucketName, importFileName, offset, maxResults);
   }
 
-  /**
-   * Creates a new instance of {@link RdeParser}
-   */
   private RdeParser newParser() {
     GcsUtils utils = new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize());
     GcsFilename filename = new GcsFilename(importBucketName, importFileName);

java/google/registry/rde/imports/RdeContactReader.java

@@ -21,8 +21,9 @@ import com.google.appengine.tools.cloudstorage.RetryParams;
 import com.google.appengine.tools.mapreduce.InputReader;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
-import google.registry.model.contact.ContactResource;
 import google.registry.util.FormattingLogger;
+import google.registry.xjc.JaxbFragment;
+import google.registry.xjc.rdecontact.XjcRdeContactElement;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Serializable;
@@ -31,7 +32,8 @@ import javax.annotation.concurrent.NotThreadSafe;
 
 /** Mapreduce {@link InputReader} for reading contacts from escrow files */
 @NotThreadSafe
-public class RdeContactReader extends InputReader<ContactResource> implements Serializable {
+public class RdeContactReader extends InputReader<JaxbFragment<XjcRdeContactElement>>
+    implements Serializable {
 
   private static final long serialVersionUID = -3688793834175577691L;
 
@@ -80,23 +82,26 @@ public class RdeContactReader extends InputReader<ContactResource> implements Se
   }
 
   @Override
-  public ContactResource next() throws IOException {
+  public JaxbFragment<XjcRdeContactElement> next() throws IOException {
     if (count < maxResults) {
       if (parser == null) {
         parser = newParser();
         if (parser.isAtContact()) {
-          count++;
-          return XjcToContactResourceConverter.convertContact(parser.getContact());
+          return readContact();
         }
       }
       if (parser.nextContact()) {
-        count++;
-        return XjcToContactResourceConverter.convertContact(parser.getContact());
+        return readContact();
       }
     }
     throw new NoSuchElementException();
   }
 
+  private JaxbFragment<XjcRdeContactElement> readContact() {
+    count++;
+    return JaxbFragment.create(new XjcRdeContactElement(parser.getContact()));
+  }
+
   @Override
   public void endSlice() throws IOException {
     super.endSlice();

java/google/registry/rde/imports/RdeDomainImportAction.java

@@ -0,0 +1,170 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.rde.imports;
+
+import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.rde.imports.RdeImportsModule.PATH;
+import static google.registry.util.PipelineUtils.createJobPath;
+
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.appengine.tools.cloudstorage.RetryParams;
+import com.google.appengine.tools.mapreduce.Mapper;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.VoidWork;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.mapreduce.MapreduceRunner;
+import google.registry.model.domain.DomainResource;
+import google.registry.request.Action;
+import google.registry.request.Parameter;
+import google.registry.request.Response;
+import google.registry.util.FormattingLogger;
+import google.registry.util.SystemClock;
+import google.registry.xjc.JaxbFragment;
+import google.registry.xjc.rdedomain.XjcRdeDomain;
+import google.registry.xjc.rdedomain.XjcRdeDomainElement;
+import javax.inject.Inject;
+
+/**
+ * A mapreduce that imports domains from an escrow file.
+ *
+ * <p>Specify the escrow file to import with the "path" parameter.
+ */
+@Action(path = "/_dr/task/importRdeDomains")
+public class RdeDomainImportAction implements Runnable {
+
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+  private static final GcsService GCS_SERVICE =
+      GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
+
+  protected final MapreduceRunner mrRunner;
+  protected final Response response;
+  protected final String importBucketName;
+  protected final String importFileName;
+  protected final Optional<Integer> mapShards;
+
+  @Inject
+  public RdeDomainImportAction(
+      MapreduceRunner mrRunner,
+      Response response,
+      @Config("rdeImportBucket") String importBucketName,
+      @Parameter(PATH) String importFileName,
+      @Parameter(PARAM_MAP_SHARDS) Optional<Integer> mapShards) {
+    this.mrRunner = mrRunner;
+    this.response = response;
+    this.importBucketName = importBucketName;
+    this.importFileName = importFileName;
+    this.mapShards = mapShards;
+  }
+
+  @Override
+  public void run() {
+    logger.infofmt(
+        "Launching domains import mapreduce: bucket=%s, filename=%s",
+        this.importBucketName,
+        this.importFileName);
+    response.sendJavaScriptRedirect(createJobPath(mrRunner
+        .setJobName("Import domains from escrow file")
+        .setModuleName("backend")
+        .runMapOnly(
+            createMapper(),
+            ImmutableList.of(createInput()))));
+  }
+
+  /**
+   * Creates a new {@link RdeDomainInput}
+   */
+  private RdeDomainInput createInput() {
+    return new RdeDomainInput(mapShards, importBucketName, importFileName);
+  }
+
+  /**
+   * Creates a new {@link RdeDomainImportMapper}
+   */
+  private RdeDomainImportMapper createMapper() {
+    return new RdeDomainImportMapper(importBucketName);
+  }
+
+  /** Mapper to import domains from an escrow file. */
+  public static class RdeDomainImportMapper
+      extends Mapper<JaxbFragment<XjcRdeDomainElement>, Void, Void> {
+
+    private static final long serialVersionUID = -7645091075256589374L;
+
+    private final String importBucketName;
+    private transient RdeImportUtils importUtils;
+
+    public RdeDomainImportMapper(String importBucketName) {
+      this.importBucketName = importBucketName;
+    }
+
+    private RdeImportUtils getImportUtils() {
+      if (importUtils == null) {
+        importUtils = createRdeImportUtils();
+      }
+      return importUtils;
+    }
+
+    /**
+     * Creates a new instance of RdeImportUtils.
+     */
+    private RdeImportUtils createRdeImportUtils() {
+      return new RdeImportUtils(
+          ofy(),
+          new SystemClock(),
+          importBucketName,
+          new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize()));
+    }
+
+    @Override
+    public void map(JaxbFragment<XjcRdeDomainElement> fragment) {
+      final XjcRdeDomain xjcDomain = fragment.getInstance().getValue();
+      try {
+        logger.infofmt("Converting xml for domain %s", xjcDomain.getName());
+        // Record number of attempted map operations
+        getContext().incrementCounter("domain imports attempted");
+        logger.infofmt("Saving domain %s", xjcDomain.getName());
+        ofy().transact(new VoidWork() {
+          @Override
+          public void vrun() {
+            DomainResource domain =
+                XjcToDomainResourceConverter.convertDomain(xjcDomain);
+            getImportUtils().importDomain(domain);
+          }
+        });
+        // Record the number of domains imported
+        getContext().incrementCounter("domains saved");
+        logger.infofmt("Domain %s was imported successfully", xjcDomain.getName());
+      } catch (ResourceExistsException e) {
+        // Record the number of domains already in the registry
+        getContext().incrementCounter("domains skipped");
+        logger.infofmt("Domain %s already exists", xjcDomain.getName());
+      } catch (Exception e) {
+        getContext().incrementCounter("domain import errors");
+        throw new DomainImportException(xjcDomain.getName(), xjcDomain.toString(), e);
+      }
+    }
+  }
+
+  private static class DomainImportException extends RuntimeException {
+    DomainImportException(String domainName, String xml, Throwable cause) {
+      super(String.format("Error processing domain %s; xml=%s", domainName, xml), cause);
+    }
+  }
+}

java/google/registry/rde/imports/RdeDomainInput.java

@@ -0,0 +1,122 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.rde.imports;
+
+import static com.google.common.math.IntMath.divide;
+import static java.math.RoundingMode.CEILING;
+import static java.math.RoundingMode.FLOOR;
+
+import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.appengine.tools.cloudstorage.RetryParams;
+import com.google.appengine.tools.mapreduce.Input;
+import com.google.appengine.tools.mapreduce.InputReader;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.model.domain.DomainResource;
+import google.registry.rde.imports.RdeParser.RdeHeader;
+import google.registry.xjc.JaxbFragment;
+import google.registry.xjc.rdedomain.XjcRdeDomainElement;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+/**
+ * A MapReduce {@link Input} that imports {@link DomainResource} objects from an escrow file.
+ *
+ * <p>If a mapShards parameter has been specified, up to that many readers will be created
+ * so that each map shard has one reader. If a mapShards parameter has not been specified, a
+ * default number of readers will be created.
+ */
+public class RdeDomainInput extends Input<JaxbFragment<XjcRdeDomainElement>> {
+
+  private static final long serialVersionUID = -366966393494008712L;
+  private static final GcsService GCS_SERVICE =
+      GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
+
+  /**
+   * Default number of readers if map shards are not specified.
+   */
+  private static final int DEFAULT_READERS = 50;
+
+  /**
+   * Minimum number of records per reader.
+   */
+  private static final int MINIMUM_RECORDS_PER_READER = 100;
+
+  /**
+   * Optional argument to explicitly specify the number of readers.
+   */
+  private final int numReaders;
+  private final String importBucketName;
+  private final String importFileName;
+
+  public RdeDomainInput(
+      Optional<Integer> mapShards, String importBucketName, String importFileName) {
+    this.numReaders = mapShards.or(DEFAULT_READERS);
+    this.importBucketName = importBucketName;
+    this.importFileName = importFileName;
+  }
+
+  @Override
+  public List<? extends InputReader<JaxbFragment<XjcRdeDomainElement>>> createReaders()
+      throws IOException {
+    int numReaders = this.numReaders;
+    RdeHeader header = newParser().getHeader();
+    int numberOfDomains = header.getDomainCount().intValue();
+    if (numberOfDomains / numReaders < MINIMUM_RECORDS_PER_READER) {
+      numReaders = divide(numberOfDomains, MINIMUM_RECORDS_PER_READER, FLOOR);
+      // use at least one reader
+      numReaders = Math.max(numReaders, 1);
+    }
+    ImmutableList.Builder<RdeDomainReader> builder = new ImmutableList.Builder<>();
+    int domainsPerReader =
+        Math.max(MINIMUM_RECORDS_PER_READER, divide(numberOfDomains, numReaders, CEILING));
+    int offset = 0;
+    for (int i = 0; i < numReaders; i++) {
+      builder = builder.add(newReader(offset, domainsPerReader));
+      offset += domainsPerReader;
+    }
+    return builder.build();
+  }
+
+  private RdeDomainReader newReader(int offset, int maxResults) {
+    return new RdeDomainReader(importBucketName, importFileName, offset, maxResults);
+  }
+
+  private RdeParser newParser() {
+    GcsUtils utils = new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize());
+    GcsFilename filename = new GcsFilename(importBucketName, importFileName);
+    try (InputStream xmlInput = utils.openInputStream(filename)) {
+      return new RdeParser(xmlInput);
+    } catch (Exception e) {
+      throw new InitializationException(
+          String.format("Error opening rde file %s/%s", importBucketName, importFileName), e);
+    }
+  }
+
+  /**
+   * Thrown when the input cannot initialize properly.
+   */
+  private static class InitializationException extends RuntimeException {
+
+    public InitializationException(String message, Throwable cause) {
+      super(message, cause);
+    }
+  }
+}

java/google/registry/rde/imports/RdeDomainReader.java

@@ -0,0 +1,109 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.rde.imports;
+
+import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.appengine.tools.cloudstorage.RetryParams;
+import com.google.appengine.tools.mapreduce.InputReader;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.util.FormattingLogger;
+import google.registry.xjc.JaxbFragment;
+import google.registry.xjc.rdedomain.XjcRdeDomainElement;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.util.NoSuchElementException;
+
+/** Mapreduce {@link InputReader} for reading domains from escrow files */
+public class RdeDomainReader extends InputReader<JaxbFragment<XjcRdeDomainElement>>
+    implements Serializable {
+
+  private static final long serialVersionUID = -2175777052970160122L;
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+  private static final GcsService GCS_SERVICE =
+      GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
+
+  final String importBucketName;
+  final String importFileName;
+  final int offset;
+  final int maxResults;
+
+  private int count = 0;
+
+  transient RdeParser parser;
+
+  /**
+   * Creates a new instance of {@link RdeParser}
+   */
+  private RdeParser newParser() {
+    GcsUtils utils = new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize());
+    GcsFilename filename = new GcsFilename(importBucketName, importFileName);
+    InputStream xmlInput = utils.openInputStream(filename);
+    try {
+      RdeParser parser = new RdeParser(xmlInput);
+      // skip the file offset and count
+      // if count is greater than 0, the reader has been rehydrated after doing some work.
+      // skip any already processed records.
+      parser.skipDomains(offset + count);
+      return parser;
+    } catch (Exception e) {
+      logger.severefmt(e, "Error opening rde file %s/%s", importBucketName, importFileName);
+      throw new RuntimeException(e);
+    }
+  }
+
+  public RdeDomainReader(
+      String importBucketName,
+      String importFileName,
+      int offset,
+      int maxResults) {
+    this.importBucketName = importBucketName;
+    this.importFileName = importFileName;
+    this.offset = offset;
+    this.maxResults = maxResults;
+  }
+
+  @Override
+  public JaxbFragment<XjcRdeDomainElement> next() throws IOException {
+    if (count < maxResults) {
+      if (parser == null) {
+        parser = newParser();
+        if (parser.isAtDomain()) {
+          return readDomain();
+        }
+      }
+      if (parser.nextDomain()) {
+        return readDomain();
+      }
+    }
+    throw new NoSuchElementException();
+  }
+
+  private JaxbFragment<XjcRdeDomainElement> readDomain() {
+    count++;
+    return JaxbFragment.create(new XjcRdeDomainElement(parser.getDomain()));
+  }
+
+  @Override
+  public void endSlice() throws IOException {
+    super.endSlice();
+    if (parser != null) {
+      parser.close();
+    }
+  }
+}

java/google/registry/rde/imports/RdeHostImportAction.java

@@ -24,6 +24,7 @@ import com.google.appengine.tools.cloudstorage.RetryParams;
 import com.google.appengine.tools.mapreduce.Mapper;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.VoidWork;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
@@ -32,7 +33,11 @@ import google.registry.model.host.HostResource;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
+import google.registry.util.FormattingLogger;
 import google.registry.util.SystemClock;
+import google.registry.xjc.JaxbFragment;
+import google.registry.xjc.rdehost.XjcRdeHost;
+import google.registry.xjc.rdehost.XjcRdeHostElement;
 import javax.inject.Inject;
 
 /**
@@ -43,6 +48,7 @@ import javax.inject.Inject;
 @Action(path = "/_dr/task/importRdeHosts")
 public class RdeHostImportAction implements Runnable {
 
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
   private static final GcsService GCS_SERVICE =
       GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
 
@@ -77,9 +83,11 @@ public class RdeHostImportAction implements Runnable {
   }
 
   /** Mapper to import hosts from an escrow file. */
-  public static class RdeHostImportMapper extends Mapper<HostResource, Void, Void> {
+  public static class RdeHostImportMapper
+      extends Mapper<JaxbFragment<XjcRdeHostElement>, Void, Void> {
 
     private static final long serialVersionUID = -2898753709127134419L;
+
     private final String importBucketName;
     private transient RdeImportUtils importUtils;
 
@@ -106,8 +114,39 @@ public class RdeHostImportAction implements Runnable {
     }
 
     @Override
-    public void map(HostResource host) {
-      getImportUtils().importHost(host);
+    public void map(JaxbFragment<XjcRdeHostElement> fragment) {
+      final XjcRdeHost xjcHost = fragment.getInstance().getValue();
+      try {
+        logger.infofmt("Converting xml for host %s", xjcHost.getName());
+        // Record number of attempted map operations
+        getContext().incrementCounter("host imports attempted");
+        logger.infofmt("Saving host %s", xjcHost.getName());
+        ofy().transact(new VoidWork() {
+
+          @Override
+          public void vrun() {
+            HostResource host = XjcToHostResourceConverter.convert(xjcHost);
+            getImportUtils().importHost(host);
+          }
+        });
+        // Record number of hosts imported
+        getContext().incrementCounter("hosts saved");
+        logger.infofmt("Host %s was imported successfully", xjcHost.getName());
+      } catch (ResourceExistsException e) {
+        // Record the number of hosts already in the registry
+        getContext().incrementCounter("hosts skipped");
+        logger.infofmt("Host %s already exists", xjcHost.getName());
+      } catch (Exception e) {
+        // Record the number of hosts with unexpected errors
+        getContext().incrementCounter("host import errors");
+        throw new HostImportException(xjcHost.getName(), xjcHost.toString(), e);
+      }
+    }
+  }
+
+  private static class HostImportException extends RuntimeException {
+    HostImportException(String hostName, String xml, Throwable cause) {
+      super(String.format("Error processing host %s; xml=%s", hostName, xml), cause);
     }
   }
 }

java/google/registry/rde/imports/RdeHostInput.java

@@ -28,6 +28,8 @@ import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.host.HostResource;
 import google.registry.rde.imports.RdeParser.RdeHeader;
+import google.registry.xjc.JaxbFragment;
+import google.registry.xjc.rdehost.XjcRdeHostElement;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
@@ -39,7 +41,7 @@ import java.util.List;
  * so that each map shard has one reader. If a mapShards parameter has not been specified, a
  * default number of readers will be created.
  */
-public class RdeHostInput extends Input<HostResource> {
+public class RdeHostInput extends Input<JaxbFragment<XjcRdeHostElement>> {
 
   private static final long serialVersionUID = 9218225041307602452L;
 
@@ -63,13 +65,6 @@ public class RdeHostInput extends Input<HostResource> {
   private final String importBucketName;
   private final String importFileName;
 
-  /**
-   * Creates a new {@link RdeHostInput}
-   *
-   * @param mapShards Number of readers that should be created
-   * @param importBucketName Name of GCS bucket for escrow file imports
-   * @param importFileName Name of escrow file in GCS
-   */
   public RdeHostInput(Optional<Integer> mapShards, String importBucketName,
       String importFileName) {
     this.numReaders = mapShards.or(DEFAULT_READERS);
@@ -79,7 +74,8 @@ public class RdeHostInput extends Input<HostResource> {
   }
 
   @Override
-  public List<? extends InputReader<HostResource>> createReaders() throws IOException {
+  public List<? extends InputReader<JaxbFragment<XjcRdeHostElement>>> createReaders()
+      throws IOException {
     int numReaders = this.numReaders;
     RdeHeader header = createParser().getHeader();
     int numberOfHosts = header.getHostCount().intValue();
@@ -99,16 +95,10 @@ public class RdeHostInput extends Input<HostResource> {
     return builder.build();
   }
 
-  /**
-   * Creates a new instance of {@link RdeHostReader}
-   */
   private RdeHostReader createReader(int offset, int maxResults) {
     return new RdeHostReader(importBucketName, importFileName, offset, maxResults);
   }
 
-  /**
-   * Creates a new instance of {@link RdeParser}
-   */
   private RdeParser createParser() {
     GcsUtils utils = new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize());
     GcsFilename filename = new GcsFilename(importBucketName, importFileName);

java/google/registry/rde/imports/RdeHostReader.java

@@ -21,8 +21,9 @@ import com.google.appengine.tools.cloudstorage.RetryParams;
 import com.google.appengine.tools.mapreduce.InputReader;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
-import google.registry.model.host.HostResource;
 import google.registry.util.FormattingLogger;
+import google.registry.xjc.JaxbFragment;
+import google.registry.xjc.rdehost.XjcRdeHostElement;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Serializable;
@@ -31,7 +32,8 @@ import javax.annotation.concurrent.NotThreadSafe;
 
 /** Mapreduce {@link InputReader} for reading hosts from escrow files */
 @NotThreadSafe
-public class RdeHostReader extends InputReader<HostResource> implements Serializable {
+public class RdeHostReader extends InputReader<JaxbFragment<XjcRdeHostElement>>
+    implements Serializable {
 
   private static final long serialVersionUID = 3037264959150412846L;
 
@@ -80,23 +82,26 @@ public class RdeHostReader extends InputReader<HostResource> implements Serializ
   }
 
   @Override
-  public HostResource next() throws IOException {
+  public JaxbFragment<XjcRdeHostElement> next() throws IOException {
     if (count < maxResults) {
       if (parser == null) {
         parser = newParser();
         if (parser.isAtHost()) {
-          count++;
-          return XjcToHostResourceConverter.convert(parser.getHost());
+          return readHost();
         }
       }
       if (parser.nextHost()) {
-        count++;
-        return XjcToHostResourceConverter.convert(parser.getHost());
+        return readHost();
       }
     }
     throw new NoSuchElementException();
   }
 
+  private JaxbFragment<XjcRdeHostElement> readHost() {
+    count++;
+    return JaxbFragment.create(new XjcRdeHostElement(parser.getHost()));
+  }
+
   @Override
   public void endSlice() throws IOException {
     super.endSlice();

java/google/registry/rde/imports/RdeImportUtils.java

@@ -16,14 +16,17 @@ package google.registry.rde.imports;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.io.BaseEncoding;
 import com.googlecode.objectify.Key;
-import com.googlecode.objectify.Work;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.EppResource;
 import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainResource;
+import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.index.ForeignKeyIndex;
@@ -37,6 +40,7 @@ import google.registry.util.FormattingLogger;
 import google.registry.xjc.rderegistrar.XjcRdeRegistrar;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.UUID;
 import javax.inject.Inject;
 import javax.xml.bind.JAXBException;
 import javax.xml.stream.XMLStreamException;
@@ -62,41 +66,29 @@ public class RdeImportUtils {
     this.escrowBucketName = escrowBucketName;
   }
 
-  private <T extends EppResource> boolean importEppResource(final T resource, final String type) {
+  private <T extends EppResource> void importEppResource(final T resource, final String type) {
     @SuppressWarnings("unchecked")
-    final Class<T> resourceClass = (Class<T>) resource.getClass();
-    return ofy.transact(
-        new Work<Boolean>() {
-          @Override
-          public Boolean run() {
-            EppResource existing = ofy.load().key(Key.create(resource)).now();
-            if (existing == null) {
-              ForeignKeyIndex<T> existingForeignKeyIndex =
-                  ForeignKeyIndex.load(
-                      resourceClass, resource.getForeignKey(), clock.nowUtc());
-              // foreign key index should not exist, since existing resource was not found.
-              checkState(
-                  existingForeignKeyIndex == null,
-                  "New %s resource has existing foreign key index; foreignKey=%s, repoId=%s",
-                  type,
-                  resource.getForeignKey(),
-                  resource.getRepoId());
-              ofy.save().entity(resource);
-              ofy.save().entity(ForeignKeyIndex.create(resource, resource.getDeletionTime()));
-              ofy.save().entity(EppResourceIndex.create(Key.create(resource)));
-              logger.infofmt(
-                  "Imported %s resource - ROID=%s, id=%s",
-                  type, resource.getRepoId(), resource.getForeignKey());
-              return true;
-            } else if (!existing.getRepoId().equals(resource.getRepoId())) {
-              logger.warningfmt(
-                  "Existing %s with same id but different ROID. "
-                      + "id=%s, existing ROID=%s, new ROID=%s",
-                  type, resource.getForeignKey(), existing.getRepoId(), resource.getRepoId());
-            }
-            return false;
-          }
-        });
+    Class<T> resourceClass = (Class<T>) resource.getClass();
+    EppResource existing = ofy.load().key(Key.create(resource)).now();
+    if (existing != null) {
+      // This will roll back the transaction and prevent duplicate history entries from being saved.
+      throw new ResourceExistsException();
+    }
+    ForeignKeyIndex<T> existingForeignKeyIndex =
+        ForeignKeyIndex.load(resourceClass, resource.getForeignKey(), START_OF_TIME);
+    // ForeignKeyIndex should never have existed, since existing resource was not found.
+    checkState(
+        existingForeignKeyIndex == null,
+        "New %s resource has existing foreign key index; foreignKey=%s, repoId=%s",
+        type,
+        resource.getForeignKey(),
+        resource.getRepoId());
+    ofy.save().entity(resource);
+    ofy.save().entity(ForeignKeyIndex.create(resource, resource.getDeletionTime()));
+    ofy.save().entity(EppResourceIndex.create(Key.create(resource)));
+    logger.infofmt(
+        "Imported %s resource - ROID=%s, id=%s",
+        type, resource.getRepoId(), resource.getForeignKey());
   }
 
   /**
@@ -106,11 +98,9 @@ public class RdeImportUtils {
    *
    * <p>If the host is imported, {@link ForeignKeyIndex} and {@link EppResourceIndex} are also
    * created.
-   *
-   * @return true if the host was created or updated, false otherwise.
    */
-  public boolean importHost(final HostResource resource) {
-    return importEppResource(resource, "host");
+  public void importHost(final HostResource resource) {
+    importEppResource(resource, "host");
   }
 
   /**
@@ -120,11 +110,21 @@ public class RdeImportUtils {
    *
    * <p>If the contact is imported, {@link ForeignKeyIndex} and {@link EppResourceIndex} are also
    * created.
-   *
-   * @return true if the contact was created or updated, false otherwise.
    */
-  public boolean importContact(final ContactResource resource) {
-    return importEppResource(resource, "contact");
+  public void importContact(final ContactResource resource) {
+    importEppResource(resource, "contact");
+  }
+
+  /**
+   * Imports a domain from an escrow file.
+   *
+   * <p>The domain will only be imported if it has not been previously imported.
+   *
+   * <p>If the domain is imported, {@link ForeignKeyIndex} and {@link EppResourceIndex} are also
+   * created.
+   */
+  public void importDomain(final DomainResource resource) {
+    importEppResource(resource, "domain");
   }
 
   /**
@@ -146,7 +146,7 @@ public class RdeImportUtils {
    * @throws IllegalArgumentException if the escrow file cannot be imported
    */
   public void validateEscrowFileForImport(String escrowFilePath) throws IOException {
-    // TODO (wolfgang): Add validation method for IDN tables
+    // TODO (wolfgang@donuts.co): Add validation method for IDN tables
     try (InputStream input =
         gcsUtils.openInputStream(new GcsFilename(escrowBucketName, escrowFilePath))) {
       try (RdeParser parser = new RdeParser(input)) {
@@ -176,4 +176,12 @@ public class RdeImportUtils {
       }
     }
   }
+
+  /** Generates a random {@link Trid} for rde import. */
+  public static Trid generateTridForImport() {
+    // Client trids must be a token between 3 and 64 characters long
+    // Base64 encoded UUID string meets this requirement
+    return Trid.create(
+        "Import_" + BaseEncoding.base64().encode(UUID.randomUUID().toString().getBytes()));
+  }
 }

java/google/registry/rde/imports/RdeImportsModule.java

@@ -28,9 +28,11 @@ import javax.servlet.http.HttpServletRequest;
 @Module
 public final class RdeImportsModule {
 
+  static final String PATH = "path";
+
   @Provides
-  @Parameter("path")
+  @Parameter(PATH)
   static String providePath(HttpServletRequest req) {
-    return RequestParameters.extractRequiredParameter(req, "path");
+    return RequestParameters.extractRequiredParameter(req, PATH);
   }
 }

java/google/registry/rde/imports/ResourceExistsException.java

@@ -0,0 +1,23 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.rde.imports;
+
+/**
+ * Indicates that a resource already exists and was not imported.
+ */
+public class ResourceExistsException extends RuntimeException {
+
+  private static final long serialVersionUID = -9180381693364904061L;
+}

java/google/registry/rde/imports/XjcToContactResourceConverter.java

@@ -16,11 +16,14 @@ package google.registry.rde.imports;
 
 import static com.google.common.base.Predicates.equalTo;
 import static com.google.common.base.Predicates.not;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.rde.imports.RdeImportUtils.generateTridForImport;
 
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
+import com.googlecode.objectify.Key;
 import google.registry.model.contact.ContactAddress;
 import google.registry.model.contact.ContactPhoneNumber;
 import google.registry.model.contact.ContactResource;
@@ -28,6 +31,7 @@ import google.registry.model.contact.Disclose;
 import google.registry.model.contact.Disclose.PostalInfoChoice;
 import google.registry.model.contact.PostalInfo;
 import google.registry.model.eppcommon.StatusValue;
+import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.util.XmlToEnumMapper;
@@ -39,11 +43,12 @@ import google.registry.xjc.contact.XjcContactPostalInfoEnumType;
 import google.registry.xjc.contact.XjcContactPostalInfoType;
 import google.registry.xjc.contact.XjcContactStatusType;
 import google.registry.xjc.rdecontact.XjcRdeContact;
+import google.registry.xjc.rdecontact.XjcRdeContactElement;
 import google.registry.xjc.rdecontact.XjcRdeContactTransferDataType;
 import javax.annotation.Nullable;
 
 /** Utility class that converts an {@link XjcRdeContact} into a {@link ContactResource}. */
-final class XjcToContactResourceConverter {
+final class XjcToContactResourceConverter extends XjcToEppResourceConverter {
 
   private static final XmlToEnumMapper<PostalInfo.Type> POSTAL_INFO_TYPE_MAPPER =
       XmlToEnumMapper.create(PostalInfo.Type.values());
@@ -68,6 +73,18 @@ final class XjcToContactResourceConverter {
 
   /** Converts {@link XjcRdeContact} to {@link ContactResource}. */
   static ContactResource convertContact(XjcRdeContact contact) {
+    ofy().save().entity(
+        new HistoryEntry.Builder()
+            .setType(HistoryEntry.Type.RDE_IMPORT)
+            .setClientId(contact.getClID())
+            .setTrid(generateTridForImport())
+            .setModificationTime(ofy().getTransactionTime())
+            .setXmlBytes(getObjectXml(new XjcRdeContactElement(contact)))
+            .setBySuperuser(true)
+            .setReason("RDE Import")
+            .setRequestedByRegistrar(false)
+            .setParent(Key.create(null, ContactResource.class, contact.getRoid()))
+            .build());
     return new ContactResource.Builder()
         .setRepoId(contact.getRoid())
         .setStatusValues(

java/google/registry/rde/imports/XjcToDomainResourceConverter.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.Iterables.transform;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.rde.imports.RdeImportUtils.generateTridForImport;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
@@ -38,7 +39,6 @@ import google.registry.model.domain.GracePeriod;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.domain.secdns.DelegationSignerData;
 import google.registry.model.eppcommon.StatusValue;
-import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.poll.PollMessage;
@@ -120,14 +120,12 @@ final class XjcToDomainResourceConverter extends XjcToEppResourceConverter {
 
     @Override
     public GracePeriod apply(XjcRgpStatusType gracePeriodStatus) {
-      // TODO: (wolfgang) address these items in code review:
-      // verify that this logic is correct
       switch (gracePeriodStatus.getS()) {
         case ADD_PERIOD:
           return GracePeriod.createWithoutBillingEvent(
               GracePeriodStatus.ADD,
               domain.getCrDate().plus(this.tld.getAddGracePeriodLength()),
-              domain.getCrRr().getClient());
+              domain.getCrRr().getValue());
         case AUTO_RENEW_PERIOD:
           return GracePeriod.createForRecurring(
               GracePeriodStatus.AUTO_RENEW,
@@ -180,6 +178,8 @@ final class XjcToDomainResourceConverter extends XjcToEppResourceConverter {
             .setCurrentSponsorClientId(domain.getClID())
             .setCreationClientId(domain.getCrRr().getValue())
             .setCreationTime(domain.getCrDate())
+            .setAutorenewPollMessage(Key.create(pollMessage))
+            .setAutorenewBillingEvent(Key.create(autoRenewBillingEvent))
             .setRegistrationExpirationTime(domain.getExDate())
             .setLastEppUpdateTime(domain.getUpDate())
             .setLastEppUpdateClientId(domain.getUpRr() == null ? null : domain.getUpRr().getValue())
@@ -235,7 +235,7 @@ final class XjcToDomainResourceConverter extends XjcToEppResourceConverter {
         new HistoryEntry.Builder()
             .setType(HistoryEntry.Type.RDE_IMPORT)
             .setClientId(domain.getClID())
-            .setTrid(Trid.create(null))
+            .setTrid(generateTridForImport())
             .setModificationTime(DateTime.now())
             .setXmlBytes(getObjectXml(element))
             .setBySuperuser(true)

java/google/registry/rde/imports/XjcToEppResourceConverter.java

@@ -14,49 +14,24 @@
 
 package google.registry.rde.imports;
 
-import com.google.common.base.Joiner;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import google.registry.model.EppResource;
+import google.registry.xjc.XjcXmlTransformer;
+import google.registry.xml.XmlException;
 import java.io.ByteArrayOutputStream;
-import java.util.Arrays;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBElement;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Marshaller;
 
 /**
  * Base class for Jaxb object to {@link EppResource} converters
  */
 public abstract class XjcToEppResourceConverter {
 
-  /** List of packages to initialize JAXBContext. **/
-  private static final String JAXB_CONTEXT_PACKAGES = Joiner.on(":")
-      .join(Arrays.asList(
-          "google.registry.xjc.contact",
-          "google.registry.xjc.domain",
-          "google.registry.xjc.host",
-          "google.registry.xjc.mark",
-          "google.registry.xjc.rde",
-          "google.registry.xjc.rdecontact",
-          "google.registry.xjc.rdedomain",
-          "google.registry.xjc.rdeeppparams",
-          "google.registry.xjc.rdeheader",
-          "google.registry.xjc.rdeidn",
-          "google.registry.xjc.rdenndn",
-          "google.registry.xjc.rderegistrar",
-          "google.registry.xjc.smd"));
-
-  /** Creates a {@link Marshaller} for serializing Jaxb objects */
-  private static Marshaller createMarshaller() throws JAXBException {
-    return JAXBContext.newInstance(JAXB_CONTEXT_PACKAGES).createMarshaller();
-  }
-
-  protected static byte[] getObjectXml(JAXBElement<?> jaxbElement) {
+  protected static byte[] getObjectXml(Object jaxbElement) {
     try {
-      Marshaller marshaller = createMarshaller();
       ByteArrayOutputStream bout = new ByteArrayOutputStream();
-      marshaller.marshal(jaxbElement, bout);
+      XjcXmlTransformer.marshalLenient(jaxbElement, bout, UTF_8);
       return bout.toByteArray();
-    } catch (JAXBException e) {
+    } catch (XmlException e) {
       throw new RuntimeException(e);
     }
   }

java/google/registry/rde/imports/XjcToHostResourceConverter.java

@@ -16,21 +16,27 @@ package google.registry.rde.imports;
 
 import static com.google.common.base.Predicates.equalTo;
 import static com.google.common.base.Predicates.not;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.rde.imports.RdeImportUtils.generateTridForImport;
 
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.net.InetAddresses;
+import com.googlecode.objectify.Key;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
+import google.registry.model.reporting.HistoryEntry;
 import google.registry.xjc.host.XjcHostAddrType;
 import google.registry.xjc.host.XjcHostStatusType;
 import google.registry.xjc.rdehost.XjcRdeHost;
+import google.registry.xjc.rdehost.XjcRdeHostElement;
 import java.net.InetAddress;
+import org.joda.time.DateTime;
 
 /** Utility class that converts an {@link XjcRdeHost} into a {@link HostResource}. */
-public class XjcToHostResourceConverter {
+public class XjcToHostResourceConverter extends XjcToEppResourceConverter {
 
   private static final Function<XjcHostStatusType, StatusValue> STATUS_VALUE_CONVERTER =
       new Function<XjcHostStatusType, StatusValue>() {
@@ -49,6 +55,19 @@ public class XjcToHostResourceConverter {
       };
 
   static HostResource convert(XjcRdeHost host) {
+    // First create and save history entry
+    ofy().save().entity(
+        new HistoryEntry.Builder()
+            .setType(HistoryEntry.Type.RDE_IMPORT)
+            .setClientId(host.getClID())
+            .setTrid(generateTridForImport())
+            .setModificationTime(DateTime.now())
+            .setXmlBytes(getObjectXml(new XjcRdeHostElement(host)))
+            .setBySuperuser(true)
+            .setReason("RDE Import")
+            .setRequestedByRegistrar(false)
+            .setParent(Key.create(null, HostResource.class, host.getRoid()))
+            .build());
     return new HostResource.Builder()
         .setFullyQualifiedHostName(host.getName())
         .setRepoId(host.getRoid())

java/google/registry/xjc/JaxbFragment.java

@@ -0,0 +1,106 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.xjc;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import google.registry.xml.XmlException;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
+/**
+ * JAXB element wrapper for java object serialization.
+ *
+ * Instances of {@link JaxbFragment} wrap a non-serializable JAXB element instance, and provide
+ * hooks into the java object serialization process that allow the elements to be safely
+ * marshalled and unmarshalled using {@link ObjectOutputStream} and {@link ObjectInputStream},
+ * respectively.
+ */
+public class JaxbFragment<T> implements Serializable {
+
+  private static final long serialVersionUID = 5651243983008818813L;
+
+  private T instance;
+
+  /** Stores a JAXB element in a {@link JaxbFragment} */
+  public static <T> JaxbFragment<T> create(T object) {
+    JaxbFragment<T> fragment = new JaxbFragment<>();
+    fragment.instance = object;
+    return fragment;
+  }
+
+  /** Serializes a JAXB element into xml bytes. */
+  private static <T> byte[] freezeInstance(T instance) throws IOException {
+    try {
+      ByteArrayOutputStream bout = new ByteArrayOutputStream();
+      XjcXmlTransformer.marshalLenient(instance, bout, UTF_8);
+      return bout.toByteArray();
+    } catch (XmlException e) {
+      throw new IOException(e);
+    }
+  }
+
+  /** Deserializes a JAXB element from xml bytes. */
+  private static <T> T unfreezeInstance(byte[] instanceData, Class<? extends Object> instanceType)
+      throws IOException {
+    try {
+      ByteArrayInputStream bin = new ByteArrayInputStream(instanceData);
+      @SuppressWarnings("unchecked")
+      T instance = (T) XjcXmlTransformer.unmarshal(instanceType, bin);
+      return instance;
+    } catch (XmlException e) {
+      throw new IOException(e);
+    }
+  }
+
+  /**
+   * Retrieves the JAXB element that is wrapped by this fragment.
+   */
+  public T getInstance() {
+    return instance;
+  }
+
+  @Override
+  public String toString() {
+    try {
+      return new String(freezeInstance(instance), UTF_8);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void writeObject(ObjectOutputStream out) throws IOException {
+    // write instanceType, then instanceData
+    out.writeObject(instance.getClass());
+    out.writeObject(freezeInstance(instance));
+  }
+
+  private void readObject(ObjectInputStream in) throws IOException {
+    // read instanceType, then instanceData
+    Class<?> instanceType;
+    byte[] instanceData;
+    try {
+      instanceType = (Class<?>) in.readObject();
+      instanceData = (byte[]) in.readObject();
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e);
+    }
+    instance = unfreezeInstance(instanceData, instanceType);
+  }
+}