Create ICANN report upload action

This is the first step in moving the current []cron-Python reporting scripts
into App Engine, as an official part of the Nomulus package. This copies the
structure of RDE uploads, with a few changes specific to monthly reporting.

I've left some TODOs related to actually testing it on the ICANN endpoint, as we're still not sure how files to be uploaded will be staged, and whether we can actually ping their endpoint on valid ports (80 or 443).

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=160408703

java/google/registry/config/RegistryConfig.java

@@ -451,6 +451,40 @@ public final class RegistryConfig {
       return config.gSuite.outgoingEmailDisplayName;
     }
 
+    /**
+     * Returns the Google Cloud Storage bucket for ICANN transaction and activity reports to
+     * be uploaded.
+     *
+     * @see google.registry.reporting.IcannReportingUploadAction
+     */
+    @Provides
+    @Config("icannReportingBucket")
+    public static String provideIcannReportingBucket(@Config("projectId") String projectId) {
+      return projectId + "-reporting";
+    }
+
+    /**
+     * Returns the URL we send HTTP PUT requests for ICANN monthly transactions reports.
+     *
+     * @see google.registry.reporting.IcannHttpReporter
+     */
+    @Provides
+    @Config("icannTransactionsReportingUploadUrl")
+    public static String provideIcannTransactionsReportingUploadUrl(RegistryConfigSettings config) {
+      return config.icannReporting.icannTransactionsReportingUploadUrl;
+    }
+
+    /**
+     * Returns the URL we send HTTP PUT requests for ICANN monthly activity reports.
+     *
+     * @see google.registry.reporting.IcannHttpReporter
+     */
+    @Provides
+    @Config("icannActivityReportingUploadUrl")
+    public static String provideIcannActivityReportingUploadUrl(RegistryConfigSettings config) {
+      return config.icannReporting.icannActivityReportingUploadUrl;
+    }
+
     /**
      * Returns the Google Cloud Storage bucket for staging escrow deposits pending upload.
      *

java/google/registry/config/RegistryConfigSettings.java

@@ -26,6 +26,7 @@ public class RegistryConfigSettings {
   public RegistryPolicy registryPolicy;
   public Datastore datastore;
   public Caching caching;
+  public IcannReporting icannReporting;
   public Rde rde;
   public RegistrarConsole registrarConsole;
   public Monitoring monitoring;
@@ -102,6 +103,12 @@ public class RegistryConfigSettings {
     public int staticPremiumListMaxCachedEntries;
   }
 
+  /** Configuration for ICANN monthly reporting. */
+  public static class IcannReporting {
+    public String icannTransactionsReportingUploadUrl;
+    public String icannActivityReportingUploadUrl;
+  }
+
   /** Configuration for Registry Data Escrow (RDE). */
   public static class Rde {
     public String reportUrlPrefix;

java/google/registry/config/files/default-config.yaml

@@ -136,6 +136,13 @@ oAuth:
   # backend services, e. g. nomulus tool, EPP proxy, etc.
   allowedOauthClientIds: []
 
+icannReporting:
+  # URL we PUT monthly ICANN transactions reports to.
+  icannTransactionsReportingUploadUrl: https://ry-api.example.org/report/registrar-transactions
+
+  # URL we PUT monthly ICANN activity reports to.
+  icannActivityReportingUploadUrl: https://ry-api.example.org/report/registry-functions-activity
+
 rde:
   # URL prefix of ICANN's server to upload RDE reports to. Nomulus adds /TLD/ID
   # to the end of this to construct the full URL.

java/google/registry/config/files/nomulus-config-production-sample.yaml

@@ -35,6 +35,10 @@ registryPolicy:
     multi-line
     placeholder
 
+icannReporting:
+  icannTransactionsReportingUploadUrl: https://ry-api.icann.org/report/registrar-transactions
+  icannActivityReportingUploadUrl: https://ry-api.icann.org/report/registry-functions-activity
+
 rde:
   reportUrlPrefix: https://ry-api.icann.org/report/registry-escrow-report
   uploadUrl: sftp://placeholder@sftpipm2.ironmountain.com/Outbox

java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -57,6 +57,15 @@
     <url-pattern>/_dr/task/brdaCopy</url-pattern>
   </servlet-mapping>
 
+  <!--
+    Monthly ICANN transaction and activity reports. This task uploads the generated
+    report files (in CSV format) via an HTTP PUT to ICANN's endpoint.
+  -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/icannReportingUpload</url-pattern>
+  </servlet-mapping>
+
   <!-- Trademark Clearinghouse -->
 
   <!-- Downloads TMCH DNL data from MarksDB. -->

java/google/registry/module/backend/BUILD

@@ -30,10 +30,10 @@ java_library(
         "//java/google/registry/monitoring/whitebox",
         "//java/google/registry/rde",
         "//java/google/registry/rde/imports",
+        "//java/google/registry/reporting",
         "//java/google/registry/request",
         "//java/google/registry/request:modules",
         "//java/google/registry/request/auth",
-        "//java/google/registry/security",
         "//java/google/registry/tmch",
         "//java/google/registry/util",
         "@com_google_appengine_api_1_0_sdk",

java/google/registry/module/backend/BackendRequestComponent.java

@@ -63,6 +63,8 @@ import google.registry.rde.imports.RdeDomainImportAction;
 import google.registry.rde.imports.RdeHostImportAction;
 import google.registry.rde.imports.RdeHostLinkAction;
 import google.registry.rde.imports.RdeImportsModule;
+import google.registry.reporting.IcannReportingModule;
+import google.registry.reporting.IcannReportingUploadAction;
 import google.registry.request.RequestComponentBuilder;
 import google.registry.request.RequestModule;
 import google.registry.request.RequestScope;
@@ -87,6 +89,7 @@ import google.registry.tmch.TmchSmdrlAction;
         DnsUpdateConfigModule.class,
         DnsUpdateWriterModule.class,
         ExportRequestModule.class,
+        IcannReportingModule.class,
         MapreduceModule.class,
         RdeModule.class,
         RdeImportsModule.class,
@@ -109,6 +112,7 @@ interface BackendRequestComponent {
   ExportDomainListsAction exportDomainListsAction();
   ExportReservedTermsAction exportReservedTermsAction();
   ExportSnapshotAction exportSnapshotAction();
+  IcannReportingUploadAction icannReportingUploadAction();
   LoadSnapshotAction loadSnapshotAction();
   MapreduceEntityCleanupAction mapreduceEntityCleanupAction();
   MetricsExportAction metricsExportAction();

java/google/registry/reporting/BUILD

@@ -0,0 +1,27 @@
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+java_library(
+    name = "reporting",
+    srcs = glob(["*.java"]),
+    deps = [
+        "//java/google/registry/config",
+        "//java/google/registry/gcs",
+        "//java/google/registry/keyring/api",
+        "//java/google/registry/model",
+        "//java/google/registry/request",
+        "//java/google/registry/request/auth",
+        "//java/google/registry/util",
+        "//java/google/registry/xjc",
+        "//java/google/registry/xml",
+        "@com_google_appengine_tools_appengine_gcs_client",
+        "@com_google_code_findbugs_jsr305",
+        "@com_google_dagger",
+        "@com_google_guava",
+        "@com_google_http_client",
+        "@javax_servlet_api",
+    ],
+)

java/google/registry/reporting/IcannHttpReporter.java

@@ -0,0 +1,138 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.net.MediaType.CSV_UTF_8;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.api.client.http.ByteArrayContent;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpTransport;
+import com.google.common.io.ByteStreams;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.keyring.api.KeyModule.Key;
+import google.registry.reporting.IcannReportingModule.ReportType;
+import google.registry.request.HttpException.InternalServerErrorException;
+import google.registry.util.FormattingLogger;
+import google.registry.xjc.XjcXmlTransformer;
+import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
+import google.registry.xjc.iirdea.XjcIirdeaResult;
+import google.registry.xml.XmlException;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import javax.inject.Inject;
+
+/**
+ * Class that uploads a CSV file to ICANN's endpoint via an HTTP PUT call.
+ *
+ * <p> It uses basic authorization credentials as specified in the "Registry Interfaces" draft.
+ *
+ * @see IcannReportingUploadAction
+ * @see <a href=https://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-07#page-9>
+ *   ICANN Reporting Specification</a>
+ */
+public class IcannHttpReporter {
+
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+
+  @Inject HttpTransport httpTransport;
+  @Inject @Key("icannReportingPassword") String password;
+  @Inject @Config("icannTransactionsReportingUploadUrl") String icannTransactionsUrl;
+  @Inject @Config("icannActivityReportingUploadUrl") String icannActivityUrl;
+  @Inject IcannHttpReporter() {}
+
+  /** Uploads {@code reportBytes} to ICANN. */
+  public void send(
+      byte[] reportBytes,
+      String tld,
+      String yearMonth,
+      ReportType reportType) throws XmlException, IOException {
+    GenericUrl uploadUrl = new GenericUrl(makeUrl(tld, yearMonth, reportType));
+    HttpRequest request =
+        httpTransport
+            .createRequestFactory()
+            .buildPutRequest(uploadUrl, new ByteArrayContent(CSV_UTF_8.toString(), reportBytes));
+
+    HttpHeaders headers = request.getHeaders();
+    headers.setBasicAuthentication(tld + "_ry", password);
+    headers.setContentType(CSV_UTF_8.toString());
+    request.setHeaders(headers);
+    request.setFollowRedirects(false);
+
+    HttpResponse response = null;
+    logger.infofmt(
+        "Sending %s report to %s with content length %s",
+        reportType,
+        uploadUrl.toString(),
+        request.getContent().getLength());
+    try {
+      response = request.execute();
+      byte[] content;
+      try {
+        content = ByteStreams.toByteArray(response.getContent());
+      } finally {
+        response.getContent().close();
+      }
+      logger.infofmt("Received response code %s", response.getStatusCode());
+      logger.infofmt("Response content: %s", new String(content, UTF_8));
+      XjcIirdeaResult result = parseResult(content);
+      if (result.getCode().getValue() != 1000) {
+        logger.warningfmt(
+            "PUT rejected, status code %s:\n%s\n%s",
+            result.getCode(),
+            result.getMsg(),
+            result.getDescription());
+        throw new InternalServerErrorException(result.getMsg());
+      }
+    } finally {
+      if (response != null) {
+        response.disconnect();
+      } else {
+        logger.warningfmt(
+            "Received null response from ICANN server at %s", uploadUrl.toString());
+      }
+    }
+  }
+
+  private XjcIirdeaResult parseResult(byte[] content) throws XmlException, IOException {
+    XjcIirdeaResponseElement response =
+        XjcXmlTransformer.unmarshal(
+            XjcIirdeaResponseElement.class, new ByteArrayInputStream(content));
+    XjcIirdeaResult result = response.getResult();
+    return result;
+  }
+
+  private String makeUrl(String tld, String yearMonth, ReportType reportType) {
+    String urlPrefix = getUrlPrefix(reportType);
+    return String.format("%s/%s/%s", urlPrefix, tld, yearMonth);
+  }
+
+  private String getUrlPrefix(ReportType reportType) {
+    switch (reportType) {
+      case TRANSACTIONS:
+        return icannTransactionsUrl;
+      case ACTIVITY:
+        return icannActivityUrl;
+      default:
+        throw new IllegalStateException(
+            String.format(
+                "Received invalid reportType! Expected ACTIVITY or TRANSACTIONS, got %s.",
+                reportType));
+    }
+  }
+}

java/google/registry/reporting/IcannReportingModule.java

@@ -0,0 +1,58 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static google.registry.request.RequestParameters.extractEnumParameter;
+import static google.registry.request.RequestParameters.extractOptionalParameter;
+import static google.registry.request.RequestParameters.extractRequiredParameter;
+
+import com.google.common.base.Optional;
+import dagger.Module;
+import dagger.Provides;
+import google.registry.request.Parameter;
+import javax.servlet.http.HttpServletRequest;
+
+/** Module for dependencies required by ICANN monthly transactions/activity reporting. */
+@Module
+public final class IcannReportingModule {
+
+  /** Enum determining the type of report to generate or upload. */
+  public enum ReportType {
+    TRANSACTIONS,
+    ACTIVITY
+  }
+
+  static final String PARAM_YEAR_MONTH = "yearMonth";
+  static final String PARAM_REPORT_TYPE = "reportType";
+  static final String PARAM_SUBDIR = "subdir";
+
+  @Provides
+  @Parameter(PARAM_YEAR_MONTH)
+  static String provideYearMonth(HttpServletRequest req) {
+    return extractRequiredParameter(req, PARAM_YEAR_MONTH);
+  }
+
+  @Provides
+  @Parameter(PARAM_REPORT_TYPE)
+  static ReportType provideReportType(HttpServletRequest req) {
+    return extractEnumParameter(req, ReportType.class, PARAM_REPORT_TYPE);
+  }
+
+  @Provides
+  @Parameter(PARAM_SUBDIR)
+  static Optional<String> provideSubdir(HttpServletRequest req) {
+    return extractOptionalParameter(req, PARAM_SUBDIR);
+  }
+}

java/google/registry/reporting/IcannReportingUploadAction.java

@@ -0,0 +1,133 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static google.registry.model.registry.Registries.assertTldExists;
+import static google.registry.request.Action.Method.POST;
+
+import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.base.Optional;
+import com.google.common.io.ByteStreams;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
+import google.registry.reporting.IcannReportingModule.ReportType;
+import google.registry.request.Action;
+import google.registry.request.Parameter;
+import google.registry.request.RequestParameters;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.request.auth.Auth.AuthMethod;
+import google.registry.request.auth.Auth.UserPolicy;
+import google.registry.request.auth.AuthLevel;
+import google.registry.util.FormattingLogger;
+import google.registry.util.Retrier;
+import google.registry.xml.XmlException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.Callable;
+import javax.inject.Inject;
+
+/**
+ * Action that uploads the monthly transaction and activity reports from Cloud Storage to ICANN via
+ * an HTTP PUT.
+ *
+ */
+@Action(
+    path = IcannReportingUploadAction.PATH,
+    method = POST,
+    auth =
+    @Auth(
+        methods = {AuthMethod.INTERNAL, AuthMethod.API},
+        minimumLevel = AuthLevel.APP,
+        userPolicy = UserPolicy.ADMIN
+    )
+)
+public final class IcannReportingUploadAction implements Runnable {
+
+  static final String PATH = "/_dr/task/icannReportingUpload";
+  static final String DEFAULT_SUBDIR = "icann/monthly";
+
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+
+  @Inject GcsUtils gcsUtils;
+  @Inject IcannHttpReporter icannReporter;
+  @Inject Retrier retrier;
+  @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
+  @Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
+  @Inject @Parameter(IcannReportingModule.PARAM_REPORT_TYPE) ReportType reportType;
+  @Inject @Parameter(IcannReportingModule.PARAM_SUBDIR) Optional<String> subdir;
+  @Inject @Config("icannReportingBucket") String reportingBucket;
+  @Inject Response response;
+
+  @Inject
+  IcannReportingUploadAction() {}
+
+  @Override
+  public void run() {
+    validateParams();
+    String reportFilename = createFilename(tld, yearMonth, reportType);
+    logger.infofmt("Reading ICANN report %s from bucket %s", reportFilename, reportingBucket);
+    final GcsFilename gcsFilename =
+        new GcsFilename(
+            reportingBucket + "/" + (subdir.isPresent() ? subdir.get() : DEFAULT_SUBDIR),
+            reportFilename);
+    checkState(
+        gcsUtils.existsAndNotEmpty(gcsFilename),
+        "ICANN report object %s in bucket %s not found",
+        gcsFilename.getObjectName(),
+        gcsFilename.getBucketName());
+
+    retrier.callWithRetry(new Callable<Void>() {
+      @Override
+      public Void call() throws IOException, XmlException {
+        final byte[] payload = readReportFromGcs(gcsFilename);
+        icannReporter.send(payload, tld, yearMonth, reportType);
+        response.setContentType(PLAIN_TEXT_UTF_8);
+        response.setPayload(
+            String.format("OK, sending: %s", new String(payload, StandardCharsets.UTF_8)));
+        return null;
+      }}, IOException.class);
+  }
+
+  private byte[] readReportFromGcs(GcsFilename reportFilename) throws IOException {
+    try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) {
+      return ByteStreams.toByteArray(gcsInput);
+    }
+  }
+
+  static String createFilename(String tld, String yearMonth, ReportType reportType) {
+    // TODO(b/62585428): Change file naming date format to YYYY-MM for consistency with URL.
+    // Report files use YYYYMM naming instead of standard YYYY-MM.
+    String fileYearMonth = yearMonth.substring(0, 4) + yearMonth.substring(5, 7);
+    return String.format("%s-%s-%s.csv", tld, reportType.toString().toLowerCase(), fileYearMonth);
+  }
+
+  private void validateParams() {
+    assertTldExists(tld);
+    checkState(
+        yearMonth.matches("[0-9]{4}-[0-9]{2}"),
+        "yearMonth must be in YYYY-MM format, got %s instead.",
+        yearMonth);
+    if (subdir.isPresent()) {
+      checkState(
+          !subdir.get().startsWith("/") && !subdir.get().endsWith("/"),
+          "subdir must not start or end with a \"/\", got %s instead.",
+          subdir.get());
+    }
+  }
+}

javatests/google/registry/module/backend/testdata/backend_routing.txt

@@ -14,6 +14,7 @@ PATH                                   CLASS                              METHOD
 /_dr/task/exportDomainLists            ExportDomainListsAction            POST     n  INTERNAL     APP IGNORED
 /_dr/task/exportReservedTerms          ExportReservedTermsAction          POST     n  INTERNAL     APP IGNORED
 /_dr/task/exportSnapshot               ExportSnapshotAction               POST     y  INTERNAL     APP IGNORED
+/_dr/task/icannReportingUpload         IcannReportingUploadAction         POST     n  INTERNAL,API APP ADMIN
 /_dr/task/importRdeContacts            RdeContactImportAction             GET      n  INTERNAL     APP IGNORED
 /_dr/task/importRdeDomains             RdeDomainImportAction              GET      n  INTERNAL     APP IGNORED
 /_dr/task/importRdeHosts               RdeHostImportAction                GET      n  INTERNAL     APP IGNORED

javatests/google/registry/reporting/BUILD

@@ -0,0 +1,37 @@
+package(
+    default_testonly = 1,
+    default_visibility = ["//java/google/registry:registry_project"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//java/com/google/testing/builddefs:GenTestRules.bzl", "GenTestRules")
+
+java_library(
+    name = "reporting",
+    srcs = glob(["*.java"]),
+    resources = glob(["testdata/*"]),
+    deps = [
+        "//java/google/registry/gcs",
+        "//java/google/registry/reporting",
+        "//java/google/registry/request",
+        "//java/google/registry/util",
+        "//javatests/google/registry/testing",
+        "@com_google_appengine_tools_appengine_gcs_client",
+        "@com_google_code_findbugs_jsr305",
+        "@com_google_dagger",
+        "@com_google_guava",
+        "@com_google_http_client",
+        "@com_google_truth",
+        "@javax_servlet_api",
+        "@junit",
+        "@org_mockito_all",
+    ],
+)
+
+GenTestRules(
+    name = "GeneratedTestRules",
+    default_test_size = "small",
+    test_files = glob(["*Test.java"]),
+    deps = [":reporting"],
+)

javatests/google/registry/reporting/IcannHttpReporterTest.java

@@ -0,0 +1,107 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.net.MediaType.CSV_UTF_8;
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.api.client.http.LowLevelHttpRequest;
+import com.google.api.client.http.LowLevelHttpResponse;
+import com.google.api.client.testing.http.MockHttpTransport;
+import com.google.api.client.testing.http.MockLowLevelHttpRequest;
+import com.google.api.client.testing.http.MockLowLevelHttpResponse;
+import com.google.api.client.util.Base64;
+import com.google.api.client.util.StringUtils;
+import com.google.common.io.ByteSource;
+import google.registry.reporting.IcannReportingModule.ReportType;
+import google.registry.request.HttpException.InternalServerErrorException;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Unit tests for {@link google.registry.reporting.IcannHttpReporter}.
+ */
+@RunWith(JUnit4.class)
+public class IcannHttpReporterTest {
+
+  private static final ByteSource IIRDEA_GOOD_XML = ReportingTestData.get("iirdea_good.xml");
+  private static final ByteSource IIRDEA_BAD_XML = ReportingTestData.get("iirdea_bad.xml");
+
+  private MockLowLevelHttpRequest mockRequest;
+
+  private MockHttpTransport createMockTransport (ByteSource iirdeaResponse) {
+    return new MockHttpTransport() {
+      @Override
+      public LowLevelHttpRequest buildRequest(String method, String url) throws IOException {
+        mockRequest = new MockLowLevelHttpRequest() {
+          @Override
+          public LowLevelHttpResponse execute() throws IOException {
+            MockLowLevelHttpResponse response = new MockLowLevelHttpResponse();
+            response.setStatusCode(200);
+            response.setContentType(PLAIN_TEXT_UTF_8.toString());
+            response.setContent(iirdeaResponse.read());
+            return response;
+          }
+        };
+        mockRequest.setUrl(url);
+        return mockRequest;
+      }
+    };
+  }
+
+  private static final byte[] FAKE_PAYLOAD = "test,csv\n1,2".getBytes(UTF_8);
+
+  private IcannHttpReporter createReporter() {
+    IcannHttpReporter reporter = new IcannHttpReporter();
+    reporter.httpTransport = createMockTransport(IIRDEA_GOOD_XML);
+    reporter.password = "fakePass";
+    reporter.icannTransactionsUrl = "https://fake-transactions.url";
+    reporter.icannActivityUrl = "https://fake-activity.url";
+    return reporter;
+  }
+
+  @Test
+  public void testSuccess() throws Exception {
+    IcannHttpReporter reporter = createReporter();
+    reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS);
+
+    assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2016-06");
+    Map<String, List<String>> headers = mockRequest.getHeaders();
+    String userPass = "test_ry:fakePass";
+    String expectedAuth =
+        String.format("Basic %s", Base64.encodeBase64String(StringUtils.getBytesUtf8(userPass)));
+    assertThat(headers.get("authorization")).containsExactly(expectedAuth);
+    assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString());
+  }
+
+  @Test
+  public void testFail_BadIirdeaResponse() throws Exception {
+    IcannHttpReporter reporter = createReporter();
+    reporter.httpTransport = createMockTransport(IIRDEA_BAD_XML);
+    try {
+      reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS);
+      assertWithMessage("Expected InternalServerErrorException to be thrown").fail();
+    } catch (InternalServerErrorException expected) {
+      assertThat(expected).hasMessageThat().isEqualTo("The structure of the report is invalid.");
+    }
+  }
+}

javatests/google/registry/reporting/IcannReportingUploadActionTest.java

@@ -0,0 +1,161 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static google.registry.reporting.IcannReportingModule.ReportType.TRANSACTIONS;
+import static google.registry.testing.DatastoreHelper.createTld;
+import static google.registry.testing.GcsTestingUtils.writeGcsFile;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.common.base.Optional;
+import google.registry.gcs.GcsUtils;
+import google.registry.testing.AppEngineRule;
+import google.registry.testing.FakeClock;
+import google.registry.testing.FakeResponse;
+import google.registry.testing.FakeSleeper;
+import google.registry.util.Retrier;
+import java.io.IOException;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link google.registry.reporting.IcannReportingUploadAction} */
+@RunWith(JUnit4.class)
+public class IcannReportingUploadActionTest {
+
+  @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build();
+
+  private static final byte[] FAKE_PAYLOAD = "test,csv\n13,37".getBytes(UTF_8);
+  private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class);
+  private final FakeResponse response = new FakeResponse();
+  private final GcsService gcsService = GcsServiceFactory.createGcsService();
+  private final GcsFilename reportFile =
+      new GcsFilename("basin/icann/monthly", "test-transactions-201706.csv");
+
+  private IcannReportingUploadAction createAction() {
+    IcannReportingUploadAction action = new IcannReportingUploadAction();
+    action.icannReporter = mockReporter;
+    action.gcsUtils = new GcsUtils(gcsService, 1024);
+    action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
+    action.yearMonth = "2017-06";
+    action.reportType = TRANSACTIONS;
+    action.subdir = Optional.absent();
+    action.tld = "test";
+    action.reportingBucket = "basin";
+    action.response = response;
+    return action;
+  }
+
+  @Before
+  public void before() throws Exception {
+    createTld("test");
+    writeGcsFile(gcsService, reportFile, FAKE_PAYLOAD);
+  }
+
+  @Test
+  public void testSuccess() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    action.run();
+    verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+    verifyNoMoreInteractions(mockReporter);
+    assertThat(((FakeResponse) action.response).getPayload())
+        .isEqualTo("OK, sending: test,csv\n13,37");
+  }
+
+  @Test
+  public void testSuccess_WithRetry() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    doThrow(new IOException("Expected exception."))
+        .doNothing()
+        .when(mockReporter)
+        .send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+    action.run();
+    verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+    verifyNoMoreInteractions(mockReporter);
+    assertThat(((FakeResponse) action.response).getPayload())
+        .isEqualTo("OK, sending: test,csv\n13,37");
+  }
+
+  @Test
+  public void testFail_NonexisistentTld() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    action.tld = "invalidTld";
+    try {
+      action.run();
+      assertWithMessage("Expected IllegalArgumentException to be thrown").fail();
+    } catch (IllegalArgumentException expected) {
+      assertThat(expected)
+          .hasMessageThat()
+          .isEqualTo("TLD invalidTld does not exist");
+    }
+  }
+
+  @Test
+  public void testFail_InvalidYearMonth() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    action.yearMonth = "2017-3";
+    try {
+      action.run();
+      assertWithMessage("Expected IllegalStateException to be thrown").fail();
+    } catch (IllegalStateException expected) {
+      assertThat(expected)
+          .hasMessageThat()
+          .isEqualTo("yearMonth must be in YYYY-MM format, got 2017-3 instead.");
+    }
+  }
+
+  @Test
+  public void testFail_InvalidSubdir() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    action.subdir = Optional.of("/subdir/with/slash");
+    try {
+      action.run();
+      assertWithMessage("Expected IllegalStateException to be thrown").fail();
+    } catch (IllegalStateException expected) {
+      assertThat(expected)
+          .hasMessageThat()
+          .isEqualTo(
+              "subdir must not start or end with a \"/\", got /subdir/with/slash instead.");
+    }
+  }
+
+  @Test
+  public void testFail_FileNotFound() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    action.yearMonth = "1234-56";
+    try {
+      action.run();
+      assertWithMessage("Expected IllegalStateException to be thrown").fail();
+    } catch (IllegalStateException expected) {
+      assertThat(expected)
+          .hasMessageThat()
+          .isEqualTo(
+              "ICANN report object test-transactions-123456.csv "
+                  + "in bucket basin/icann/monthly not found");
+    }
+  }
+}

javatests/google/registry/reporting/ReportingTestData.java

@@ -0,0 +1,32 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import com.google.common.io.ByteSource;
+import com.google.common.io.Resources;
+import java.net.URL;
+
+/** Utility class providing easy access to contents of the {@code testdata/} directory. */
+public final class ReportingTestData {
+
+  /** Returns {@link ByteSource} for file in {@code reporting/testdata/} directory. */
+  public static ByteSource get(String filename) {
+    return Resources.asByteSource(getUrl(filename));
+  }
+
+  private static URL getUrl(String filename) {
+    return Resources.getResource(ReportingTestData.class, "testdata/" + filename);
+  }
+}

javatests/google/registry/reporting/testdata/iirdea_bad.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<response xmlns="urn:ietf:params:xml:ns:iirdea-1.0">
+  <result code="2001">
+    <msg>The structure of the report is invalid.</msg>
+    <description>
+      'XX' could not be parsed as a number (line: 2 column:3)
+    </description>
+  </result>
+</response>

javatests/google/registry/reporting/testdata/iirdea_good.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<response xmlns="urn:ietf:params:xml:ns:iirdea-1.0">
+  <result code="1000">
+    <msg>You done well.</msg>
+  </result>
+</response>