Create ICANN report upload action

This is the first step in moving the current []cron-Python reporting scripts
into App Engine, as an official part of the Nomulus package. This copies the
structure of RDE uploads, with a few changes specific to monthly reporting.

I've left some TODOs related to actually testing it on the ICANN endpoint, as we're still not sure how files to be uploaded will be staged, and whether we can actually ping their endpoint on valid ports (80 or 443).

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=160408703

javatests/google/registry/module/backend/testdata/backend_routing.txt

@@ -14,6 +14,7 @@ PATH                                   CLASS                              METHOD
 /_dr/task/exportDomainLists            ExportDomainListsAction            POST     n  INTERNAL     APP IGNORED
 /_dr/task/exportReservedTerms          ExportReservedTermsAction          POST     n  INTERNAL     APP IGNORED
 /_dr/task/exportSnapshot               ExportSnapshotAction               POST     y  INTERNAL     APP IGNORED
+/_dr/task/icannReportingUpload         IcannReportingUploadAction         POST     n  INTERNAL,API APP ADMIN
 /_dr/task/importRdeContacts            RdeContactImportAction             GET      n  INTERNAL     APP IGNORED
 /_dr/task/importRdeDomains             RdeDomainImportAction              GET      n  INTERNAL     APP IGNORED
 /_dr/task/importRdeHosts               RdeHostImportAction                GET      n  INTERNAL     APP IGNORED

javatests/google/registry/reporting/BUILD

@@ -0,0 +1,37 @@
+package(
+    default_testonly = 1,
+    default_visibility = ["//java/google/registry:registry_project"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//java/com/google/testing/builddefs:GenTestRules.bzl", "GenTestRules")
+
+java_library(
+    name = "reporting",
+    srcs = glob(["*.java"]),
+    resources = glob(["testdata/*"]),
+    deps = [
+        "//java/google/registry/gcs",
+        "//java/google/registry/reporting",
+        "//java/google/registry/request",
+        "//java/google/registry/util",
+        "//javatests/google/registry/testing",
+        "@com_google_appengine_tools_appengine_gcs_client",
+        "@com_google_code_findbugs_jsr305",
+        "@com_google_dagger",
+        "@com_google_guava",
+        "@com_google_http_client",
+        "@com_google_truth",
+        "@javax_servlet_api",
+        "@junit",
+        "@org_mockito_all",
+    ],
+)
+
+GenTestRules(
+    name = "GeneratedTestRules",
+    default_test_size = "small",
+    test_files = glob(["*Test.java"]),
+    deps = [":reporting"],
+)

javatests/google/registry/reporting/IcannHttpReporterTest.java

@@ -0,0 +1,107 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.net.MediaType.CSV_UTF_8;
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.api.client.http.LowLevelHttpRequest;
+import com.google.api.client.http.LowLevelHttpResponse;
+import com.google.api.client.testing.http.MockHttpTransport;
+import com.google.api.client.testing.http.MockLowLevelHttpRequest;
+import com.google.api.client.testing.http.MockLowLevelHttpResponse;
+import com.google.api.client.util.Base64;
+import com.google.api.client.util.StringUtils;
+import com.google.common.io.ByteSource;
+import google.registry.reporting.IcannReportingModule.ReportType;
+import google.registry.request.HttpException.InternalServerErrorException;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Unit tests for {@link google.registry.reporting.IcannHttpReporter}.
+ */
+@RunWith(JUnit4.class)
+public class IcannHttpReporterTest {
+
+  private static final ByteSource IIRDEA_GOOD_XML = ReportingTestData.get("iirdea_good.xml");
+  private static final ByteSource IIRDEA_BAD_XML = ReportingTestData.get("iirdea_bad.xml");
+
+  private MockLowLevelHttpRequest mockRequest;
+
+  private MockHttpTransport createMockTransport (ByteSource iirdeaResponse) {
+    return new MockHttpTransport() {
+      @Override
+      public LowLevelHttpRequest buildRequest(String method, String url) throws IOException {
+        mockRequest = new MockLowLevelHttpRequest() {
+          @Override
+          public LowLevelHttpResponse execute() throws IOException {
+            MockLowLevelHttpResponse response = new MockLowLevelHttpResponse();
+            response.setStatusCode(200);
+            response.setContentType(PLAIN_TEXT_UTF_8.toString());
+            response.setContent(iirdeaResponse.read());
+            return response;
+          }
+        };
+        mockRequest.setUrl(url);
+        return mockRequest;
+      }
+    };
+  }
+
+  private static final byte[] FAKE_PAYLOAD = "test,csv\n1,2".getBytes(UTF_8);
+
+  private IcannHttpReporter createReporter() {
+    IcannHttpReporter reporter = new IcannHttpReporter();
+    reporter.httpTransport = createMockTransport(IIRDEA_GOOD_XML);
+    reporter.password = "fakePass";
+    reporter.icannTransactionsUrl = "https://fake-transactions.url";
+    reporter.icannActivityUrl = "https://fake-activity.url";
+    return reporter;
+  }
+
+  @Test
+  public void testSuccess() throws Exception {
+    IcannHttpReporter reporter = createReporter();
+    reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS);
+
+    assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2016-06");
+    Map<String, List<String>> headers = mockRequest.getHeaders();
+    String userPass = "test_ry:fakePass";
+    String expectedAuth =
+        String.format("Basic %s", Base64.encodeBase64String(StringUtils.getBytesUtf8(userPass)));
+    assertThat(headers.get("authorization")).containsExactly(expectedAuth);
+    assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString());
+  }
+
+  @Test
+  public void testFail_BadIirdeaResponse() throws Exception {
+    IcannHttpReporter reporter = createReporter();
+    reporter.httpTransport = createMockTransport(IIRDEA_BAD_XML);
+    try {
+      reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS);
+      assertWithMessage("Expected InternalServerErrorException to be thrown").fail();
+    } catch (InternalServerErrorException expected) {
+      assertThat(expected).hasMessageThat().isEqualTo("The structure of the report is invalid.");
+    }
+  }
+}

javatests/google/registry/reporting/IcannReportingUploadActionTest.java

@@ -0,0 +1,161 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static google.registry.reporting.IcannReportingModule.ReportType.TRANSACTIONS;
+import static google.registry.testing.DatastoreHelper.createTld;
+import static google.registry.testing.GcsTestingUtils.writeGcsFile;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.common.base.Optional;
+import google.registry.gcs.GcsUtils;
+import google.registry.testing.AppEngineRule;
+import google.registry.testing.FakeClock;
+import google.registry.testing.FakeResponse;
+import google.registry.testing.FakeSleeper;
+import google.registry.util.Retrier;
+import java.io.IOException;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link google.registry.reporting.IcannReportingUploadAction} */
+@RunWith(JUnit4.class)
+public class IcannReportingUploadActionTest {
+
+  @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build();
+
+  private static final byte[] FAKE_PAYLOAD = "test,csv\n13,37".getBytes(UTF_8);
+  private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class);
+  private final FakeResponse response = new FakeResponse();
+  private final GcsService gcsService = GcsServiceFactory.createGcsService();
+  private final GcsFilename reportFile =
+      new GcsFilename("basin/icann/monthly", "test-transactions-201706.csv");
+
+  private IcannReportingUploadAction createAction() {
+    IcannReportingUploadAction action = new IcannReportingUploadAction();
+    action.icannReporter = mockReporter;
+    action.gcsUtils = new GcsUtils(gcsService, 1024);
+    action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
+    action.yearMonth = "2017-06";
+    action.reportType = TRANSACTIONS;
+    action.subdir = Optional.absent();
+    action.tld = "test";
+    action.reportingBucket = "basin";
+    action.response = response;
+    return action;
+  }
+
+  @Before
+  public void before() throws Exception {
+    createTld("test");
+    writeGcsFile(gcsService, reportFile, FAKE_PAYLOAD);
+  }
+
+  @Test
+  public void testSuccess() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    action.run();
+    verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+    verifyNoMoreInteractions(mockReporter);
+    assertThat(((FakeResponse) action.response).getPayload())
+        .isEqualTo("OK, sending: test,csv\n13,37");
+  }
+
+  @Test
+  public void testSuccess_WithRetry() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    doThrow(new IOException("Expected exception."))
+        .doNothing()
+        .when(mockReporter)
+        .send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+    action.run();
+    verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+    verifyNoMoreInteractions(mockReporter);
+    assertThat(((FakeResponse) action.response).getPayload())
+        .isEqualTo("OK, sending: test,csv\n13,37");
+  }
+
+  @Test
+  public void testFail_NonexisistentTld() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    action.tld = "invalidTld";
+    try {
+      action.run();
+      assertWithMessage("Expected IllegalArgumentException to be thrown").fail();
+    } catch (IllegalArgumentException expected) {
+      assertThat(expected)
+          .hasMessageThat()
+          .isEqualTo("TLD invalidTld does not exist");
+    }
+  }
+
+  @Test
+  public void testFail_InvalidYearMonth() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    action.yearMonth = "2017-3";
+    try {
+      action.run();
+      assertWithMessage("Expected IllegalStateException to be thrown").fail();
+    } catch (IllegalStateException expected) {
+      assertThat(expected)
+          .hasMessageThat()
+          .isEqualTo("yearMonth must be in YYYY-MM format, got 2017-3 instead.");
+    }
+  }
+
+  @Test
+  public void testFail_InvalidSubdir() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    action.subdir = Optional.of("/subdir/with/slash");
+    try {
+      action.run();
+      assertWithMessage("Expected IllegalStateException to be thrown").fail();
+    } catch (IllegalStateException expected) {
+      assertThat(expected)
+          .hasMessageThat()
+          .isEqualTo(
+              "subdir must not start or end with a \"/\", got /subdir/with/slash instead.");
+    }
+  }
+
+  @Test
+  public void testFail_FileNotFound() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    action.yearMonth = "1234-56";
+    try {
+      action.run();
+      assertWithMessage("Expected IllegalStateException to be thrown").fail();
+    } catch (IllegalStateException expected) {
+      assertThat(expected)
+          .hasMessageThat()
+          .isEqualTo(
+              "ICANN report object test-transactions-123456.csv "
+                  + "in bucket basin/icann/monthly not found");
+    }
+  }
+}

javatests/google/registry/reporting/ReportingTestData.java

@@ -0,0 +1,32 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import com.google.common.io.ByteSource;
+import com.google.common.io.Resources;
+import java.net.URL;
+
+/** Utility class providing easy access to contents of the {@code testdata/} directory. */
+public final class ReportingTestData {
+
+  /** Returns {@link ByteSource} for file in {@code reporting/testdata/} directory. */
+  public static ByteSource get(String filename) {
+    return Resources.asByteSource(getUrl(filename));
+  }
+
+  private static URL getUrl(String filename) {
+    return Resources.getResource(ReportingTestData.class, "testdata/" + filename);
+  }
+}

javatests/google/registry/reporting/testdata/iirdea_bad.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<response xmlns="urn:ietf:params:xml:ns:iirdea-1.0">
+  <result code="2001">
+    <msg>The structure of the report is invalid.</msg>
+    <description>
+      'XX' could not be parsed as a number (line: 2 column:3)
+    </description>
+  </result>
+</response>

javatests/google/registry/reporting/testdata/iirdea_good.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<response xmlns="urn:ietf:params:xml:ns:iirdea-1.0">
+  <result code="1000">
+    <msg>You done well.</msg>
+  </result>
+</response>