diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index 6ee928584..a40e91632 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -451,6 +451,40 @@ public final class RegistryConfig { return config.gSuite.outgoingEmailDisplayName; } + /** + * Returns the Google Cloud Storage bucket for ICANN transaction and activity reports to + * be uploaded. + * + * @see google.registry.reporting.IcannReportingUploadAction + */ + @Provides + @Config("icannReportingBucket") + public static String provideIcannReportingBucket(@Config("projectId") String projectId) { + return projectId + "-reporting"; + } + + /** + * Returns the URL we send HTTP PUT requests for ICANN monthly transactions reports. + * + * @see google.registry.reporting.IcannHttpReporter + */ + @Provides + @Config("icannTransactionsReportingUploadUrl") + public static String provideIcannTransactionsReportingUploadUrl(RegistryConfigSettings config) { + return config.icannReporting.icannTransactionsReportingUploadUrl; + } + + /** + * Returns the URL we send HTTP PUT requests for ICANN monthly activity reports. + * + * @see google.registry.reporting.IcannHttpReporter + */ + @Provides + @Config("icannActivityReportingUploadUrl") + public static String provideIcannActivityReportingUploadUrl(RegistryConfigSettings config) { + return config.icannReporting.icannActivityReportingUploadUrl; + } + /** * Returns the Google Cloud Storage bucket for staging escrow deposits pending upload. * diff --git a/java/google/registry/config/RegistryConfigSettings.java b/java/google/registry/config/RegistryConfigSettings.java index b91336f80..d1d90f85f 100644 --- a/java/google/registry/config/RegistryConfigSettings.java +++ b/java/google/registry/config/RegistryConfigSettings.java @@ -26,6 +26,7 @@ public class RegistryConfigSettings { public RegistryPolicy registryPolicy; public Datastore datastore; public Caching caching; + public IcannReporting icannReporting; public Rde rde; public RegistrarConsole registrarConsole; public Monitoring monitoring; @@ -102,6 +103,12 @@ public class RegistryConfigSettings { public int staticPremiumListMaxCachedEntries; } + /** Configuration for ICANN monthly reporting. */ + public static class IcannReporting { + public String icannTransactionsReportingUploadUrl; + public String icannActivityReportingUploadUrl; + } + /** Configuration for Registry Data Escrow (RDE). */ public static class Rde { public String reportUrlPrefix; diff --git a/java/google/registry/config/files/default-config.yaml b/java/google/registry/config/files/default-config.yaml index b2569e749..c1f505398 100644 --- a/java/google/registry/config/files/default-config.yaml +++ b/java/google/registry/config/files/default-config.yaml @@ -136,6 +136,13 @@ oAuth: # backend services, e. g. nomulus tool, EPP proxy, etc. allowedOauthClientIds: [] +icannReporting: + # URL we PUT monthly ICANN transactions reports to. + icannTransactionsReportingUploadUrl: https://ry-api.example.org/report/registrar-transactions + + # URL we PUT monthly ICANN activity reports to. + icannActivityReportingUploadUrl: https://ry-api.example.org/report/registry-functions-activity + rde: # URL prefix of ICANN's server to upload RDE reports to. Nomulus adds /TLD/ID # to the end of this to construct the full URL. diff --git a/java/google/registry/config/files/nomulus-config-production-sample.yaml b/java/google/registry/config/files/nomulus-config-production-sample.yaml index ea4348e51..ce24579ca 100644 --- a/java/google/registry/config/files/nomulus-config-production-sample.yaml +++ b/java/google/registry/config/files/nomulus-config-production-sample.yaml @@ -35,6 +35,10 @@ registryPolicy: multi-line placeholder +icannReporting: + icannTransactionsReportingUploadUrl: https://ry-api.icann.org/report/registrar-transactions + icannActivityReportingUploadUrl: https://ry-api.icann.org/report/registry-functions-activity + rde: reportUrlPrefix: https://ry-api.icann.org/report/registry-escrow-report uploadUrl: sftp://placeholder@sftpipm2.ironmountain.com/Outbox diff --git a/java/google/registry/env/common/backend/WEB-INF/web.xml b/java/google/registry/env/common/backend/WEB-INF/web.xml index 16f5840b6..716ab146c 100644 --- a/java/google/registry/env/common/backend/WEB-INF/web.xml +++ b/java/google/registry/env/common/backend/WEB-INF/web.xml @@ -57,6 +57,15 @@ /_dr/task/brdaCopy + + + backend-servlet + /_dr/task/icannReportingUpload + + diff --git a/java/google/registry/module/backend/BUILD b/java/google/registry/module/backend/BUILD index 980c1f8aa..7485f2277 100644 --- a/java/google/registry/module/backend/BUILD +++ b/java/google/registry/module/backend/BUILD @@ -30,10 +30,10 @@ java_library( "//java/google/registry/monitoring/whitebox", "//java/google/registry/rde", "//java/google/registry/rde/imports", + "//java/google/registry/reporting", "//java/google/registry/request", "//java/google/registry/request:modules", "//java/google/registry/request/auth", - "//java/google/registry/security", "//java/google/registry/tmch", "//java/google/registry/util", "@com_google_appengine_api_1_0_sdk", diff --git a/java/google/registry/module/backend/BackendRequestComponent.java b/java/google/registry/module/backend/BackendRequestComponent.java index 7eb860729..177664104 100644 --- a/java/google/registry/module/backend/BackendRequestComponent.java +++ b/java/google/registry/module/backend/BackendRequestComponent.java @@ -63,6 +63,8 @@ import google.registry.rde.imports.RdeDomainImportAction; import google.registry.rde.imports.RdeHostImportAction; import google.registry.rde.imports.RdeHostLinkAction; import google.registry.rde.imports.RdeImportsModule; +import google.registry.reporting.IcannReportingModule; +import google.registry.reporting.IcannReportingUploadAction; import google.registry.request.RequestComponentBuilder; import google.registry.request.RequestModule; import google.registry.request.RequestScope; @@ -87,6 +89,7 @@ import google.registry.tmch.TmchSmdrlAction; DnsUpdateConfigModule.class, DnsUpdateWriterModule.class, ExportRequestModule.class, + IcannReportingModule.class, MapreduceModule.class, RdeModule.class, RdeImportsModule.class, @@ -109,6 +112,7 @@ interface BackendRequestComponent { ExportDomainListsAction exportDomainListsAction(); ExportReservedTermsAction exportReservedTermsAction(); ExportSnapshotAction exportSnapshotAction(); + IcannReportingUploadAction icannReportingUploadAction(); LoadSnapshotAction loadSnapshotAction(); MapreduceEntityCleanupAction mapreduceEntityCleanupAction(); MetricsExportAction metricsExportAction(); diff --git a/java/google/registry/reporting/BUILD b/java/google/registry/reporting/BUILD new file mode 100644 index 000000000..75e85ddb6 --- /dev/null +++ b/java/google/registry/reporting/BUILD @@ -0,0 +1,27 @@ +package( + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) # Apache 2.0 + +java_library( + name = "reporting", + srcs = glob(["*.java"]), + deps = [ + "//java/google/registry/config", + "//java/google/registry/gcs", + "//java/google/registry/keyring/api", + "//java/google/registry/model", + "//java/google/registry/request", + "//java/google/registry/request/auth", + "//java/google/registry/util", + "//java/google/registry/xjc", + "//java/google/registry/xml", + "@com_google_appengine_tools_appengine_gcs_client", + "@com_google_code_findbugs_jsr305", + "@com_google_dagger", + "@com_google_guava", + "@com_google_http_client", + "@javax_servlet_api", + ], +) diff --git a/java/google/registry/reporting/IcannHttpReporter.java b/java/google/registry/reporting/IcannHttpReporter.java new file mode 100644 index 000000000..ea7842d6d --- /dev/null +++ b/java/google/registry/reporting/IcannHttpReporter.java @@ -0,0 +1,138 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting; + +import static com.google.common.net.MediaType.CSV_UTF_8; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.api.client.http.ByteArrayContent; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpHeaders; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpTransport; +import com.google.common.io.ByteStreams; +import google.registry.config.RegistryConfig.Config; +import google.registry.keyring.api.KeyModule.Key; +import google.registry.reporting.IcannReportingModule.ReportType; +import google.registry.request.HttpException.InternalServerErrorException; +import google.registry.util.FormattingLogger; +import google.registry.xjc.XjcXmlTransformer; +import google.registry.xjc.iirdea.XjcIirdeaResponseElement; +import google.registry.xjc.iirdea.XjcIirdeaResult; +import google.registry.xml.XmlException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import javax.inject.Inject; + +/** + * Class that uploads a CSV file to ICANN's endpoint via an HTTP PUT call. + * + *

It uses basic authorization credentials as specified in the "Registry Interfaces" draft. + * + * @see IcannReportingUploadAction + * @see + * ICANN Reporting Specification + */ +public class IcannHttpReporter { + + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); + + @Inject HttpTransport httpTransport; + @Inject @Key("icannReportingPassword") String password; + @Inject @Config("icannTransactionsReportingUploadUrl") String icannTransactionsUrl; + @Inject @Config("icannActivityReportingUploadUrl") String icannActivityUrl; + @Inject IcannHttpReporter() {} + + /** Uploads {@code reportBytes} to ICANN. */ + public void send( + byte[] reportBytes, + String tld, + String yearMonth, + ReportType reportType) throws XmlException, IOException { + GenericUrl uploadUrl = new GenericUrl(makeUrl(tld, yearMonth, reportType)); + HttpRequest request = + httpTransport + .createRequestFactory() + .buildPutRequest(uploadUrl, new ByteArrayContent(CSV_UTF_8.toString(), reportBytes)); + + HttpHeaders headers = request.getHeaders(); + headers.setBasicAuthentication(tld + "_ry", password); + headers.setContentType(CSV_UTF_8.toString()); + request.setHeaders(headers); + request.setFollowRedirects(false); + + HttpResponse response = null; + logger.infofmt( + "Sending %s report to %s with content length %s", + reportType, + uploadUrl.toString(), + request.getContent().getLength()); + try { + response = request.execute(); + byte[] content; + try { + content = ByteStreams.toByteArray(response.getContent()); + } finally { + response.getContent().close(); + } + logger.infofmt("Received response code %s", response.getStatusCode()); + logger.infofmt("Response content: %s", new String(content, UTF_8)); + XjcIirdeaResult result = parseResult(content); + if (result.getCode().getValue() != 1000) { + logger.warningfmt( + "PUT rejected, status code %s:\n%s\n%s", + result.getCode(), + result.getMsg(), + result.getDescription()); + throw new InternalServerErrorException(result.getMsg()); + } + } finally { + if (response != null) { + response.disconnect(); + } else { + logger.warningfmt( + "Received null response from ICANN server at %s", uploadUrl.toString()); + } + } + } + + private XjcIirdeaResult parseResult(byte[] content) throws XmlException, IOException { + XjcIirdeaResponseElement response = + XjcXmlTransformer.unmarshal( + XjcIirdeaResponseElement.class, new ByteArrayInputStream(content)); + XjcIirdeaResult result = response.getResult(); + return result; + } + + private String makeUrl(String tld, String yearMonth, ReportType reportType) { + String urlPrefix = getUrlPrefix(reportType); + return String.format("%s/%s/%s", urlPrefix, tld, yearMonth); + } + + private String getUrlPrefix(ReportType reportType) { + switch (reportType) { + case TRANSACTIONS: + return icannTransactionsUrl; + case ACTIVITY: + return icannActivityUrl; + default: + throw new IllegalStateException( + String.format( + "Received invalid reportType! Expected ACTIVITY or TRANSACTIONS, got %s.", + reportType)); + } + } +} diff --git a/java/google/registry/reporting/IcannReportingModule.java b/java/google/registry/reporting/IcannReportingModule.java new file mode 100644 index 000000000..592eaabf0 --- /dev/null +++ b/java/google/registry/reporting/IcannReportingModule.java @@ -0,0 +1,58 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting; + +import static google.registry.request.RequestParameters.extractEnumParameter; +import static google.registry.request.RequestParameters.extractOptionalParameter; +import static google.registry.request.RequestParameters.extractRequiredParameter; + +import com.google.common.base.Optional; +import dagger.Module; +import dagger.Provides; +import google.registry.request.Parameter; +import javax.servlet.http.HttpServletRequest; + +/** Module for dependencies required by ICANN monthly transactions/activity reporting. */ +@Module +public final class IcannReportingModule { + + /** Enum determining the type of report to generate or upload. */ + public enum ReportType { + TRANSACTIONS, + ACTIVITY + } + + static final String PARAM_YEAR_MONTH = "yearMonth"; + static final String PARAM_REPORT_TYPE = "reportType"; + static final String PARAM_SUBDIR = "subdir"; + + @Provides + @Parameter(PARAM_YEAR_MONTH) + static String provideYearMonth(HttpServletRequest req) { + return extractRequiredParameter(req, PARAM_YEAR_MONTH); + } + + @Provides + @Parameter(PARAM_REPORT_TYPE) + static ReportType provideReportType(HttpServletRequest req) { + return extractEnumParameter(req, ReportType.class, PARAM_REPORT_TYPE); + } + + @Provides + @Parameter(PARAM_SUBDIR) + static Optional provideSubdir(HttpServletRequest req) { + return extractOptionalParameter(req, PARAM_SUBDIR); + } +} diff --git a/java/google/registry/reporting/IcannReportingUploadAction.java b/java/google/registry/reporting/IcannReportingUploadAction.java new file mode 100644 index 000000000..00c8bda27 --- /dev/null +++ b/java/google/registry/reporting/IcannReportingUploadAction.java @@ -0,0 +1,133 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; +import static google.registry.model.registry.Registries.assertTldExists; +import static google.registry.request.Action.Method.POST; + +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.common.base.Optional; +import com.google.common.io.ByteStreams; +import google.registry.config.RegistryConfig.Config; +import google.registry.gcs.GcsUtils; +import google.registry.reporting.IcannReportingModule.ReportType; +import google.registry.request.Action; +import google.registry.request.Parameter; +import google.registry.request.RequestParameters; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import google.registry.request.auth.Auth.AuthMethod; +import google.registry.request.auth.Auth.UserPolicy; +import google.registry.request.auth.AuthLevel; +import google.registry.util.FormattingLogger; +import google.registry.util.Retrier; +import google.registry.xml.XmlException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.Callable; +import javax.inject.Inject; + +/** + * Action that uploads the monthly transaction and activity reports from Cloud Storage to ICANN via + * an HTTP PUT. + * + */ +@Action( + path = IcannReportingUploadAction.PATH, + method = POST, + auth = + @Auth( + methods = {AuthMethod.INTERNAL, AuthMethod.API}, + minimumLevel = AuthLevel.APP, + userPolicy = UserPolicy.ADMIN + ) +) +public final class IcannReportingUploadAction implements Runnable { + + static final String PATH = "/_dr/task/icannReportingUpload"; + static final String DEFAULT_SUBDIR = "icann/monthly"; + + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); + + @Inject GcsUtils gcsUtils; + @Inject IcannHttpReporter icannReporter; + @Inject Retrier retrier; + @Inject @Parameter(RequestParameters.PARAM_TLD) String tld; + @Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth; + @Inject @Parameter(IcannReportingModule.PARAM_REPORT_TYPE) ReportType reportType; + @Inject @Parameter(IcannReportingModule.PARAM_SUBDIR) Optional subdir; + @Inject @Config("icannReportingBucket") String reportingBucket; + @Inject Response response; + + @Inject + IcannReportingUploadAction() {} + + @Override + public void run() { + validateParams(); + String reportFilename = createFilename(tld, yearMonth, reportType); + logger.infofmt("Reading ICANN report %s from bucket %s", reportFilename, reportingBucket); + final GcsFilename gcsFilename = + new GcsFilename( + reportingBucket + "/" + (subdir.isPresent() ? subdir.get() : DEFAULT_SUBDIR), + reportFilename); + checkState( + gcsUtils.existsAndNotEmpty(gcsFilename), + "ICANN report object %s in bucket %s not found", + gcsFilename.getObjectName(), + gcsFilename.getBucketName()); + + retrier.callWithRetry(new Callable() { + @Override + public Void call() throws IOException, XmlException { + final byte[] payload = readReportFromGcs(gcsFilename); + icannReporter.send(payload, tld, yearMonth, reportType); + response.setContentType(PLAIN_TEXT_UTF_8); + response.setPayload( + String.format("OK, sending: %s", new String(payload, StandardCharsets.UTF_8))); + return null; + }}, IOException.class); + } + + private byte[] readReportFromGcs(GcsFilename reportFilename) throws IOException { + try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) { + return ByteStreams.toByteArray(gcsInput); + } + } + + static String createFilename(String tld, String yearMonth, ReportType reportType) { + // TODO(b/62585428): Change file naming date format to YYYY-MM for consistency with URL. + // Report files use YYYYMM naming instead of standard YYYY-MM. + String fileYearMonth = yearMonth.substring(0, 4) + yearMonth.substring(5, 7); + return String.format("%s-%s-%s.csv", tld, reportType.toString().toLowerCase(), fileYearMonth); + } + + private void validateParams() { + assertTldExists(tld); + checkState( + yearMonth.matches("[0-9]{4}-[0-9]{2}"), + "yearMonth must be in YYYY-MM format, got %s instead.", + yearMonth); + if (subdir.isPresent()) { + checkState( + !subdir.get().startsWith("/") && !subdir.get().endsWith("/"), + "subdir must not start or end with a \"/\", got %s instead.", + subdir.get()); + } + } +} diff --git a/javatests/google/registry/module/backend/testdata/backend_routing.txt b/javatests/google/registry/module/backend/testdata/backend_routing.txt index 5c41f1567..1e9be530d 100644 --- a/javatests/google/registry/module/backend/testdata/backend_routing.txt +++ b/javatests/google/registry/module/backend/testdata/backend_routing.txt @@ -14,6 +14,7 @@ PATH CLASS METHOD /_dr/task/exportDomainLists ExportDomainListsAction POST n INTERNAL APP IGNORED /_dr/task/exportReservedTerms ExportReservedTermsAction POST n INTERNAL APP IGNORED /_dr/task/exportSnapshot ExportSnapshotAction POST y INTERNAL APP IGNORED +/_dr/task/icannReportingUpload IcannReportingUploadAction POST n INTERNAL,API APP ADMIN /_dr/task/importRdeContacts RdeContactImportAction GET n INTERNAL APP IGNORED /_dr/task/importRdeDomains RdeDomainImportAction GET n INTERNAL APP IGNORED /_dr/task/importRdeHosts RdeHostImportAction GET n INTERNAL APP IGNORED diff --git a/javatests/google/registry/reporting/BUILD b/javatests/google/registry/reporting/BUILD new file mode 100644 index 000000000..f60b83c80 --- /dev/null +++ b/javatests/google/registry/reporting/BUILD @@ -0,0 +1,37 @@ +package( + default_testonly = 1, + default_visibility = ["//java/google/registry:registry_project"], +) + +licenses(["notice"]) # Apache 2.0 + +load("//java/com/google/testing/builddefs:GenTestRules.bzl", "GenTestRules") + +java_library( + name = "reporting", + srcs = glob(["*.java"]), + resources = glob(["testdata/*"]), + deps = [ + "//java/google/registry/gcs", + "//java/google/registry/reporting", + "//java/google/registry/request", + "//java/google/registry/util", + "//javatests/google/registry/testing", + "@com_google_appengine_tools_appengine_gcs_client", + "@com_google_code_findbugs_jsr305", + "@com_google_dagger", + "@com_google_guava", + "@com_google_http_client", + "@com_google_truth", + "@javax_servlet_api", + "@junit", + "@org_mockito_all", + ], +) + +GenTestRules( + name = "GeneratedTestRules", + default_test_size = "small", + test_files = glob(["*Test.java"]), + deps = [":reporting"], +) diff --git a/javatests/google/registry/reporting/IcannHttpReporterTest.java b/javatests/google/registry/reporting/IcannHttpReporterTest.java new file mode 100644 index 000000000..8cbbed41f --- /dev/null +++ b/javatests/google/registry/reporting/IcannHttpReporterTest.java @@ -0,0 +1,107 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting; + +import static com.google.common.net.MediaType.CSV_UTF_8; +import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.api.client.util.Base64; +import com.google.api.client.util.StringUtils; +import com.google.common.io.ByteSource; +import google.registry.reporting.IcannReportingModule.ReportType; +import google.registry.request.HttpException.InternalServerErrorException; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Unit tests for {@link google.registry.reporting.IcannHttpReporter}. + */ +@RunWith(JUnit4.class) +public class IcannHttpReporterTest { + + private static final ByteSource IIRDEA_GOOD_XML = ReportingTestData.get("iirdea_good.xml"); + private static final ByteSource IIRDEA_BAD_XML = ReportingTestData.get("iirdea_bad.xml"); + + private MockLowLevelHttpRequest mockRequest; + + private MockHttpTransport createMockTransport (ByteSource iirdeaResponse) { + return new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) throws IOException { + mockRequest = new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() throws IOException { + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + response.setStatusCode(200); + response.setContentType(PLAIN_TEXT_UTF_8.toString()); + response.setContent(iirdeaResponse.read()); + return response; + } + }; + mockRequest.setUrl(url); + return mockRequest; + } + }; + } + + private static final byte[] FAKE_PAYLOAD = "test,csv\n1,2".getBytes(UTF_8); + + private IcannHttpReporter createReporter() { + IcannHttpReporter reporter = new IcannHttpReporter(); + reporter.httpTransport = createMockTransport(IIRDEA_GOOD_XML); + reporter.password = "fakePass"; + reporter.icannTransactionsUrl = "https://fake-transactions.url"; + reporter.icannActivityUrl = "https://fake-activity.url"; + return reporter; + } + + @Test + public void testSuccess() throws Exception { + IcannHttpReporter reporter = createReporter(); + reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS); + + assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2016-06"); + Map> headers = mockRequest.getHeaders(); + String userPass = "test_ry:fakePass"; + String expectedAuth = + String.format("Basic %s", Base64.encodeBase64String(StringUtils.getBytesUtf8(userPass))); + assertThat(headers.get("authorization")).containsExactly(expectedAuth); + assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString()); + } + + @Test + public void testFail_BadIirdeaResponse() throws Exception { + IcannHttpReporter reporter = createReporter(); + reporter.httpTransport = createMockTransport(IIRDEA_BAD_XML); + try { + reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS); + assertWithMessage("Expected InternalServerErrorException to be thrown").fail(); + } catch (InternalServerErrorException expected) { + assertThat(expected).hasMessageThat().isEqualTo("The structure of the report is invalid."); + } + } +} diff --git a/javatests/google/registry/reporting/IcannReportingUploadActionTest.java b/javatests/google/registry/reporting/IcannReportingUploadActionTest.java new file mode 100644 index 000000000..aa63ba92d --- /dev/null +++ b/javatests/google/registry/reporting/IcannReportingUploadActionTest.java @@ -0,0 +1,161 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static google.registry.reporting.IcannReportingModule.ReportType.TRANSACTIONS; +import static google.registry.testing.DatastoreHelper.createTld; +import static google.registry.testing.GcsTestingUtils.writeGcsFile; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.common.base.Optional; +import google.registry.gcs.GcsUtils; +import google.registry.testing.AppEngineRule; +import google.registry.testing.FakeClock; +import google.registry.testing.FakeResponse; +import google.registry.testing.FakeSleeper; +import google.registry.util.Retrier; +import java.io.IOException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link google.registry.reporting.IcannReportingUploadAction} */ +@RunWith(JUnit4.class) +public class IcannReportingUploadActionTest { + + @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build(); + + private static final byte[] FAKE_PAYLOAD = "test,csv\n13,37".getBytes(UTF_8); + private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class); + private final FakeResponse response = new FakeResponse(); + private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final GcsFilename reportFile = + new GcsFilename("basin/icann/monthly", "test-transactions-201706.csv"); + + private IcannReportingUploadAction createAction() { + IcannReportingUploadAction action = new IcannReportingUploadAction(); + action.icannReporter = mockReporter; + action.gcsUtils = new GcsUtils(gcsService, 1024); + action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); + action.yearMonth = "2017-06"; + action.reportType = TRANSACTIONS; + action.subdir = Optional.absent(); + action.tld = "test"; + action.reportingBucket = "basin"; + action.response = response; + return action; + } + + @Before + public void before() throws Exception { + createTld("test"); + writeGcsFile(gcsService, reportFile, FAKE_PAYLOAD); + } + + @Test + public void testSuccess() throws Exception { + IcannReportingUploadAction action = createAction(); + action.run(); + verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS); + verifyNoMoreInteractions(mockReporter); + assertThat(((FakeResponse) action.response).getPayload()) + .isEqualTo("OK, sending: test,csv\n13,37"); + } + + @Test + public void testSuccess_WithRetry() throws Exception { + IcannReportingUploadAction action = createAction(); + doThrow(new IOException("Expected exception.")) + .doNothing() + .when(mockReporter) + .send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS); + action.run(); + verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS); + verifyNoMoreInteractions(mockReporter); + assertThat(((FakeResponse) action.response).getPayload()) + .isEqualTo("OK, sending: test,csv\n13,37"); + } + + @Test + public void testFail_NonexisistentTld() throws Exception { + IcannReportingUploadAction action = createAction(); + action.tld = "invalidTld"; + try { + action.run(); + assertWithMessage("Expected IllegalArgumentException to be thrown").fail(); + } catch (IllegalArgumentException expected) { + assertThat(expected) + .hasMessageThat() + .isEqualTo("TLD invalidTld does not exist"); + } + } + + @Test + public void testFail_InvalidYearMonth() throws Exception { + IcannReportingUploadAction action = createAction(); + action.yearMonth = "2017-3"; + try { + action.run(); + assertWithMessage("Expected IllegalStateException to be thrown").fail(); + } catch (IllegalStateException expected) { + assertThat(expected) + .hasMessageThat() + .isEqualTo("yearMonth must be in YYYY-MM format, got 2017-3 instead."); + } + } + + @Test + public void testFail_InvalidSubdir() throws Exception { + IcannReportingUploadAction action = createAction(); + action.subdir = Optional.of("/subdir/with/slash"); + try { + action.run(); + assertWithMessage("Expected IllegalStateException to be thrown").fail(); + } catch (IllegalStateException expected) { + assertThat(expected) + .hasMessageThat() + .isEqualTo( + "subdir must not start or end with a \"/\", got /subdir/with/slash instead."); + } + } + + @Test + public void testFail_FileNotFound() throws Exception { + IcannReportingUploadAction action = createAction(); + action.yearMonth = "1234-56"; + try { + action.run(); + assertWithMessage("Expected IllegalStateException to be thrown").fail(); + } catch (IllegalStateException expected) { + assertThat(expected) + .hasMessageThat() + .isEqualTo( + "ICANN report object test-transactions-123456.csv " + + "in bucket basin/icann/monthly not found"); + } + } +} diff --git a/javatests/google/registry/reporting/ReportingTestData.java b/javatests/google/registry/reporting/ReportingTestData.java new file mode 100644 index 000000000..0eb919942 --- /dev/null +++ b/javatests/google/registry/reporting/ReportingTestData.java @@ -0,0 +1,32 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting; + +import com.google.common.io.ByteSource; +import com.google.common.io.Resources; +import java.net.URL; + +/** Utility class providing easy access to contents of the {@code testdata/} directory. */ +public final class ReportingTestData { + + /** Returns {@link ByteSource} for file in {@code reporting/testdata/} directory. */ + public static ByteSource get(String filename) { + return Resources.asByteSource(getUrl(filename)); + } + + private static URL getUrl(String filename) { + return Resources.getResource(ReportingTestData.class, "testdata/" + filename); + } +} diff --git a/javatests/google/registry/reporting/testdata/iirdea_bad.xml b/javatests/google/registry/reporting/testdata/iirdea_bad.xml new file mode 100644 index 000000000..98a4d1d7c --- /dev/null +++ b/javatests/google/registry/reporting/testdata/iirdea_bad.xml @@ -0,0 +1,9 @@ + + + + The structure of the report is invalid. + + 'XX' could not be parsed as a number (line: 2 column:3) + + + diff --git a/javatests/google/registry/reporting/testdata/iirdea_good.xml b/javatests/google/registry/reporting/testdata/iirdea_good.xml new file mode 100644 index 000000000..2af64b3e3 --- /dev/null +++ b/javatests/google/registry/reporting/testdata/iirdea_good.xml @@ -0,0 +1,6 @@ + + + + You done well. + +