diff --git a/java/google/registry/bigquery/BigqueryJobFailureException.java b/java/google/registry/bigquery/BigqueryJobFailureException.java index b9f2041e4..503238d61 100644 --- a/java/google/registry/bigquery/BigqueryJobFailureException.java +++ b/java/google/registry/bigquery/BigqueryJobFailureException.java @@ -54,7 +54,7 @@ public final class BigqueryJobFailureException extends RuntimeException { @Nullable private final GoogleJsonError jsonError; - private BigqueryJobFailureException( + public BigqueryJobFailureException( String message, @Nullable Throwable cause, @Nullable JobStatus jobStatus, diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index 1ea3822c8..160896cf1 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -292,7 +292,7 @@ public final class RegistryConfig { @Provides @Config("dnsDefaultATtl") public static Duration provideDnsDefaultATtl() { - return Duration.standardSeconds(180); + return Duration.standardMinutes(3); } /** @@ -303,7 +303,7 @@ public final class RegistryConfig { @Provides @Config("dnsDefaultNsTtl") public static Duration provideDnsDefaultNsTtl() { - return Duration.standardSeconds(180); + return Duration.standardMinutes(3); } /** @@ -314,7 +314,7 @@ public final class RegistryConfig { @Provides @Config("dnsDefaultDsTtl") public static Duration provideDnsDefaultDsTtl() { - return Duration.standardSeconds(180); + return Duration.standardMinutes(3); } /** @@ -497,6 +497,30 @@ public final class RegistryConfig { return config.icannReporting.icannActivityReportingUploadUrl; } + /** + * Returns the email address from which we send ICANN reporting email summaries from. + * + * @see google.registry.reporting.ReportingEmailUtils + */ + @Provides + @Config("icannReportingSenderEmailAddress") + public static String provideIcannReportingEmailSenderAddress( + @Config("projectId") String projectId, RegistryConfigSettings config) { + return String.format( + "%s@%s", projectId, config.icannReporting.icannReportingEmailSenderDomain); + } + + /** + * Returns the email address from which we send ICANN reporting email summaries to. + * + * @see google.registry.reporting.ReportingEmailUtils + */ + @Provides + @Config("icannReportingRecipientEmailAddress") + public static String provideIcannReportingEmailRecipientAddress(RegistryConfigSettings config) { + return config.icannReporting.icannReportingEmailRecipient; + } + /** * Returns the Google Cloud Storage bucket for staging escrow deposits pending upload. * @@ -552,7 +576,7 @@ public final class RegistryConfig { @Provides @Config("rdeReportLockTimeout") public static Duration provideRdeReportLockTimeout() { - return Duration.standardSeconds(60); + return Duration.standardMinutes(1); } /** diff --git a/java/google/registry/config/RegistryConfigSettings.java b/java/google/registry/config/RegistryConfigSettings.java index c6aa08647..742ac3183 100644 --- a/java/google/registry/config/RegistryConfigSettings.java +++ b/java/google/registry/config/RegistryConfigSettings.java @@ -115,6 +115,8 @@ public class RegistryConfigSettings { public static class IcannReporting { public String icannTransactionsReportingUploadUrl; public String icannActivityReportingUploadUrl; + public String icannReportingEmailSenderDomain; + public String icannReportingEmailRecipient; } /** Configuration for Registry Data Escrow (RDE). */ diff --git a/java/google/registry/config/files/default-config.yaml b/java/google/registry/config/files/default-config.yaml index b4b3577ac..5d12a67b0 100644 --- a/java/google/registry/config/files/default-config.yaml +++ b/java/google/registry/config/files/default-config.yaml @@ -160,6 +160,12 @@ icannReporting: # URL we PUT monthly ICANN activity reports to. icannActivityReportingUploadUrl: https://ry-api.icann.org/report/registry-functions-activity + # Domain for the email address we send reporting pipeline summary emails from. + icannReportingEmailSenderDomain: appspotmail.com + + # Address we send reporting pipeline summary emails to. + icannReportingEmailRecipient: email@example.com + rde: # URL prefix of ICANN's server to upload RDE reports to. Nomulus adds /TLD/ID # to the end of this to construct the full URL. diff --git a/java/google/registry/reporting/BUILD b/java/google/registry/reporting/BUILD index 315e737dc..dac5e8e81 100644 --- a/java/google/registry/reporting/BUILD +++ b/java/google/registry/reporting/BUILD @@ -21,6 +21,7 @@ java_library( "//java/google/registry/xml", "@com_google_api_client", "@com_google_apis_google_api_services_bigquery", + "@com_google_appengine_api_1_0_sdk", "@com_google_appengine_tools_appengine_gcs_client", "@com_google_code_findbugs_jsr305", "@com_google_dagger", diff --git a/java/google/registry/reporting/IcannHttpReporter.java b/java/google/registry/reporting/IcannHttpReporter.java index fca16bea9..5eb2250fe 100644 --- a/java/google/registry/reporting/IcannHttpReporter.java +++ b/java/google/registry/reporting/IcannHttpReporter.java @@ -31,7 +31,6 @@ import com.google.common.io.ByteStreams; import google.registry.config.RegistryConfig.Config; import google.registry.keyring.api.KeyModule.Key; import google.registry.reporting.IcannReportingModule.ReportType; -import google.registry.request.HttpException.InternalServerErrorException; import google.registry.util.FormattingLogger; import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.iirdea.XjcIirdeaResponseElement; @@ -66,8 +65,8 @@ public class IcannHttpReporter { @Inject @Config("icannActivityReportingUploadUrl") String icannActivityUrl; @Inject IcannHttpReporter() {} - /** Uploads {@code reportBytes} to ICANN. */ - public void send(byte[] reportBytes, String reportFilename) throws XmlException, IOException { + /** Uploads {@code reportBytes} to ICANN, returning whether or not it succeeded. */ + public boolean send(byte[] reportBytes, String reportFilename) throws XmlException, IOException { validateReportFilename(reportFilename); GenericUrl uploadUrl = new GenericUrl(makeUrl(reportFilename)); HttpRequest request = @@ -85,6 +84,7 @@ public class IcannHttpReporter { logger.infofmt( "Sending report to %s with content length %s", uploadUrl.toString(), request.getContent().getLength()); + boolean success = true; try { response = request.execute(); byte[] content; @@ -93,25 +93,28 @@ public class IcannHttpReporter { } finally { response.getContent().close(); } - logger.infofmt("Received response code %s", response.getStatusCode()); - logger.infofmt("Response content: %s", new String(content, UTF_8)); + logger.infofmt( + "Received response code %s with content %s", + response.getStatusCode(), new String(content, UTF_8)); XjcIirdeaResult result = parseResult(content); if (result.getCode().getValue() != 1000) { + success = false; logger.warningfmt( "PUT rejected, status code %s:\n%s\n%s", result.getCode(), result.getMsg(), result.getDescription()); - throw new InternalServerErrorException(result.getMsg()); } } finally { if (response != null) { response.disconnect(); } else { + success = false; logger.warningfmt( "Received null response from ICANN server at %s", uploadUrl.toString()); } } + return success; } private XjcIirdeaResult parseResult(byte[] content) throws XmlException, IOException { diff --git a/java/google/registry/reporting/IcannReportingModule.java b/java/google/registry/reporting/IcannReportingModule.java index 77aa5370e..6790c0331 100644 --- a/java/google/registry/reporting/IcannReportingModule.java +++ b/java/google/registry/reporting/IcannReportingModule.java @@ -28,6 +28,7 @@ import google.registry.bigquery.BigqueryConnection; import google.registry.request.HttpException.BadRequestException; import google.registry.request.Parameter; import google.registry.util.Clock; +import google.registry.util.SendEmailService; import java.util.Optional; import javax.servlet.http.HttpServletRequest; import org.joda.time.Duration; @@ -138,5 +139,10 @@ public final class IcannReportingModule { throw new RuntimeException("Could not initialize BigqueryConnection!", e); } } + + @Provides + static SendEmailService provideSendEmailService() { + return new SendEmailService(); + } } diff --git a/java/google/registry/reporting/IcannReportingStager.java b/java/google/registry/reporting/IcannReportingStager.java index 04ab07ec0..9d34e9c57 100644 --- a/java/google/registry/reporting/IcannReportingStager.java +++ b/java/google/registry/reporting/IcannReportingStager.java @@ -21,6 +21,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.common.base.Ascii; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; @@ -93,7 +94,6 @@ public class IcannReportingStager { // Get report headers from the table schema and convert into CSV format String headerRow = constructRow(getHeaders(reportTable.columnKeySet())); - logger.infofmt("Headers: %s", headerRow); return (reportType == ReportType.ACTIVITY) ? stageActivityReports(headerRow, reportTable.rowMap().values()) @@ -231,7 +231,6 @@ public class IcannReportingStager { reportCsv.append("\r\n"); reportCsv.append(row); } - logger.infofmt("Created report:\n%s", reportCsv.toString()); return reportCsv.toString(); } @@ -240,8 +239,11 @@ public class IcannReportingStager { throws IOException { // Upload resulting CSV file to GCS byte[] reportBytes = reportCsv.getBytes(UTF_8); - String reportFilename = ReportingUtils.createFilename(tld, yearMonth, reportType); - String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir); + String reportFilename = + String.format( + "%s-%s-%s.csv", + tld, Ascii.toLowerCase(reportType.toString()), yearMonth.replace("-", "")); + String reportBucketname = String.format("%s/%s", reportingBucket, subdir); final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename); gcsUtils.createFromBytes(gcsFilename, reportBytes); logger.infofmt( @@ -253,7 +255,7 @@ public class IcannReportingStager { /** Creates and stores a manifest file on GCS, indicating which reports were generated. */ void createAndUploadManifest(ImmutableList filenames) throws IOException { - String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir); + String reportBucketname = String.format("%s/%s", reportingBucket, subdir); final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME); StringBuilder manifestString = new StringBuilder(); filenames.forEach((filename) -> manifestString.append(filename).append("\n")); diff --git a/java/google/registry/reporting/IcannReportingStagingAction.java b/java/google/registry/reporting/IcannReportingStagingAction.java index c1e47bc44..1afdb836a 100644 --- a/java/google/registry/reporting/IcannReportingStagingAction.java +++ b/java/google/registry/reporting/IcannReportingStagingAction.java @@ -18,16 +18,17 @@ import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_OK; -import com.google.common.base.Throwables; +import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.net.MediaType; +import google.registry.bigquery.BigqueryJobFailureException; import google.registry.reporting.IcannReportingModule.ReportType; import google.registry.request.Action; import google.registry.request.Parameter; import google.registry.request.Response; import google.registry.request.auth.Auth; import google.registry.util.FormattingLogger; -import java.util.Arrays; +import google.registry.util.Retrier; import javax.inject.Inject; /** @@ -60,31 +61,50 @@ public final class IcannReportingStagingAction implements Runnable { ImmutableList reportTypes; @Inject IcannReportingStager stager; + @Inject Retrier retrier; @Inject Response response; + @Inject ReportingEmailUtils emailUtils; @Inject IcannReportingStagingAction() {} @Override public void run() { - try { - ImmutableList.Builder manifestedFilesBuilder = new ImmutableList.Builder<>(); - for (ReportType reportType : reportTypes) { - manifestedFilesBuilder.addAll(stager.stageReports(reportType)); - } - ImmutableList manifestedFiles = manifestedFilesBuilder.build(); - stager.createAndUploadManifest(manifestedFiles); + retrier.callWithRetry( + () -> { + ImmutableList.Builder manifestedFilesBuilder = new ImmutableList.Builder<>(); + for (ReportType reportType : reportTypes) { + manifestedFilesBuilder.addAll(stager.stageReports(reportType)); + } + ImmutableList manifestedFiles = manifestedFilesBuilder.build(); + stager.createAndUploadManifest(manifestedFiles); - logger.infofmt("Completed staging %d report files.", manifestedFiles.size()); - response.setStatus(SC_OK); - response.setContentType(MediaType.PLAIN_TEXT_UTF_8); - response.setPayload("Completed staging action."); - } catch (Exception e) { - logger.severe("Reporting staging action failed!"); - logger.severe(Throwables.getStackTraceAsString(e)); - response.setStatus(SC_INTERNAL_SERVER_ERROR); - response.setContentType(MediaType.PLAIN_TEXT_UTF_8); - response.setPayload( - String.format("Caught exception:\n%s\n%s", e.getMessage(), - Arrays.toString(e.getStackTrace()))); - } + logger.infofmt("Completed staging %d report files.", manifestedFiles.size()); + emailUtils.emailResults( + "ICANN Monthly report staging summary [SUCCESS]", + String.format( + "Completed staging the following %d ICANN reports:\n%s", + manifestedFiles.size(), Joiner.on('\n').join(manifestedFiles))); + + response.setStatus(SC_OK); + response.setContentType(MediaType.PLAIN_TEXT_UTF_8); + response.setPayload("Completed staging action."); + return null; + }, + new Retrier.FailureReporter() { + @Override + public void beforeRetry(Throwable thrown, int failures, int maxAttempts) {} + + @Override + public void afterFinalFailure(Throwable thrown, int failures) { + emailUtils.emailResults( + "ICANN Monthly report staging summary [FAILURE]", + String.format( + "Staging failed due to %s, check logs for more details.", thrown.toString())); + logger.severefmt("Staging action failed due to %s", thrown.toString()); + response.setStatus(SC_INTERNAL_SERVER_ERROR); + response.setContentType(MediaType.PLAIN_TEXT_UTF_8); + response.setPayload(String.format("Staging failed due to %s", thrown.toString())); + } + }, + BigqueryJobFailureException.class); } } diff --git a/java/google/registry/reporting/IcannReportingUploadAction.java b/java/google/registry/reporting/IcannReportingUploadAction.java index 6b6a2e646..aa221f211 100644 --- a/java/google/registry/reporting/IcannReportingUploadAction.java +++ b/java/google/registry/reporting/IcannReportingUploadAction.java @@ -23,6 +23,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import google.registry.config.RegistryConfig.Config; import google.registry.gcs.GcsUtils; @@ -34,6 +35,7 @@ import google.registry.util.FormattingLogger; import google.registry.util.Retrier; import java.io.IOException; import java.io.InputStream; +import java.util.stream.Collectors; import javax.inject.Inject; /** @@ -68,29 +70,55 @@ public final class IcannReportingUploadAction implements Runnable { @Inject IcannHttpReporter icannReporter; @Inject Retrier retrier; @Inject Response response; + @Inject ReportingEmailUtils emailUtils; @Inject IcannReportingUploadAction() {} @Override public void run() { - String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir); + String reportBucketname = String.format("%s/%s", reportingBucket, subdir); ImmutableList manifestedFiles = getManifestedFiles(reportBucketname); + ImmutableMap.Builder reportSummaryBuilder = new ImmutableMap.Builder<>(); // Report on all manifested files for (String reportFilename : manifestedFiles) { logger.infofmt("Reading ICANN report %s from bucket %s", reportFilename, reportBucketname); final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename); verifyFileExists(gcsFilename); - retrier.callWithRetry( - () -> { - final byte[] payload = readBytesFromGcs(gcsFilename); - icannReporter.send(payload, reportFilename); - response.setContentType(PLAIN_TEXT_UTF_8); - response.setPayload(String.format("OK, sending: %s", new String(payload, UTF_8))); - return null; - }, - IOException.class); + boolean success = false; + try { + success = + retrier.callWithRetry( + () -> { + final byte[] payload = readBytesFromGcs(gcsFilename); + return icannReporter.send(payload, reportFilename); + }, + IOException.class); + } catch (RuntimeException e) { + logger.warningfmt("Upload to %s failed due to %s", gcsFilename.toString(), e.toString()); + } + reportSummaryBuilder.put(reportFilename, success); } + emailUploadResults(reportSummaryBuilder.build()); + response.setContentType(PLAIN_TEXT_UTF_8); + response.setPayload( + String.format("OK, attempted uploading %d reports", manifestedFiles.size())); + } + + private void emailUploadResults(ImmutableMap reportSummary) { + emailUtils.emailResults( + String.format( + "ICANN Monthly report upload summary: %d/%d succeeded", + reportSummary.values().stream().filter((b) -> b).count(), reportSummary.size()), + String.format( + "Report Filename - Upload status:\n%s", + reportSummary + .entrySet() + .stream() + .map( + (e) -> + String.format("%s - %s", e.getKey(), e.getValue() ? "SUCCESS" : "FAILURE")) + .collect(Collectors.joining("\n")))); } private ImmutableList getManifestedFiles(String reportBucketname) { diff --git a/java/google/registry/reporting/ReportingEmailUtils.java b/java/google/registry/reporting/ReportingEmailUtils.java new file mode 100644 index 000000000..11a2ecb30 --- /dev/null +++ b/java/google/registry/reporting/ReportingEmailUtils.java @@ -0,0 +1,48 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting; + +import google.registry.config.RegistryConfig.Config; +import google.registry.util.FormattingLogger; +import google.registry.util.SendEmailService; +import javax.inject.Inject; +import javax.mail.Message; +import javax.mail.Message.RecipientType; +import javax.mail.internet.InternetAddress; + +/** Static utils for emailing reporting results. */ +public class ReportingEmailUtils { + + @Inject @Config("icannReportingSenderEmailAddress") String sender; + @Inject @Config("icannReportingRecipientEmailAddress") String recipient; + @Inject SendEmailService emailService; + @Inject ReportingEmailUtils() {} + + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); + + void emailResults(String subject, String body) { + try { + Message msg = emailService.createMessage(); + logger.infofmt("Emailing %s", recipient); + msg.setFrom(new InternetAddress(sender)); + msg.addRecipient(RecipientType.TO, new InternetAddress(recipient)); + msg.setSubject(subject); + msg.setText(body); + emailService.sendMessage(msg); + } catch (Exception e) { + logger.warningfmt("E-mail service failed due to %s", e.toString()); + } + } +} diff --git a/java/google/registry/reporting/ReportingUtils.java b/java/google/registry/reporting/ReportingUtils.java deleted file mode 100644 index 7303b4f94..000000000 --- a/java/google/registry/reporting/ReportingUtils.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.reporting; - -import com.google.common.base.Ascii; -import google.registry.reporting.IcannReportingModule.ReportType; - -/** Static utils for reporting. */ -public final class ReportingUtils { - - /** Generates a report filename in accord with ICANN's specifications. */ - static String createFilename(String tld, String yearMonth, ReportType reportType) { - // Report files use YYYYMM naming instead of standard YYYY-MM, per ICANN requirements. - return String.format( - "%s-%s-%s.csv", tld, Ascii.toLowerCase(reportType.toString()), yearMonth.replace("-", "")); - } - - /** Constructs the bucket name to store/upload reports to. */ - static String createReportingBucketName(String reportingBucket, String subdir) { - return String.format("%s/%s", reportingBucket, subdir); - } -} diff --git a/javatests/google/registry/reporting/BUILD b/javatests/google/registry/reporting/BUILD index 7a8d3384b..91ff836e0 100644 --- a/javatests/google/registry/reporting/BUILD +++ b/javatests/google/registry/reporting/BUILD @@ -19,6 +19,7 @@ java_library( "//java/google/registry/util", "//javatests/google/registry/testing", "@com_google_apis_google_api_services_bigquery", + "@com_google_appengine_api_1_0_sdk", "@com_google_appengine_tools_appengine_gcs_client", "@com_google_code_findbugs_jsr305", "@com_google_dagger", diff --git a/javatests/google/registry/reporting/IcannHttpReporterTest.java b/javatests/google/registry/reporting/IcannHttpReporterTest.java index 7952a6e45..01bf05ae8 100644 --- a/javatests/google/registry/reporting/IcannHttpReporterTest.java +++ b/javatests/google/registry/reporting/IcannHttpReporterTest.java @@ -17,7 +17,6 @@ package google.registry.reporting; import static com.google.common.net.MediaType.CSV_UTF_8; import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth.assertWithMessage; import static google.registry.testing.DatastoreHelper.createTld; import static java.nio.charset.StandardCharsets.UTF_8; @@ -29,7 +28,6 @@ import com.google.api.client.testing.http.MockLowLevelHttpResponse; import com.google.api.client.util.Base64; import com.google.api.client.util.StringUtils; import com.google.common.io.ByteSource; -import google.registry.request.HttpException.InternalServerErrorException; import google.registry.testing.AppEngineRule; import google.registry.testing.ExceptionRule; import java.io.IOException; @@ -124,12 +122,7 @@ public class IcannHttpReporterTest { public void testFail_BadIirdeaResponse() throws Exception { IcannHttpReporter reporter = createReporter(); reporter.httpTransport = createMockTransport(IIRDEA_BAD_XML); - try { - reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv"); - assertWithMessage("Expected InternalServerErrorException to be thrown").fail(); - } catch (InternalServerErrorException expected) { - assertThat(expected).hasMessageThat().isEqualTo("The structure of the report is invalid."); - } + assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isFalse(); } @Test diff --git a/javatests/google/registry/reporting/IcannReportingStagingActionTest.java b/javatests/google/registry/reporting/IcannReportingStagingActionTest.java index 482a6cde4..7db5020a8 100644 --- a/javatests/google/registry/reporting/IcannReportingStagingActionTest.java +++ b/javatests/google/registry/reporting/IcannReportingStagingActionTest.java @@ -14,14 +14,21 @@ package google.registry.reporting; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; +import google.registry.bigquery.BigqueryJobFailureException; import google.registry.reporting.IcannReportingModule.ReportType; import google.registry.testing.AppEngineRule; +import google.registry.testing.FakeClock; import google.registry.testing.FakeResponse; +import google.registry.testing.FakeSleeper; +import google.registry.util.Retrier; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -36,6 +43,7 @@ public class IcannReportingStagingActionTest { FakeResponse response = new FakeResponse(); IcannReportingStager stager = mock(IcannReportingStager.class); + ReportingEmailUtils emailUtils = mock(ReportingEmailUtils.class); @Rule public final AppEngineRule appEngine = AppEngineRule.builder() @@ -54,6 +62,8 @@ public class IcannReportingStagingActionTest { action.reportTypes = reportingMode; action.response = response; action.stager = stager; + action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); + action.emailUtils = emailUtils; return action; } @@ -63,6 +73,10 @@ public class IcannReportingStagingActionTest { action.run(); verify(stager).stageReports(ReportType.ACTIVITY); verify(stager).createAndUploadManifest(ImmutableList.of("a", "b")); + verify(emailUtils) + .emailResults( + "ICANN Monthly report staging summary [SUCCESS]", + "Completed staging the following 2 ICANN reports:\na\nb"); } @Test @@ -73,6 +87,48 @@ public class IcannReportingStagingActionTest { verify(stager).stageReports(ReportType.ACTIVITY); verify(stager).stageReports(ReportType.TRANSACTIONS); verify(stager).createAndUploadManifest(ImmutableList.of("a", "b", "c", "d")); + verify(emailUtils) + .emailResults( + "ICANN Monthly report staging summary [SUCCESS]", + "Completed staging the following 4 ICANN reports:\na\nb\nc\nd"); + } + + @Test + public void testRetryOnBigqueryException() throws Exception { + IcannReportingStagingAction action = + createAction(ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS)); + when(stager.stageReports(ReportType.TRANSACTIONS)) + .thenThrow(new BigqueryJobFailureException("Expected failure", null, null, null)) + .thenReturn(ImmutableList.of("c", "d")); + action.run(); + verify(stager, times(2)).stageReports(ReportType.ACTIVITY); + verify(stager, times(2)).stageReports(ReportType.TRANSACTIONS); + verify(stager).createAndUploadManifest(ImmutableList.of("a", "b", "c", "d")); + verify(emailUtils) + .emailResults( + "ICANN Monthly report staging summary [SUCCESS]", + "Completed staging the following 4 ICANN reports:\na\nb\nc\nd"); + } + + @Test + public void testEmailEng_onMoreThanRetriableFailure() throws Exception { + IcannReportingStagingAction action = + createAction(ImmutableList.of(ReportType.ACTIVITY)); + when(stager.stageReports(ReportType.ACTIVITY)) + .thenThrow(new BigqueryJobFailureException("Expected failure", null, null, null)); + try { + action.run(); + assertWithMessage("Expected to encounter a BigqueryJobFailureException").fail(); + } catch (BigqueryJobFailureException expected) { + // Expect the exception. + assertThat(expected).hasMessageThat().isEqualTo("Expected failure"); + } + verify(stager, times(3)).stageReports(ReportType.ACTIVITY); + verify(emailUtils) + .emailResults( + "ICANN Monthly report staging summary [FAILURE]", + "Staging failed due to BigqueryJobFailureException: Expected failure," + + " check logs for more details."); } } diff --git a/javatests/google/registry/reporting/IcannReportingUploadActionTest.java b/javatests/google/registry/reporting/IcannReportingUploadActionTest.java index 8e9f5d8f0..6ad2bd5ff 100644 --- a/javatests/google/registry/reporting/IcannReportingUploadActionTest.java +++ b/javatests/google/registry/reporting/IcannReportingUploadActionTest.java @@ -18,11 +18,11 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static google.registry.testing.GcsTestingUtils.writeGcsFile; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.appengine.tools.cloudstorage.GcsService; @@ -46,15 +46,14 @@ public class IcannReportingUploadActionTest { @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build(); - private static final byte[] FAKE_PAYLOAD = "test,csv\n13,37".getBytes(UTF_8); - private static final byte[] MANIFEST_PAYLOAD = "test-transactions-201706.csv\n".getBytes(UTF_8); + private static final byte[] PAYLOAD_SUCCESS = "test,csv\n13,37".getBytes(UTF_8); + private static final byte[] PAYLOAD_FAIL = "ahah,csv\n12,34".getBytes(UTF_8); + private static final byte[] MANIFEST_PAYLOAD = + "test-transactions-201706.csv\na-activity-201706.csv\n".getBytes(UTF_8); private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class); + private final ReportingEmailUtils emailUtils = mock(ReportingEmailUtils.class); private final FakeResponse response = new FakeResponse(); private final GcsService gcsService = GcsServiceFactory.createGcsService(); - private final GcsFilename reportFile = - new GcsFilename("basin/icann/monthly/2017-06", "test-transactions-201706.csv"); - private final GcsFilename manifestFile = - new GcsFilename("basin/icann/monthly/2017-06", "MANIFEST.txt"); private IcannReportingUploadAction createAction() { IcannReportingUploadAction action = new IcannReportingUploadAction(); @@ -63,40 +62,84 @@ public class IcannReportingUploadActionTest { action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); action.subdir = "icann/monthly/2017-06"; action.reportingBucket = "basin"; + action.emailUtils = emailUtils; action.response = response; return action; } @Before public void before() throws Exception { - writeGcsFile(gcsService, reportFile, FAKE_PAYLOAD); - writeGcsFile(gcsService, manifestFile, MANIFEST_PAYLOAD); + writeGcsFile( + gcsService, + new GcsFilename("basin/icann/monthly/2017-06", "test-transactions-201706.csv"), + PAYLOAD_SUCCESS); + writeGcsFile( + gcsService, + new GcsFilename("basin/icann/monthly/2017-06", "a-activity-201706.csv"), + PAYLOAD_FAIL); + writeGcsFile( + gcsService, + new GcsFilename("basin/icann/monthly/2017-06", "MANIFEST.txt"), + MANIFEST_PAYLOAD); + when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv")).thenReturn(true); + when(mockReporter.send(PAYLOAD_FAIL, "a-activity-201706.csv")).thenReturn(false); } @Test public void testSuccess() throws Exception { IcannReportingUploadAction action = createAction(); action.run(); - verify(mockReporter).send(FAKE_PAYLOAD, "test-transactions-201706.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv"); + verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv"); verifyNoMoreInteractions(mockReporter); assertThat(((FakeResponse) action.response).getPayload()) - .isEqualTo("OK, sending: test,csv\n13,37"); + .isEqualTo("OK, attempted uploading 2 reports"); + verify(emailUtils) + .emailResults( + "ICANN Monthly report upload summary: 1/2 succeeded", + "Report Filename - Upload status:\n" + + "test-transactions-201706.csv - SUCCESS\n" + + "a-activity-201706.csv - FAILURE"); } @Test public void testSuccess_WithRetry() throws Exception { IcannReportingUploadAction action = createAction(); - doThrow(new IOException("Expected exception.")) - .doNothing() - .when(mockReporter) - .send(FAKE_PAYLOAD, "test-transactions-201706.csv"); + when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv")) + .thenThrow(new IOException("Expected exception.")) + .thenReturn(true); action.run(); - verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test-transactions-201706.csv"); + verify(mockReporter, times(2)).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv"); + verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv"); verifyNoMoreInteractions(mockReporter); assertThat(((FakeResponse) action.response).getPayload()) - .isEqualTo("OK, sending: test,csv\n13,37"); + .isEqualTo("OK, attempted uploading 2 reports"); + verify(emailUtils) + .emailResults( + "ICANN Monthly report upload summary: 1/2 succeeded", + "Report Filename - Upload status:\n" + + "test-transactions-201706.csv - SUCCESS\n" + + "a-activity-201706.csv - FAILURE"); } + @Test + public void testFailure_firstUnrecoverable_stillAttemptsUploadingBoth() throws Exception { + IcannReportingUploadAction action = createAction(); + when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv")) + .thenThrow(new IOException("Expected exception")); + action.run(); + verify(mockReporter, times(3)).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv"); + verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv"); + verifyNoMoreInteractions(mockReporter); + assertThat(((FakeResponse) action.response).getPayload()) + .isEqualTo("OK, attempted uploading 2 reports"); + verify(emailUtils) + .emailResults( + "ICANN Monthly report upload summary: 0/2 succeeded", + "Report Filename - Upload status:\n" + + "test-transactions-201706.csv - FAILURE\n" + + "a-activity-201706.csv - FAILURE"); + } @Test public void testFail_FileNotFound() throws Exception { IcannReportingUploadAction action = createAction(); diff --git a/javatests/google/registry/reporting/ReportingEmailUtilsTest.java b/javatests/google/registry/reporting/ReportingEmailUtilsTest.java new file mode 100644 index 000000000..422af5549 --- /dev/null +++ b/javatests/google/registry/reporting/ReportingEmailUtilsTest.java @@ -0,0 +1,68 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import google.registry.util.SendEmailService; +import java.util.Properties; +import javax.mail.Message; +import javax.mail.Message.RecipientType; +import javax.mail.Session; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link ReportingEmailUtils}. */ +@RunWith(JUnit4.class) +public class ReportingEmailUtilsTest { + private Message msg; + private final SendEmailService emailService = mock(SendEmailService.class); + + @Before + public void setUp() { + msg = new MimeMessage(Session.getDefaultInstance(new Properties(), null)); + when(emailService.createMessage()).thenReturn(msg); + } + + private ReportingEmailUtils createEmailUtil() { + ReportingEmailUtils emailUtils = new ReportingEmailUtils(); + emailUtils.sender = "test-project.appspotmail.com"; + emailUtils.recipient = "email@example.com"; + emailUtils.emailService = emailService; + return emailUtils; + } + + @Test + public void testSuccess_sendsEmail() throws Exception { + ReportingEmailUtils emailUtils = createEmailUtil(); + emailUtils.emailResults("Subject", "Body"); + + assertThat(msg.getFrom()).hasLength(1); + assertThat(msg.getFrom()[0]) + .isEqualTo(new InternetAddress("test-project.appspotmail.com")); + assertThat(msg.getRecipients(RecipientType.TO)).hasLength(1); + assertThat(msg.getRecipients(RecipientType.TO)[0]) + .isEqualTo(new InternetAddress("email@example.com")); + assertThat(msg.getSubject()).isEqualTo("Subject"); + assertThat(msg.getContentType()).isEqualTo("text/plain"); + assertThat(msg.getContent().toString()).isEqualTo("Body"); + } +} diff --git a/javatests/google/registry/reporting/ReportingUtilsTest.java b/javatests/google/registry/reporting/ReportingUtilsTest.java deleted file mode 100644 index b30906fcf..000000000 --- a/javatests/google/registry/reporting/ReportingUtilsTest.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.reporting; - -import static com.google.common.truth.Truth.assertThat; - -import google.registry.reporting.IcannReportingModule.ReportType; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link google.registry.reporting.ReportingUtils}. */ -@RunWith(JUnit4.class) -public class ReportingUtilsTest { - @Test - public void testCreateFilename_success() { - assertThat(ReportingUtils.createFilename("test", "2017-06", ReportType.ACTIVITY)) - .isEqualTo("test-activity-201706.csv"); - assertThat(ReportingUtils.createFilename("foo", "2017-06", ReportType.TRANSACTIONS)) - .isEqualTo("foo-transactions-201706.csv"); - } - - @Test - public void testCreateBucketName_success() { - assertThat(ReportingUtils.createReportingBucketName("gs://domain-registry-basin", "my/subdir")) - .isEqualTo("gs://domain-registry-basin/my/subdir"); - } -}