Add reporting retry, emailing and better logging

This change:
- Adds retries to the staging action
- Emails domain-registry-eng@ upon completion of either action
- Simplifies logging to be more useful

TODO: fix up Module @Inject naming conventions and yearMonth injection

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=173294822

java/google/registry/bigquery/BigqueryJobFailureException.java

@@ -54,7 +54,7 @@ public final class BigqueryJobFailureException extends RuntimeException {
   @Nullable
   private final GoogleJsonError jsonError;
 
-  private BigqueryJobFailureException(
+  public BigqueryJobFailureException(
       String message,
       @Nullable Throwable cause,
       @Nullable JobStatus jobStatus,

java/google/registry/config/RegistryConfig.java

@@ -292,7 +292,7 @@ public final class RegistryConfig {
     @Provides
     @Config("dnsDefaultATtl")
     public static Duration provideDnsDefaultATtl() {
-      return Duration.standardSeconds(180);
+      return Duration.standardMinutes(3);
     }
 
     /**
@@ -303,7 +303,7 @@ public final class RegistryConfig {
     @Provides
     @Config("dnsDefaultNsTtl")
     public static Duration provideDnsDefaultNsTtl() {
-      return Duration.standardSeconds(180);
+      return Duration.standardMinutes(3);
     }
 
     /**
@@ -314,7 +314,7 @@ public final class RegistryConfig {
     @Provides
     @Config("dnsDefaultDsTtl")
     public static Duration provideDnsDefaultDsTtl() {
-      return Duration.standardSeconds(180);
+      return Duration.standardMinutes(3);
     }
 
     /**
@@ -497,6 +497,30 @@ public final class RegistryConfig {
       return config.icannReporting.icannActivityReportingUploadUrl;
     }
 
+    /**
+     * Returns the email address from which we send ICANN reporting email summaries from.
+     *
+     * @see google.registry.reporting.ReportingEmailUtils
+     */
+    @Provides
+    @Config("icannReportingSenderEmailAddress")
+    public static String provideIcannReportingEmailSenderAddress(
+        @Config("projectId") String projectId, RegistryConfigSettings config) {
+      return String.format(
+          "%s@%s", projectId, config.icannReporting.icannReportingEmailSenderDomain);
+    }
+
+    /**
+     * Returns the email address from which we send ICANN reporting email summaries to.
+     *
+     * @see google.registry.reporting.ReportingEmailUtils
+     */
+    @Provides
+    @Config("icannReportingRecipientEmailAddress")
+    public static String provideIcannReportingEmailRecipientAddress(RegistryConfigSettings config) {
+      return config.icannReporting.icannReportingEmailRecipient;
+    }
+
     /**
      * Returns the Google Cloud Storage bucket for staging escrow deposits pending upload.
      *
@@ -552,7 +576,7 @@ public final class RegistryConfig {
     @Provides
     @Config("rdeReportLockTimeout")
     public static Duration provideRdeReportLockTimeout() {
-      return Duration.standardSeconds(60);
+      return Duration.standardMinutes(1);
     }
 
     /**

java/google/registry/config/RegistryConfigSettings.java

@@ -115,6 +115,8 @@ public class RegistryConfigSettings {
   public static class IcannReporting {
     public String icannTransactionsReportingUploadUrl;
     public String icannActivityReportingUploadUrl;
+    public String icannReportingEmailSenderDomain;
+    public String icannReportingEmailRecipient;
   }
 
   /** Configuration for Registry Data Escrow (RDE). */

java/google/registry/config/files/default-config.yaml

@@ -160,6 +160,12 @@ icannReporting:
   # URL we PUT monthly ICANN activity reports to.
   icannActivityReportingUploadUrl: https://ry-api.icann.org/report/registry-functions-activity
 
+  # Domain for the email address we send reporting pipeline summary emails from.
+  icannReportingEmailSenderDomain: appspotmail.com
+
+  # Address we send reporting pipeline summary emails to.
+  icannReportingEmailRecipient: email@example.com
+
 rde:
   # URL prefix of ICANN's server to upload RDE reports to. Nomulus adds /TLD/ID
   # to the end of this to construct the full URL.

java/google/registry/reporting/BUILD

@@ -21,6 +21,7 @@ java_library(
         "//java/google/registry/xml",
         "@com_google_api_client",
         "@com_google_apis_google_api_services_bigquery",
+        "@com_google_appengine_api_1_0_sdk",
         "@com_google_appengine_tools_appengine_gcs_client",
         "@com_google_code_findbugs_jsr305",
         "@com_google_dagger",

java/google/registry/reporting/IcannHttpReporter.java

@@ -31,7 +31,6 @@ import com.google.common.io.ByteStreams;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.reporting.IcannReportingModule.ReportType;
-import google.registry.request.HttpException.InternalServerErrorException;
 import google.registry.util.FormattingLogger;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
@@ -66,8 +65,8 @@ public class IcannHttpReporter {
   @Inject @Config("icannActivityReportingUploadUrl") String icannActivityUrl;
   @Inject IcannHttpReporter() {}
 
-  /** Uploads {@code reportBytes} to ICANN. */
+  /** Uploads {@code reportBytes} to ICANN, returning whether or not it succeeded. */
-  public void send(byte[] reportBytes, String reportFilename) throws XmlException, IOException {
+  public boolean send(byte[] reportBytes, String reportFilename) throws XmlException, IOException {
     validateReportFilename(reportFilename);
     GenericUrl uploadUrl = new GenericUrl(makeUrl(reportFilename));
     HttpRequest request =
@@ -85,6 +84,7 @@ public class IcannHttpReporter {
     logger.infofmt(
         "Sending report to %s with content length %s",
         uploadUrl.toString(), request.getContent().getLength());
+    boolean success = true;
     try {
       response = request.execute();
       byte[] content;
@@ -93,25 +93,28 @@ public class IcannHttpReporter {
       } finally {
         response.getContent().close();
       }
-      logger.infofmt("Received response code %s", response.getStatusCode());
+      logger.infofmt(
-      logger.infofmt("Response content: %s", new String(content, UTF_8));
+          "Received response code %s with content %s",
+          response.getStatusCode(), new String(content, UTF_8));
       XjcIirdeaResult result = parseResult(content);
       if (result.getCode().getValue() != 1000) {
+        success = false;
         logger.warningfmt(
             "PUT rejected, status code %s:\n%s\n%s",
             result.getCode(),
             result.getMsg(),
             result.getDescription());
-        throw new InternalServerErrorException(result.getMsg());
       }
     } finally {
       if (response != null) {
         response.disconnect();
       } else {
+        success = false;
         logger.warningfmt(
             "Received null response from ICANN server at %s", uploadUrl.toString());
       }
     }
+    return success;
   }
 
   private XjcIirdeaResult parseResult(byte[] content) throws XmlException, IOException {

java/google/registry/reporting/IcannReportingModule.java

@@ -28,6 +28,7 @@ import google.registry.bigquery.BigqueryConnection;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.Parameter;
 import google.registry.util.Clock;
+import google.registry.util.SendEmailService;
 import java.util.Optional;
 import javax.servlet.http.HttpServletRequest;
 import org.joda.time.Duration;
@@ -138,5 +139,10 @@ public final class IcannReportingModule {
       throw new RuntimeException("Could not initialize BigqueryConnection!", e);
     }
   }
+
+  @Provides
+  static SendEmailService provideSendEmailService() {
+    return new SendEmailService();
+  }
 }
 

java/google/registry/reporting/IcannReportingStager.java

@@ -21,6 +21,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.services.bigquery.model.TableFieldSchema;
 import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.base.Ascii;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
@@ -93,7 +94,6 @@ public class IcannReportingStager {
 
     // Get report headers from the table schema and convert into CSV format
     String headerRow = constructRow(getHeaders(reportTable.columnKeySet()));
-    logger.infofmt("Headers: %s", headerRow);
 
     return (reportType == ReportType.ACTIVITY)
         ? stageActivityReports(headerRow, reportTable.rowMap().values())
@@ -231,7 +231,6 @@ public class IcannReportingStager {
       reportCsv.append("\r\n");
       reportCsv.append(row);
     }
-    logger.infofmt("Created report:\n%s", reportCsv.toString());
     return reportCsv.toString();
   }
 
@@ -240,8 +239,11 @@ public class IcannReportingStager {
       throws IOException {
     // Upload resulting CSV file to GCS
     byte[] reportBytes = reportCsv.getBytes(UTF_8);
-    String reportFilename = ReportingUtils.createFilename(tld, yearMonth, reportType);
+    String reportFilename =
-    String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
+        String.format(
+            "%s-%s-%s.csv",
+            tld, Ascii.toLowerCase(reportType.toString()), yearMonth.replace("-", ""));
+    String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
     final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
     gcsUtils.createFromBytes(gcsFilename, reportBytes);
     logger.infofmt(
@@ -253,7 +255,7 @@ public class IcannReportingStager {
 
   /** Creates and stores a manifest file on GCS, indicating which reports were generated. */
   void createAndUploadManifest(ImmutableList<String> filenames) throws IOException {
-    String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
+    String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
     final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
     StringBuilder manifestString = new StringBuilder();
     filenames.forEach((filename) -> manifestString.append(filename).append("\n"));

java/google/registry/reporting/IcannReportingStagingAction.java

@@ -18,16 +18,17 @@ import static google.registry.request.Action.Method.POST;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.common.base.Throwables;
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.net.MediaType;
+import google.registry.bigquery.BigqueryJobFailureException;
 import google.registry.reporting.IcannReportingModule.ReportType;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.FormattingLogger;
-import java.util.Arrays;
+import google.registry.util.Retrier;
 import javax.inject.Inject;
 
 /**
@@ -60,12 +61,15 @@ public final class IcannReportingStagingAction implements Runnable {
   ImmutableList<ReportType> reportTypes;
 
   @Inject IcannReportingStager stager;
+  @Inject Retrier retrier;
   @Inject Response response;
+  @Inject ReportingEmailUtils emailUtils;
   @Inject IcannReportingStagingAction() {}
 
   @Override
   public void run() {
-    try {
+    retrier.callWithRetry(
+        () -> {
           ImmutableList.Builder<String> manifestedFilesBuilder = new ImmutableList.Builder<>();
           for (ReportType reportType : reportTypes) {
             manifestedFilesBuilder.addAll(stager.stageReports(reportType));
@@ -74,17 +78,33 @@ public final class IcannReportingStagingAction implements Runnable {
           stager.createAndUploadManifest(manifestedFiles);
 
           logger.infofmt("Completed staging %d report files.", manifestedFiles.size());
+          emailUtils.emailResults(
+              "ICANN Monthly report staging summary [SUCCESS]",
+              String.format(
+                  "Completed staging the following %d ICANN reports:\n%s",
+                  manifestedFiles.size(), Joiner.on('\n').join(manifestedFiles)));
+
           response.setStatus(SC_OK);
           response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
           response.setPayload("Completed staging action.");
-    } catch (Exception e) {
+          return null;
-      logger.severe("Reporting staging action failed!");
+        },
-      logger.severe(Throwables.getStackTraceAsString(e));
+        new Retrier.FailureReporter() {
+          @Override
+          public void beforeRetry(Throwable thrown, int failures, int maxAttempts) {}
+
+          @Override
+          public void afterFinalFailure(Throwable thrown, int failures) {
+            emailUtils.emailResults(
+                "ICANN Monthly report staging summary [FAILURE]",
+                String.format(
+                    "Staging failed due to %s, check logs for more details.", thrown.toString()));
+            logger.severefmt("Staging action failed due to %s", thrown.toString());
             response.setStatus(SC_INTERNAL_SERVER_ERROR);
             response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
-      response.setPayload(
+            response.setPayload(String.format("Staging failed due to %s", thrown.toString()));
-          String.format("Caught exception:\n%s\n%s", e.getMessage(),
-              Arrays.toString(e.getStackTrace())));
           }
+        },
+        BigqueryJobFailureException.class);
   }
 }

java/google/registry/reporting/IcannReportingUploadAction.java

@@ -23,6 +23,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 import com.google.appengine.tools.cloudstorage.GcsFilename;
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteStreams;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
@@ -34,6 +35,7 @@ import google.registry.util.FormattingLogger;
 import google.registry.util.Retrier;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.stream.Collectors;
 import javax.inject.Inject;
 
 /**
@@ -68,29 +70,55 @@ public final class IcannReportingUploadAction implements Runnable {
   @Inject IcannHttpReporter icannReporter;
   @Inject Retrier retrier;
   @Inject Response response;
+  @Inject ReportingEmailUtils emailUtils;
 
   @Inject
   IcannReportingUploadAction() {}
 
   @Override
   public void run() {
-    String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
+    String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
     ImmutableList<String> manifestedFiles = getManifestedFiles(reportBucketname);
+    ImmutableMap.Builder<String, Boolean> reportSummaryBuilder = new ImmutableMap.Builder<>();
     // Report on all manifested files
     for (String reportFilename : manifestedFiles) {
       logger.infofmt("Reading ICANN report %s from bucket %s", reportFilename, reportBucketname);
       final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
       verifyFileExists(gcsFilename);
+      boolean success = false;
+      try {
+        success =
             retrier.callWithRetry(
                 () -> {
                   final byte[] payload = readBytesFromGcs(gcsFilename);
-            icannReporter.send(payload, reportFilename);
+                  return icannReporter.send(payload, reportFilename);
-            response.setContentType(PLAIN_TEXT_UTF_8);
-            response.setPayload(String.format("OK, sending: %s", new String(payload, UTF_8)));
-            return null;
                 },
                 IOException.class);
+      } catch (RuntimeException e) {
+        logger.warningfmt("Upload to %s failed due to %s", gcsFilename.toString(), e.toString());
       }
+      reportSummaryBuilder.put(reportFilename, success);
+    }
+    emailUploadResults(reportSummaryBuilder.build());
+    response.setContentType(PLAIN_TEXT_UTF_8);
+    response.setPayload(
+        String.format("OK, attempted uploading %d reports", manifestedFiles.size()));
+  }
+
+  private void emailUploadResults(ImmutableMap<String, Boolean> reportSummary) {
+    emailUtils.emailResults(
+        String.format(
+            "ICANN Monthly report upload summary: %d/%d succeeded",
+            reportSummary.values().stream().filter((b) -> b).count(), reportSummary.size()),
+        String.format(
+            "Report Filename - Upload status:\n%s",
+            reportSummary
+                .entrySet()
+                .stream()
+                .map(
+                    (e) ->
+                        String.format("%s - %s", e.getKey(), e.getValue() ? "SUCCESS" : "FAILURE"))
+                .collect(Collectors.joining("\n"))));
   }
 
   private ImmutableList<String> getManifestedFiles(String reportBucketname) {

java/google/registry/reporting/ReportingEmailUtils.java

@@ -0,0 +1,48 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.FormattingLogger;
+import google.registry.util.SendEmailService;
+import javax.inject.Inject;
+import javax.mail.Message;
+import javax.mail.Message.RecipientType;
+import javax.mail.internet.InternetAddress;
+
+/** Static utils for emailing reporting results. */
+public class ReportingEmailUtils {
+
+  @Inject @Config("icannReportingSenderEmailAddress") String sender;
+  @Inject @Config("icannReportingRecipientEmailAddress") String recipient;
+  @Inject SendEmailService emailService;
+  @Inject ReportingEmailUtils() {}
+
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+
+  void emailResults(String subject, String body) {
+    try {
+      Message msg = emailService.createMessage();
+      logger.infofmt("Emailing %s", recipient);
+      msg.setFrom(new InternetAddress(sender));
+      msg.addRecipient(RecipientType.TO, new InternetAddress(recipient));
+      msg.setSubject(subject);
+      msg.setText(body);
+      emailService.sendMessage(msg);
+    } catch (Exception e) {
+      logger.warningfmt("E-mail service failed due to %s", e.toString());
+    }
+  }
+}

java/google/registry/reporting/ReportingUtils.java

@@ -1,34 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.reporting;
-
-import com.google.common.base.Ascii;
-import google.registry.reporting.IcannReportingModule.ReportType;
-
-/** Static utils for reporting. */
-public final class ReportingUtils {
-
-  /** Generates a report filename in accord with ICANN's specifications. */
-  static String createFilename(String tld, String yearMonth, ReportType reportType) {
-    // Report files use YYYYMM naming instead of standard YYYY-MM, per ICANN requirements.
-    return String.format(
-        "%s-%s-%s.csv", tld, Ascii.toLowerCase(reportType.toString()), yearMonth.replace("-", ""));
-  }
-
-  /** Constructs the bucket name to store/upload reports to. */
-  static String createReportingBucketName(String reportingBucket, String subdir) {
-    return String.format("%s/%s", reportingBucket, subdir);
-  }
-}

javatests/google/registry/reporting/BUILD

@@ -19,6 +19,7 @@ java_library(
         "//java/google/registry/util",
         "//javatests/google/registry/testing",
         "@com_google_apis_google_api_services_bigquery",
+        "@com_google_appengine_api_1_0_sdk",
         "@com_google_appengine_tools_appengine_gcs_client",
         "@com_google_code_findbugs_jsr305",
         "@com_google_dagger",

javatests/google/registry/reporting/IcannHttpReporterTest.java

@@ -17,7 +17,6 @@ package google.registry.reporting;
 import static com.google.common.net.MediaType.CSV_UTF_8;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth.assertWithMessage;
 import static google.registry.testing.DatastoreHelper.createTld;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
@@ -29,7 +28,6 @@ import com.google.api.client.testing.http.MockLowLevelHttpResponse;
 import com.google.api.client.util.Base64;
 import com.google.api.client.util.StringUtils;
 import com.google.common.io.ByteSource;
-import google.registry.request.HttpException.InternalServerErrorException;
 import google.registry.testing.AppEngineRule;
 import google.registry.testing.ExceptionRule;
 import java.io.IOException;
@@ -124,12 +122,7 @@ public class IcannHttpReporterTest {
   public void testFail_BadIirdeaResponse() throws Exception {
     IcannHttpReporter reporter = createReporter();
     reporter.httpTransport = createMockTransport(IIRDEA_BAD_XML);
-    try {
+    assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isFalse();
-      reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv");
-      assertWithMessage("Expected InternalServerErrorException to be thrown").fail();
-    } catch (InternalServerErrorException expected) {
-      assertThat(expected).hasMessageThat().isEqualTo("The structure of the report is invalid.");
-    }
   }
 
   @Test

javatests/google/registry/reporting/IcannReportingStagingActionTest.java

@@ -14,14 +14,21 @@
 
 package google.registry.reporting;
 
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableList;
+import google.registry.bigquery.BigqueryJobFailureException;
 import google.registry.reporting.IcannReportingModule.ReportType;
 import google.registry.testing.AppEngineRule;
+import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
+import google.registry.testing.FakeSleeper;
+import google.registry.util.Retrier;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -36,6 +43,7 @@ public class IcannReportingStagingActionTest {
 
   FakeResponse response = new FakeResponse();
   IcannReportingStager stager = mock(IcannReportingStager.class);
+  ReportingEmailUtils emailUtils = mock(ReportingEmailUtils.class);
 
   @Rule
   public final AppEngineRule appEngine = AppEngineRule.builder()
@@ -54,6 +62,8 @@ public class IcannReportingStagingActionTest {
     action.reportTypes = reportingMode;
     action.response = response;
     action.stager = stager;
+    action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
+    action.emailUtils = emailUtils;
     return action;
   }
 
@@ -63,6 +73,10 @@ public class IcannReportingStagingActionTest {
     action.run();
     verify(stager).stageReports(ReportType.ACTIVITY);
     verify(stager).createAndUploadManifest(ImmutableList.of("a", "b"));
+    verify(emailUtils)
+        .emailResults(
+            "ICANN Monthly report staging summary [SUCCESS]",
+            "Completed staging the following 2 ICANN reports:\na\nb");
   }
 
   @Test
@@ -73,6 +87,48 @@ public class IcannReportingStagingActionTest {
     verify(stager).stageReports(ReportType.ACTIVITY);
     verify(stager).stageReports(ReportType.TRANSACTIONS);
     verify(stager).createAndUploadManifest(ImmutableList.of("a", "b", "c", "d"));
+    verify(emailUtils)
+        .emailResults(
+            "ICANN Monthly report staging summary [SUCCESS]",
+            "Completed staging the following 4 ICANN reports:\na\nb\nc\nd");
+  }
+
+  @Test
+  public void testRetryOnBigqueryException() throws Exception {
+    IcannReportingStagingAction action =
+        createAction(ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS));
+    when(stager.stageReports(ReportType.TRANSACTIONS))
+        .thenThrow(new BigqueryJobFailureException("Expected failure", null, null, null))
+        .thenReturn(ImmutableList.of("c", "d"));
+    action.run();
+    verify(stager, times(2)).stageReports(ReportType.ACTIVITY);
+    verify(stager, times(2)).stageReports(ReportType.TRANSACTIONS);
+    verify(stager).createAndUploadManifest(ImmutableList.of("a", "b", "c", "d"));
+    verify(emailUtils)
+        .emailResults(
+            "ICANN Monthly report staging summary [SUCCESS]",
+            "Completed staging the following 4 ICANN reports:\na\nb\nc\nd");
+  }
+
+  @Test
+  public void testEmailEng_onMoreThanRetriableFailure() throws Exception {
+    IcannReportingStagingAction action =
+        createAction(ImmutableList.of(ReportType.ACTIVITY));
+    when(stager.stageReports(ReportType.ACTIVITY))
+        .thenThrow(new BigqueryJobFailureException("Expected failure", null, null, null));
+    try {
+      action.run();
+      assertWithMessage("Expected to encounter a BigqueryJobFailureException").fail();
+    } catch (BigqueryJobFailureException expected) {
+      // Expect the exception.
+      assertThat(expected).hasMessageThat().isEqualTo("Expected failure");
+    }
+    verify(stager, times(3)).stageReports(ReportType.ACTIVITY);
+    verify(emailUtils)
+        .emailResults(
+            "ICANN Monthly report staging summary [FAILURE]",
+            "Staging failed due to BigqueryJobFailureException: Expected failure,"
+                + " check logs for more details.");
   }
 }
 

javatests/google/registry/reporting/IcannReportingUploadActionTest.java

@@ -18,11 +18,11 @@ import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
 import static google.registry.testing.GcsTestingUtils.writeGcsFile;
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
 import com.google.appengine.tools.cloudstorage.GcsService;
@@ -46,15 +46,14 @@ public class IcannReportingUploadActionTest {
 
   @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build();
 
-  private static final byte[] FAKE_PAYLOAD = "test,csv\n13,37".getBytes(UTF_8);
+  private static final byte[] PAYLOAD_SUCCESS = "test,csv\n13,37".getBytes(UTF_8);
-  private static final byte[] MANIFEST_PAYLOAD = "test-transactions-201706.csv\n".getBytes(UTF_8);
+  private static final byte[] PAYLOAD_FAIL = "ahah,csv\n12,34".getBytes(UTF_8);
+  private static final byte[] MANIFEST_PAYLOAD =
+      "test-transactions-201706.csv\na-activity-201706.csv\n".getBytes(UTF_8);
   private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class);
+  private final ReportingEmailUtils emailUtils = mock(ReportingEmailUtils.class);
   private final FakeResponse response = new FakeResponse();
   private final GcsService gcsService = GcsServiceFactory.createGcsService();
-  private final GcsFilename reportFile =
-      new GcsFilename("basin/icann/monthly/2017-06", "test-transactions-201706.csv");
-  private final GcsFilename manifestFile =
-      new GcsFilename("basin/icann/monthly/2017-06", "MANIFEST.txt");
 
   private IcannReportingUploadAction createAction() {
     IcannReportingUploadAction action = new IcannReportingUploadAction();
@@ -63,40 +62,84 @@ public class IcannReportingUploadActionTest {
     action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
     action.subdir = "icann/monthly/2017-06";
     action.reportingBucket = "basin";
+    action.emailUtils = emailUtils;
     action.response = response;
     return action;
   }
 
   @Before
   public void before() throws Exception {
-    writeGcsFile(gcsService, reportFile, FAKE_PAYLOAD);
+    writeGcsFile(
-    writeGcsFile(gcsService, manifestFile, MANIFEST_PAYLOAD);
+        gcsService,
+        new GcsFilename("basin/icann/monthly/2017-06", "test-transactions-201706.csv"),
+        PAYLOAD_SUCCESS);
+    writeGcsFile(
+        gcsService,
+        new GcsFilename("basin/icann/monthly/2017-06", "a-activity-201706.csv"),
+        PAYLOAD_FAIL);
+    writeGcsFile(
+        gcsService,
+        new GcsFilename("basin/icann/monthly/2017-06", "MANIFEST.txt"),
+        MANIFEST_PAYLOAD);
+    when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv")).thenReturn(true);
+    when(mockReporter.send(PAYLOAD_FAIL, "a-activity-201706.csv")).thenReturn(false);
   }
 
   @Test
   public void testSuccess() throws Exception {
     IcannReportingUploadAction action = createAction();
     action.run();
-    verify(mockReporter).send(FAKE_PAYLOAD, "test-transactions-201706.csv");
+    verify(mockReporter).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv");
+    verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv");
     verifyNoMoreInteractions(mockReporter);
     assertThat(((FakeResponse) action.response).getPayload())
-        .isEqualTo("OK, sending: test,csv\n13,37");
+        .isEqualTo("OK, attempted uploading 2 reports");
+    verify(emailUtils)
+        .emailResults(
+            "ICANN Monthly report upload summary: 1/2 succeeded",
+            "Report Filename - Upload status:\n"
+                + "test-transactions-201706.csv - SUCCESS\n"
+                + "a-activity-201706.csv - FAILURE");
   }
 
   @Test
   public void testSuccess_WithRetry() throws Exception {
     IcannReportingUploadAction action = createAction();
-    doThrow(new IOException("Expected exception."))
+    when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv"))
-        .doNothing()
+        .thenThrow(new IOException("Expected exception."))
-        .when(mockReporter)
+        .thenReturn(true);
-        .send(FAKE_PAYLOAD, "test-transactions-201706.csv");
     action.run();
-    verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test-transactions-201706.csv");
+    verify(mockReporter, times(2)).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv");
+    verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv");
     verifyNoMoreInteractions(mockReporter);
     assertThat(((FakeResponse) action.response).getPayload())
-        .isEqualTo("OK, sending: test,csv\n13,37");
+        .isEqualTo("OK, attempted uploading 2 reports");
+    verify(emailUtils)
+        .emailResults(
+            "ICANN Monthly report upload summary: 1/2 succeeded",
+            "Report Filename - Upload status:\n"
+                + "test-transactions-201706.csv - SUCCESS\n"
+                + "a-activity-201706.csv - FAILURE");
   }
 
+  @Test
+  public void testFailure_firstUnrecoverable_stillAttemptsUploadingBoth() throws Exception {
+    IcannReportingUploadAction action = createAction();
+    when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv"))
+        .thenThrow(new IOException("Expected exception"));
+    action.run();
+    verify(mockReporter, times(3)).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv");
+    verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv");
+    verifyNoMoreInteractions(mockReporter);
+    assertThat(((FakeResponse) action.response).getPayload())
+        .isEqualTo("OK, attempted uploading 2 reports");
+    verify(emailUtils)
+        .emailResults(
+            "ICANN Monthly report upload summary: 0/2 succeeded",
+            "Report Filename - Upload status:\n"
+                + "test-transactions-201706.csv - FAILURE\n"
+                + "a-activity-201706.csv - FAILURE");
+  }
   @Test
   public void testFail_FileNotFound() throws Exception {
     IcannReportingUploadAction action = createAction();

javatests/google/registry/reporting/ReportingEmailUtilsTest.java

@@ -0,0 +1,68 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import google.registry.util.SendEmailService;
+import java.util.Properties;
+import javax.mail.Message;
+import javax.mail.Message.RecipientType;
+import javax.mail.Session;
+import javax.mail.internet.InternetAddress;
+import javax.mail.internet.MimeMessage;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link ReportingEmailUtils}. */
+@RunWith(JUnit4.class)
+public class ReportingEmailUtilsTest {
+  private Message msg;
+  private final SendEmailService emailService = mock(SendEmailService.class);
+
+  @Before
+  public void setUp() {
+    msg = new MimeMessage(Session.getDefaultInstance(new Properties(), null));
+    when(emailService.createMessage()).thenReturn(msg);
+  }
+
+  private ReportingEmailUtils createEmailUtil() {
+    ReportingEmailUtils emailUtils = new ReportingEmailUtils();
+    emailUtils.sender = "test-project.appspotmail.com";
+    emailUtils.recipient = "email@example.com";
+    emailUtils.emailService = emailService;
+    return emailUtils;
+  }
+
+  @Test
+  public void testSuccess_sendsEmail() throws Exception {
+    ReportingEmailUtils emailUtils = createEmailUtil();
+    emailUtils.emailResults("Subject", "Body");
+
+    assertThat(msg.getFrom()).hasLength(1);
+    assertThat(msg.getFrom()[0])
+      .isEqualTo(new InternetAddress("test-project.appspotmail.com"));
+    assertThat(msg.getRecipients(RecipientType.TO)).hasLength(1);
+    assertThat(msg.getRecipients(RecipientType.TO)[0])
+        .isEqualTo(new InternetAddress("email@example.com"));
+    assertThat(msg.getSubject()).isEqualTo("Subject");
+    assertThat(msg.getContentType()).isEqualTo("text/plain");
+    assertThat(msg.getContent().toString()).isEqualTo("Body");
+  }
+}

javatests/google/registry/reporting/ReportingUtilsTest.java

@@ -1,40 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.reporting;
-
-import static com.google.common.truth.Truth.assertThat;
-
-import google.registry.reporting.IcannReportingModule.ReportType;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Unit tests for {@link google.registry.reporting.ReportingUtils}. */
-@RunWith(JUnit4.class)
-public class ReportingUtilsTest {
-  @Test
-  public void testCreateFilename_success() {
-    assertThat(ReportingUtils.createFilename("test", "2017-06", ReportType.ACTIVITY))
-        .isEqualTo("test-activity-201706.csv");
-    assertThat(ReportingUtils.createFilename("foo", "2017-06", ReportType.TRANSACTIONS))
-        .isEqualTo("foo-transactions-201706.csv");
-  }
-
-  @Test
-  public void testCreateBucketName_success() {
-    assertThat(ReportingUtils.createReportingBucketName("gs://domain-registry-basin", "my/subdir"))
-        .isEqualTo("gs://domain-registry-basin/my/subdir");
-  }
-}