Add reporting retry, emailing and better logging

This change:
- Adds retries to the staging action
- Emails domain-registry-eng@ upon completion of either action
- Simplifies logging to be more useful

TODO: fix up Module @Inject naming conventions and yearMonth injection

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=173294822

java/google/registry/bigquery/BigqueryJobFailureException.java

@@ -54,7 +54,7 @@ public final class BigqueryJobFailureException extends RuntimeException {
   @Nullable
   private final GoogleJsonError jsonError;
 
-  private BigqueryJobFailureException(
+  public BigqueryJobFailureException(
       String message,
       @Nullable Throwable cause,
       @Nullable JobStatus jobStatus,

java/google/registry/config/RegistryConfig.java

@@ -292,7 +292,7 @@ public final class RegistryConfig {
     @Provides
     @Config("dnsDefaultATtl")
     public static Duration provideDnsDefaultATtl() {
-      return Duration.standardSeconds(180);
+      return Duration.standardMinutes(3);
     }
 
     /**
@@ -303,7 +303,7 @@ public final class RegistryConfig {
     @Provides
     @Config("dnsDefaultNsTtl")
     public static Duration provideDnsDefaultNsTtl() {
-      return Duration.standardSeconds(180);
+      return Duration.standardMinutes(3);
     }
 
     /**
@@ -314,7 +314,7 @@ public final class RegistryConfig {
     @Provides
     @Config("dnsDefaultDsTtl")
     public static Duration provideDnsDefaultDsTtl() {
-      return Duration.standardSeconds(180);
+      return Duration.standardMinutes(3);
     }
 
     /**
@@ -497,6 +497,30 @@ public final class RegistryConfig {
       return config.icannReporting.icannActivityReportingUploadUrl;
     }
 
+    /**
+     * Returns the email address from which we send ICANN reporting email summaries from.
+     *
+     * @see google.registry.reporting.ReportingEmailUtils
+     */
+    @Provides
+    @Config("icannReportingSenderEmailAddress")
+    public static String provideIcannReportingEmailSenderAddress(
+        @Config("projectId") String projectId, RegistryConfigSettings config) {
+      return String.format(
+          "%s@%s", projectId, config.icannReporting.icannReportingEmailSenderDomain);
+    }
+
+    /**
+     * Returns the email address from which we send ICANN reporting email summaries to.
+     *
+     * @see google.registry.reporting.ReportingEmailUtils
+     */
+    @Provides
+    @Config("icannReportingRecipientEmailAddress")
+    public static String provideIcannReportingEmailRecipientAddress(RegistryConfigSettings config) {
+      return config.icannReporting.icannReportingEmailRecipient;
+    }
+
     /**
      * Returns the Google Cloud Storage bucket for staging escrow deposits pending upload.
      *
@@ -552,7 +576,7 @@ public final class RegistryConfig {
     @Provides
     @Config("rdeReportLockTimeout")
     public static Duration provideRdeReportLockTimeout() {
-      return Duration.standardSeconds(60);
+      return Duration.standardMinutes(1);
     }
 
     /**

java/google/registry/config/RegistryConfigSettings.java

@@ -115,6 +115,8 @@ public class RegistryConfigSettings {
   public static class IcannReporting {
     public String icannTransactionsReportingUploadUrl;
     public String icannActivityReportingUploadUrl;
+    public String icannReportingEmailSenderDomain;
+    public String icannReportingEmailRecipient;
   }
 
   /** Configuration for Registry Data Escrow (RDE). */

java/google/registry/config/files/default-config.yaml

@@ -160,6 +160,12 @@ icannReporting:
   # URL we PUT monthly ICANN activity reports to.
   icannActivityReportingUploadUrl: https://ry-api.icann.org/report/registry-functions-activity
 
+  # Domain for the email address we send reporting pipeline summary emails from.
+  icannReportingEmailSenderDomain: appspotmail.com
+
+  # Address we send reporting pipeline summary emails to.
+  icannReportingEmailRecipient: email@example.com
+
 rde:
   # URL prefix of ICANN's server to upload RDE reports to. Nomulus adds /TLD/ID
   # to the end of this to construct the full URL.

java/google/registry/reporting/BUILD

@@ -21,6 +21,7 @@ java_library(
         "//java/google/registry/xml",
         "@com_google_api_client",
         "@com_google_apis_google_api_services_bigquery",
+        "@com_google_appengine_api_1_0_sdk",
         "@com_google_appengine_tools_appengine_gcs_client",
         "@com_google_code_findbugs_jsr305",
         "@com_google_dagger",

java/google/registry/reporting/IcannHttpReporter.java

@@ -31,7 +31,6 @@ import com.google.common.io.ByteStreams;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.reporting.IcannReportingModule.ReportType;
-import google.registry.request.HttpException.InternalServerErrorException;
 import google.registry.util.FormattingLogger;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
@@ -66,8 +65,8 @@ public class IcannHttpReporter {
   @Inject @Config("icannActivityReportingUploadUrl") String icannActivityUrl;
   @Inject IcannHttpReporter() {}
 
-  /** Uploads {@code reportBytes} to ICANN. */
-  public void send(byte[] reportBytes, String reportFilename) throws XmlException, IOException {
+  /** Uploads {@code reportBytes} to ICANN, returning whether or not it succeeded. */
+  public boolean send(byte[] reportBytes, String reportFilename) throws XmlException, IOException {
     validateReportFilename(reportFilename);
     GenericUrl uploadUrl = new GenericUrl(makeUrl(reportFilename));
     HttpRequest request =
@@ -85,6 +84,7 @@ public class IcannHttpReporter {
     logger.infofmt(
         "Sending report to %s with content length %s",
         uploadUrl.toString(), request.getContent().getLength());
+    boolean success = true;
     try {
       response = request.execute();
       byte[] content;
@@ -93,25 +93,28 @@ public class IcannHttpReporter {
       } finally {
         response.getContent().close();
       }
-      logger.infofmt("Received response code %s", response.getStatusCode());
-      logger.infofmt("Response content: %s", new String(content, UTF_8));
+      logger.infofmt(
+          "Received response code %s with content %s",
+          response.getStatusCode(), new String(content, UTF_8));
       XjcIirdeaResult result = parseResult(content);
       if (result.getCode().getValue() != 1000) {
+        success = false;
         logger.warningfmt(
             "PUT rejected, status code %s:\n%s\n%s",
             result.getCode(),
             result.getMsg(),
             result.getDescription());
-        throw new InternalServerErrorException(result.getMsg());
       }
     } finally {
       if (response != null) {
         response.disconnect();
       } else {
+        success = false;
         logger.warningfmt(
             "Received null response from ICANN server at %s", uploadUrl.toString());
       }
     }
+    return success;
   }
 
   private XjcIirdeaResult parseResult(byte[] content) throws XmlException, IOException {

java/google/registry/reporting/IcannReportingModule.java

@@ -28,6 +28,7 @@ import google.registry.bigquery.BigqueryConnection;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.Parameter;
 import google.registry.util.Clock;
+import google.registry.util.SendEmailService;
 import java.util.Optional;
 import javax.servlet.http.HttpServletRequest;
 import org.joda.time.Duration;
@@ -138,5 +139,10 @@ public final class IcannReportingModule {
       throw new RuntimeException("Could not initialize BigqueryConnection!", e);
     }
   }
+
+  @Provides
+  static SendEmailService provideSendEmailService() {
+    return new SendEmailService();
+  }
 }
 

java/google/registry/reporting/IcannReportingStager.java

@@ -21,6 +21,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.services.bigquery.model.TableFieldSchema;
 import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.base.Ascii;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
@@ -93,7 +94,6 @@ public class IcannReportingStager {
 
     // Get report headers from the table schema and convert into CSV format
     String headerRow = constructRow(getHeaders(reportTable.columnKeySet()));
-    logger.infofmt("Headers: %s", headerRow);
 
     return (reportType == ReportType.ACTIVITY)
         ? stageActivityReports(headerRow, reportTable.rowMap().values())
@@ -231,7 +231,6 @@ public class IcannReportingStager {
       reportCsv.append("\r\n");
       reportCsv.append(row);
     }
-    logger.infofmt("Created report:\n%s", reportCsv.toString());
     return reportCsv.toString();
   }
 
@@ -240,8 +239,11 @@ public class IcannReportingStager {
       throws IOException {
     // Upload resulting CSV file to GCS
     byte[] reportBytes = reportCsv.getBytes(UTF_8);
-    String reportFilename = ReportingUtils.createFilename(tld, yearMonth, reportType);
-    String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
+    String reportFilename =
+        String.format(
+            "%s-%s-%s.csv",
+            tld, Ascii.toLowerCase(reportType.toString()), yearMonth.replace("-", ""));
+    String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
     final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
     gcsUtils.createFromBytes(gcsFilename, reportBytes);
     logger.infofmt(
@@ -253,7 +255,7 @@ public class IcannReportingStager {
 
   /** Creates and stores a manifest file on GCS, indicating which reports were generated. */
   void createAndUploadManifest(ImmutableList<String> filenames) throws IOException {
-    String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
+    String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
     final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
     StringBuilder manifestString = new StringBuilder();
     filenames.forEach((filename) -> manifestString.append(filename).append("\n"));

java/google/registry/reporting/IcannReportingStagingAction.java

@@ -18,16 +18,17 @@ import static google.registry.request.Action.Method.POST;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.common.base.Throwables;
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.net.MediaType;
+import google.registry.bigquery.BigqueryJobFailureException;
 import google.registry.reporting.IcannReportingModule.ReportType;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.FormattingLogger;
-import java.util.Arrays;
+import google.registry.util.Retrier;
 import javax.inject.Inject;
 
 /**
@@ -60,31 +61,50 @@ public final class IcannReportingStagingAction implements Runnable {
   ImmutableList<ReportType> reportTypes;
 
   @Inject IcannReportingStager stager;
+  @Inject Retrier retrier;
   @Inject Response response;
+  @Inject ReportingEmailUtils emailUtils;
   @Inject IcannReportingStagingAction() {}
 
   @Override
   public void run() {
-    try {
-      ImmutableList.Builder<String> manifestedFilesBuilder = new ImmutableList.Builder<>();
-      for (ReportType reportType : reportTypes) {
-        manifestedFilesBuilder.addAll(stager.stageReports(reportType));
-      }
-      ImmutableList<String> manifestedFiles = manifestedFilesBuilder.build();
-      stager.createAndUploadManifest(manifestedFiles);
+    retrier.callWithRetry(
+        () -> {
+          ImmutableList.Builder<String> manifestedFilesBuilder = new ImmutableList.Builder<>();
+          for (ReportType reportType : reportTypes) {
+            manifestedFilesBuilder.addAll(stager.stageReports(reportType));
+          }
+          ImmutableList<String> manifestedFiles = manifestedFilesBuilder.build();
+          stager.createAndUploadManifest(manifestedFiles);
 
-      logger.infofmt("Completed staging %d report files.", manifestedFiles.size());
-      response.setStatus(SC_OK);
-      response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
-      response.setPayload("Completed staging action.");
-    } catch (Exception e) {
-      logger.severe("Reporting staging action failed!");
-      logger.severe(Throwables.getStackTraceAsString(e));
-      response.setStatus(SC_INTERNAL_SERVER_ERROR);
-      response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
-      response.setPayload(
-          String.format("Caught exception:\n%s\n%s", e.getMessage(),
-              Arrays.toString(e.getStackTrace())));
-    }
+          logger.infofmt("Completed staging %d report files.", manifestedFiles.size());
+          emailUtils.emailResults(
+              "ICANN Monthly report staging summary [SUCCESS]",
+              String.format(
+                  "Completed staging the following %d ICANN reports:\n%s",
+                  manifestedFiles.size(), Joiner.on('\n').join(manifestedFiles)));
+
+          response.setStatus(SC_OK);
+          response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+          response.setPayload("Completed staging action.");
+          return null;
+        },
+        new Retrier.FailureReporter() {
+          @Override
+          public void beforeRetry(Throwable thrown, int failures, int maxAttempts) {}
+
+          @Override
+          public void afterFinalFailure(Throwable thrown, int failures) {
+            emailUtils.emailResults(
+                "ICANN Monthly report staging summary [FAILURE]",
+                String.format(
+                    "Staging failed due to %s, check logs for more details.", thrown.toString()));
+            logger.severefmt("Staging action failed due to %s", thrown.toString());
+            response.setStatus(SC_INTERNAL_SERVER_ERROR);
+            response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+            response.setPayload(String.format("Staging failed due to %s", thrown.toString()));
+          }
+        },
+        BigqueryJobFailureException.class);
   }
 }

java/google/registry/reporting/IcannReportingUploadAction.java

@@ -23,6 +23,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 import com.google.appengine.tools.cloudstorage.GcsFilename;
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteStreams;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
@@ -34,6 +35,7 @@ import google.registry.util.FormattingLogger;
 import google.registry.util.Retrier;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.stream.Collectors;
 import javax.inject.Inject;
 
 /**
@@ -68,29 +70,55 @@ public final class IcannReportingUploadAction implements Runnable {
   @Inject IcannHttpReporter icannReporter;
   @Inject Retrier retrier;
   @Inject Response response;
+  @Inject ReportingEmailUtils emailUtils;
 
   @Inject
   IcannReportingUploadAction() {}
 
   @Override
   public void run() {
-    String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
+    String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
     ImmutableList<String> manifestedFiles = getManifestedFiles(reportBucketname);
+    ImmutableMap.Builder<String, Boolean> reportSummaryBuilder = new ImmutableMap.Builder<>();
     // Report on all manifested files
     for (String reportFilename : manifestedFiles) {
       logger.infofmt("Reading ICANN report %s from bucket %s", reportFilename, reportBucketname);
       final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
       verifyFileExists(gcsFilename);
-      retrier.callWithRetry(
-          () -> {
-            final byte[] payload = readBytesFromGcs(gcsFilename);
-            icannReporter.send(payload, reportFilename);
-            response.setContentType(PLAIN_TEXT_UTF_8);
-            response.setPayload(String.format("OK, sending: %s", new String(payload, UTF_8)));
-            return null;
-          },
-          IOException.class);
+      boolean success = false;
+      try {
+        success =
+            retrier.callWithRetry(
+                () -> {
+                  final byte[] payload = readBytesFromGcs(gcsFilename);
+                  return icannReporter.send(payload, reportFilename);
+                },
+                IOException.class);
+      } catch (RuntimeException e) {
+        logger.warningfmt("Upload to %s failed due to %s", gcsFilename.toString(), e.toString());
+      }
+      reportSummaryBuilder.put(reportFilename, success);
     }
+    emailUploadResults(reportSummaryBuilder.build());
+    response.setContentType(PLAIN_TEXT_UTF_8);
+    response.setPayload(
+        String.format("OK, attempted uploading %d reports", manifestedFiles.size()));
+  }
+
+  private void emailUploadResults(ImmutableMap<String, Boolean> reportSummary) {
+    emailUtils.emailResults(
+        String.format(
+            "ICANN Monthly report upload summary: %d/%d succeeded",
+            reportSummary.values().stream().filter((b) -> b).count(), reportSummary.size()),
+        String.format(
+            "Report Filename - Upload status:\n%s",
+            reportSummary
+                .entrySet()
+                .stream()
+                .map(
+                    (e) ->
+                        String.format("%s - %s", e.getKey(), e.getValue() ? "SUCCESS" : "FAILURE"))
+                .collect(Collectors.joining("\n"))));
   }
 
   private ImmutableList<String> getManifestedFiles(String reportBucketname) {

java/google/registry/reporting/ReportingEmailUtils.java

@@ -0,0 +1,48 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.FormattingLogger;
+import google.registry.util.SendEmailService;
+import javax.inject.Inject;
+import javax.mail.Message;
+import javax.mail.Message.RecipientType;
+import javax.mail.internet.InternetAddress;
+
+/** Static utils for emailing reporting results. */
+public class ReportingEmailUtils {
+
+  @Inject @Config("icannReportingSenderEmailAddress") String sender;
+  @Inject @Config("icannReportingRecipientEmailAddress") String recipient;
+  @Inject SendEmailService emailService;
+  @Inject ReportingEmailUtils() {}
+
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+
+  void emailResults(String subject, String body) {
+    try {
+      Message msg = emailService.createMessage();
+      logger.infofmt("Emailing %s", recipient);
+      msg.setFrom(new InternetAddress(sender));
+      msg.addRecipient(RecipientType.TO, new InternetAddress(recipient));
+      msg.setSubject(subject);
+      msg.setText(body);
+      emailService.sendMessage(msg);
+    } catch (Exception e) {
+      logger.warningfmt("E-mail service failed due to %s", e.toString());
+    }
+  }
+}

java/google/registry/reporting/ReportingUtils.java

@@ -1,34 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.reporting;
-
-import com.google.common.base.Ascii;
-import google.registry.reporting.IcannReportingModule.ReportType;
-
-/** Static utils for reporting. */
-public final class ReportingUtils {
-
-  /** Generates a report filename in accord with ICANN's specifications. */
-  static String createFilename(String tld, String yearMonth, ReportType reportType) {
-    // Report files use YYYYMM naming instead of standard YYYY-MM, per ICANN requirements.
-    return String.format(
-        "%s-%s-%s.csv", tld, Ascii.toLowerCase(reportType.toString()), yearMonth.replace("-", ""));
-  }
-
-  /** Constructs the bucket name to store/upload reports to. */
-  static String createReportingBucketName(String reportingBucket, String subdir) {
-    return String.format("%s/%s", reportingBucket, subdir);
-  }
-}