Add diff logic and send daily Spec11 emails with new threats

For each registrar, the daily email will only include threats that did not appear
in the prior run's email.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=228889972

gradle/core/build.gradle

@@ -354,6 +354,12 @@ task soyToJava {
         }.filter {
           it.name.endsWith(".soy")
         })
+
+    soyToJava('google.registry.reporting.spec11.soy',
+        "${generatedDir}/google/registry/reporting/spec11/soy",
+        fileTree(
+            dir: "${javaDir}/google/registry/reporting/spec11/soy",
+            include: ['**/*.soy']))
   }
 }
 

java/google/registry/config/RegistryConfig.java

@@ -900,14 +900,25 @@ public final class RegistryConfig {
     }
 
     /**
-     * Returns the template for the body of the spec 11 email to the registrars.
+     * Returns the name of the registry, for use in spec 11 emails.
      *
      * @see google.registry.reporting.spec11.Spec11EmailUtils
      */
     @Provides
-    @Config("spec11EmailBodyTemplate")
+    @Config("registryName")
-    public static String provideSpec11EmailBodyTemplate(RegistryConfigSettings config) {
+    public static String provideRegistryName(RegistryConfigSettings config) {
-      return config.registryPolicy.spec11EmailBodyTemplate;
+      return config.registryPolicy.registryName;
+    }
+
+    /**
+     * Returns a list of resources we send to registrars when informing them of spec 11 threats.
+     *
+     * @see google.registry.reporting.spec11.Spec11EmailUtils
+     */
+    @Provides
+    @Config("spec11WebResources")
+    public static ImmutableList<String> provideSpec11WebResources(RegistryConfigSettings config) {
+      return ImmutableList.copyOf(config.registryPolicy.spec11WebResources);
     }
 
     /**

java/google/registry/config/RegistryConfigSettings.java

@@ -92,7 +92,8 @@ public class RegistryConfigSettings {
     public String whoisDisclaimer;
     public String rdapTos;
     public String rdapTosStaticUrl;
-    public String spec11EmailBodyTemplate;
+    public String registryName;
+    public List<String> spec11WebResources;
     public boolean requireSslCertificates;
   }
 

java/google/registry/config/files/default-config.yaml

@@ -160,32 +160,12 @@ registryPolicy:
   # responses. If null, no static Web page link is generated.
   rdapTosStaticUrl: null
 
-  # Body of the spec 11 email sent to registrars.
+  # Name of the registry for use in spec 11 emails
-  # Items in braces are to be replaced.
+  registryName: Example Registry
-  spec11EmailBodyTemplate: |
-    Dear registrar partner,
 
-    The registry conducts periodic technical analyses of all domains registered
+  # A list of resources we send to registrars when informing them of
-    in its TLDs. As part of this analysis, the following domains that you
+  # spec 11 threats
-    manage were flagged for potential security concerns:
+  spec11WebResources: []
-
-    {LIST_OF_THREATS}
-
-    Please communicate these findings to the registrant and work with the
-    registrant to mitigate any security issues and have the domains delisted.
-
-    Some helpful sites for getting off a blocked list include:
-
-      - Google Search Console (https://search.google.com/search-console/about)
-        -- includes information and tools for webmasters to learn about and
-        mitigate security threats and have their websites delisted
-      - first.org -- a registry of Computer Emergency Response Teams (CERTs)
-        that may be able to assist in mitigation
-      - stopbadware.org -- a non-profit anti-malware organization that provides
-        support and information for webmasters dealing with security threats
-
-    If you have any questions regarding this notice, please contact
-    {REPLY_TO_EMAIL}.
 
   # Whether to require an SSL certificate hash in order to be able to log in
   # via EPP and run commands. This can be false for testing environments but

java/google/registry/reporting/spec11/BUILD

@@ -13,6 +13,7 @@ java_library(
         "//java/google/registry/gcs",
         "//java/google/registry/keyring/api",
         "//java/google/registry/reporting",
+        "//java/google/registry/reporting/spec11/soy:soy_java_wrappers",
         "//java/google/registry/request",
         "//java/google/registry/request/auth",
         "//java/google/registry/util",
@@ -26,6 +27,7 @@ java_library(
         "@com_google_flogger_system_backend",
         "@com_google_guava",
         "@com_google_http_client",
+        "@io_bazel_rules_closure//closure/templates",
         "@javax_inject",
         "@javax_servlet_api",
         "@joda_time",

java/google/registry/reporting/spec11/PublishSpec11ReportAction.java

@@ -14,6 +14,7 @@
 
 package google.registry.reporting.spec11;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.request.Action.Method.POST;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED;
@@ -22,17 +23,24 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 
 import com.google.api.services.dataflow.Dataflow;
 import com.google.api.services.dataflow.model.Job;
-import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
+import com.google.template.soy.parseinfo.SoyTemplateInfo;
+import google.registry.beam.spec11.ThreatMatch;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.reporting.ReportingModule;
+import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 import javax.inject.Inject;
 import org.joda.time.LocalDate;
 import org.json.JSONException;
@@ -41,8 +49,8 @@ import org.json.JSONException;
  * Retries until a {@code Dataflow} job with a given {@code jobId} completes, continuing the Spec11
  * pipeline accordingly.
  *
- * <p>This calls {@link Spec11EmailUtils#emailSpec11Reports(String, String, List)} ()} on success or
+ * <p>This calls {@link Spec11EmailUtils#emailSpec11Reports(SoyTemplateInfo, String, Set)} on
- * {@link Spec11EmailUtils#sendAlertEmail(String, String)} on failure.
+ * success or {@link Spec11EmailUtils#sendAlertEmail(String, String)} on failure.
  */
 @Action(path = PublishSpec11ReportAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 public class PublishSpec11ReportAction implements Runnable {
@@ -54,7 +62,7 @@ public class PublishSpec11ReportAction implements Runnable {
   private static final String JOB_FAILED = "JOB_STATE_FAILED";
 
   private final String projectId;
-  private final String spec11EmailBodyTemplate;
+  private final String registryName;
   private final String jobId;
   private final Spec11EmailUtils emailUtils;
   private final Spec11RegistrarThreatMatchesParser spec11RegistrarThreatMatchesParser;
@@ -65,7 +73,7 @@ public class PublishSpec11ReportAction implements Runnable {
   @Inject
   PublishSpec11ReportAction(
       @Config("projectId") String projectId,
-      @Config("spec11EmailBodyTemplate") String spec11EmailBodyTemplate,
+      @Config("registryName") String registryName,
       @Parameter(ReportingModule.PARAM_JOB_ID) String jobId,
       Spec11EmailUtils emailUtils,
       Spec11RegistrarThreatMatchesParser spec11RegistrarThreatMatchesParser,
@@ -73,7 +81,7 @@ public class PublishSpec11ReportAction implements Runnable {
       Response response,
       LocalDate date) {
     this.projectId = projectId;
-    this.spec11EmailBodyTemplate = spec11EmailBodyTemplate;
+    this.registryName = registryName;
     this.jobId = jobId;
     this.emailUtils = emailUtils;
     this.spec11RegistrarThreatMatchesParser = spec11RegistrarThreatMatchesParser;
@@ -90,14 +98,22 @@ public class PublishSpec11ReportAction implements Runnable {
       String state = job.getCurrentState();
       switch (state) {
         case JOB_DONE:
-          logger.atInfo().log(
+          logger.atInfo().log("Dataflow job %s finished successfully, publishing results.", jobId);
-              "Dataflow job %s finished successfully, publishing results if appropriate.", jobId);
           response.setStatus(SC_OK);
-          if (shouldSendSpec11Email()) {
+          if (shouldSendMonthlySpec11Email()) {
-            ImmutableList<RegistrarThreatMatches> matchesList =
+            sendMonthlyEmail();
-                spec11RegistrarThreatMatchesParser.getRegistrarThreatMatches();
+          } else {
-            String subject = String.format("Google Registry Monthly Threat Detector [%s]", date);
+            Optional<LocalDate> previousDate =
-            emailUtils.emailSpec11Reports(spec11EmailBodyTemplate, subject, matchesList);
+                spec11RegistrarThreatMatchesParser.getPreviousDateWithMatches(date);
+            if (previousDate.isPresent()) {
+              processDailyDiff(previousDate.get());
+            } else {
+              emailUtils.sendAlertEmail(
+                  String.format("Spec11 Diff Error %s", date),
+                  String.format(
+                      "Could not find a previous file within the past month of %s", date));
+              response.setStatus(SC_NO_CONTENT);
+            }
           }
           break;
         case JOB_FAILED:
@@ -123,8 +139,51 @@ public class PublishSpec11ReportAction implements Runnable {
     }
   }
 
-  private boolean shouldSendSpec11Email() {
+  private void sendMonthlyEmail() throws IOException, JSONException {
-    // TODO(b/120496893): send emails every day with the diff content
+    ImmutableSet<RegistrarThreatMatches> monthlyMatchesSet =
+        spec11RegistrarThreatMatchesParser.getRegistrarThreatMatches(date);
+    String subject = String.format("%s Monthly Threat Detector [%s]", registryName, date);
+    emailUtils.emailSpec11Reports(
+        Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL, subject, monthlyMatchesSet);
+  }
+
+  private void processDailyDiff(LocalDate previousDate) throws IOException, JSONException {
+    ImmutableSet<RegistrarThreatMatches> previousMatches =
+        spec11RegistrarThreatMatchesParser.getRegistrarThreatMatches(previousDate);
+    ImmutableSet<RegistrarThreatMatches> currentMatches =
+        spec11RegistrarThreatMatchesParser.getRegistrarThreatMatches(date);
+    String dailySubject = String.format("%s Daily Threat Detector [%s]", registryName, date);
+    emailUtils.emailSpec11Reports(
+        Spec11EmailSoyInfo.DAILY_SPEC_11_EMAIL,
+        dailySubject,
+        getNewMatches(previousMatches, currentMatches));
+  }
+
+  private ImmutableSet<RegistrarThreatMatches> getNewMatches(
+      Set<RegistrarThreatMatches> previousMatchesSet,
+      Set<RegistrarThreatMatches> currentMatchesSet) {
+    Map<String, List<ThreatMatch>> currentMatchMap =
+        currentMatchesSet.stream()
+            .collect(
+                Collectors.toMap(
+                    RegistrarThreatMatches::registrarEmailAddress,
+                    RegistrarThreatMatches::threatMatches));
+    previousMatchesSet.forEach(
+        previousMatches ->
+            currentMatchMap.computeIfPresent(
+                previousMatches.registrarEmailAddress(),
+                (email, currentMatches) ->
+                    currentMatches.stream()
+                        .filter(
+                            currentMatch -> !previousMatches.threatMatches().contains(currentMatch))
+                        .collect(Collectors.toList())));
+    return currentMatchMap.entrySet().stream()
+        .filter(entry -> !entry.getValue().isEmpty())
+        .map(entry -> RegistrarThreatMatches.create(entry.getKey(), entry.getValue()))
+        .collect(toImmutableSet());
+  }
+
+  private boolean shouldSendMonthlySpec11Email() {
     return date.getDayOfMonth() == 2;
   }
 }

java/google/registry/reporting/spec11/Spec11EmailUtils.java

@@ -15,13 +15,23 @@
 package google.registry.reporting.spec11;
 
 import static com.google.common.base.Throwables.getRootCause;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.io.Resources.getResource;
 
-import google.registry.beam.spec11.ThreatMatch;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.template.soy.SoyFileSet;
+import com.google.template.soy.parseinfo.SoyTemplateInfo;
+import com.google.template.soy.tofu.SoyTofu;
+import com.google.template.soy.tofu.SoyTofu.Renderer;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo;
 import google.registry.util.Retrier;
 import google.registry.util.SendEmailService;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import javax.inject.Inject;
 import javax.mail.Message;
 import javax.mail.Message.RecipientType;
@@ -32,11 +42,22 @@ import org.joda.time.LocalDate;
 /** Provides e-mail functionality for Spec11 tasks, such as sending Spec11 reports to registrars. */
 public class Spec11EmailUtils {
 
+  private static final SoyTofu SOY_SAUCE =
+      SoyFileSet.builder()
+          .add(
+              getResource(
+                  Spec11EmailSoyInfo.getInstance().getClass(),
+                  Spec11EmailSoyInfo.getInstance().getFileName()))
+          .build()
+          .compileToTofu();
+
   private final SendEmailService emailService;
   private final LocalDate date;
   private final String outgoingEmailAddress;
   private final String alertRecipientAddress;
   private final String spec11ReplyToAddress;
+  private final ImmutableList<String> spec11WebResources;
+  private final String registryName;
   private final Retrier retrier;
 
   @Inject
@@ -46,12 +67,16 @@ public class Spec11EmailUtils {
       @Config("gSuiteOutgoingEmailAddress") String outgoingEmailAddress,
       @Config("alertRecipientEmailAddress") String alertRecipientAddress,
       @Config("spec11ReplyToEmailAddress") String spec11ReplyToAddress,
+      @Config("spec11WebResources") ImmutableList<String> spec11WebResources,
+      @Config("registryName") String registryName,
       Retrier retrier) {
     this.emailService = emailService;
     this.date = date;
     this.outgoingEmailAddress = outgoingEmailAddress;
     this.alertRecipientAddress = alertRecipientAddress;
     this.spec11ReplyToAddress = spec11ReplyToAddress;
+    this.spec11WebResources = spec11WebResources;
+    this.registryName = registryName;
     this.retrier = retrier;
   }
 
@@ -60,14 +85,14 @@ public class Spec11EmailUtils {
    * appropriate address.
    */
   void emailSpec11Reports(
-      String spec11EmailBodyTemplate,
+      SoyTemplateInfo soyTemplateInfo,
       String subject,
-      List<RegistrarThreatMatches> registrarThreatMatchesList) {
+      Set<RegistrarThreatMatches> registrarThreatMatchesSet) {
     try {
       retrier.callWithRetry(
           () -> {
-            for (RegistrarThreatMatches registrarThreatMatches : registrarThreatMatchesList) {
+            for (RegistrarThreatMatches registrarThreatMatches : registrarThreatMatchesSet) {
-              emailRegistrar(spec11EmailBodyTemplate, subject, registrarThreatMatches);
+              emailRegistrar(soyTemplateInfo, subject, registrarThreatMatches);
             }
           },
           IOException.class,
@@ -85,28 +110,45 @@ public class Spec11EmailUtils {
   }
 
   private void emailRegistrar(
-      String spec11EmailBodyTemplate, String subject, RegistrarThreatMatches registrarThreatMatches)
+      SoyTemplateInfo soyTemplateInfo,
+      String subject,
+      RegistrarThreatMatches registrarThreatMatches)
       throws MessagingException {
-    String registrarEmail = registrarThreatMatches.registrarEmailAddress();
-    StringBuilder threatList = new StringBuilder();
-    for (ThreatMatch threatMatch : registrarThreatMatches.threatMatches()) {
-      threatList.append(
-          String.format(
-              "%s - %s\n", threatMatch.fullyQualifiedDomainName(), threatMatch.threatType()));
-    }
-    String body =
-        spec11EmailBodyTemplate
-            .replace("{REPLY_TO_EMAIL}", spec11ReplyToAddress)
-            .replace("{LIST_OF_THREATS}", threatList.toString());
     Message msg = emailService.createMessage();
     msg.setSubject(subject);
-    msg.setText(body);
+    String content = getContent(soyTemplateInfo, registrarThreatMatches);
+    msg.setContent(content, "text/html");
+    msg.setHeader("Content-Type", "text/html");
     msg.setFrom(new InternetAddress(outgoingEmailAddress));
-    msg.addRecipient(RecipientType.TO, new InternetAddress(registrarEmail));
+    msg.addRecipient(
+        RecipientType.TO, new InternetAddress(registrarThreatMatches.registrarEmailAddress()));
     msg.addRecipient(RecipientType.BCC, new InternetAddress(spec11ReplyToAddress));
     emailService.sendMessage(msg);
   }
 
+  private String getContent(
+      SoyTemplateInfo soyTemplateInfo, RegistrarThreatMatches registrarThreatMatches) {
+    Renderer renderer = SOY_SAUCE.newRenderer(soyTemplateInfo);
+    // Soy templates require that data be in raw map/list form.
+    List<Map<String, String>> threatMatchMap =
+        registrarThreatMatches.threatMatches().stream()
+            .map(
+                threatMatch ->
+                    ImmutableMap.of(
+                        "fullyQualifiedDomainName", threatMatch.fullyQualifiedDomainName(),
+                        "threatType", threatMatch.threatType()))
+            .collect(toImmutableList());
+
+    Map<String, Object> data =
+        ImmutableMap.of(
+            "registry", registryName,
+            "replyToEmail", spec11ReplyToAddress,
+            "threats", threatMatchMap,
+            "resources", spec11WebResources);
+    renderer.setData(data);
+    return renderer.render();
+  }
+
   /** Sends an e-mail indicating the state of the spec11 pipeline, with a given subject and body. */
   void sendAlertEmail(String subject, String body) {
     try {

java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java

@@ -18,6 +18,8 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
 import com.google.common.io.CharStreams;
 import google.registry.beam.spec11.Spec11Pipeline;
 import google.registry.beam.spec11.ThreatMatch;
@@ -26,6 +28,7 @@ import google.registry.gcs.GcsUtils;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.LocalDate;
 import org.json.JSONArray;
@@ -35,25 +38,53 @@ import org.json.JSONObject;
 /** Parser to retrieve which registrar-threat matches we should notify via email */
 public class Spec11RegistrarThreatMatchesParser {
 
-  private final LocalDate date;
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   private final GcsUtils gcsUtils;
   private final String reportingBucket;
 
   @Inject
   public Spec11RegistrarThreatMatchesParser(
-      LocalDate date, GcsUtils gcsUtils, @Config("reportingBucket") String reportingBucket) {
+      GcsUtils gcsUtils, @Config("reportingBucket") String reportingBucket) {
-    this.date = date;
     this.gcsUtils = gcsUtils;
     this.reportingBucket = reportingBucket;
   }
 
-  /** Gets the list of registrar:set-of-threat-match pairings from the file in GCS. */
+  /**
-  public ImmutableList<RegistrarThreatMatches> getRegistrarThreatMatches()
+   * Gets the entire set of registrar:set-of-threat-match pairings from the most recent report file
+   * in GCS.
+   */
+  public ImmutableSet<RegistrarThreatMatches> getRegistrarThreatMatches(LocalDate date)
       throws IOException, JSONException {
-    // TODO(b/120078223): this should only be the diff of this run and the prior run.
+    return getFromFile(getGcsFilename(date));
-    GcsFilename spec11ReportFilename =
+  }
-        new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(date));
+
-    ImmutableList.Builder<RegistrarThreatMatches> builder = ImmutableList.builder();
+  public Optional<LocalDate> getPreviousDateWithMatches(LocalDate date) {
+    LocalDate yesterday = date.minusDays(1);
+    GcsFilename gcsFilename = getGcsFilename(yesterday);
+    if (gcsUtils.existsAndNotEmpty(gcsFilename)) {
+      return Optional.of(yesterday);
+    }
+    logger.atWarning().log("Could not find previous file from date %s", yesterday);
+
+    for (LocalDate dateToCheck = yesterday.minusDays(1);
+        !dateToCheck.isBefore(date.minusMonths(1));
+        dateToCheck = dateToCheck.minusDays(1)) {
+      gcsFilename = getGcsFilename(dateToCheck);
+      if (gcsUtils.existsAndNotEmpty(gcsFilename)) {
+        return Optional.of(dateToCheck);
+      }
+    }
+    return Optional.empty();
+  }
+
+  private GcsFilename getGcsFilename(LocalDate localDate) {
+    return new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate));
+  }
+
+  private ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
+      throws IOException, JSONException {
+    ImmutableSet.Builder<RegistrarThreatMatches> builder = ImmutableSet.builder();
     try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) {
       ImmutableList<String> reportLines =
           ImmutableList.copyOf(CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n"));

java/google/registry/reporting/spec11/soy/BUILD

@@ -0,0 +1,13 @@
+package(
+    default_visibility = ["//java/google/registry:registry_project"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("@io_bazel_rules_closure//closure:defs.bzl", "closure_java_template_library")
+
+closure_java_template_library(
+    name = "soy_java_wrappers",
+    srcs = glob(["*.soy"]),
+    java_package = "google.registry.reporting.spec11.soy",
+)

java/google/registry/reporting/spec11/soy/Spec11Email.soy

@@ -0,0 +1,126 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{namespace registry.soy.reporting.spec11}
+
+/**
+ * Template for the content of the monthly spec11 email
+ */
+{template .monthlySpec11Email}
+  {@param threats: list<map<string, string>>}
+  {@param resources: list<string>}
+  {@param registry: string}
+  {@param replyToEmail: string}
+
+  Dear registrar partner,
+
+  <p>{$registry} previously notified you when the following domains managed by your
+    registrar were flagged for potential security concerns.</p>
+
+  <p>The following domains that you manage continue to be flagged by our analysis for potential
+    security concerns. This may be because the registrants have not completed the requisite steps
+    to mitigate the potential security abuse and/or have it reviewed and delisted.</p>
+
+  {call .threatMatchTable}
+    {param threats: $threats /}
+  {/call}
+
+  <p>Please work with the registrant to mitigate any security issues and have the
+    domains delisted.</p>
+
+  {call .resourceList}
+    {param resources: $resources /}
+  {/call}
+
+  <p>You will continue to receive a monthly summary of all domains managed by your registrar
+    that remain on our lists of potential security threats. You will additionally receive a daily
+    notice when any new domains that are added to these lists. Once the registrant has resolved
+    the security issues and followed the steps to have his or her domain reviewed and delisted
+    it will automatically be removed from our monthly reporting.</p>
+
+  <p>If you have any questions regarding this notice, please contact {$replyToEmail}.</p>
+{/template}
+
+/**
+ * Template for the content of the daily spec11 email
+ */
+{template .dailySpec11Email}
+  {@param threats: list<map<string, string>>}
+  {@param resources: list<string>}
+  {@param date: string}
+  {@param registry: string}
+  {@param replyToEmail: string}
+
+  Dear registrar partner,
+
+  <p>{$registry} conducts a daily analysis of all domains registered in its TLDs to
+    identify potential security concerns. On {$date}, the following domains that your
+    registrar manages were flagged for potential security concerns:</p>
+
+  {call .threatMatchTable}
+    {param threats: $threats /}
+  {/call}
+
+  <p><b>Please communicate these findings to the registrant and work with the
+    registrant to mitigate any security issues and have the domains delisted.</b></p>
+
+  {call .resourceList}
+    {param resources: $resources /}
+  {/call}
+
+  <p>You will continue to receive daily notices when new domains managed by your registrar
+    are flagged for abuse, as well as a monthly summary of all of your domains under management
+    that remain flagged for abuse. Once the registrant has resolved the security issues and
+    followed the steps to have his or her domain reviewed and delisted it will automatically
+    be removed from our reporting.</p>
+
+  <p>If you would like to change the email to which these notices are sent please update your
+    abuse contact using your registrar portal account.</p>
+
+  <p>If you have any questions regarding this notice, please contact {$replyToEmail}.</p>
+{/template}
+
+/**
+ * Template for the list of potentially-useful resources
+ */
+{template .resourceList}
+  {@param resources: list<string>}
+  {if length($resources) > 0}
+    Some helpful resources for getting off a blocked list include:
+    <ul>
+      {for $resource in $resources}
+        <li>{$resource}</li>
+      {/for}
+    </ul>
+  {/if}
+{/template}
+
+/**
+ * Template for the table containing the threats themselves
+ */
+{template .threatMatchTable}
+  {@param threats: list<map<string, string>>}
+  <table>
+    <tr>
+      <th>Domain Name</th>
+      <th>Threat Type</th>
+    </tr>
+    {for $threat in $threats}
+      <tr>
+        <td>{$threat['fullyQualifiedDomainName']}</td>
+        <td>{$threat['threatType']}</td>
+      </tr>
+    {/for}
+  </table>
+{/template}

javatests/google/registry/reporting/spec11/BUILD

@@ -15,6 +15,7 @@ java_library(
         "//java/google/registry/beam/spec11",
         "//java/google/registry/gcs",
         "//java/google/registry/reporting/spec11",
+        "//java/google/registry/reporting/spec11/soy:soy_java_wrappers",
         "//java/google/registry/util",
         "//javatests/google/registry/testing",
         "@com_google_apis_google_api_services_dataflow",

javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java

@@ -30,9 +30,12 @@ import com.google.api.services.dataflow.Dataflow.Projects;
 import com.google.api.services.dataflow.Dataflow.Projects.Jobs;
 import com.google.api.services.dataflow.Dataflow.Projects.Jobs.Get;
 import com.google.api.services.dataflow.model.Job;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.net.MediaType;
+import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo;
 import google.registry.testing.FakeResponse;
 import java.io.IOException;
+import java.util.Optional;
 import org.joda.time.LocalDate;
 import org.junit.Before;
 import org.junit.Test;
@@ -43,7 +46,6 @@ import org.junit.runners.JUnit4;
 @RunWith(JUnit4.class)
 public class PublishSpec11ReportActionTest {
 
-  private final String spec11BodyTemplate = "{LIST_OF_THREATS}\n{REPLY_TO_EMAIL}";
   private final LocalDate date = new LocalDate(2018, 6, 5);
 
   private Dataflow dataflow;
@@ -69,41 +71,44 @@ public class PublishSpec11ReportActionTest {
     expectedJob = new Job();
     when(get.execute()).thenReturn(expectedJob);
     emailUtils = mock(Spec11EmailUtils.class);
+    parser = mock(Spec11RegistrarThreatMatchesParser.class);
     response = new FakeResponse();
     parser = mock(Spec11RegistrarThreatMatchesParser.class);
-    when(parser.getRegistrarThreatMatches()).thenReturn(sampleThreatMatches());
     publishAction =
         new PublishSpec11ReportAction(
             "test-project",
-            spec11BodyTemplate,
+            "Super Cool Registry",
             "12345",
             emailUtils,
-            mock(Spec11RegistrarThreatMatchesParser.class),
+            parser,
             dataflow,
             response,
             date);
   }
 
   @Test
-  public void testJobDone_emailsResultsOnSecondOfMonth() throws Exception {
+  public void testJobDone_emailsOnlyMonthlyResultsOnSecondOfMonth() throws Exception {
+    LocalDate secondOfMonth = date.withDayOfMonth(2);
+    when(parser.getRegistrarThreatMatches(secondOfMonth)).thenReturn(sampleThreatMatches());
     expectedJob.setCurrentState("JOB_STATE_DONE");
     publishAction =
         new PublishSpec11ReportAction(
             "test-project",
-            spec11BodyTemplate,
+            "Super Cool Registry",
             "12345",
             emailUtils,
             parser,
             dataflow,
             response,
-            date.withDayOfMonth(2));
+            secondOfMonth);
     publishAction.run();
     assertThat(response.getStatus()).isEqualTo(SC_OK);
     verify(emailUtils)
         .emailSpec11Reports(
-            spec11BodyTemplate,
+            Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL,
-            "Google Registry Monthly Threat Detector [2018-06-02]",
+            "Super Cool Registry Monthly Threat Detector [2018-06-02]",
             sampleThreatMatches());
+    verifyNoMoreInteractions(emailUtils);
   }
 
   @Test
@@ -139,10 +144,31 @@ public class PublishSpec11ReportActionTest {
   }
 
   @Test
-  public void testJobDone_doesNotEmailResults() {
+  public void testJobDone_onlyDailyResults() throws Exception {
+    LocalDate yesterday = date.minusDays(1);
+    when(parser.getPreviousDateWithMatches(date)).thenReturn(Optional.of(yesterday));
+    when(parser.getRegistrarThreatMatches(date)).thenReturn(sampleThreatMatches());
+    when(parser.getRegistrarThreatMatches(yesterday)).thenReturn(ImmutableSet.of());
     expectedJob.setCurrentState("JOB_STATE_DONE");
     publishAction.run();
     assertThat(response.getStatus()).isEqualTo(SC_OK);
+    verify(emailUtils)
+        .emailSpec11Reports(
+            Spec11EmailSoyInfo.DAILY_SPEC_11_EMAIL,
+            "Super Cool Registry Daily Threat Detector [2018-06-05]",
+            sampleThreatMatches());
     verifyNoMoreInteractions(emailUtils);
   }
+
+  @Test
+  public void testJobDone_failsDueToNoPreviousResults() {
+    when(parser.getPreviousDateWithMatches(date)).thenReturn(Optional.empty());
+    expectedJob.setCurrentState("JOB_STATE_DONE");
+    publishAction.run();
+    assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
+    verify(emailUtils)
+        .sendAlertEmail(
+            String.format("Spec11 Diff Error %s", date),
+            String.format("Could not find a previous file within the past month of %s", date));
+  }
 }

javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java

@@ -25,6 +25,8 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import com.google.common.collect.ImmutableList;
+import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
 import google.registry.util.Retrier;
@@ -53,6 +55,7 @@ import org.mockito.stubbing.Answer;
 public class Spec11EmailUtilsTest {
 
   private static final int RETRY_COUNT = 2;
+  private static final ImmutableList<String> FAKE_RESOURCES = ImmutableList.of("foo");
 
   private SendEmailService emailService;
   private Spec11EmailUtils emailUtils;
@@ -66,50 +69,75 @@ public class Spec11EmailUtilsTest {
         .thenAnswer((args) -> new MimeMessage(Session.getInstance(new Properties(), null)));
 
     parser = mock(Spec11RegistrarThreatMatchesParser.class);
-    when(parser.getRegistrarThreatMatches()).thenReturn(sampleThreatMatches());
+    LocalDate date = new LocalDate(2018, 7, 15);
+    when(parser.getRegistrarThreatMatches(date)).thenReturn(sampleThreatMatches());
 
     gotMessage = ArgumentCaptor.forClass(Message.class);
 
     emailUtils =
         new Spec11EmailUtils(
             emailService,
-            new LocalDate(2018, 7, 15),
+            date,
             "my-sender@test.com",
             "my-receiver@test.com",
             "my-reply-to@test.com",
+            FAKE_RESOURCES,
+            "Super Cool Registry",
             new Retrier(new FakeSleeper(new FakeClock()), RETRY_COUNT));
   }
 
   @Test
   public void testSuccess_emailSpec11Reports() throws Exception {
     emailUtils.emailSpec11Reports(
-        "{LIST_OF_THREATS}\n{REPLY_TO_EMAIL}",
+        Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL,
-        "Google Registry Monthly Threat Detector [2018-07-15]",
+        "Super Cool Registry Monthly Threat Detector [2018-07-15]",
         sampleThreatMatches());
     // We inspect individual parameters because Message doesn't implement equals().
     verify(emailService, times(3)).sendMessage(gotMessage.capture());
     List<Message> capturedMessages = gotMessage.getAllValues();
+    String emailFormat =
+        "Dear registrar partner,<p>Super Cool Registry previously notified you when the following "
+            + "domains managed by your registrar were flagged for potential security concerns."
+            + "</p><p>The following domains that you manage continue to be flagged by our analysis "
+            + "for potential security concerns. This may be because the registrants have not "
+            + "completed the requisite steps to mitigate the potential security abuse and/or have "
+            + "it reviewed and delisted.</p><table><tr><th>Domain Name</th><th>Threat Type</th>"
+            + "</tr>%s</table><p>Please work with the registrant to mitigate any security issues "
+            + "and have the domains delisted.</p>Some helpful resources for getting off a blocked "
+            + "list include:<ul><li>foo</li></ul><p>You will continue to receive a monthly summary "
+            + "of all domains managed by your registrar that remain on our lists of potential "
+            + "security threats. You will additionally receive a daily notice when any new domains "
+            + "that are added to these lists. Once the registrant has resolved the security issues "
+            + "and followed the steps to have his or her domain reviewed and delisted it will "
+            + "automatically be removed from our monthly reporting.</p><p>If you have any q"
+            + "uestions regarding this notice, please contact my-reply-to@test.com.</p>";
+
     validateMessage(
         capturedMessages.get(0),
         "my-sender@test.com",
         "a@fake.com",
         "my-reply-to@test.com",
-        "Google Registry Monthly Threat Detector [2018-07-15]",
+        "Super Cool Registry Monthly Threat Detector [2018-07-15]",
-        "a.com - MALWARE\n\nmy-reply-to@test.com");
+        String.format(emailFormat, "<tr><td>a.com</td><td>MALWARE</td></tr>"),
+        "text/html");
     validateMessage(
         capturedMessages.get(1),
         "my-sender@test.com",
         "b@fake.com",
         "my-reply-to@test.com",
-        "Google Registry Monthly Threat Detector [2018-07-15]",
+        "Super Cool Registry Monthly Threat Detector [2018-07-15]",
-        "b.com - MALWARE\nc.com - MALWARE\n\nmy-reply-to@test.com");
+        String.format(
+            emailFormat,
+            "<tr><td>b.com</td><td>MALWARE</td></tr><tr><td>c.com</td><td>MALWARE</td></tr>"),
+        "text/html");
     validateMessage(
         capturedMessages.get(2),
         "my-sender@test.com",
         "my-receiver@test.com",
         null,
         "Spec11 Pipeline Success 2018-07-15",
-        "Spec11 reporting completed successfully.");
+        "Spec11 reporting completed successfully.",
+        "text/plain");
   }
 
   @Test
@@ -139,7 +167,9 @@ public class Spec11EmailUtilsTest {
     RuntimeException thrown =
         assertThrows(
             RuntimeException.class,
-            () -> emailUtils.emailSpec11Reports("foo", "bar", sampleThreatMatches()));
+            () ->
+                emailUtils.emailSpec11Reports(
+                    Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL, "bar", sampleThreatMatches()));
     assertThat(thrown).hasMessageThat().isEqualTo("Emailing spec11 report failed");
     assertThat(thrown)
         .hasCauseThat()
@@ -155,7 +185,8 @@ public class Spec11EmailUtilsTest {
         "my-receiver@test.com",
         null,
         "Spec11 Emailing Failure 2018-07-15",
-        "Emailing spec11 reports failed due to expected");
+        "Emailing spec11 reports failed due to expected",
+        "text/plain");
   }
 
   @Test
@@ -168,7 +199,8 @@ public class Spec11EmailUtilsTest {
         "my-receiver@test.com",
         null,
         "Spec11 Pipeline Alert: 2018-07",
-        "Alert!");
+        "Alert!",
+        "text/plain");
   }
 
   private void validateMessage(
@@ -177,7 +209,8 @@ public class Spec11EmailUtilsTest {
       String recipient,
       @Nullable String replyTo,
       String subject,
-      String body)
+      String body,
+      String contentType)
       throws MessagingException, IOException {
     assertThat(message.getFrom()).asList().containsExactly(new InternetAddress(from));
     assertThat(message.getRecipients(RecipientType.TO))
@@ -192,7 +225,7 @@ public class Spec11EmailUtilsTest {
     }
     assertThat(message.getRecipients(RecipientType.CC)).isNull();
     assertThat(message.getSubject()).isEqualTo(subject);
-    assertThat(message.getContentType()).isEqualTo("text/plain");
+    assertThat(message.getContentType()).isEqualTo(contentType);
-    assertThat(message.getContent().toString()).isEqualTo(body);
+    assertThat(message.getContent()).isEqualTo(body);
   }
 }

javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java

@@ -15,20 +15,20 @@
 package google.registry.reporting.spec11;
 
 import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth8.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 import google.registry.beam.spec11.ThreatMatch;
 import google.registry.gcs.GcsUtils;
 import google.registry.testing.TestDataHelper;
 import java.io.ByteArrayInputStream;
 import java.nio.charset.StandardCharsets;
-import java.util.List;
 import org.joda.time.LocalDate;
-import org.json.JSONException;
 import org.json.JSONObject;
 import org.junit.Before;
 import org.junit.Test;
@@ -39,36 +39,68 @@ import org.junit.runners.JUnit4;
 @RunWith(JUnit4.class)
 public class Spec11RegistrarThreatMatchesParserTest {
 
+  private static final String TODAY = "2018-07-21";
+  private static final String YESTERDAY = "2018-07-20";
+
   private final GcsUtils gcsUtils = mock(GcsUtils.class);
   private final Spec11RegistrarThreatMatchesParser parser =
-      new Spec11RegistrarThreatMatchesParser(new LocalDate(2018, 7, 21), gcsUtils, "test-bucket");
+      new Spec11RegistrarThreatMatchesParser(gcsUtils, "test-bucket");
 
   @Before
   public void setUp() {
-    when(gcsUtils.openInputStream(
+    setupFile("spec11_fake_report", TODAY);
-            new GcsFilename(
-                "test-bucket", "icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_2018-07-21")))
-        .thenAnswer(
-            (args) ->
-                new ByteArrayInputStream(
-                    loadFile("spec11_fake_report").getBytes(StandardCharsets.UTF_8)));
   }
 
   @Test
   public void testSuccess_retrievesReport() throws Exception {
-    List<RegistrarThreatMatches> matches = parser.getRegistrarThreatMatches();
+    assertThat(parser.getRegistrarThreatMatches(LocalDate.parse(TODAY)))
-    assertThat(matches).isEqualTo(sampleThreatMatches());
+        .isEqualTo(sampleThreatMatches());
   }
 
-  /** Returns a {@link String} from a file in the {@code spec11/testdata/} directory. */
+  @Test
-  public static String loadFile(String filename) {
+  public void testFindPrevious_exists() throws Exception {
-    return TestDataHelper.loadFile(Spec11EmailUtils.class, filename);
+    setupFile("spec11_fake_report_previous_day", YESTERDAY);
+    assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY)))
+        .hasValue(LocalDate.parse(YESTERDAY));
+  }
+
+  @Test
+  public void testFindPrevious_notFound() {
+    assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY))).isEmpty();
+  }
+
+  @Test
+  public void testFindPrevious_olderThanYesterdayFound() throws Exception {
+    setupFile("spec11_fake_report_previous_day", "2018-07-14");
+
+    assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY)))
+        .hasValue(LocalDate.parse("2018-07-14"));
   }
 
   /** The expected contents of the sample spec11 report file */
-  public static ImmutableList<RegistrarThreatMatches> sampleThreatMatches() throws JSONException {
+  public static ImmutableSet<RegistrarThreatMatches> sampleThreatMatches() throws Exception {
-    return ImmutableList.of(
+    return ImmutableSet.of(getMatchA(), getMatchB());
-        RegistrarThreatMatches.create(
+  }
+
+  private void setupFile(String fileWithContent, String fileDate) {
+    GcsFilename gcsFilename =
+        new GcsFilename(
+            "test-bucket",
+            String.format("icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_%s", fileDate));
+    when(gcsUtils.existsAndNotEmpty(gcsFilename)).thenReturn(true);
+    when(gcsUtils.openInputStream(gcsFilename))
+        .thenAnswer(
+            (args) ->
+                new ByteArrayInputStream(
+                    loadFile(fileWithContent).getBytes(StandardCharsets.UTF_8)));
+  }
+
+  private static String loadFile(String filename) {
+    return TestDataHelper.loadFile(Spec11EmailUtils.class, filename);
+  }
+
+  private static RegistrarThreatMatches getMatchA() throws Exception {
+    return RegistrarThreatMatches.create(
         "a@fake.com",
         ImmutableList.of(
             ThreatMatch.fromJSON(
@@ -77,8 +109,11 @@ public class Spec11RegistrarThreatMatchesParserTest {
                         "threatType", "MALWARE",
                         "platformType", "ANY_PLATFORM",
                         "threatEntryMetadata", "NONE",
-                            "fullyQualifiedDomainName", "a.com"))))),
+                        "fullyQualifiedDomainName", "a.com")))));
-        RegistrarThreatMatches.create(
+  }
+
+  private static RegistrarThreatMatches getMatchB() throws Exception {
+    return RegistrarThreatMatches.create(
         "b@fake.com",
         ImmutableList.of(
             ThreatMatch.fromJSON(
@@ -94,6 +129,6 @@ public class Spec11RegistrarThreatMatchesParserTest {
                         "threatType", "MALWARE",
                         "platformType", "ANY_PLATFORM",
                         "threatEntryMetadata", "NONE",
-                            "fullyQualifiedDomainName", "c.com"))))));
+                        "fullyQualifiedDomainName", "c.com")))));
   }
 }

javatests/google/registry/reporting/spec11/testdata/spec11_fake_report_previous_day

@@ -0,0 +1,3 @@
+Map from registrar email to detected subdomain threats:
+{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"a.dev","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"a@fake.com"}
+{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"b.com","platformType":"ANY_PLATFORM"},{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"c.com","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"b@fake.com"}