Use the registrar client ID and abuse email address in Spec11

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=244899045

java/google/registry/reporting/spec11/BUILD

@@ -12,6 +12,7 @@ java_library(
         "//java/google/registry/config",
         "//java/google/registry/gcs",
         "//java/google/registry/keyring/api",
+        "//java/google/registry/model",
         "//java/google/registry/reporting",
         "//java/google/registry/reporting/spec11/soy:soy_java_wrappers",
         "//java/google/registry/request",

java/google/registry/reporting/spec11/PublishSpec11ReportAction.java

@@ -52,8 +52,8 @@ import org.json.JSONException;
  * Retries until a {@code Dataflow} job with a given {@code jobId} completes, continuing the Spec11
  * pipeline accordingly.
  *
- * <p>This calls {@link Spec11EmailUtils#emailSpec11Reports(SoyTemplateInfo, String, Set)} on
+ * <p>This calls {@link Spec11EmailUtils#emailSpec11Reports(LocalDate, SoyTemplateInfo, String,
- * success or {@link Spec11EmailUtils#sendAlertEmail(String, String)} on failure.
+ * Set)} on success or {@link Spec11EmailUtils#sendAlertEmail(String, String)} on failure.
  */
 @Action(
     service = Action.Service.BACKEND,
@@ -193,7 +193,7 @@ public class PublishSpec11ReportAction implements Runnable {
     // Group by email address then flat-map all of the ThreatMatch objects together
     return ImmutableMap.copyOf(
         Maps.transformValues(
-            Multimaps.index(registrarThreatMatches, RegistrarThreatMatches::registrarEmailAddress)
+            Multimaps.index(registrarThreatMatches, RegistrarThreatMatches::clientId)
                 .asMap(),
             registrarThreatMatchesCollection ->
                 registrarThreatMatchesCollection.stream()

java/google/registry/reporting/spec11/RegistrarThreatMatches.java

@@ -23,13 +23,11 @@ import java.util.List;
 @AutoValue
 public abstract class RegistrarThreatMatches {
 
-  public abstract String registrarEmailAddress();
+  public abstract String clientId();
 
   public abstract ImmutableList<ThreatMatch> threatMatches();
 
-  static RegistrarThreatMatches create(
+  static RegistrarThreatMatches create(String clientId, List<ThreatMatch> threatMatches) {
-      String registrarEmailAddress, List<ThreatMatch> threatMatches) {
+    return new AutoValue_RegistrarThreatMatches(clientId, ImmutableList.copyOf(threatMatches));
-    return new AutoValue_RegistrarThreatMatches(
-        registrarEmailAddress, ImmutableList.copyOf(threatMatches));
   }
 }

java/google/registry/reporting/spec11/Spec11EmailUtils.java

@@ -27,6 +27,8 @@ import com.google.template.soy.parseinfo.SoyTemplateInfo;
 import com.google.template.soy.tofu.SoyTofu;
 import com.google.template.soy.tofu.SoyTofu.Renderer;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
 import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo;
 import google.registry.util.EmailMessage;
 import google.registry.util.SendEmailService;
@@ -105,11 +107,10 @@ public class Spec11EmailUtils {
           String.format("Spec11 Emailing Failure %s", date),
           String.format("Emailing Spec11 reports failed due to %s", firstThrowable.getMessage()));
       for (int i = 1; i < failedMatches.size(); i++) {
-        // TODO(b/129401965): Use only client IDs in this message
         logger.atSevere().withCause(failedMatchesList.get(i).getValue()).log(
             "Additional exception thrown when sending email to registrar %s, in addition to the"
                 + " re-thrown exception",
-            failedMatchesList.get(i).getKey().registrarEmailAddress());
+            failedMatchesList.get(i).getKey().clientId());
       }
       throw new RuntimeException(
           "Emailing Spec11 reports failed, first exception:", firstThrowable);
@@ -131,7 +132,7 @@ public class Spec11EmailUtils {
             .setBody(getContent(date, soyTemplateInfo, registrarThreatMatches))
             .setContentType(MediaType.HTML_UTF_8)
             .setFrom(outgoingEmailAddress)
-            .addRecipient(new InternetAddress(registrarThreatMatches.registrarEmailAddress()))
+            .addRecipient(getEmailAddressForRegistrar(registrarThreatMatches.clientId()))
             .setBcc(spec11ReplyToAddress)
             .build());
   }
@@ -176,4 +177,19 @@ public class Spec11EmailUtils {
       throw new RuntimeException("The spec11 alert e-mail system failed.", e);
     }
   }
+
+  private InternetAddress getEmailAddressForRegistrar(String clientId) throws MessagingException {
+    // Attempt to use the registrar's WHOIS abuse contact, then fall back to the regular address.
+    Registrar registrar =
+        Registrar.loadByClientIdCached(clientId)
+            .orElseThrow(
+                () ->
+                    new IllegalArgumentException(
+                        String.format("Could not find registrar %s", clientId)));
+    return new InternetAddress(
+        registrar
+            .getWhoisAbuseContact()
+            .map(RegistrarContact::getEmailAddress)
+            .orElse(registrar.getEmailAddress()));
+  }
 }

java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java

@@ -98,12 +98,12 @@ public class Spec11RegistrarThreatMatchesParser {
 
   private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException {
     JSONObject reportJSON = new JSONObject(line);
-    String registrarEmail = reportJSON.getString(Spec11Pipeline.REGISTRAR_EMAIL_FIELD);
+    String clientId = reportJSON.getString(Spec11Pipeline.REGISTRAR_CLIENT_ID_FIELD);
     JSONArray threatMatchesArray = reportJSON.getJSONArray(Spec11Pipeline.THREAT_MATCHES_FIELD);
     ImmutableList.Builder<ThreatMatch> threatMatches = ImmutableList.builder();
     for (int i = 0; i < threatMatchesArray.length(); i++) {
       threatMatches.add(ThreatMatch.fromJSON(threatMatchesArray.getJSONObject(i)));
     }
-    return RegistrarThreatMatches.create(registrarEmail, threatMatches.build());
+    return RegistrarThreatMatches.create(clientId, threatMatches.build());
   }
 }

javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java

@@ -177,7 +177,7 @@ public class PublishSpec11ReportActionTest {
     RegistrarThreatMatches firstMatches = getMatchA();
     ImmutableList<ThreatMatch> secondMatchList = getMatchB().threatMatches();
     RegistrarThreatMatches secondMatches =
-        RegistrarThreatMatches.create("a@fake.com", secondMatchList);
+        RegistrarThreatMatches.create("TheRegistrar", secondMatchList);
     when(parser.getRegistrarThreatMatches(date))
         .thenReturn(ImmutableSet.of(firstMatches, secondMatches));
     expectedJob.setCurrentState("JOB_STATE_DONE");
@@ -185,7 +185,7 @@ public class PublishSpec11ReportActionTest {
     ImmutableSet<RegistrarThreatMatches> expectedMatchSet =
         ImmutableSet.of(
             RegistrarThreatMatches.create(
-                "a@fake.com",
+                "TheRegistrar",
                 ImmutableList.<ThreatMatch>builder()
                     .addAll(firstMatches.threatMatches())
                     .addAll(secondMatchList)

javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java

@@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat;
 import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.getMatchA;
 import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.getMatchB;
 import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches;
+import static google.registry.testing.DatastoreHelper.persistResource;
 import static google.registry.testing.JUnitBackports.assertThrows;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
@@ -26,8 +27,10 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.net.MediaType;
 import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo;
+import google.registry.testing.AppEngineRule;
 import google.registry.util.EmailMessage;
 import google.registry.util.SendEmailService;
 import java.util.LinkedHashSet;
@@ -37,6 +40,7 @@ import javax.mail.MessagingException;
 import javax.mail.internet.InternetAddress;
 import org.joda.time.LocalDate;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -106,6 +110,8 @@ public class Spec11EmailUtilsTest {
           + "<p>If you have any questions regarding this notice, please contact "
           + "my-reply-to@test.com.</p>";
 
+  @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build();
+
   private SendEmailService emailService;
   private Spec11EmailUtils emailUtils;
   private Spec11RegistrarThreatMatchesParser parser;
@@ -141,7 +147,7 @@ public class Spec11EmailUtilsTest {
     validateMessage(
         capturedContents.get(0),
         "my-sender@test.com",
-        "a@fake.com",
+        "the.registrar@example.com",
         Optional.of("my-reply-to@test.com"),
         "Super Cool Registry Monthly Threat Detector [2018-07-15]",
         String.format(MONTHLY_EMAIL_FORMAT, "<tr><td>a.com</td><td>MALWARE</td></tr>"),
@@ -149,7 +155,7 @@ public class Spec11EmailUtilsTest {
     validateMessage(
         capturedContents.get(1),
         "my-sender@test.com",
-        "b@fake.com",
+        "new.registrar@example.com",
         Optional.of("my-reply-to@test.com"),
         "Super Cool Registry Monthly Threat Detector [2018-07-15]",
         String.format(
@@ -179,7 +185,7 @@ public class Spec11EmailUtilsTest {
     validateMessage(
         capturedMessages.get(0),
         "my-sender@test.com",
-        "a@fake.com",
+        "the.registrar@example.com",
         Optional.of("my-reply-to@test.com"),
         "Super Cool Registry Daily Threat Detector [2018-07-15]",
         String.format(DAILY_EMAIL_FORMAT, "<tr><td>a.com</td><td>MALWARE</td></tr>"),
@@ -187,7 +193,7 @@ public class Spec11EmailUtilsTest {
     validateMessage(
         capturedMessages.get(1),
         "my-sender@test.com",
-        "b@fake.com",
+        "new.registrar@example.com",
         Optional.of("my-reply-to@test.com"),
         "Super Cool Registry Daily Threat Detector [2018-07-15]",
         String.format(
@@ -234,7 +240,7 @@ public class Spec11EmailUtilsTest {
     validateMessage(
         capturedMessages.get(0),
         "my-sender@test.com",
-        "a@fake.com",
+        "the.registrar@example.com",
         Optional.of("my-reply-to@test.com"),
         "Super Cool Registry Monthly Threat Detector [2018-07-15]",
         String.format(MONTHLY_EMAIL_FORMAT, "<tr><td>a.com</td><td>MALWARE</td></tr>"),
@@ -242,7 +248,7 @@ public class Spec11EmailUtilsTest {
     validateMessage(
         capturedMessages.get(1),
         "my-sender@test.com",
-        "b@fake.com",
+        "new.registrar@example.com",
         Optional.of("my-reply-to@test.com"),
         "Super Cool Registry Monthly Threat Detector [2018-07-15]",
         String.format(
@@ -260,7 +266,7 @@ public class Spec11EmailUtilsTest {
   }
 
   @Test
-  public void testSuccess_sendAlertEmail() throws MessagingException {
+  public void testSuccess_sendAlertEmail() throws Exception {
     emailUtils.sendAlertEmail("Spec11 Pipeline Alert: 2018-07", "Alert!");
     verify(emailService).sendEmail(contentCaptor.capture());
     validateMessage(
@@ -273,6 +279,45 @@ public class Spec11EmailUtilsTest {
         Optional.empty());
   }
 
+  @Test
+  public void testSuccess_useWhoisAbuseEmailIfAvailable() throws Exception {
+    // if John Doe is the whois abuse contact, email them instead of the regular email
+    persistResource(
+        AppEngineRule.makeRegistrarContact2()
+            .asBuilder()
+            .setEmailAddress("johndoe@theregistrar.com")
+            .setVisibleInDomainWhoisAsAbuse(true)
+            .build());
+    emailUtils.emailSpec11Reports(
+        date,
+        Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL,
+        "Super Cool Registry Monthly Threat Detector [2018-07-15]",
+        sampleThreatMatches());
+    verify(emailService, times(3)).sendEmail(contentCaptor.capture());
+    assertThat(contentCaptor.getAllValues().get(0).recipients())
+        .containsExactly(new InternetAddress("johndoe@theregistrar.com"));
+  }
+
+  @Test
+  public void testFailure_badClientId() {
+    RuntimeException thrown =
+        assertThrows(
+            RuntimeException.class,
+            () ->
+                emailUtils.emailSpec11Reports(
+                    date,
+                    Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL,
+                    "Super Cool Registry Monthly Threat Detector [2018-07-15]",
+                    ImmutableSet.of(
+                        RegistrarThreatMatches.create(
+                            "badClientId", getMatchA().threatMatches()))));
+    assertThat(thrown)
+        .hasCauseThat()
+        .hasMessageThat()
+        .isEqualTo("Could not find registrar badClientId");
+    assertThat(thrown).hasCauseThat().isInstanceOf(IllegalArgumentException.class);
+  }
+
   private void validateMessage(
       EmailMessage message,
       String from,

javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java

@@ -84,7 +84,7 @@ public class Spec11RegistrarThreatMatchesParserTest {
 
   static RegistrarThreatMatches getMatchA() throws Exception {
     return RegistrarThreatMatches.create(
-        "a@fake.com",
+        "TheRegistrar",
         ImmutableList.of(
             ThreatMatch.fromJSON(
                 new JSONObject(
@@ -97,7 +97,7 @@ public class Spec11RegistrarThreatMatchesParserTest {
 
   static RegistrarThreatMatches getMatchB() throws Exception {
     return RegistrarThreatMatches.create(
-        "b@fake.com",
+        "NewRegistrar",
         ImmutableList.of(
             ThreatMatch.fromJSON(
                 new JSONObject(

javatests/google/registry/reporting/spec11/testdata/spec11_fake_report

@@ -1,3 +1,3 @@
-Map from registrar email to detected subdomain threats:
+Map from registrar email / name to detected subdomain threats:
-{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"a.com","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"a@fake.com"}
+{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"a.com","platformType":"ANY_PLATFORM"}],"registrarClientId":"TheRegistrar","registrarEmailAddress":"the.registrar@example.com"}
-{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"b.com","platformType":"ANY_PLATFORM"},{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"c.com","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"b@fake.com"}
+{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"b.com","platformType":"ANY_PLATFORM"},{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"c.com","platformType":"ANY_PLATFORM"}],"registrarClientId":"NewRegistrar","registrarEmailAddress":"new.registrar@example.com"}