diff --git a/java/google/registry/reporting/spec11/BUILD b/java/google/registry/reporting/spec11/BUILD index 96d18495e..bcc899193 100644 --- a/java/google/registry/reporting/spec11/BUILD +++ b/java/google/registry/reporting/spec11/BUILD @@ -12,6 +12,7 @@ java_library( "//java/google/registry/config", "//java/google/registry/gcs", "//java/google/registry/keyring/api", + "//java/google/registry/model", "//java/google/registry/reporting", "//java/google/registry/reporting/spec11/soy:soy_java_wrappers", "//java/google/registry/request", diff --git a/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java b/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java index a6e2e380e..89ee7ebf1 100644 --- a/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java +++ b/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java @@ -52,8 +52,8 @@ import org.json.JSONException; * Retries until a {@code Dataflow} job with a given {@code jobId} completes, continuing the Spec11 * pipeline accordingly. * - *

This calls {@link Spec11EmailUtils#emailSpec11Reports(SoyTemplateInfo, String, Set)} on - * success or {@link Spec11EmailUtils#sendAlertEmail(String, String)} on failure. + *

This calls {@link Spec11EmailUtils#emailSpec11Reports(LocalDate, SoyTemplateInfo, String, + * Set)} on success or {@link Spec11EmailUtils#sendAlertEmail(String, String)} on failure. */ @Action( service = Action.Service.BACKEND, @@ -193,7 +193,7 @@ public class PublishSpec11ReportAction implements Runnable { // Group by email address then flat-map all of the ThreatMatch objects together return ImmutableMap.copyOf( Maps.transformValues( - Multimaps.index(registrarThreatMatches, RegistrarThreatMatches::registrarEmailAddress) + Multimaps.index(registrarThreatMatches, RegistrarThreatMatches::clientId) .asMap(), registrarThreatMatchesCollection -> registrarThreatMatchesCollection.stream() diff --git a/java/google/registry/reporting/spec11/RegistrarThreatMatches.java b/java/google/registry/reporting/spec11/RegistrarThreatMatches.java index 04e32615e..64259cfcc 100644 --- a/java/google/registry/reporting/spec11/RegistrarThreatMatches.java +++ b/java/google/registry/reporting/spec11/RegistrarThreatMatches.java @@ -23,13 +23,11 @@ import java.util.List; @AutoValue public abstract class RegistrarThreatMatches { - public abstract String registrarEmailAddress(); + public abstract String clientId(); public abstract ImmutableList threatMatches(); - static RegistrarThreatMatches create( - String registrarEmailAddress, List threatMatches) { - return new AutoValue_RegistrarThreatMatches( - registrarEmailAddress, ImmutableList.copyOf(threatMatches)); + static RegistrarThreatMatches create(String clientId, List threatMatches) { + return new AutoValue_RegistrarThreatMatches(clientId, ImmutableList.copyOf(threatMatches)); } } diff --git a/java/google/registry/reporting/spec11/Spec11EmailUtils.java b/java/google/registry/reporting/spec11/Spec11EmailUtils.java index 5d2561019..77d5ca034 100644 --- a/java/google/registry/reporting/spec11/Spec11EmailUtils.java +++ b/java/google/registry/reporting/spec11/Spec11EmailUtils.java @@ -27,6 +27,8 @@ import com.google.template.soy.parseinfo.SoyTemplateInfo; import com.google.template.soy.tofu.SoyTofu; import com.google.template.soy.tofu.SoyTofu.Renderer; import google.registry.config.RegistryConfig.Config; +import google.registry.model.registrar.Registrar; +import google.registry.model.registrar.RegistrarContact; import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo; import google.registry.util.EmailMessage; import google.registry.util.SendEmailService; @@ -105,11 +107,10 @@ public class Spec11EmailUtils { String.format("Spec11 Emailing Failure %s", date), String.format("Emailing Spec11 reports failed due to %s", firstThrowable.getMessage())); for (int i = 1; i < failedMatches.size(); i++) { - // TODO(b/129401965): Use only client IDs in this message logger.atSevere().withCause(failedMatchesList.get(i).getValue()).log( "Additional exception thrown when sending email to registrar %s, in addition to the" + " re-thrown exception", - failedMatchesList.get(i).getKey().registrarEmailAddress()); + failedMatchesList.get(i).getKey().clientId()); } throw new RuntimeException( "Emailing Spec11 reports failed, first exception:", firstThrowable); @@ -131,7 +132,7 @@ public class Spec11EmailUtils { .setBody(getContent(date, soyTemplateInfo, registrarThreatMatches)) .setContentType(MediaType.HTML_UTF_8) .setFrom(outgoingEmailAddress) - .addRecipient(new InternetAddress(registrarThreatMatches.registrarEmailAddress())) + .addRecipient(getEmailAddressForRegistrar(registrarThreatMatches.clientId())) .setBcc(spec11ReplyToAddress) .build()); } @@ -176,4 +177,19 @@ public class Spec11EmailUtils { throw new RuntimeException("The spec11 alert e-mail system failed.", e); } } + + private InternetAddress getEmailAddressForRegistrar(String clientId) throws MessagingException { + // Attempt to use the registrar's WHOIS abuse contact, then fall back to the regular address. + Registrar registrar = + Registrar.loadByClientIdCached(clientId) + .orElseThrow( + () -> + new IllegalArgumentException( + String.format("Could not find registrar %s", clientId))); + return new InternetAddress( + registrar + .getWhoisAbuseContact() + .map(RegistrarContact::getEmailAddress) + .orElse(registrar.getEmailAddress())); + } } diff --git a/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java b/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java index ef139b30d..b99de5bcb 100644 --- a/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java +++ b/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java @@ -98,12 +98,12 @@ public class Spec11RegistrarThreatMatchesParser { private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException { JSONObject reportJSON = new JSONObject(line); - String registrarEmail = reportJSON.getString(Spec11Pipeline.REGISTRAR_EMAIL_FIELD); + String clientId = reportJSON.getString(Spec11Pipeline.REGISTRAR_CLIENT_ID_FIELD); JSONArray threatMatchesArray = reportJSON.getJSONArray(Spec11Pipeline.THREAT_MATCHES_FIELD); ImmutableList.Builder threatMatches = ImmutableList.builder(); for (int i = 0; i < threatMatchesArray.length(); i++) { threatMatches.add(ThreatMatch.fromJSON(threatMatchesArray.getJSONObject(i))); } - return RegistrarThreatMatches.create(registrarEmail, threatMatches.build()); + return RegistrarThreatMatches.create(clientId, threatMatches.build()); } } diff --git a/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java b/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java index 6d81d5c67..dd09b63b7 100644 --- a/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java +++ b/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java @@ -177,7 +177,7 @@ public class PublishSpec11ReportActionTest { RegistrarThreatMatches firstMatches = getMatchA(); ImmutableList secondMatchList = getMatchB().threatMatches(); RegistrarThreatMatches secondMatches = - RegistrarThreatMatches.create("a@fake.com", secondMatchList); + RegistrarThreatMatches.create("TheRegistrar", secondMatchList); when(parser.getRegistrarThreatMatches(date)) .thenReturn(ImmutableSet.of(firstMatches, secondMatches)); expectedJob.setCurrentState("JOB_STATE_DONE"); @@ -185,7 +185,7 @@ public class PublishSpec11ReportActionTest { ImmutableSet expectedMatchSet = ImmutableSet.of( RegistrarThreatMatches.create( - "a@fake.com", + "TheRegistrar", ImmutableList.builder() .addAll(firstMatches.threatMatches()) .addAll(secondMatchList) diff --git a/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java b/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java index 11085e1b2..39414bae1 100644 --- a/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java +++ b/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java @@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat; import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.getMatchA; import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.getMatchB; import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches; +import static google.registry.testing.DatastoreHelper.persistResource; import static google.registry.testing.JUnitBackports.assertThrows; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -26,8 +27,10 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.net.MediaType; import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo; +import google.registry.testing.AppEngineRule; import google.registry.util.EmailMessage; import google.registry.util.SendEmailService; import java.util.LinkedHashSet; @@ -37,6 +40,7 @@ import javax.mail.MessagingException; import javax.mail.internet.InternetAddress; import org.joda.time.LocalDate; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -106,6 +110,8 @@ public class Spec11EmailUtilsTest { + "

If you have any questions regarding this notice, please contact " + "my-reply-to@test.com.

"; + @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build(); + private SendEmailService emailService; private Spec11EmailUtils emailUtils; private Spec11RegistrarThreatMatchesParser parser; @@ -141,7 +147,7 @@ public class Spec11EmailUtilsTest { validateMessage( capturedContents.get(0), "my-sender@test.com", - "a@fake.com", + "the.registrar@example.com", Optional.of("my-reply-to@test.com"), "Super Cool Registry Monthly Threat Detector [2018-07-15]", String.format(MONTHLY_EMAIL_FORMAT, "a.comMALWARE"), @@ -149,7 +155,7 @@ public class Spec11EmailUtilsTest { validateMessage( capturedContents.get(1), "my-sender@test.com", - "b@fake.com", + "new.registrar@example.com", Optional.of("my-reply-to@test.com"), "Super Cool Registry Monthly Threat Detector [2018-07-15]", String.format( @@ -179,7 +185,7 @@ public class Spec11EmailUtilsTest { validateMessage( capturedMessages.get(0), "my-sender@test.com", - "a@fake.com", + "the.registrar@example.com", Optional.of("my-reply-to@test.com"), "Super Cool Registry Daily Threat Detector [2018-07-15]", String.format(DAILY_EMAIL_FORMAT, "a.comMALWARE"), @@ -187,7 +193,7 @@ public class Spec11EmailUtilsTest { validateMessage( capturedMessages.get(1), "my-sender@test.com", - "b@fake.com", + "new.registrar@example.com", Optional.of("my-reply-to@test.com"), "Super Cool Registry Daily Threat Detector [2018-07-15]", String.format( @@ -234,7 +240,7 @@ public class Spec11EmailUtilsTest { validateMessage( capturedMessages.get(0), "my-sender@test.com", - "a@fake.com", + "the.registrar@example.com", Optional.of("my-reply-to@test.com"), "Super Cool Registry Monthly Threat Detector [2018-07-15]", String.format(MONTHLY_EMAIL_FORMAT, "a.comMALWARE"), @@ -242,7 +248,7 @@ public class Spec11EmailUtilsTest { validateMessage( capturedMessages.get(1), "my-sender@test.com", - "b@fake.com", + "new.registrar@example.com", Optional.of("my-reply-to@test.com"), "Super Cool Registry Monthly Threat Detector [2018-07-15]", String.format( @@ -260,7 +266,7 @@ public class Spec11EmailUtilsTest { } @Test - public void testSuccess_sendAlertEmail() throws MessagingException { + public void testSuccess_sendAlertEmail() throws Exception { emailUtils.sendAlertEmail("Spec11 Pipeline Alert: 2018-07", "Alert!"); verify(emailService).sendEmail(contentCaptor.capture()); validateMessage( @@ -273,6 +279,45 @@ public class Spec11EmailUtilsTest { Optional.empty()); } + @Test + public void testSuccess_useWhoisAbuseEmailIfAvailable() throws Exception { + // if John Doe is the whois abuse contact, email them instead of the regular email + persistResource( + AppEngineRule.makeRegistrarContact2() + .asBuilder() + .setEmailAddress("johndoe@theregistrar.com") + .setVisibleInDomainWhoisAsAbuse(true) + .build()); + emailUtils.emailSpec11Reports( + date, + Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL, + "Super Cool Registry Monthly Threat Detector [2018-07-15]", + sampleThreatMatches()); + verify(emailService, times(3)).sendEmail(contentCaptor.capture()); + assertThat(contentCaptor.getAllValues().get(0).recipients()) + .containsExactly(new InternetAddress("johndoe@theregistrar.com")); + } + + @Test + public void testFailure_badClientId() { + RuntimeException thrown = + assertThrows( + RuntimeException.class, + () -> + emailUtils.emailSpec11Reports( + date, + Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL, + "Super Cool Registry Monthly Threat Detector [2018-07-15]", + ImmutableSet.of( + RegistrarThreatMatches.create( + "badClientId", getMatchA().threatMatches())))); + assertThat(thrown) + .hasCauseThat() + .hasMessageThat() + .isEqualTo("Could not find registrar badClientId"); + assertThat(thrown).hasCauseThat().isInstanceOf(IllegalArgumentException.class); + } + private void validateMessage( EmailMessage message, String from, diff --git a/javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java b/javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java index dc8712102..4849f4d69 100644 --- a/javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java +++ b/javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java @@ -84,7 +84,7 @@ public class Spec11RegistrarThreatMatchesParserTest { static RegistrarThreatMatches getMatchA() throws Exception { return RegistrarThreatMatches.create( - "a@fake.com", + "TheRegistrar", ImmutableList.of( ThreatMatch.fromJSON( new JSONObject( @@ -97,7 +97,7 @@ public class Spec11RegistrarThreatMatchesParserTest { static RegistrarThreatMatches getMatchB() throws Exception { return RegistrarThreatMatches.create( - "b@fake.com", + "NewRegistrar", ImmutableList.of( ThreatMatch.fromJSON( new JSONObject( diff --git a/javatests/google/registry/reporting/spec11/testdata/spec11_fake_report b/javatests/google/registry/reporting/spec11/testdata/spec11_fake_report index ce9ae9cfe..d08c14455 100644 --- a/javatests/google/registry/reporting/spec11/testdata/spec11_fake_report +++ b/javatests/google/registry/reporting/spec11/testdata/spec11_fake_report @@ -1,3 +1,3 @@ -Map from registrar email to detected subdomain threats: -{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"a.com","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"a@fake.com"} -{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"b.com","platformType":"ANY_PLATFORM"},{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"c.com","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"b@fake.com"} +Map from registrar email / name to detected subdomain threats: +{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"a.com","platformType":"ANY_PLATFORM"}],"registrarClientId":"TheRegistrar","registrarEmailAddress":"the.registrar@example.com"} +{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"b.com","platformType":"ANY_PLATFORM"},{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"c.com","platformType":"ANY_PLATFORM"}],"registrarClientId":"NewRegistrar","registrarEmailAddress":"new.registrar@example.com"}