mirror of
https://github.com/google/nomulus.git
synced 2025-08-03 16:32:11 +02:00
Add diff logic and send daily Spec11 emails with new threats
For each registrar, the daily email will only include threats that did not appear in the prior run's email. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=228889972
This commit is contained in:
gbrodman
Ben McIlwain
parent
a80a44cd06
commit
f017798162
15 changed files with 529 additions and 160 deletions
|
|
@ -15,6 +15,7 @@ java_library(
|
|||
"//java/google/registry/beam/spec11",
|
||||
"//java/google/registry/gcs",
|
||||
"//java/google/registry/reporting/spec11",
|
||||
"//java/google/registry/reporting/spec11/soy:soy_java_wrappers",
|
||||
"//java/google/registry/util",
|
||||
"//javatests/google/registry/testing",
|
||||
"@com_google_apis_google_api_services_dataflow",
|
||||
|
|
|
|||
|
|
@ -30,9 +30,12 @@ import com.google.api.services.dataflow.Dataflow.Projects;
|
|||
import com.google.api.services.dataflow.Dataflow.Projects.Jobs;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Jobs.Get;
|
||||
import com.google.api.services.dataflow.model.Job;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import java.io.IOException;
|
||||
import java.util.Optional;
|
||||
import org.joda.time.LocalDate;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
|
@ -43,7 +46,6 @@ import org.junit.runners.JUnit4;
|
|||
@RunWith(JUnit4.class)
|
||||
public class PublishSpec11ReportActionTest {
|
||||
|
||||
private final String spec11BodyTemplate = "{LIST_OF_THREATS}\n{REPLY_TO_EMAIL}";
|
||||
private final LocalDate date = new LocalDate(2018, 6, 5);
|
||||
|
||||
private Dataflow dataflow;
|
||||
|
|
@ -69,41 +71,44 @@ public class PublishSpec11ReportActionTest {
|
|||
expectedJob = new Job();
|
||||
when(get.execute()).thenReturn(expectedJob);
|
||||
emailUtils = mock(Spec11EmailUtils.class);
|
||||
parser = mock(Spec11RegistrarThreatMatchesParser.class);
|
||||
response = new FakeResponse();
|
||||
parser = mock(Spec11RegistrarThreatMatchesParser.class);
|
||||
when(parser.getRegistrarThreatMatches()).thenReturn(sampleThreatMatches());
|
||||
publishAction =
|
||||
new PublishSpec11ReportAction(
|
||||
"test-project",
|
||||
spec11BodyTemplate,
|
||||
"Super Cool Registry",
|
||||
"12345",
|
||||
emailUtils,
|
||||
mock(Spec11RegistrarThreatMatchesParser.class),
|
||||
parser,
|
||||
dataflow,
|
||||
response,
|
||||
date);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testJobDone_emailsResultsOnSecondOfMonth() throws Exception {
|
||||
public void testJobDone_emailsOnlyMonthlyResultsOnSecondOfMonth() throws Exception {
|
||||
LocalDate secondOfMonth = date.withDayOfMonth(2);
|
||||
when(parser.getRegistrarThreatMatches(secondOfMonth)).thenReturn(sampleThreatMatches());
|
||||
expectedJob.setCurrentState("JOB_STATE_DONE");
|
||||
publishAction =
|
||||
new PublishSpec11ReportAction(
|
||||
"test-project",
|
||||
spec11BodyTemplate,
|
||||
"Super Cool Registry",
|
||||
"12345",
|
||||
emailUtils,
|
||||
parser,
|
||||
dataflow,
|
||||
response,
|
||||
date.withDayOfMonth(2));
|
||||
secondOfMonth);
|
||||
publishAction.run();
|
||||
assertThat(response.getStatus()).isEqualTo(SC_OK);
|
||||
verify(emailUtils)
|
||||
.emailSpec11Reports(
|
||||
spec11BodyTemplate,
|
||||
"Google Registry Monthly Threat Detector [2018-06-02]",
|
||||
Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL,
|
||||
"Super Cool Registry Monthly Threat Detector [2018-06-02]",
|
||||
sampleThreatMatches());
|
||||
verifyNoMoreInteractions(emailUtils);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -139,10 +144,31 @@ public class PublishSpec11ReportActionTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testJobDone_doesNotEmailResults() {
|
||||
public void testJobDone_onlyDailyResults() throws Exception {
|
||||
LocalDate yesterday = date.minusDays(1);
|
||||
when(parser.getPreviousDateWithMatches(date)).thenReturn(Optional.of(yesterday));
|
||||
when(parser.getRegistrarThreatMatches(date)).thenReturn(sampleThreatMatches());
|
||||
when(parser.getRegistrarThreatMatches(yesterday)).thenReturn(ImmutableSet.of());
|
||||
expectedJob.setCurrentState("JOB_STATE_DONE");
|
||||
publishAction.run();
|
||||
assertThat(response.getStatus()).isEqualTo(SC_OK);
|
||||
verify(emailUtils)
|
||||
.emailSpec11Reports(
|
||||
Spec11EmailSoyInfo.DAILY_SPEC_11_EMAIL,
|
||||
"Super Cool Registry Daily Threat Detector [2018-06-05]",
|
||||
sampleThreatMatches());
|
||||
verifyNoMoreInteractions(emailUtils);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testJobDone_failsDueToNoPreviousResults() {
|
||||
when(parser.getPreviousDateWithMatches(date)).thenReturn(Optional.empty());
|
||||
expectedJob.setCurrentState("JOB_STATE_DONE");
|
||||
publishAction.run();
|
||||
assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
|
||||
verify(emailUtils)
|
||||
.sendAlertEmail(
|
||||
String.format("Spec11 Diff Error %s", date),
|
||||
String.format("Could not find a previous file within the past month of %s", date));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ import static org.mockito.Mockito.times;
|
|||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeSleeper;
|
||||
import google.registry.util.Retrier;
|
||||
|
|
@ -53,6 +55,7 @@ import org.mockito.stubbing.Answer;
|
|||
public class Spec11EmailUtilsTest {
|
||||
|
||||
private static final int RETRY_COUNT = 2;
|
||||
private static final ImmutableList<String> FAKE_RESOURCES = ImmutableList.of("foo");
|
||||
|
||||
private SendEmailService emailService;
|
||||
private Spec11EmailUtils emailUtils;
|
||||
|
|
@ -66,50 +69,75 @@ public class Spec11EmailUtilsTest {
|
|||
.thenAnswer((args) -> new MimeMessage(Session.getInstance(new Properties(), null)));
|
||||
|
||||
parser = mock(Spec11RegistrarThreatMatchesParser.class);
|
||||
when(parser.getRegistrarThreatMatches()).thenReturn(sampleThreatMatches());
|
||||
LocalDate date = new LocalDate(2018, 7, 15);
|
||||
when(parser.getRegistrarThreatMatches(date)).thenReturn(sampleThreatMatches());
|
||||
|
||||
gotMessage = ArgumentCaptor.forClass(Message.class);
|
||||
|
||||
emailUtils =
|
||||
new Spec11EmailUtils(
|
||||
emailService,
|
||||
new LocalDate(2018, 7, 15),
|
||||
date,
|
||||
"my-sender@test.com",
|
||||
"my-receiver@test.com",
|
||||
"my-reply-to@test.com",
|
||||
FAKE_RESOURCES,
|
||||
"Super Cool Registry",
|
||||
new Retrier(new FakeSleeper(new FakeClock()), RETRY_COUNT));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSuccess_emailSpec11Reports() throws Exception {
|
||||
emailUtils.emailSpec11Reports(
|
||||
"{LIST_OF_THREATS}\n{REPLY_TO_EMAIL}",
|
||||
"Google Registry Monthly Threat Detector [2018-07-15]",
|
||||
Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL,
|
||||
"Super Cool Registry Monthly Threat Detector [2018-07-15]",
|
||||
sampleThreatMatches());
|
||||
// We inspect individual parameters because Message doesn't implement equals().
|
||||
verify(emailService, times(3)).sendMessage(gotMessage.capture());
|
||||
List<Message> capturedMessages = gotMessage.getAllValues();
|
||||
String emailFormat =
|
||||
"Dear registrar partner,<p>Super Cool Registry previously notified you when the following "
|
||||
+ "domains managed by your registrar were flagged for potential security concerns."
|
||||
+ "</p><p>The following domains that you manage continue to be flagged by our analysis "
|
||||
+ "for potential security concerns. This may be because the registrants have not "
|
||||
+ "completed the requisite steps to mitigate the potential security abuse and/or have "
|
||||
+ "it reviewed and delisted.</p><table><tr><th>Domain Name</th><th>Threat Type</th>"
|
||||
+ "</tr>%s</table><p>Please work with the registrant to mitigate any security issues "
|
||||
+ "and have the domains delisted.</p>Some helpful resources for getting off a blocked "
|
||||
+ "list include:<ul><li>foo</li></ul><p>You will continue to receive a monthly summary "
|
||||
+ "of all domains managed by your registrar that remain on our lists of potential "
|
||||
+ "security threats. You will additionally receive a daily notice when any new domains "
|
||||
+ "that are added to these lists. Once the registrant has resolved the security issues "
|
||||
+ "and followed the steps to have his or her domain reviewed and delisted it will "
|
||||
+ "automatically be removed from our monthly reporting.</p><p>If you have any q"
|
||||
+ "uestions regarding this notice, please contact my-reply-to@test.com.</p>";
|
||||
|
||||
validateMessage(
|
||||
capturedMessages.get(0),
|
||||
"my-sender@test.com",
|
||||
"a@fake.com",
|
||||
"my-reply-to@test.com",
|
||||
"Google Registry Monthly Threat Detector [2018-07-15]",
|
||||
"a.com - MALWARE\n\nmy-reply-to@test.com");
|
||||
"Super Cool Registry Monthly Threat Detector [2018-07-15]",
|
||||
String.format(emailFormat, "<tr><td>a.com</td><td>MALWARE</td></tr>"),
|
||||
"text/html");
|
||||
validateMessage(
|
||||
capturedMessages.get(1),
|
||||
"my-sender@test.com",
|
||||
"b@fake.com",
|
||||
"my-reply-to@test.com",
|
||||
"Google Registry Monthly Threat Detector [2018-07-15]",
|
||||
"b.com - MALWARE\nc.com - MALWARE\n\nmy-reply-to@test.com");
|
||||
"Super Cool Registry Monthly Threat Detector [2018-07-15]",
|
||||
String.format(
|
||||
emailFormat,
|
||||
"<tr><td>b.com</td><td>MALWARE</td></tr><tr><td>c.com</td><td>MALWARE</td></tr>"),
|
||||
"text/html");
|
||||
validateMessage(
|
||||
capturedMessages.get(2),
|
||||
"my-sender@test.com",
|
||||
"my-receiver@test.com",
|
||||
null,
|
||||
"Spec11 Pipeline Success 2018-07-15",
|
||||
"Spec11 reporting completed successfully.");
|
||||
"Spec11 reporting completed successfully.",
|
||||
"text/plain");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -139,7 +167,9 @@ public class Spec11EmailUtilsTest {
|
|||
RuntimeException thrown =
|
||||
assertThrows(
|
||||
RuntimeException.class,
|
||||
() -> emailUtils.emailSpec11Reports("foo", "bar", sampleThreatMatches()));
|
||||
() ->
|
||||
emailUtils.emailSpec11Reports(
|
||||
Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL, "bar", sampleThreatMatches()));
|
||||
assertThat(thrown).hasMessageThat().isEqualTo("Emailing spec11 report failed");
|
||||
assertThat(thrown)
|
||||
.hasCauseThat()
|
||||
|
|
@ -155,7 +185,8 @@ public class Spec11EmailUtilsTest {
|
|||
"my-receiver@test.com",
|
||||
null,
|
||||
"Spec11 Emailing Failure 2018-07-15",
|
||||
"Emailing spec11 reports failed due to expected");
|
||||
"Emailing spec11 reports failed due to expected",
|
||||
"text/plain");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -168,7 +199,8 @@ public class Spec11EmailUtilsTest {
|
|||
"my-receiver@test.com",
|
||||
null,
|
||||
"Spec11 Pipeline Alert: 2018-07",
|
||||
"Alert!");
|
||||
"Alert!",
|
||||
"text/plain");
|
||||
}
|
||||
|
||||
private void validateMessage(
|
||||
|
|
@ -177,7 +209,8 @@ public class Spec11EmailUtilsTest {
|
|||
String recipient,
|
||||
@Nullable String replyTo,
|
||||
String subject,
|
||||
String body)
|
||||
String body,
|
||||
String contentType)
|
||||
throws MessagingException, IOException {
|
||||
assertThat(message.getFrom()).asList().containsExactly(new InternetAddress(from));
|
||||
assertThat(message.getRecipients(RecipientType.TO))
|
||||
|
|
@ -192,7 +225,7 @@ public class Spec11EmailUtilsTest {
|
|||
}
|
||||
assertThat(message.getRecipients(RecipientType.CC)).isNull();
|
||||
assertThat(message.getSubject()).isEqualTo(subject);
|
||||
assertThat(message.getContentType()).isEqualTo("text/plain");
|
||||
assertThat(message.getContent().toString()).isEqualTo(body);
|
||||
assertThat(message.getContentType()).isEqualTo(contentType);
|
||||
assertThat(message.getContent()).isEqualTo(body);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,20 +15,20 @@
|
|||
package google.registry.reporting.spec11;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth8.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.beam.spec11.ThreatMatch;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.testing.TestDataHelper;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import org.joda.time.LocalDate;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
|
@ -39,61 +39,96 @@ import org.junit.runners.JUnit4;
|
|||
@RunWith(JUnit4.class)
|
||||
public class Spec11RegistrarThreatMatchesParserTest {
|
||||
|
||||
private static final String TODAY = "2018-07-21";
|
||||
private static final String YESTERDAY = "2018-07-20";
|
||||
|
||||
private final GcsUtils gcsUtils = mock(GcsUtils.class);
|
||||
private final Spec11RegistrarThreatMatchesParser parser =
|
||||
new Spec11RegistrarThreatMatchesParser(new LocalDate(2018, 7, 21), gcsUtils, "test-bucket");
|
||||
new Spec11RegistrarThreatMatchesParser(gcsUtils, "test-bucket");
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
when(gcsUtils.openInputStream(
|
||||
new GcsFilename(
|
||||
"test-bucket", "icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_2018-07-21")))
|
||||
.thenAnswer(
|
||||
(args) ->
|
||||
new ByteArrayInputStream(
|
||||
loadFile("spec11_fake_report").getBytes(StandardCharsets.UTF_8)));
|
||||
setupFile("spec11_fake_report", TODAY);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSuccess_retrievesReport() throws Exception {
|
||||
List<RegistrarThreatMatches> matches = parser.getRegistrarThreatMatches();
|
||||
assertThat(matches).isEqualTo(sampleThreatMatches());
|
||||
assertThat(parser.getRegistrarThreatMatches(LocalDate.parse(TODAY)))
|
||||
.isEqualTo(sampleThreatMatches());
|
||||
}
|
||||
|
||||
/** Returns a {@link String} from a file in the {@code spec11/testdata/} directory. */
|
||||
public static String loadFile(String filename) {
|
||||
return TestDataHelper.loadFile(Spec11EmailUtils.class, filename);
|
||||
@Test
|
||||
public void testFindPrevious_exists() throws Exception {
|
||||
setupFile("spec11_fake_report_previous_day", YESTERDAY);
|
||||
assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY)))
|
||||
.hasValue(LocalDate.parse(YESTERDAY));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFindPrevious_notFound() {
|
||||
assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY))).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFindPrevious_olderThanYesterdayFound() throws Exception {
|
||||
setupFile("spec11_fake_report_previous_day", "2018-07-14");
|
||||
|
||||
assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY)))
|
||||
.hasValue(LocalDate.parse("2018-07-14"));
|
||||
}
|
||||
|
||||
/** The expected contents of the sample spec11 report file */
|
||||
public static ImmutableList<RegistrarThreatMatches> sampleThreatMatches() throws JSONException {
|
||||
return ImmutableList.of(
|
||||
RegistrarThreatMatches.create(
|
||||
"a@fake.com",
|
||||
ImmutableList.of(
|
||||
ThreatMatch.fromJSON(
|
||||
new JSONObject(
|
||||
ImmutableMap.of(
|
||||
"threatType", "MALWARE",
|
||||
"platformType", "ANY_PLATFORM",
|
||||
"threatEntryMetadata", "NONE",
|
||||
"fullyQualifiedDomainName", "a.com"))))),
|
||||
RegistrarThreatMatches.create(
|
||||
"b@fake.com",
|
||||
ImmutableList.of(
|
||||
ThreatMatch.fromJSON(
|
||||
new JSONObject(
|
||||
ImmutableMap.of(
|
||||
"threatType", "MALWARE",
|
||||
"platformType", "ANY_PLATFORM",
|
||||
"threatEntryMetadata", "NONE",
|
||||
"fullyQualifiedDomainName", "b.com"))),
|
||||
ThreatMatch.fromJSON(
|
||||
new JSONObject(
|
||||
ImmutableMap.of(
|
||||
"threatType", "MALWARE",
|
||||
"platformType", "ANY_PLATFORM",
|
||||
"threatEntryMetadata", "NONE",
|
||||
"fullyQualifiedDomainName", "c.com"))))));
|
||||
public static ImmutableSet<RegistrarThreatMatches> sampleThreatMatches() throws Exception {
|
||||
return ImmutableSet.of(getMatchA(), getMatchB());
|
||||
}
|
||||
|
||||
private void setupFile(String fileWithContent, String fileDate) {
|
||||
GcsFilename gcsFilename =
|
||||
new GcsFilename(
|
||||
"test-bucket",
|
||||
String.format("icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_%s", fileDate));
|
||||
when(gcsUtils.existsAndNotEmpty(gcsFilename)).thenReturn(true);
|
||||
when(gcsUtils.openInputStream(gcsFilename))
|
||||
.thenAnswer(
|
||||
(args) ->
|
||||
new ByteArrayInputStream(
|
||||
loadFile(fileWithContent).getBytes(StandardCharsets.UTF_8)));
|
||||
}
|
||||
|
||||
private static String loadFile(String filename) {
|
||||
return TestDataHelper.loadFile(Spec11EmailUtils.class, filename);
|
||||
}
|
||||
|
||||
private static RegistrarThreatMatches getMatchA() throws Exception {
|
||||
return RegistrarThreatMatches.create(
|
||||
"a@fake.com",
|
||||
ImmutableList.of(
|
||||
ThreatMatch.fromJSON(
|
||||
new JSONObject(
|
||||
ImmutableMap.of(
|
||||
"threatType", "MALWARE",
|
||||
"platformType", "ANY_PLATFORM",
|
||||
"threatEntryMetadata", "NONE",
|
||||
"fullyQualifiedDomainName", "a.com")))));
|
||||
}
|
||||
|
||||
private static RegistrarThreatMatches getMatchB() throws Exception {
|
||||
return RegistrarThreatMatches.create(
|
||||
"b@fake.com",
|
||||
ImmutableList.of(
|
||||
ThreatMatch.fromJSON(
|
||||
new JSONObject(
|
||||
ImmutableMap.of(
|
||||
"threatType", "MALWARE",
|
||||
"platformType", "ANY_PLATFORM",
|
||||
"threatEntryMetadata", "NONE",
|
||||
"fullyQualifiedDomainName", "b.com"))),
|
||||
ThreatMatch.fromJSON(
|
||||
new JSONObject(
|
||||
ImmutableMap.of(
|
||||
"threatType", "MALWARE",
|
||||
"platformType", "ANY_PLATFORM",
|
||||
"threatEntryMetadata", "NONE",
|
||||
"fullyQualifiedDomainName", "c.com")))));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
3
javatests/google/registry/reporting/spec11/testdata/spec11_fake_report_previous_day
vendored
Normal file
3
javatests/google/registry/reporting/spec11/testdata/spec11_fake_report_previous_day
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
Map from registrar email to detected subdomain threats:
|
||||
{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"a.dev","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"a@fake.com"}
|
||||
{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"b.com","platformType":"ANY_PLATFORM"},{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"c.com","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"b@fake.com"}
|
||||
Loading…
Add table
Add a link
Reference in a new issue