zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-08-03 08:22:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

Run the Spec11 pipeline daily without sending emails

Browse source 
Add a sendSpec11Email parameter that allows us to only send the email on
one run per month. Next, we will compute the diffs between the daily runs
and send daily emails with those diffs.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=224404653
This commit is contained in:
jianglai 2018-12-06 14:10:25 -08:00
parent 3ef8cd692d
commit ec26e3a96a
20 changed files with 434 additions and 158 deletions
Download patch file Download diff file Expand all files Collapse all files

4
javatests/google/registry/beam/spec11/Spec11PipelineTest.java
View file

@ -132,7 +132,7 @@ public class Spec11PipelineTest {
// Apply input and evaluation transforms
PCollection<Subdomain> input = p.apply(Create.of(inputRows));
spec11Pipeline.evaluateUrlHealth(input, evalFn, StaticValueProvider.of("2018-06"));
spec11Pipeline.evaluateUrlHealth(input, evalFn, StaticValueProvider.of("2018-06-01"));
p.run();
// Verify header and 3 threat matches for 2 registrars are found
@ -277,7 +277,7 @@ public class Spec11PipelineTest {
File resultFile =
new File(
String.format(
"%s/icann/spec11/2018-06/SPEC11_MONTHLY_REPORT",
"%s/icann/spec11/2018-06/SPEC11_MONTHLY_REPORT_2018-06-01",
tempFolder.getRoot().getAbsolutePath()));
return ImmutableList.copyOf(
ResourceUtils.readResourceUtf8(resultFile.toURI().toURL()).split("\n"));

52
javatests/google/registry/reporting/ReportingModuleTest.java
View file

@ -19,9 +19,11 @@ import static google.registry.testing.JUnitBackports.assertThrows;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.google.common.truth.Truth8;
import google.registry.request.HttpException.BadRequestException;
import google.registry.testing.FakeClock;
import google.registry.util.Clock;
import java.time.LocalDate;
import java.util.Optional;
import javax.servlet.http.HttpServletRequest;
import org.joda.time.DateTime;
@ -37,6 +39,7 @@ public class ReportingModuleTest {
private HttpServletRequest req = mock(HttpServletRequest.class);
private Clock clock;
@Before
public void setUp() {
clock = new FakeClock(DateTime.parse("2017-07-01TZ"));
@ -45,7 +48,14 @@ public class ReportingModuleTest {
@Test
public void testEmptyYearMonthParameter_returnsEmptyYearMonthOptional() {
when(req.getParameter("yearMonth")).thenReturn("");
assertThat(ReportingModule.provideYearMonthOptional(req)).isEqualTo(Optional.empty());
Truth8.assertThat(ReportingModule.provideYearMonthOptional(req)).isEmpty();
}
@Test
public void testValidYearMonthParameter_returnsThatMonth() {
when(req.getParameter("yearMonth")).thenReturn("2017-05");
Truth8.assertThat(ReportingModule.provideYearMonthOptional(req))
.hasValue(new YearMonth(2017, 5));
}
@Test
@ -61,14 +71,50 @@ public class ReportingModuleTest {
@Test
public void testEmptyYearMonth_returnsLastMonth() {
assertThat(ReportingModule.provideYearMonth(Optional.empty(), clock))
assertThat(ReportingModule.provideYearMonth(Optional.empty(), LocalDate.of(2017, 7, 6)))
.isEqualTo(new YearMonth(2017, 6));
}
@Test
public void testGivenYearMonth_returnsThatMonth() {
assertThat(ReportingModule.provideYearMonth(Optional.of(new YearMonth(2017, 5)), clock))
assertThat(
ReportingModule.provideYearMonth(
Optional.of(new YearMonth(2017, 5)), LocalDate.of(2017, 7, 6)))
.isEqualTo(new YearMonth(2017, 5));
}
@Test
public void testEmptyDateParameter_returnsEmptyDateOptional() {
when(req.getParameter("date")).thenReturn("");
Truth8.assertThat(ReportingModule.provideDateOptional(req)).isEmpty();
}
@Test
public void testValidDateParameter_returnsThatDate() {
when(req.getParameter("date")).thenReturn("2017-05-13");
Truth8.assertThat(ReportingModule.provideDateOptional(req))
.hasValue(LocalDate.of(2017, 5, 13));
}
@Test
public void testInvalidDateParameter_throwsException() {
when(req.getParameter("date")).thenReturn("20170513");
BadRequestException thrown =
assertThrows(BadRequestException.class, () -> ReportingModule.provideDateOptional(req));
assertThat(thrown)
.hasMessageThat()
.contains("date must be in yyyy-MM-dd format, got 20170513 instead");
}
@Test
public void testEmptyDate_returnsToday() {
assertThat(ReportingModule.provideDate(Optional.empty(), clock))
.isEqualTo(LocalDate.of(2017, 7, 1));
}
@Test
public void testGivenDate_returnsThatDate() {
assertThat(ReportingModule.provideDate(Optional.of(LocalDate.of(2017, 7, 2)), clock))
.isEqualTo(LocalDate.of(2017, 7, 2));
}
}

1
javatests/google/registry/reporting/spec11/BUILD
View file

@ -12,6 +12,7 @@ java_library(
srcs = glob(["*.java"]),
resources = glob(["testdata/*"]),
deps = [
"//java/google/registry/beam/spec11",
"//java/google/registry/gcs",
"//java/google/registry/reporting/spec11",
"//java/google/registry/util",

10
javatests/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java
View file

@ -35,7 +35,7 @@ import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeResponse;
import google.registry.testing.TaskQueueHelper.TaskMatcher;
import java.io.IOException;
import org.joda.time.YearMonth;
import java.time.LocalDate;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@ -87,20 +87,20 @@ public class GenerateSpec11ReportActionTest {
"gs://template",
"us-east1-c",
"api_key/a",
YearMonth.parse("2018-06"),
LocalDate.parse("2018-06-11"),
response,
dataflow);
action.run();
LaunchTemplateParameters expectedLaunchTemplateParameters =
new LaunchTemplateParameters()
.setJobName("spec11_2018-06")
.setJobName("spec11_2018-06-11")
.setEnvironment(
new RuntimeEnvironment()
.setZone("us-east1-c")
.setTempLocation("gs://my-bucket-beam/temporary"))
.setParameters(
ImmutableMap.of("safeBrowsingApiKey", "api_key/a", "yearMonth", "2018-06"));
ImmutableMap.of("safeBrowsingApiKey", "api_key/a", "date", "2018-06-11"));
verify(dataflowTemplates).launch("test", expectedLaunchTemplateParameters);
verify(dataflowLaunch).setGcsPath("gs://template");
assertThat(response.getStatus()).isEqualTo(200);
@ -112,7 +112,7 @@ public class GenerateSpec11ReportActionTest {
.url("/_dr/task/publishSpec11")
.method("POST")
.param("jobId", "jobid")
.param("yearMonth", "2018-06");
.param("date", "2018-06-11");
assertTasksEnqueued("beam-reporting", matcher);
}
}

25
javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java
View file

@ -32,7 +32,7 @@ import com.google.api.services.dataflow.model.Job;
import com.google.common.net.MediaType;
import google.registry.testing.FakeResponse;
import java.io.IOException;
import org.joda.time.YearMonth;
import java.time.LocalDate;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -67,12 +67,15 @@ public class PublishSpec11ReportActionTest {
response = new FakeResponse();
publishAction =
new PublishSpec11ReportAction(
"test-project", "12345", emailUtils, dataflow, response, new YearMonth(2018, 6));
"test-project", "12345", emailUtils, dataflow, response, LocalDate.of(2018, 6, 5));
}
@Test
public void testJobDone_emailsResults() {
public void testJobDone_emailsResultsOnSecondOfMonth() {
expectedJob.setCurrentState("JOB_STATE_DONE");
publishAction =
new PublishSpec11ReportAction(
"test-project", "12345", emailUtils, dataflow, response, LocalDate.of(2018, 6, 2));
publishAction.run();
assertThat(response.getStatus()).isEqualTo(SC_OK);
verify(emailUtils).emailSpec11Reports();
@ -85,8 +88,8 @@ public class PublishSpec11ReportActionTest {
assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
verify(emailUtils)
.sendAlertEmail(
"Spec11 Dataflow Pipeline Failure 2018-06",
"Spec11 2018-06 job 12345 ended in status failure.");
"Spec11 Dataflow Pipeline Failure 2018-06-05",
"Spec11 2018-06-05 job 12345 ended in status failure.");
}
@Test
@ -106,7 +109,15 @@ public class PublishSpec11ReportActionTest {
assertThat(response.getPayload()).isEqualTo("Template launch failed: expected");
verify(emailUtils)
.sendAlertEmail(
"Spec11 Publish Failure 2018-06",
"Spec11 2018-06 publish action failed due to expected");
"Spec11 Publish Failure 2018-06-05",
"Spec11 2018-06-05 publish action failed due to expected");
}
@Test
public void testJobDone_doesNotEmailResults() {
expectedJob.setCurrentState("JOB_STATE_DONE");
publishAction.run();
assertThat(response.getStatus()).isEqualTo(SC_OK);
verifyNoMoreInteractions(emailUtils);
}
}

42
javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java
View file

@ -16,6 +16,7 @@ package google.registry.reporting.spec11;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches;
import static google.registry.testing.JUnitBackports.assertThrows;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doThrow;
@ -24,16 +25,11 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import google.registry.gcs.GcsUtils;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeSleeper;
import google.registry.testing.TestDataHelper;
import google.registry.util.Retrier;
import google.registry.util.SendEmailService;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Properties;
import javax.annotation.Nullable;
@ -60,36 +56,29 @@ public class Spec11EmailUtilsTest {
private SendEmailService emailService;
private Spec11EmailUtils emailUtils;
private GcsUtils gcsUtils;
private Spec11RegistrarThreatMatchesParser parser;
private ArgumentCaptor<Message> gotMessage;
@Before
public void setUp() {
public void setUp() throws Exception {
emailService = mock(SendEmailService.class);
when(emailService.createMessage())
.thenAnswer((args) -> new MimeMessage(Session.getInstance(new Properties(), null)));
gcsUtils = mock(GcsUtils.class);
when(gcsUtils.openInputStream(
new GcsFilename("test-bucket", "icann/spec11/2018-06/SPEC11_MONTHLY_REPORT")))
.thenAnswer(
(args) ->
new ByteArrayInputStream(
loadFile("spec11_fake_report").getBytes(StandardCharsets.UTF_8)));
parser = mock(Spec11RegistrarThreatMatchesParser.class);
when(parser.getRegistrarThreatMatches()).thenReturn(sampleThreatMatches());
gotMessage = ArgumentCaptor.forClass(Message.class);
emailUtils =
new Spec11EmailUtils(
emailService,
new YearMonth(2018, 6),
new YearMonth(2018, 7),
"my-sender@test.com",
"my-receiver@test.com",
"my-reply-to@test.com",
"{LIST_OF_THREATS}\n{REPLY_TO_EMAIL}",
"test-bucket",
"icann/spec11/2018-06/SPEC11_MONTHLY_REPORT",
gcsUtils,
parser,
new Retrier(new FakeSleeper(new FakeClock()), RETRY_COUNT));
}
@ -104,21 +93,21 @@ public class Spec11EmailUtilsTest {
"my-sender@test.com",
"a@fake.com",
"my-reply-to@test.com",
"Google Registry Monthly Threat Detector [2018-06]",
"Google Registry Monthly Threat Detector [2018-07]",
"a.com - MALWARE\n\nmy-reply-to@test.com");
validateMessage(
capturedMessages.get(1),
"my-sender@test.com",
"b@fake.com",
"my-reply-to@test.com",
"Google Registry Monthly Threat Detector [2018-06]",
"Google Registry Monthly Threat Detector [2018-07]",
"b.com - MALWARE\nc.com - MALWARE\n\nmy-reply-to@test.com");
validateMessage(
capturedMessages.get(2),
"my-sender@test.com",
"my-receiver@test.com",
null,
"Spec11 Pipeline Success 2018-06",
"Spec11 Pipeline Success 2018-07",
"Spec11 reporting completed successfully.");
}
@ -162,20 +151,20 @@ public class Spec11EmailUtilsTest {
"my-sender@test.com",
"my-receiver@test.com",
null,
"Spec11 Emailing Failure 2018-06",
"Spec11 Emailing Failure 2018-07",
"Emailing spec11 reports failed due to expected");
}
@Test
public void testSuccess_sendAlertEmail() throws MessagingException, IOException {
emailUtils.sendAlertEmail("Spec11 Pipeline Alert: 2018-06", "Alert!");
emailUtils.sendAlertEmail("Spec11 Pipeline Alert: 2018-07", "Alert!");
verify(emailService).sendMessage(gotMessage.capture());
validateMessage(
gotMessage.getValue(),
"my-sender@test.com",
"my-receiver@test.com",
null,
"Spec11 Pipeline Alert: 2018-06",
"Spec11 Pipeline Alert: 2018-07",
"Alert!");
}
@ -203,9 +192,4 @@ public class Spec11EmailUtilsTest {
assertThat(message.getContentType()).isEqualTo("text/plain");
assertThat(message.getContent().toString()).isEqualTo(body);
}
/** Returns a {@link String} from a file in the {@code spec11/testdata/} directory. */
public static String loadFile(String filename) {
return TestDataHelper.loadFile(Spec11EmailUtils.class, filename);
}
}

99
javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java Normal file
View file

@ -0,0 +1,99 @@
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.reporting.spec11;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import google.registry.beam.spec11.ThreatMatch;
import google.registry.gcs.GcsUtils;
import google.registry.testing.TestDataHelper;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.util.List;
import org.json.JSONException;
import org.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link Spec11RegistrarThreatMatchesParser}. */
@RunWith(JUnit4.class)
public class Spec11RegistrarThreatMatchesParserTest {
private final GcsUtils gcsUtils = mock(GcsUtils.class);
private final Spec11RegistrarThreatMatchesParser parser =
new Spec11RegistrarThreatMatchesParser(LocalDate.of(2018, 7, 21), gcsUtils, "test-bucket");
@Before
public void setUp() {
when(gcsUtils.openInputStream(
new GcsFilename(
"test-bucket", "icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_2018-07-21")))
.thenAnswer(
(args) ->
new ByteArrayInputStream(
loadFile("spec11_fake_report").getBytes(StandardCharsets.UTF_8)));
}
@Test
public void testSuccess_retrievesReport() throws Exception {
List<RegistrarThreatMatches> matches = parser.getRegistrarThreatMatches();
assertThat(matches).isEqualTo(sampleThreatMatches());
}
/** Returns a {@link String} from a file in the {@code spec11/testdata/} directory. */
public static String loadFile(String filename) {
return TestDataHelper.loadFile(Spec11EmailUtils.class, filename);
}
/** The expected contents of the sample spec11 report file */
public static ImmutableList<RegistrarThreatMatches> sampleThreatMatches() throws JSONException {
return ImmutableList.of(
RegistrarThreatMatches.create(
"a@fake.com",
ImmutableList.of(
ThreatMatch.fromJSON(
new JSONObject(
ImmutableMap.of(
"threatType", "MALWARE",
"platformType", "ANY_PLATFORM",
"threatEntryMetadata", "NONE",
"fullyQualifiedDomainName", "a.com"))))),
RegistrarThreatMatches.create(
"b@fake.com",
ImmutableList.of(
ThreatMatch.fromJSON(
new JSONObject(
ImmutableMap.of(
"threatType", "MALWARE",
"platformType", "ANY_PLATFORM",
"threatEntryMetadata", "NONE",
"fullyQualifiedDomainName", "b.com"))),
ThreatMatch.fromJSON(
new JSONObject(
ImmutableMap.of(
"threatType", "MALWARE",
"platformType", "ANY_PLATFORM",
"threatEntryMetadata", "NONE",
"fullyQualifiedDomainName", "c.com"))))));
}
}