zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-13 07:57:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

Run the Spec11 pipeline daily without sending emails

Browse source 
Add a sendSpec11Email parameter that allows us to only send the email on
one run per month. Next, we will compute the diffs between the daily runs
and send daily emails with those diffs.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=224404653
This commit is contained in:
jianglai 2018-12-06 14:10:25 -08:00
parent 3ef8cd692d
commit ec26e3a96a
20 changed files with 434 additions and 158 deletions
Download patch file Download diff file Expand all files Collapse all files

1
java/google/registry/beam/spec11/BUILD
View file

@ -9,6 +9,7 @@ java_library(
srcs = glob(["*.java"]),
resources = glob(["sql/*"]),
deps = [
"//java/com/google/common/time",
"//java/google/registry/beam",
"//java/google/registry/config",
"//java/google/registry/util",

49
java/google/registry/beam/spec11/Spec11Pipeline.java
View file

@ -15,17 +15,14 @@
package google.registry.beam.spec11;
import static google.registry.beam.BeamUtils.getQueryFromFile;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.auth.oauth2.GoogleCredentials;
import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn;
import google.registry.config.CredentialModule.LocalCredentialJson;
import google.registry.config.RegistryConfig.Config;
import google.registry.util.Retrier;
import google.registry.util.SqlTemplate;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.Serializable;
import java.time.LocalDate;
import java.time.YearMonth;
import javax.inject.Inject;
import org.apache.beam.runners.dataflow.DataflowRunner;
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
@ -61,13 +58,14 @@ import org.json.JSONObject;
public class Spec11Pipeline implements Serializable {
/**
* Returns the subdirectory spec11 reports reside in for a given yearMonth in yyyy-MM format.
* Returns the subdirectory spec11 reports reside in for a given local date in yyyy-MM-dd format.
*
* @see google.registry.beam.spec11.Spec11Pipeline
* @see google.registry.reporting.spec11.Spec11EmailUtils
*/
public static String getSpec11Subdirectory(String yearMonth) {
return String.format("icann/spec11/%s/SPEC11_MONTHLY_REPORT", yearMonth);
public static String getSpec11ReportFilePath(LocalDate localDate) {
YearMonth yearMonth = YearMonth.of(localDate.getYear(), localDate.getMonth());
return String.format("icann/spec11/%s/SPEC11_MONTHLY_REPORT_%s", yearMonth, localDate);
}
/** The JSON object field we put the registrar's e-mail address for Spec11 reports. */
@ -91,27 +89,24 @@ public class Spec11Pipeline implements Serializable {
@Config("reportingBucketUrl")
String reportingBucketUrl;
@Inject
Retrier retrier;
@Inject @LocalCredentialJson String credentialJson;
@Inject Retrier retrier;
@Inject
Spec11Pipeline() {}
/** Custom options for running the spec11 pipeline. */
interface Spec11PipelineOptions extends DataflowPipelineOptions {
/** Returns the yearMonth we're generating the report for, in yyyy-MM format. */
@Description("The yearMonth we generate the report for, in yyyy-MM format.")
ValueProvider<String> getYearMonth();
/** Returns the local date we're generating the report for, in yyyy-MM-dd format. */
@Description("The local date we generate the report for, in yyyy-MM-dd format.")
ValueProvider<String> getDate();
/**
* Sets the yearMonth we generate invoices for.
* Sets the local date we generate invoices for.
*
* <p>This is implicitly set when executing the Dataflow template, by specifying the "yearMonth"
* <p>This is implicitly set when executing the Dataflow template, by specifying the "date"
* parameter.
*/
void setYearMonth(ValueProvider<String> value);
void setDate(ValueProvider<String> value);
/** Returns the SafeBrowsing API key we use to evaluate subdomain health. */
@Description("The API key we use to access the SafeBrowsing API.")
@ -130,12 +125,6 @@ public class Spec11Pipeline implements Serializable {
public void deploy() {
// We can't store options as a member variable due to serialization concerns.
Spec11PipelineOptions options = PipelineOptionsFactory.as(Spec11PipelineOptions.class);
try {
options.setGcpCredential(
GoogleCredentials.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8))));
} catch (IOException e) {
throw new RuntimeException("Cannot obtain local credential to deploy the spec11 pipeline", e);
}
options.setProject(projectId);
options.setRunner(DataflowRunner.class);
// This causes p.run() to stage the pipeline as a template on GCS, as opposed to running it.
@ -162,7 +151,7 @@ public class Spec11Pipeline implements Serializable {
evaluateUrlHealth(
domains,
new EvaluateSafeBrowsingFn(options.getSafeBrowsingApiKey(), retrier),
options.getYearMonth());
options.getDate());
p.run();
}
@ -174,7 +163,7 @@ public class Spec11Pipeline implements Serializable {
void evaluateUrlHealth(
PCollection<Subdomain> domains,
EvaluateSafeBrowsingFn evaluateSafeBrowsingFn,
ValueProvider<String> yearMonthProvider) {
ValueProvider<String> dateProvider) {
domains
.apply("Run through SafeBrowsingAPI", ParDo.of(evaluateSafeBrowsingFn))
.apply(
@ -212,13 +201,13 @@ public class Spec11Pipeline implements Serializable {
TextIO.write()
.to(
NestedValueProvider.of(
yearMonthProvider,
yearMonth ->
dateProvider,
date ->
String.format(
"%s/%s",
reportingBucketUrl, getSpec11Subdirectory(yearMonth))))
reportingBucketUrl,
getSpec11ReportFilePath(LocalDate.parse(date)))))
.withoutSharding()
.withHeader("Map from registrar email to detected subdomain threats:"));
}
}

9
java/google/registry/env/production/default/WEB-INF/cron.xml vendored
View file

@ -289,13 +289,12 @@
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/generateSpec11&runInEmpty]]></url>
<description>
Starts the beam/spec11/Spec11Pipeline Dataflow template, which creates last month's Spec11
report. This report is stored in gs://[PROJECT-ID]-reporting/icann/spec11/yyyy-MM.
Upon Dataflow job completion, sends an e-mail to all registrars with domain registrations
flagged by the SafeBrowsing API.
Starts the beam/spec11/Spec11Pipeline Dataflow template, which creates today's Spec11
report. This report is stored in gs://[PROJECT-ID]-reporting/icann/spec11/yyyy-MM/.
This job will only send email notifications on the second of every month.
See GenerateSpec11ReportAction for more details.
</description>
<schedule>2 of month 15:00</schedule>
<schedule>every day 15:00</schedule>
<target>backend</target>
</cron>
</cronentries>

51
java/google/registry/reporting/ReportingModule.java
View file

@ -26,8 +26,11 @@ import google.registry.config.RegistryConfig.Config;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.Parameter;
import google.registry.util.Clock;
import java.time.LocalDate;
import java.time.format.DateTimeParseException;
import java.util.Optional;
import javax.servlet.http.HttpServletRequest;
import org.joda.time.DateTime;
import org.joda.time.YearMonth;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
@ -37,11 +40,20 @@ import org.joda.time.format.DateTimeFormatter;
public class ReportingModule {
public static final String BEAM_QUEUE = "beam-reporting";
/**
* The request parameter name used by reporting actions that takes a year/month parameter, which
* defaults to the last month.
*/
// TODO(b/120497263): remove this and replace with the date
public static final String PARAM_YEAR_MONTH = "yearMonth";
/**
* The request parameter name used by reporting actions that take a local date as a parameter,
* which defaults to the current date.
*/
public static final String PARAM_DATE = "date";
/** The request parameter specifying the jobId for a running Dataflow pipeline. */
public static final String PARAM_JOB_ID = "jobId";
@ -74,18 +86,49 @@ public class ReportingModule {
*/
@Provides
static YearMonth provideYearMonth(
@Parameter(PARAM_YEAR_MONTH) Optional<YearMonth> yearMonthOptional, Clock clock) {
return yearMonthOptional.orElseGet(() -> new YearMonth(clock.nowUtc().minusMonths(1)));
@Parameter(PARAM_YEAR_MONTH) Optional<YearMonth> yearMonthOptional, LocalDate date) {
return yearMonthOptional.orElseGet(
() -> new YearMonth(date.getYear(), date.getMonthValue() - 1));
}
/** Extracts an optional date in yyyy-MM-dd format from the request. */
@Provides
@Parameter(PARAM_DATE)
static Optional<LocalDate> provideDateOptional(HttpServletRequest req) {
java.time.format.DateTimeFormatter formatter =
java.time.format.DateTimeFormatter.ISO_LOCAL_DATE;
Optional<String> optionalDateString = extractOptionalParameter(req, PARAM_DATE);
try {
return optionalDateString.map(s -> LocalDate.parse(s, formatter));
} catch (DateTimeParseException e) {
throw new BadRequestException(
String.format(
"date must be in yyyy-MM-dd format, got %s instead",
optionalDateString.orElse("UNSPECIFIED LOCAL DATE")),
e);
}
}
/**
* Provides the local date in yyyy-MM-dd format, if not specified in the request, defaults to the
* current date.
*/
@Provides
static LocalDate provideDate(
@Parameter(PARAM_DATE) Optional<LocalDate> dateOptional, Clock clock) {
return dateOptional.orElseGet(
() -> {
DateTime now = clock.nowUtc();
return LocalDate.of(now.getYear(), now.getMonthOfYear(), now.getDayOfMonth());
});
}
/** Constructs a {@link Dataflow} API client with default settings. */
@Provides
static Dataflow provideDataflow(
@DefaultCredential GoogleCredential credential, @Config("projectId") String projectId) {
return new Dataflow.Builder(credential.getTransport(), credential.getJsonFactory(), credential)
.setApplicationName(String.format("%s billing", projectId))
.build();
}
}

9
java/google/registry/reporting/ReportingUtils.java
View file

@ -16,6 +16,7 @@ package google.registry.reporting;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskOptions;
import java.util.Map;
import org.joda.time.Duration;
import org.joda.time.YearMonth;
@ -25,15 +26,13 @@ public class ReportingUtils {
private static final int ENQUEUE_DELAY_MINUTES = 10;
/** Enqueues a task that takes a Beam jobId and the {@link YearMonth} as parameters. */
public static void enqueueBeamReportingTask(String path, String jobId, YearMonth yearMonth) {
public static void enqueueBeamReportingTask(String path, Map<String, String> parameters) {
TaskOptions publishTask =
TaskOptions.Builder.withUrl(path)
.method(TaskOptions.Method.POST)
// Dataflow jobs tend to take about 10 minutes to complete.
.countdownMillis(Duration.standardMinutes(ENQUEUE_DELAY_MINUTES).getMillis())
.param(ReportingModule.PARAM_JOB_ID, jobId)
// Need to pass this through to ensure transitive yearMonth dependencies are satisfied.
.param(ReportingModule.PARAM_YEAR_MONTH, yearMonth.toString());
.countdownMillis(Duration.standardMinutes(ENQUEUE_DELAY_MINUTES).getMillis());
parameters.forEach(publishTask::param);
QueueFactory.getQueue(ReportingModule.BEAM_QUEUE).add(publishTask);
}
}

10
java/google/registry/reporting/billing/GenerateInvoicesAction.java
View file

@ -28,11 +28,13 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.flogger.FluentLogger;
import com.google.common.net.MediaType;
import google.registry.config.RegistryConfig.Config;
import google.registry.reporting.ReportingModule;
import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import java.io.IOException;
import java.util.Map;
import javax.inject.Inject;
import org.joda.time.YearMonth;
@ -105,7 +107,13 @@ public class GenerateInvoicesAction implements Runnable {
logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
String jobId = launchResponse.getJob().getId();
if (shouldPublish) {
enqueueBeamReportingTask(PublishInvoicesAction.PATH, jobId, yearMonth);
Map<String, String> beamTaskParameters =
ImmutableMap.of(
ReportingModule.PARAM_JOB_ID,
jobId,
ReportingModule.PARAM_YEAR_MONTH,
yearMonth.toString());
enqueueBeamReportingTask(PublishInvoicesAction.PATH, beamTaskParameters);
}
} catch (IOException e) {
logger.atWarning().withCause(e).log("Template Launch failed");

1
java/google/registry/reporting/spec11/BUILD
View file

@ -20,6 +20,7 @@ java_library(
"@com_google_apis_google_api_services_dataflow",
"@com_google_appengine_api_1_0_sdk",
"@com_google_appengine_tools_appengine_gcs_client",
"@com_google_auto_value",
"@com_google_dagger",
"@com_google_flogger",
"@com_google_flogger_system_backend",

23
java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java
View file

@ -28,12 +28,14 @@ import com.google.common.flogger.FluentLogger;
import com.google.common.net.MediaType;
import google.registry.config.RegistryConfig.Config;
import google.registry.keyring.api.KeyModule.Key;
import google.registry.reporting.ReportingModule;
import google.registry.request.Action;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import java.io.IOException;
import java.time.LocalDate;
import java.util.Map;
import javax.inject.Inject;
import org.joda.time.YearMonth;
/**
* Invokes the {@code Spec11Pipeline} Beam template via the REST api.
@ -53,7 +55,7 @@ public class GenerateSpec11ReportAction implements Runnable {
private final String spec11TemplateUrl;
private final String jobZone;
private final String apiKey;
private final YearMonth yearMonth;
private final LocalDate date;
private final Response response;
private final Dataflow dataflow;
@ -64,7 +66,7 @@ public class GenerateSpec11ReportAction implements Runnable {
@Config("spec11TemplateUrl") String spec11TemplateUrl,
@Config("defaultJobZone") String jobZone,
@Key("safeBrowsingAPIKey") String apiKey,
YearMonth yearMonth,
LocalDate date,
Response response,
Dataflow dataflow) {
this.projectId = projectId;
@ -72,7 +74,7 @@ public class GenerateSpec11ReportAction implements Runnable {
this.spec11TemplateUrl = spec11TemplateUrl;
this.jobZone = jobZone;
this.apiKey = apiKey;
this.yearMonth = yearMonth;
this.date = date;
this.response = response;
this.dataflow = dataflow;
}
@ -82,14 +84,14 @@ public class GenerateSpec11ReportAction implements Runnable {
try {
LaunchTemplateParameters params =
new LaunchTemplateParameters()
.setJobName(String.format("spec11_%s", yearMonth.toString()))
.setJobName(String.format("spec11_%s", date))
.setEnvironment(
new RuntimeEnvironment()
.setZone(jobZone)
.setTempLocation(beamBucketUrl + "/temporary"))
.setParameters(
ImmutableMap.of(
"safeBrowsingApiKey", apiKey, "yearMonth", yearMonth.toString("yyyy-MM")));
"safeBrowsingApiKey", apiKey, ReportingModule.PARAM_DATE, date.toString()));
LaunchTemplateResponse launchResponse =
dataflow
.projects()
@ -97,8 +99,13 @@ public class GenerateSpec11ReportAction implements Runnable {
.launch(projectId, params)
.setGcsPath(spec11TemplateUrl)
.execute();
enqueueBeamReportingTask(
PublishSpec11ReportAction.PATH, launchResponse.getJob().getId(), yearMonth);
Map<String, String> beamTaskParameters =
ImmutableMap.of(
ReportingModule.PARAM_JOB_ID,
launchResponse.getJob().getId(),
ReportingModule.PARAM_DATE,
date.toString());
enqueueBeamReportingTask(PublishSpec11ReportAction.PATH, beamTaskParameters);
logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
} catch (IOException e) {
logger.atWarning().withCause(e).log("Template Launch failed");

30
java/google/registry/reporting/spec11/PublishSpec11ReportAction.java
View file

@ -31,8 +31,8 @@ import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import java.io.IOException;
import java.time.LocalDate;
import javax.inject.Inject;
import org.joda.time.YearMonth;
/**
* Retries until a {@code Dataflow} job with a given {@code jobId} completes, continuing the Spec11
@ -55,7 +55,7 @@ public class PublishSpec11ReportAction implements Runnable {
private final Spec11EmailUtils emailUtils;
private final Dataflow dataflow;
private final Response response;
private final YearMonth yearMonth;
private final LocalDate date;
@Inject
PublishSpec11ReportAction(
@ -64,13 +64,13 @@ public class PublishSpec11ReportAction implements Runnable {
Spec11EmailUtils emailUtils,
Dataflow dataflow,
Response response,
YearMonth yearMonth) {
LocalDate date) {
this.projectId = projectId;
this.jobId = jobId;
this.emailUtils = emailUtils;
this.dataflow = dataflow;
this.response = response;
this.yearMonth = yearMonth;
this.date = date;
}
@Override
@ -81,17 +81,19 @@ public class PublishSpec11ReportAction implements Runnable {
String state = job.getCurrentState();
switch (state) {
case JOB_DONE:
logger.atInfo().log("Dataflow job %s finished successfully, publishing results.", jobId);
logger.atInfo().log(
"Dataflow job %s finished successfully, publishing results if appropriate.", jobId);
response.setStatus(SC_OK);
emailUtils.emailSpec11Reports();
if (shouldSendSpec11Email()) {
emailUtils.emailSpec11Reports();
}
break;
case JOB_FAILED:
logger.atSevere().log("Dataflow job %s finished unsuccessfully.", jobId);
response.setStatus(SC_NO_CONTENT);
emailUtils.sendAlertEmail(
String.format("Spec11 Dataflow Pipeline Failure %s", yearMonth.toString()),
String.format(
"Spec11 %s job %s ended in status failure.", yearMonth.toString(), jobId));
String.format("Spec11 Dataflow Pipeline Failure %s", date),
String.format("Spec11 %s job %s ended in status failure.", date, jobId));
break;
default:
logger.atInfo().log("Job in non-terminal state %s, retrying:", state);
@ -101,12 +103,16 @@ public class PublishSpec11ReportAction implements Runnable {
} catch (IOException e) {
logger.atSevere().withCause(e).log("Failed to publish Spec11 reports.");
emailUtils.sendAlertEmail(
String.format("Spec11 Publish Failure %s", yearMonth.toString()),
String.format(
"Spec11 %s publish action failed due to %s", yearMonth.toString(), e.getMessage()));
String.format("Spec11 Publish Failure %s", date),
String.format("Spec11 %s publish action failed due to %s", date, e.getMessage()));
response.setStatus(SC_INTERNAL_SERVER_ERROR);
response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
response.setPayload(String.format("Template launch failed: %s", e.getMessage()));
}
}
private boolean shouldSendSpec11Email() {
// TODO(b/120496893): send emails every day with the diff content
return date.getDayOfMonth() == 2;
}
}

33
java/google/registry/reporting/spec11/RegistrarThreatMatches.java Normal file
View file

@ -0,0 +1,33 @@
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.reporting.spec11;
import com.google.auto.value.AutoValue;
import google.registry.beam.spec11.ThreatMatch;
import java.util.List;
/** Value class representing the registrar and list-of-threat-matches pair stored in GCS. */
@AutoValue
public abstract class RegistrarThreatMatches {
public abstract String registrarEmailAddress();
public abstract List<ThreatMatch> threatMatches();
static RegistrarThreatMatches create(
String registrarEmailAddress, List<ThreatMatch> threatMatches) {
return new AutoValue_RegistrarThreatMatches(registrarEmailAddress, threatMatches);
}
}

55
java/google/registry/reporting/spec11/Spec11EmailUtils.java
View file

@ -15,30 +15,19 @@
package google.registry.reporting.spec11;
import static com.google.common.base.Throwables.getRootCause;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.collect.ImmutableList;
import com.google.common.io.CharStreams;
import google.registry.beam.spec11.Spec11Pipeline;
import google.registry.beam.spec11.ThreatMatch;
import google.registry.config.RegistryConfig.Config;
import google.registry.gcs.GcsUtils;
import google.registry.reporting.spec11.Spec11Module.Spec11ReportDirectory;
import google.registry.util.Retrier;
import google.registry.util.SendEmailService;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import javax.inject.Inject;
import javax.mail.Message;
import javax.mail.Message.RecipientType;
import javax.mail.MessagingException;
import javax.mail.internet.InternetAddress;
import org.joda.time.YearMonth;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
/** Provides e-mail functionality for Spec11 tasks, such as sending Spec11 reports to registrars. */
public class Spec11EmailUtils {
@ -48,10 +37,8 @@ public class Spec11EmailUtils {
private final String outgoingEmailAddress;
private final String alertRecipientAddress;
private final String spec11ReplyToAddress;
private final String reportingBucket;
private final String spec11ReportDirectory;
private final String spec11EmailBodyTemplate;
private final GcsUtils gcsUtils;
private final Spec11RegistrarThreatMatchesParser spec11RegistrarThreatMatchesParser;
private final Retrier retrier;
@Inject
@ -62,19 +49,15 @@ public class Spec11EmailUtils {
@Config("alertRecipientEmailAddress") String alertRecipientAddress,
@Config("spec11ReplyToEmailAddress") String spec11ReplyToAddress,
@Config("spec11EmailBodyTemplate") String spec11EmailBodyTemplate,
@Config("reportingBucket") String reportingBucket,
@Spec11ReportDirectory String spec11ReportDirectory,
GcsUtils gcsUtils,
Spec11RegistrarThreatMatchesParser spec11RegistrarThreatMatchesParser,
Retrier retrier) {
this.emailService = emailService;
this.yearMonth = yearMonth;
this.outgoingEmailAddress = outgoingEmailAddress;
this.alertRecipientAddress = alertRecipientAddress;
this.spec11ReplyToAddress = spec11ReplyToAddress;
this.reportingBucket = reportingBucket;
this.spec11ReportDirectory = spec11ReportDirectory;
this.spec11RegistrarThreatMatchesParser = spec11RegistrarThreatMatchesParser;
this.spec11EmailBodyTemplate = spec11EmailBodyTemplate;
this.gcsUtils = gcsUtils;
this.retrier = retrier;
}
@ -86,17 +69,10 @@ public class Spec11EmailUtils {
try {
retrier.callWithRetry(
() -> {
// Grab the file as an inputstream
GcsFilename spec11ReportFilename =
new GcsFilename(reportingBucket, spec11ReportDirectory);
try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) {
ImmutableList<String> reportLines =
ImmutableList.copyOf(
CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n"));
// Iterate from 1 to size() to skip the header at line 0.
for (int i = 1; i < reportLines.size(); i++) {
emailRegistrar(reportLines.get(i));
}
ImmutableList<RegistrarThreatMatches> registrarThreatMatchesList =
spec11RegistrarThreatMatchesParser.getRegistrarThreatMatches();
for (RegistrarThreatMatches registrarThreatMatches : registrarThreatMatchesList) {
emailRegistrar(registrarThreatMatches);
}
},
IOException.class,
@ -105,9 +81,7 @@ public class Spec11EmailUtils {
// Send an alert with the root cause, unwrapping the retrier's RuntimeException
sendAlertEmail(
String.format("Spec11 Emailing Failure %s", yearMonth.toString()),
String.format(
"Emailing spec11 reports failed due to %s",
getRootCause(e).getMessage()));
String.format("Emailing spec11 reports failed due to %s", getRootCause(e).getMessage()));
throw new RuntimeException("Emailing spec11 report failed", e);
}
sendAlertEmail(
@ -115,14 +89,11 @@ public class Spec11EmailUtils {
"Spec11 reporting completed successfully.");
}
private void emailRegistrar(String line) throws MessagingException, JSONException {
// Parse the Spec11 report JSON
JSONObject reportJSON = new JSONObject(line);
String registrarEmail = reportJSON.getString(Spec11Pipeline.REGISTRAR_EMAIL_FIELD);
JSONArray threatMatches = reportJSON.getJSONArray(Spec11Pipeline.THREAT_MATCHES_FIELD);
private void emailRegistrar(RegistrarThreatMatches registrarThreatMatches)
throws MessagingException {
String registrarEmail = registrarThreatMatches.registrarEmailAddress();
StringBuilder threatList = new StringBuilder();
for (int i = 0; i < threatMatches.length(); i++) {
ThreatMatch threatMatch = ThreatMatch.fromJSON(threatMatches.getJSONObject(i));
for (ThreatMatch threatMatch : registrarThreatMatches.threatMatches()) {
threatList.append(
String.format(
"%s - %s\n", threatMatch.fullyQualifiedDomainName(), threatMatch.threatType()));
@ -134,7 +105,7 @@ public class Spec11EmailUtils {
Message msg = emailService.createMessage();
msg.setSubject(
String.format("Google Registry Monthly Threat Detector [%s]", yearMonth.toString()));
msg.setText(body.toString());
msg.setText(body);
msg.setFrom(new InternetAddress(outgoingEmailAddress));
msg.addRecipient(RecipientType.TO, new InternetAddress(registrarEmail));
msg.addRecipient(RecipientType.BCC, new InternetAddress(spec11ReplyToAddress));

10
java/google/registry/reporting/spec11/Spec11Module.java
View file

@ -21,22 +21,22 @@ import dagger.Provides;
import google.registry.beam.spec11.Spec11Pipeline;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.time.LocalDate;
import javax.inject.Qualifier;
import org.joda.time.YearMonth;
/** Module for dependencies required by Spec11 reporting. */
@Module
public class Spec11Module {
@Provides
@Spec11ReportDirectory
static String provideDirectoryPrefix(YearMonth yearMonth) {
return Spec11Pipeline.getSpec11Subdirectory(yearMonth.toString("yyyy-MM"));
@Spec11ReportFilePath
static String provideSpec11ReportFilePath(LocalDate localDate) {
return Spec11Pipeline.getSpec11ReportFilePath(localDate);
}
/** Dagger qualifier for the subdirectory we stage to/upload from for Spec11 reports. */
@Qualifier
@Documented
@Retention(RUNTIME)
@interface Spec11ReportDirectory {}
@interface Spec11ReportFilePath {}
}

78
java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java Normal file
View file

@ -0,0 +1,78 @@
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.reporting.spec11;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.collect.ImmutableList;
import com.google.common.io.CharStreams;
import google.registry.beam.spec11.Spec11Pipeline;
import google.registry.beam.spec11.ThreatMatch;
import google.registry.config.RegistryConfig.Config;
import google.registry.gcs.GcsUtils;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.time.LocalDate;
import javax.inject.Inject;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
/** Parser to retrieve which registrar-threat matches we should notify via email */
public class Spec11RegistrarThreatMatchesParser {
private final LocalDate date;
private final GcsUtils gcsUtils;
private final String reportingBucket;
@Inject
public Spec11RegistrarThreatMatchesParser(
LocalDate date, GcsUtils gcsUtils, @Config("reportingBucket") String reportingBucket) {
this.date = date;
this.gcsUtils = gcsUtils;
this.reportingBucket = reportingBucket;
}
/** Gets the list of registrar:set-of-threat-match pairings from the file in GCS. */
public ImmutableList<RegistrarThreatMatches> getRegistrarThreatMatches()
throws IOException, JSONException {
// TODO(b/120078223): this should only be the diff of this run and the prior run.
GcsFilename spec11ReportFilename =
new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(date));
ImmutableList.Builder<RegistrarThreatMatches> builder = ImmutableList.builder();
try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) {
ImmutableList<String> reportLines =
ImmutableList.copyOf(CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n"));
// Iterate from 1 to size() to skip the header at line 0.
for (int i = 1; i < reportLines.size(); i++) {
builder.add(parseRegistrarThreatMatch(reportLines.get(i)));
}
return builder.build();
}
}
private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException {
JSONObject reportJSON = new JSONObject(line);
String registrarEmail = reportJSON.getString(Spec11Pipeline.REGISTRAR_EMAIL_FIELD);
JSONArray threatMatchesArray = reportJSON.getJSONArray(Spec11Pipeline.THREAT_MATCHES_FIELD);
ImmutableList.Builder<ThreatMatch> threatMatches = ImmutableList.builder();
for (int i = 0; i < threatMatchesArray.length(); i++) {
threatMatches.add(ThreatMatch.fromJSON(threatMatchesArray.getJSONObject(i)));
}
return RegistrarThreatMatches.create(registrarEmail, threatMatches.build());
}
}

4
javatests/google/registry/beam/spec11/Spec11PipelineTest.java
View file

@ -132,7 +132,7 @@ public class Spec11PipelineTest {
// Apply input and evaluation transforms
PCollection<Subdomain> input = p.apply(Create.of(inputRows));
spec11Pipeline.evaluateUrlHealth(input, evalFn, StaticValueProvider.of("2018-06"));
spec11Pipeline.evaluateUrlHealth(input, evalFn, StaticValueProvider.of("2018-06-01"));
p.run();
// Verify header and 3 threat matches for 2 registrars are found
@ -277,7 +277,7 @@ public class Spec11PipelineTest {
File resultFile =
new File(
String.format(
"%s/icann/spec11/2018-06/SPEC11_MONTHLY_REPORT",
"%s/icann/spec11/2018-06/SPEC11_MONTHLY_REPORT_2018-06-01",
tempFolder.getRoot().getAbsolutePath()));
return ImmutableList.copyOf(
ResourceUtils.readResourceUtf8(resultFile.toURI().toURL()).split("\n"));

52
javatests/google/registry/reporting/ReportingModuleTest.java
View file

@ -19,9 +19,11 @@ import static google.registry.testing.JUnitBackports.assertThrows;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.google.common.truth.Truth8;
import google.registry.request.HttpException.BadRequestException;
import google.registry.testing.FakeClock;
import google.registry.util.Clock;
import java.time.LocalDate;
import java.util.Optional;
import javax.servlet.http.HttpServletRequest;
import org.joda.time.DateTime;
@ -37,6 +39,7 @@ public class ReportingModuleTest {
private HttpServletRequest req = mock(HttpServletRequest.class);
private Clock clock;
@Before
public void setUp() {
clock = new FakeClock(DateTime.parse("2017-07-01TZ"));
@ -45,7 +48,14 @@ public class ReportingModuleTest {
@Test
public void testEmptyYearMonthParameter_returnsEmptyYearMonthOptional() {
when(req.getParameter("yearMonth")).thenReturn("");
assertThat(ReportingModule.provideYearMonthOptional(req)).isEqualTo(Optional.empty());
Truth8.assertThat(ReportingModule.provideYearMonthOptional(req)).isEmpty();
}
@Test
public void testValidYearMonthParameter_returnsThatMonth() {
when(req.getParameter("yearMonth")).thenReturn("2017-05");
Truth8.assertThat(ReportingModule.provideYearMonthOptional(req))
.hasValue(new YearMonth(2017, 5));
}
@Test
@ -61,14 +71,50 @@ public class ReportingModuleTest {
@Test
public void testEmptyYearMonth_returnsLastMonth() {
assertThat(ReportingModule.provideYearMonth(Optional.empty(), clock))
assertThat(ReportingModule.provideYearMonth(Optional.empty(), LocalDate.of(2017, 7, 6)))
.isEqualTo(new YearMonth(2017, 6));
}
@Test
public void testGivenYearMonth_returnsThatMonth() {
assertThat(ReportingModule.provideYearMonth(Optional.of(new YearMonth(2017, 5)), clock))
assertThat(
ReportingModule.provideYearMonth(
Optional.of(new YearMonth(2017, 5)), LocalDate.of(2017, 7, 6)))
.isEqualTo(new YearMonth(2017, 5));
}
@Test
public void testEmptyDateParameter_returnsEmptyDateOptional() {
when(req.getParameter("date")).thenReturn("");
Truth8.assertThat(ReportingModule.provideDateOptional(req)).isEmpty();
}
@Test
public void testValidDateParameter_returnsThatDate() {
when(req.getParameter("date")).thenReturn("2017-05-13");
Truth8.assertThat(ReportingModule.provideDateOptional(req))
.hasValue(LocalDate.of(2017, 5, 13));
}
@Test
public void testInvalidDateParameter_throwsException() {
when(req.getParameter("date")).thenReturn("20170513");
BadRequestException thrown =
assertThrows(BadRequestException.class, () -> ReportingModule.provideDateOptional(req));
assertThat(thrown)
.hasMessageThat()
.contains("date must be in yyyy-MM-dd format, got 20170513 instead");
}
@Test
public void testEmptyDate_returnsToday() {
assertThat(ReportingModule.provideDate(Optional.empty(), clock))
.isEqualTo(LocalDate.of(2017, 7, 1));
}
@Test
public void testGivenDate_returnsThatDate() {
assertThat(ReportingModule.provideDate(Optional.of(LocalDate.of(2017, 7, 2)), clock))
.isEqualTo(LocalDate.of(2017, 7, 2));
}
}

1
javatests/google/registry/reporting/spec11/BUILD
View file

@ -12,6 +12,7 @@ java_library(
srcs = glob(["*.java"]),
resources = glob(["testdata/*"]),
deps = [
"//java/google/registry/beam/spec11",
"//java/google/registry/gcs",
"//java/google/registry/reporting/spec11",
"//java/google/registry/util",

10
javatests/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java
View file

@ -35,7 +35,7 @@ import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeResponse;
import google.registry.testing.TaskQueueHelper.TaskMatcher;
import java.io.IOException;
import org.joda.time.YearMonth;
import java.time.LocalDate;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@ -87,20 +87,20 @@ public class GenerateSpec11ReportActionTest {
"gs://template",
"us-east1-c",
"api_key/a",
YearMonth.parse("2018-06"),
LocalDate.parse("2018-06-11"),
response,
dataflow);
action.run();
LaunchTemplateParameters expectedLaunchTemplateParameters =
new LaunchTemplateParameters()
.setJobName("spec11_2018-06")
.setJobName("spec11_2018-06-11")
.setEnvironment(
new RuntimeEnvironment()
.setZone("us-east1-c")
.setTempLocation("gs://my-bucket-beam/temporary"))
.setParameters(
ImmutableMap.of("safeBrowsingApiKey", "api_key/a", "yearMonth", "2018-06"));
ImmutableMap.of("safeBrowsingApiKey", "api_key/a", "date", "2018-06-11"));
verify(dataflowTemplates).launch("test", expectedLaunchTemplateParameters);
verify(dataflowLaunch).setGcsPath("gs://template");
assertThat(response.getStatus()).isEqualTo(200);
@ -112,7 +112,7 @@ public class GenerateSpec11ReportActionTest {
.url("/_dr/task/publishSpec11")
.method("POST")
.param("jobId", "jobid")
.param("yearMonth", "2018-06");
.param("date", "2018-06-11");
assertTasksEnqueued("beam-reporting", matcher);
}
}

25
javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java
View file

@ -32,7 +32,7 @@ import com.google.api.services.dataflow.model.Job;
import com.google.common.net.MediaType;
import google.registry.testing.FakeResponse;
import java.io.IOException;
import org.joda.time.YearMonth;
import java.time.LocalDate;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -67,12 +67,15 @@ public class PublishSpec11ReportActionTest {
response = new FakeResponse();
publishAction =
new PublishSpec11ReportAction(
"test-project", "12345", emailUtils, dataflow, response, new YearMonth(2018, 6));
"test-project", "12345", emailUtils, dataflow, response, LocalDate.of(2018, 6, 5));
}
@Test
public void testJobDone_emailsResults() {
public void testJobDone_emailsResultsOnSecondOfMonth() {
expectedJob.setCurrentState("JOB_STATE_DONE");
publishAction =
new PublishSpec11ReportAction(
"test-project", "12345", emailUtils, dataflow, response, LocalDate.of(2018, 6, 2));
publishAction.run();
assertThat(response.getStatus()).isEqualTo(SC_OK);
verify(emailUtils).emailSpec11Reports();
@ -85,8 +88,8 @@ public class PublishSpec11ReportActionTest {
assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
verify(emailUtils)
.sendAlertEmail(
"Spec11 Dataflow Pipeline Failure 2018-06",
"Spec11 2018-06 job 12345 ended in status failure.");
"Spec11 Dataflow Pipeline Failure 2018-06-05",
"Spec11 2018-06-05 job 12345 ended in status failure.");
}
@Test
@ -106,7 +109,15 @@ public class PublishSpec11ReportActionTest {
assertThat(response.getPayload()).isEqualTo("Template launch failed: expected");
verify(emailUtils)
.sendAlertEmail(
"Spec11 Publish Failure 2018-06",
"Spec11 2018-06 publish action failed due to expected");
"Spec11 Publish Failure 2018-06-05",
"Spec11 2018-06-05 publish action failed due to expected");
}
@Test
public void testJobDone_doesNotEmailResults() {
expectedJob.setCurrentState("JOB_STATE_DONE");
publishAction.run();
assertThat(response.getStatus()).isEqualTo(SC_OK);
verifyNoMoreInteractions(emailUtils);
}
}

42
javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java
View file

@ -16,6 +16,7 @@ package google.registry.reporting.spec11;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches;
import static google.registry.testing.JUnitBackports.assertThrows;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doThrow;
@ -24,16 +25,11 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import google.registry.gcs.GcsUtils;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeSleeper;
import google.registry.testing.TestDataHelper;
import google.registry.util.Retrier;
import google.registry.util.SendEmailService;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Properties;
import javax.annotation.Nullable;
@ -60,36 +56,29 @@ public class Spec11EmailUtilsTest {
private SendEmailService emailService;
private Spec11EmailUtils emailUtils;
private GcsUtils gcsUtils;
private Spec11RegistrarThreatMatchesParser parser;
private ArgumentCaptor<Message> gotMessage;
@Before
public void setUp() {
public void setUp() throws Exception {
emailService = mock(SendEmailService.class);
when(emailService.createMessage())
.thenAnswer((args) -> new MimeMessage(Session.getInstance(new Properties(), null)));
gcsUtils = mock(GcsUtils.class);
when(gcsUtils.openInputStream(
new GcsFilename("test-bucket", "icann/spec11/2018-06/SPEC11_MONTHLY_REPORT")))
.thenAnswer(
(args) ->
new ByteArrayInputStream(
loadFile("spec11_fake_report").getBytes(StandardCharsets.UTF_8)));
parser = mock(Spec11RegistrarThreatMatchesParser.class);
when(parser.getRegistrarThreatMatches()).thenReturn(sampleThreatMatches());
gotMessage = ArgumentCaptor.forClass(Message.class);
emailUtils =
new Spec11EmailUtils(
emailService,
new YearMonth(2018, 6),
new YearMonth(2018, 7),
"my-sender@test.com",
"my-receiver@test.com",
"my-reply-to@test.com",
"{LIST_OF_THREATS}\n{REPLY_TO_EMAIL}",
"test-bucket",
"icann/spec11/2018-06/SPEC11_MONTHLY_REPORT",
gcsUtils,
parser,
new Retrier(new FakeSleeper(new FakeClock()), RETRY_COUNT));
}
@ -104,21 +93,21 @@ public class Spec11EmailUtilsTest {
"my-sender@test.com",
"a@fake.com",
"my-reply-to@test.com",
"Google Registry Monthly Threat Detector [2018-06]",
"Google Registry Monthly Threat Detector [2018-07]",
"a.com - MALWARE\n\nmy-reply-to@test.com");
validateMessage(
capturedMessages.get(1),
"my-sender@test.com",
"b@fake.com",
"my-reply-to@test.com",
"Google Registry Monthly Threat Detector [2018-06]",
"Google Registry Monthly Threat Detector [2018-07]",
"b.com - MALWARE\nc.com - MALWARE\n\nmy-reply-to@test.com");
validateMessage(
capturedMessages.get(2),
"my-sender@test.com",
"my-receiver@test.com",
null,
"Spec11 Pipeline Success 2018-06",
"Spec11 Pipeline Success 2018-07",
"Spec11 reporting completed successfully.");
}
@ -162,20 +151,20 @@ public class Spec11EmailUtilsTest {
"my-sender@test.com",
"my-receiver@test.com",
null,
"Spec11 Emailing Failure 2018-06",
"Spec11 Emailing Failure 2018-07",
"Emailing spec11 reports failed due to expected");
}
@Test
public void testSuccess_sendAlertEmail() throws MessagingException, IOException {
emailUtils.sendAlertEmail("Spec11 Pipeline Alert: 2018-06", "Alert!");
emailUtils.sendAlertEmail("Spec11 Pipeline Alert: 2018-07", "Alert!");
verify(emailService).sendMessage(gotMessage.capture());
validateMessage(
gotMessage.getValue(),
"my-sender@test.com",
"my-receiver@test.com",
null,
"Spec11 Pipeline Alert: 2018-06",
"Spec11 Pipeline Alert: 2018-07",
"Alert!");
}
@ -203,9 +192,4 @@ public class Spec11EmailUtilsTest {
assertThat(message.getContentType()).isEqualTo("text/plain");
assertThat(message.getContent().toString()).isEqualTo(body);
}
/** Returns a {@link String} from a file in the {@code spec11/testdata/} directory. */
public static String loadFile(String filename) {
return TestDataHelper.loadFile(Spec11EmailUtils.class, filename);
}
}

99
javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java Normal file
View file

@ -0,0 +1,99 @@
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.reporting.spec11;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import google.registry.beam.spec11.ThreatMatch;
import google.registry.gcs.GcsUtils;
import google.registry.testing.TestDataHelper;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.util.List;
import org.json.JSONException;
import org.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link Spec11RegistrarThreatMatchesParser}. */
@RunWith(JUnit4.class)
public class Spec11RegistrarThreatMatchesParserTest {
private final GcsUtils gcsUtils = mock(GcsUtils.class);
private final Spec11RegistrarThreatMatchesParser parser =
new Spec11RegistrarThreatMatchesParser(LocalDate.of(2018, 7, 21), gcsUtils, "test-bucket");
@Before
public void setUp() {
when(gcsUtils.openInputStream(
new GcsFilename(
"test-bucket", "icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_2018-07-21")))
.thenAnswer(
(args) ->
new ByteArrayInputStream(
loadFile("spec11_fake_report").getBytes(StandardCharsets.UTF_8)));
}
@Test
public void testSuccess_retrievesReport() throws Exception {
List<RegistrarThreatMatches> matches = parser.getRegistrarThreatMatches();
assertThat(matches).isEqualTo(sampleThreatMatches());
}
/** Returns a {@link String} from a file in the {@code spec11/testdata/} directory. */
public static String loadFile(String filename) {
return TestDataHelper.loadFile(Spec11EmailUtils.class, filename);
}
/** The expected contents of the sample spec11 report file */
public static ImmutableList<RegistrarThreatMatches> sampleThreatMatches() throws JSONException {
return ImmutableList.of(
RegistrarThreatMatches.create(
"a@fake.com",
ImmutableList.of(
ThreatMatch.fromJSON(
new JSONObject(
ImmutableMap.of(
"threatType", "MALWARE",
"platformType", "ANY_PLATFORM",
"threatEntryMetadata", "NONE",
"fullyQualifiedDomainName", "a.com"))))),
RegistrarThreatMatches.create(
"b@fake.com",
ImmutableList.of(
ThreatMatch.fromJSON(
new JSONObject(
ImmutableMap.of(
"threatType", "MALWARE",
"platformType", "ANY_PLATFORM",
"threatEntryMetadata", "NONE",
"fullyQualifiedDomainName", "b.com"))),
ThreatMatch.fromJSON(
new JSONObject(
ImmutableMap.of(
"threatType", "MALWARE",
"platformType", "ANY_PLATFORM",
"threatEntryMetadata", "NONE",
"fullyQualifiedDomainName", "c.com"))))));
}
}