Deploy spec11 reporting to production

This turns on spec11 reporting in production by adding it to the cron.xml, generating the report and sending an e-mail with a list of all problematic registrations to the associated registrar on the 2nd of each month at 15:00Z (11am EST)

This also tweaks the e-mail template a bit according to suggestions from Bruno.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=213031440
This commit is contained in:
larryruili 2018-09-14 13:13:37 -07:00 committed by jianglai
parent cdcd5c2a0e
commit f36c72cca0
5 changed files with 49 additions and 21 deletions

View file

@ -271,7 +271,7 @@
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/generateInvoices&runInEmpty]]></url>
<description>
Starts the beam/InvoicingPipeline Dataflow template, which creates the overall invoice and
Starts the beam/invoicing/InvoicingPipeline Dataflow template, which creates the overall invoice and
detail report CSVs for last month, storing them in gs://[PROJECT-ID]-billing/invoices/yyyy-MM.
Upon success, sends an e-mail copy of the invoice to billing personnel, and copies detail
reports to the associated registrars' drive folders.
@ -285,4 +285,17 @@
<schedule>1 of month 17:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/generateSpec11&runInEmpty]]></url>
<description>
Starts the beam/spec11/Spec11Pipeline Dataflow template, which creates last month's Spec11
report. This report is stored in gs://[PROJECT-ID]-reporting/icann/spec11/yyyy-MM.
Upon Dataflow job completion, sends an e-mail to all registrars with domain registrations
flagged by the SafeBrowsing API.
See GenerateSpec11ReportAction for more details.
</description>
<schedule>2 of month 15:00</schedule>
<target>backend</target>
</cron>
</cronentries>

View file

@ -39,7 +39,7 @@ import org.joda.time.YearMonth;
* pipeline accordingly.
*
* <p>This calls {@link Spec11EmailUtils#emailSpec11Reports()} on success or {@link
* Spec11EmailUtils#sendFailureAlertEmail(String)} on failure.
* Spec11EmailUtils#sendAlertEmail(String, String)} on failure.
*/
@Action(path = PublishSpec11ReportAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_OR_ADMIN)
public class PublishSpec11ReportAction implements Runnable {
@ -88,7 +88,8 @@ public class PublishSpec11ReportAction implements Runnable {
case JOB_FAILED:
logger.atSevere().log("Dataflow job %s finished unsuccessfully.", jobId);
response.setStatus(SC_NO_CONTENT);
emailUtils.sendFailureAlertEmail(
emailUtils.sendAlertEmail(
String.format("Spec11 Dataflow Pipeline Failure %s", yearMonth.toString()),
String.format(
"Spec11 %s job %s ended in status failure.", yearMonth.toString(), jobId));
break;
@ -99,7 +100,8 @@ public class PublishSpec11ReportAction implements Runnable {
}
} catch (IOException e) {
logger.atSevere().withCause(e).log("Failed to publish Spec11 reports.");
emailUtils.sendFailureAlertEmail(
emailUtils.sendAlertEmail(
String.format("Spec11 Publish Failure %s", yearMonth.toString()),
String.format(
"Spec11 %s publish action failed due to %s", yearMonth.toString(), e.getMessage()));
response.setStatus(SC_INTERNAL_SERVER_ERROR);

View file

@ -97,12 +97,16 @@ public class Spec11EmailUtils {
MessagingException.class);
} catch (Throwable e) {
// Send an alert with the root cause, unwrapping the retrier's RuntimeException
sendFailureAlertEmail(
sendAlertEmail(
String.format("Spec11 Emailing Failure %s", yearMonth.toString()),
String.format(
"Emailing spec11 reports failed due to %s",
getRootCause(e).getMessage()));
throw new RuntimeException("Emailing spec11 report failed", e);
}
sendAlertEmail(
String.format("Spec11 Pipeline Success %s", yearMonth.toString()),
"Spec11 reporting completed successfully.");
}
private void emailRegistrar(String line) throws MessagingException, JSONException {
@ -110,10 +114,9 @@ public class Spec11EmailUtils {
JSONObject reportJSON = new JSONObject(line);
String registrarEmail = reportJSON.getString(Spec11Pipeline.REGISTRAR_EMAIL_FIELD);
JSONArray threatMatches = reportJSON.getJSONArray(Spec11Pipeline.THREAT_MATCHES_FIELD);
// TODO(b/112354588): Reword this e-mail according to business team's opinions.
StringBuilder body =
new StringBuilder("Hello registrar partner,\n")
.append("The SafeBrowsing API has detected problems with the following domains:\n");
.append("We have detected problems with the following domains:\n");
for (int i = 0; i < threatMatches.length(); i++) {
ThreatMatch threatMatch = ThreatMatch.fromJSON(threatMatches.getJSONObject(i));
body.append(
@ -124,23 +127,22 @@ public class Spec11EmailUtils {
.append("Regards,\nGoogle Registry\n");
Message msg = emailService.createMessage();
msg.setSubject(
String.format("Spec11 Monthly Threat Detector [%s]", yearMonth.toString()));
String.format("Google Registry Monthly Threat Detector [%s]", yearMonth.toString()));
msg.setText(body.toString());
msg.setFrom(new InternetAddress(alertSenderAddress));
msg.setRecipient(RecipientType.TO, new InternetAddress(registrarEmail));
emailService.sendMessage(msg);
}
/** Sends an e-mail to the provided alert e-mail address indicating a spec11 failure. */
void sendFailureAlertEmail(String body) {
/** Sends an e-mail indicating the state of the spec11 pipeline, with a given subject and body. */
void sendAlertEmail(String subject, String body) {
try {
retrier.callWithRetry(
() -> {
Message msg = emailService.createMessage();
msg.setFrom(new InternetAddress(alertSenderAddress));
msg.addRecipient(RecipientType.TO, new InternetAddress(alertRecipientAddress));
msg.setSubject(String.format("Spec11 Pipeline Alert: %s", yearMonth.toString()));
msg.setSubject(subject);
msg.setText(body);
emailService.sendMessage(msg);
return null;