diff --git a/gradle/core/build.gradle b/gradle/core/build.gradle index 27eba6897..a7ffd3e60 100644 --- a/gradle/core/build.gradle +++ b/gradle/core/build.gradle @@ -354,6 +354,12 @@ task soyToJava { }.filter { it.name.endsWith(".soy") }) + + soyToJava('google.registry.reporting.spec11.soy', + "${generatedDir}/google/registry/reporting/spec11/soy", + fileTree( + dir: "${javaDir}/google/registry/reporting/spec11/soy", + include: ['**/*.soy'])) } } diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index a00f10132..a2299f917 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -900,14 +900,25 @@ public final class RegistryConfig { } /** - * Returns the template for the body of the spec 11 email to the registrars. + * Returns the name of the registry, for use in spec 11 emails. * * @see google.registry.reporting.spec11.Spec11EmailUtils */ @Provides - @Config("spec11EmailBodyTemplate") - public static String provideSpec11EmailBodyTemplate(RegistryConfigSettings config) { - return config.registryPolicy.spec11EmailBodyTemplate; + @Config("registryName") + public static String provideRegistryName(RegistryConfigSettings config) { + return config.registryPolicy.registryName; + } + + /** + * Returns a list of resources we send to registrars when informing them of spec 11 threats. + * + * @see google.registry.reporting.spec11.Spec11EmailUtils + */ + @Provides + @Config("spec11WebResources") + public static ImmutableList provideSpec11WebResources(RegistryConfigSettings config) { + return ImmutableList.copyOf(config.registryPolicy.spec11WebResources); } /** diff --git a/java/google/registry/config/RegistryConfigSettings.java b/java/google/registry/config/RegistryConfigSettings.java index e7b1eac16..dbbf4ffc4 100644 --- a/java/google/registry/config/RegistryConfigSettings.java +++ b/java/google/registry/config/RegistryConfigSettings.java @@ -92,7 +92,8 @@ public class RegistryConfigSettings { public String whoisDisclaimer; public String rdapTos; public String rdapTosStaticUrl; - public String spec11EmailBodyTemplate; + public String registryName; + public List spec11WebResources; public boolean requireSslCertificates; } diff --git a/java/google/registry/config/files/default-config.yaml b/java/google/registry/config/files/default-config.yaml index 17e2bbf19..8062689a5 100644 --- a/java/google/registry/config/files/default-config.yaml +++ b/java/google/registry/config/files/default-config.yaml @@ -160,32 +160,12 @@ registryPolicy: # responses. If null, no static Web page link is generated. rdapTosStaticUrl: null - # Body of the spec 11 email sent to registrars. - # Items in braces are to be replaced. - spec11EmailBodyTemplate: | - Dear registrar partner, + # Name of the registry for use in spec 11 emails + registryName: Example Registry - The registry conducts periodic technical analyses of all domains registered - in its TLDs. As part of this analysis, the following domains that you - manage were flagged for potential security concerns: - - {LIST_OF_THREATS} - - Please communicate these findings to the registrant and work with the - registrant to mitigate any security issues and have the domains delisted. - - Some helpful sites for getting off a blocked list include: - - - Google Search Console (https://search.google.com/search-console/about) - -- includes information and tools for webmasters to learn about and - mitigate security threats and have their websites delisted - - first.org -- a registry of Computer Emergency Response Teams (CERTs) - that may be able to assist in mitigation - - stopbadware.org -- a non-profit anti-malware organization that provides - support and information for webmasters dealing with security threats - - If you have any questions regarding this notice, please contact - {REPLY_TO_EMAIL}. + # A list of resources we send to registrars when informing them of + # spec 11 threats + spec11WebResources: [] # Whether to require an SSL certificate hash in order to be able to log in # via EPP and run commands. This can be false for testing environments but @@ -259,11 +239,11 @@ caching: oAuth: # OAuth scopes to detect on access tokens. Superset of requiredOauthScopes. availableOauthScopes: - - https://www.googleapis.com/auth/userinfo.email + - https://www.googleapis.com/auth/userinfo.email # OAuth scopes required for authenticating. Subset of availableOauthScopes. requiredOauthScopes: - - https://www.googleapis.com/auth/userinfo.email + - https://www.googleapis.com/auth/userinfo.email # OAuth client IDs that are allowed to authenticate and communicate with # backend services, e. g. nomulus tool, EPP proxy, etc. The client_id value @@ -276,10 +256,10 @@ credentialOAuth: # OAuth scopes required for accessing Google APIs using the default # credential. defaultCredentialOauthScopes: - # View and manage data in all Google Cloud APIs. - - https://www.googleapis.com/auth/cloud-platform - # View and manage files in Google Drive, e.g., Docs and Sheets. - - https://www.googleapis.com/auth/drive + # View and manage data in all Google Cloud APIs. + - https://www.googleapis.com/auth/cloud-platform + # View and manage files in Google Drive, e.g., Docs and Sheets. + - https://www.googleapis.com/auth/drive # OAuth scopes required for delegated admin access to G Suite domain. # Deployment of changes to this list must be coordinated with G Suite admin # configuration, which can be managed in the admin console: @@ -288,20 +268,20 @@ credentialOAuth: # - Removed scopes must remain on G Suite domain configuration until the # release is deployed. delegatedCredentialOauthScopes: - # View and manage groups on your domain in Directory API. - - https://www.googleapis.com/auth/admin.directory.group - # View and manage group settings in Group Settings API. - - https://www.googleapis.com/auth/apps.groups.settings + # View and manage groups on your domain in Directory API. + - https://www.googleapis.com/auth/admin.directory.group + # View and manage group settings in Group Settings API. + - https://www.googleapis.com/auth/apps.groups.settings # OAuth scopes required to create a credential locally in for the nomulus tool. localCredentialOauthScopes: - # View and manage data in all Google Cloud APIs. - - https://www.googleapis.com/auth/cloud-platform - # Call App Engine APIs locally. - - https://www.googleapis.com/auth/appengine.apis - # View your email address. - - https://www.googleapis.com/auth/userinfo.email - # View and manage your applications deployed on Google App Engine - - https://www.googleapis.com/auth/appengine.admin + # View and manage data in all Google Cloud APIs. + - https://www.googleapis.com/auth/cloud-platform + # Call App Engine APIs locally. + - https://www.googleapis.com/auth/appengine.apis + # View your email address. + - https://www.googleapis.com/auth/userinfo.email + # View and manage your applications deployed on Google App Engine + - https://www.googleapis.com/auth/appengine.admin icannReporting: # URL we PUT monthly ICANN transactions reports to. diff --git a/java/google/registry/reporting/spec11/BUILD b/java/google/registry/reporting/spec11/BUILD index a7b4f296a..96d18495e 100644 --- a/java/google/registry/reporting/spec11/BUILD +++ b/java/google/registry/reporting/spec11/BUILD @@ -13,6 +13,7 @@ java_library( "//java/google/registry/gcs", "//java/google/registry/keyring/api", "//java/google/registry/reporting", + "//java/google/registry/reporting/spec11/soy:soy_java_wrappers", "//java/google/registry/request", "//java/google/registry/request/auth", "//java/google/registry/util", @@ -26,6 +27,7 @@ java_library( "@com_google_flogger_system_backend", "@com_google_guava", "@com_google_http_client", + "@io_bazel_rules_closure//closure/templates", "@javax_inject", "@javax_servlet_api", "@joda_time", diff --git a/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java b/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java index 7a7870062..6e6d99856 100644 --- a/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java +++ b/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java @@ -14,6 +14,7 @@ package google.registry.reporting.spec11; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED; @@ -22,17 +23,24 @@ import static javax.servlet.http.HttpServletResponse.SC_OK; import com.google.api.services.dataflow.Dataflow; import com.google.api.services.dataflow.model.Job; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; +import com.google.template.soy.parseinfo.SoyTemplateInfo; +import google.registry.beam.spec11.ThreatMatch; import google.registry.config.RegistryConfig.Config; import google.registry.reporting.ReportingModule; +import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo; import google.registry.request.Action; import google.registry.request.Parameter; import google.registry.request.Response; import google.registry.request.auth.Auth; import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import javax.inject.Inject; import org.joda.time.LocalDate; import org.json.JSONException; @@ -41,8 +49,8 @@ import org.json.JSONException; * Retries until a {@code Dataflow} job with a given {@code jobId} completes, continuing the Spec11 * pipeline accordingly. * - *

This calls {@link Spec11EmailUtils#emailSpec11Reports(String, String, List)} ()} on success or - * {@link Spec11EmailUtils#sendAlertEmail(String, String)} on failure. + *

This calls {@link Spec11EmailUtils#emailSpec11Reports(SoyTemplateInfo, String, Set)} on + * success or {@link Spec11EmailUtils#sendAlertEmail(String, String)} on failure. */ @Action(path = PublishSpec11ReportAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_OR_ADMIN) public class PublishSpec11ReportAction implements Runnable { @@ -54,7 +62,7 @@ public class PublishSpec11ReportAction implements Runnable { private static final String JOB_FAILED = "JOB_STATE_FAILED"; private final String projectId; - private final String spec11EmailBodyTemplate; + private final String registryName; private final String jobId; private final Spec11EmailUtils emailUtils; private final Spec11RegistrarThreatMatchesParser spec11RegistrarThreatMatchesParser; @@ -65,7 +73,7 @@ public class PublishSpec11ReportAction implements Runnable { @Inject PublishSpec11ReportAction( @Config("projectId") String projectId, - @Config("spec11EmailBodyTemplate") String spec11EmailBodyTemplate, + @Config("registryName") String registryName, @Parameter(ReportingModule.PARAM_JOB_ID) String jobId, Spec11EmailUtils emailUtils, Spec11RegistrarThreatMatchesParser spec11RegistrarThreatMatchesParser, @@ -73,7 +81,7 @@ public class PublishSpec11ReportAction implements Runnable { Response response, LocalDate date) { this.projectId = projectId; - this.spec11EmailBodyTemplate = spec11EmailBodyTemplate; + this.registryName = registryName; this.jobId = jobId; this.emailUtils = emailUtils; this.spec11RegistrarThreatMatchesParser = spec11RegistrarThreatMatchesParser; @@ -90,14 +98,22 @@ public class PublishSpec11ReportAction implements Runnable { String state = job.getCurrentState(); switch (state) { case JOB_DONE: - logger.atInfo().log( - "Dataflow job %s finished successfully, publishing results if appropriate.", jobId); + logger.atInfo().log("Dataflow job %s finished successfully, publishing results.", jobId); response.setStatus(SC_OK); - if (shouldSendSpec11Email()) { - ImmutableList matchesList = - spec11RegistrarThreatMatchesParser.getRegistrarThreatMatches(); - String subject = String.format("Google Registry Monthly Threat Detector [%s]", date); - emailUtils.emailSpec11Reports(spec11EmailBodyTemplate, subject, matchesList); + if (shouldSendMonthlySpec11Email()) { + sendMonthlyEmail(); + } else { + Optional previousDate = + spec11RegistrarThreatMatchesParser.getPreviousDateWithMatches(date); + if (previousDate.isPresent()) { + processDailyDiff(previousDate.get()); + } else { + emailUtils.sendAlertEmail( + String.format("Spec11 Diff Error %s", date), + String.format( + "Could not find a previous file within the past month of %s", date)); + response.setStatus(SC_NO_CONTENT); + } } break; case JOB_FAILED: @@ -123,8 +139,51 @@ public class PublishSpec11ReportAction implements Runnable { } } - private boolean shouldSendSpec11Email() { - // TODO(b/120496893): send emails every day with the diff content + private void sendMonthlyEmail() throws IOException, JSONException { + ImmutableSet monthlyMatchesSet = + spec11RegistrarThreatMatchesParser.getRegistrarThreatMatches(date); + String subject = String.format("%s Monthly Threat Detector [%s]", registryName, date); + emailUtils.emailSpec11Reports( + Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL, subject, monthlyMatchesSet); + } + + private void processDailyDiff(LocalDate previousDate) throws IOException, JSONException { + ImmutableSet previousMatches = + spec11RegistrarThreatMatchesParser.getRegistrarThreatMatches(previousDate); + ImmutableSet currentMatches = + spec11RegistrarThreatMatchesParser.getRegistrarThreatMatches(date); + String dailySubject = String.format("%s Daily Threat Detector [%s]", registryName, date); + emailUtils.emailSpec11Reports( + Spec11EmailSoyInfo.DAILY_SPEC_11_EMAIL, + dailySubject, + getNewMatches(previousMatches, currentMatches)); + } + + private ImmutableSet getNewMatches( + Set previousMatchesSet, + Set currentMatchesSet) { + Map> currentMatchMap = + currentMatchesSet.stream() + .collect( + Collectors.toMap( + RegistrarThreatMatches::registrarEmailAddress, + RegistrarThreatMatches::threatMatches)); + previousMatchesSet.forEach( + previousMatches -> + currentMatchMap.computeIfPresent( + previousMatches.registrarEmailAddress(), + (email, currentMatches) -> + currentMatches.stream() + .filter( + currentMatch -> !previousMatches.threatMatches().contains(currentMatch)) + .collect(Collectors.toList()))); + return currentMatchMap.entrySet().stream() + .filter(entry -> !entry.getValue().isEmpty()) + .map(entry -> RegistrarThreatMatches.create(entry.getKey(), entry.getValue())) + .collect(toImmutableSet()); + } + + private boolean shouldSendMonthlySpec11Email() { return date.getDayOfMonth() == 2; } } diff --git a/java/google/registry/reporting/spec11/Spec11EmailUtils.java b/java/google/registry/reporting/spec11/Spec11EmailUtils.java index 7b54068e3..89cf7f996 100644 --- a/java/google/registry/reporting/spec11/Spec11EmailUtils.java +++ b/java/google/registry/reporting/spec11/Spec11EmailUtils.java @@ -15,13 +15,23 @@ package google.registry.reporting.spec11; import static com.google.common.base.Throwables.getRootCause; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.io.Resources.getResource; -import google.registry.beam.spec11.ThreatMatch; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.template.soy.SoyFileSet; +import com.google.template.soy.parseinfo.SoyTemplateInfo; +import com.google.template.soy.tofu.SoyTofu; +import com.google.template.soy.tofu.SoyTofu.Renderer; import google.registry.config.RegistryConfig.Config; +import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo; import google.registry.util.Retrier; import google.registry.util.SendEmailService; import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.Set; import javax.inject.Inject; import javax.mail.Message; import javax.mail.Message.RecipientType; @@ -32,11 +42,22 @@ import org.joda.time.LocalDate; /** Provides e-mail functionality for Spec11 tasks, such as sending Spec11 reports to registrars. */ public class Spec11EmailUtils { + private static final SoyTofu SOY_SAUCE = + SoyFileSet.builder() + .add( + getResource( + Spec11EmailSoyInfo.getInstance().getClass(), + Spec11EmailSoyInfo.getInstance().getFileName())) + .build() + .compileToTofu(); + private final SendEmailService emailService; private final LocalDate date; private final String outgoingEmailAddress; private final String alertRecipientAddress; private final String spec11ReplyToAddress; + private final ImmutableList spec11WebResources; + private final String registryName; private final Retrier retrier; @Inject @@ -46,12 +67,16 @@ public class Spec11EmailUtils { @Config("gSuiteOutgoingEmailAddress") String outgoingEmailAddress, @Config("alertRecipientEmailAddress") String alertRecipientAddress, @Config("spec11ReplyToEmailAddress") String spec11ReplyToAddress, + @Config("spec11WebResources") ImmutableList spec11WebResources, + @Config("registryName") String registryName, Retrier retrier) { this.emailService = emailService; this.date = date; this.outgoingEmailAddress = outgoingEmailAddress; this.alertRecipientAddress = alertRecipientAddress; this.spec11ReplyToAddress = spec11ReplyToAddress; + this.spec11WebResources = spec11WebResources; + this.registryName = registryName; this.retrier = retrier; } @@ -60,14 +85,14 @@ public class Spec11EmailUtils { * appropriate address. */ void emailSpec11Reports( - String spec11EmailBodyTemplate, + SoyTemplateInfo soyTemplateInfo, String subject, - List registrarThreatMatchesList) { + Set registrarThreatMatchesSet) { try { retrier.callWithRetry( () -> { - for (RegistrarThreatMatches registrarThreatMatches : registrarThreatMatchesList) { - emailRegistrar(spec11EmailBodyTemplate, subject, registrarThreatMatches); + for (RegistrarThreatMatches registrarThreatMatches : registrarThreatMatchesSet) { + emailRegistrar(soyTemplateInfo, subject, registrarThreatMatches); } }, IOException.class, @@ -85,28 +110,45 @@ public class Spec11EmailUtils { } private void emailRegistrar( - String spec11EmailBodyTemplate, String subject, RegistrarThreatMatches registrarThreatMatches) + SoyTemplateInfo soyTemplateInfo, + String subject, + RegistrarThreatMatches registrarThreatMatches) throws MessagingException { - String registrarEmail = registrarThreatMatches.registrarEmailAddress(); - StringBuilder threatList = new StringBuilder(); - for (ThreatMatch threatMatch : registrarThreatMatches.threatMatches()) { - threatList.append( - String.format( - "%s - %s\n", threatMatch.fullyQualifiedDomainName(), threatMatch.threatType())); - } - String body = - spec11EmailBodyTemplate - .replace("{REPLY_TO_EMAIL}", spec11ReplyToAddress) - .replace("{LIST_OF_THREATS}", threatList.toString()); Message msg = emailService.createMessage(); msg.setSubject(subject); - msg.setText(body); + String content = getContent(soyTemplateInfo, registrarThreatMatches); + msg.setContent(content, "text/html"); + msg.setHeader("Content-Type", "text/html"); msg.setFrom(new InternetAddress(outgoingEmailAddress)); - msg.addRecipient(RecipientType.TO, new InternetAddress(registrarEmail)); + msg.addRecipient( + RecipientType.TO, new InternetAddress(registrarThreatMatches.registrarEmailAddress())); msg.addRecipient(RecipientType.BCC, new InternetAddress(spec11ReplyToAddress)); emailService.sendMessage(msg); } + private String getContent( + SoyTemplateInfo soyTemplateInfo, RegistrarThreatMatches registrarThreatMatches) { + Renderer renderer = SOY_SAUCE.newRenderer(soyTemplateInfo); + // Soy templates require that data be in raw map/list form. + List> threatMatchMap = + registrarThreatMatches.threatMatches().stream() + .map( + threatMatch -> + ImmutableMap.of( + "fullyQualifiedDomainName", threatMatch.fullyQualifiedDomainName(), + "threatType", threatMatch.threatType())) + .collect(toImmutableList()); + + Map data = + ImmutableMap.of( + "registry", registryName, + "replyToEmail", spec11ReplyToAddress, + "threats", threatMatchMap, + "resources", spec11WebResources); + renderer.setData(data); + return renderer.render(); + } + /** Sends an e-mail indicating the state of the spec11 pipeline, with a given subject and body. */ void sendAlertEmail(String subject, String body) { try { diff --git a/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java b/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java index 4e16ebc01..ef139b30d 100644 --- a/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java +++ b/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java @@ -18,6 +18,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.FluentLogger; import com.google.common.io.CharStreams; import google.registry.beam.spec11.Spec11Pipeline; import google.registry.beam.spec11.ThreatMatch; @@ -26,6 +28,7 @@ import google.registry.gcs.GcsUtils; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.Optional; import javax.inject.Inject; import org.joda.time.LocalDate; import org.json.JSONArray; @@ -35,25 +38,53 @@ import org.json.JSONObject; /** Parser to retrieve which registrar-threat matches we should notify via email */ public class Spec11RegistrarThreatMatchesParser { - private final LocalDate date; + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private final GcsUtils gcsUtils; private final String reportingBucket; @Inject public Spec11RegistrarThreatMatchesParser( - LocalDate date, GcsUtils gcsUtils, @Config("reportingBucket") String reportingBucket) { - this.date = date; + GcsUtils gcsUtils, @Config("reportingBucket") String reportingBucket) { this.gcsUtils = gcsUtils; this.reportingBucket = reportingBucket; } - /** Gets the list of registrar:set-of-threat-match pairings from the file in GCS. */ - public ImmutableList getRegistrarThreatMatches() + /** + * Gets the entire set of registrar:set-of-threat-match pairings from the most recent report file + * in GCS. + */ + public ImmutableSet getRegistrarThreatMatches(LocalDate date) throws IOException, JSONException { - // TODO(b/120078223): this should only be the diff of this run and the prior run. - GcsFilename spec11ReportFilename = - new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(date)); - ImmutableList.Builder builder = ImmutableList.builder(); + return getFromFile(getGcsFilename(date)); + } + + public Optional getPreviousDateWithMatches(LocalDate date) { + LocalDate yesterday = date.minusDays(1); + GcsFilename gcsFilename = getGcsFilename(yesterday); + if (gcsUtils.existsAndNotEmpty(gcsFilename)) { + return Optional.of(yesterday); + } + logger.atWarning().log("Could not find previous file from date %s", yesterday); + + for (LocalDate dateToCheck = yesterday.minusDays(1); + !dateToCheck.isBefore(date.minusMonths(1)); + dateToCheck = dateToCheck.minusDays(1)) { + gcsFilename = getGcsFilename(dateToCheck); + if (gcsUtils.existsAndNotEmpty(gcsFilename)) { + return Optional.of(dateToCheck); + } + } + return Optional.empty(); + } + + private GcsFilename getGcsFilename(LocalDate localDate) { + return new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate)); + } + + private ImmutableSet getFromFile(GcsFilename spec11ReportFilename) + throws IOException, JSONException { + ImmutableSet.Builder builder = ImmutableSet.builder(); try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) { ImmutableList reportLines = ImmutableList.copyOf(CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n")); diff --git a/java/google/registry/reporting/spec11/soy/BUILD b/java/google/registry/reporting/spec11/soy/BUILD new file mode 100644 index 000000000..e06e03688 --- /dev/null +++ b/java/google/registry/reporting/spec11/soy/BUILD @@ -0,0 +1,13 @@ +package( + default_visibility = ["//java/google/registry:registry_project"], +) + +licenses(["notice"]) # Apache 2.0 + +load("@io_bazel_rules_closure//closure:defs.bzl", "closure_java_template_library") + +closure_java_template_library( + name = "soy_java_wrappers", + srcs = glob(["*.soy"]), + java_package = "google.registry.reporting.spec11.soy", +) diff --git a/java/google/registry/reporting/spec11/soy/Spec11Email.soy b/java/google/registry/reporting/spec11/soy/Spec11Email.soy new file mode 100644 index 000000000..44c9eaf61 --- /dev/null +++ b/java/google/registry/reporting/spec11/soy/Spec11Email.soy @@ -0,0 +1,126 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{namespace registry.soy.reporting.spec11} + +/** + * Template for the content of the monthly spec11 email + */ +{template .monthlySpec11Email} + {@param threats: list>} + {@param resources: list} + {@param registry: string} + {@param replyToEmail: string} + + Dear registrar partner, + +

{$registry} previously notified you when the following domains managed by your + registrar were flagged for potential security concerns.

+ +

The following domains that you manage continue to be flagged by our analysis for potential + security concerns. This may be because the registrants have not completed the requisite steps + to mitigate the potential security abuse and/or have it reviewed and delisted.

+ + {call .threatMatchTable} + {param threats: $threats /} + {/call} + +

Please work with the registrant to mitigate any security issues and have the + domains delisted.

+ + {call .resourceList} + {param resources: $resources /} + {/call} + +

You will continue to receive a monthly summary of all domains managed by your registrar + that remain on our lists of potential security threats. You will additionally receive a daily + notice when any new domains that are added to these lists. Once the registrant has resolved + the security issues and followed the steps to have his or her domain reviewed and delisted + it will automatically be removed from our monthly reporting.

+ +

If you have any questions regarding this notice, please contact {$replyToEmail}.

+{/template} + +/** + * Template for the content of the daily spec11 email + */ +{template .dailySpec11Email} + {@param threats: list>} + {@param resources: list} + {@param date: string} + {@param registry: string} + {@param replyToEmail: string} + + Dear registrar partner, + +

{$registry} conducts a daily analysis of all domains registered in its TLDs to + identify potential security concerns. On {$date}, the following domains that your + registrar manages were flagged for potential security concerns:

+ + {call .threatMatchTable} + {param threats: $threats /} + {/call} + +

Please communicate these findings to the registrant and work with the + registrant to mitigate any security issues and have the domains delisted.

+ + {call .resourceList} + {param resources: $resources /} + {/call} + +

You will continue to receive daily notices when new domains managed by your registrar + are flagged for abuse, as well as a monthly summary of all of your domains under management + that remain flagged for abuse. Once the registrant has resolved the security issues and + followed the steps to have his or her domain reviewed and delisted it will automatically + be removed from our reporting.

+ +

If you would like to change the email to which these notices are sent please update your + abuse contact using your registrar portal account.

+ +

If you have any questions regarding this notice, please contact {$replyToEmail}.

+{/template} + +/** + * Template for the list of potentially-useful resources + */ +{template .resourceList} + {@param resources: list} + {if length($resources) > 0} + Some helpful resources for getting off a blocked list include: +
    + {for $resource in $resources} +
  • {$resource}
    • + {/for} +
+ {/if} +{/template} + +/** + * Template for the table containing the threats themselves + */ +{template .threatMatchTable} + {@param threats: list>} + + + + + + {for $threat in $threats} + + + + + {/for} +
Domain NameThreat Type
{$threat['fullyQualifiedDomainName']}{$threat['threatType']}
+{/template} diff --git a/javatests/google/registry/reporting/spec11/BUILD b/javatests/google/registry/reporting/spec11/BUILD index 6eb863e8b..42c62284d 100644 --- a/javatests/google/registry/reporting/spec11/BUILD +++ b/javatests/google/registry/reporting/spec11/BUILD @@ -15,6 +15,7 @@ java_library( "//java/google/registry/beam/spec11", "//java/google/registry/gcs", "//java/google/registry/reporting/spec11", + "//java/google/registry/reporting/spec11/soy:soy_java_wrappers", "//java/google/registry/util", "//javatests/google/registry/testing", "@com_google_apis_google_api_services_dataflow", diff --git a/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java b/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java index 3a60f3b6e..65b1d7ed4 100644 --- a/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java +++ b/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java @@ -30,9 +30,12 @@ import com.google.api.services.dataflow.Dataflow.Projects; import com.google.api.services.dataflow.Dataflow.Projects.Jobs; import com.google.api.services.dataflow.Dataflow.Projects.Jobs.Get; import com.google.api.services.dataflow.model.Job; +import com.google.common.collect.ImmutableSet; import com.google.common.net.MediaType; +import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo; import google.registry.testing.FakeResponse; import java.io.IOException; +import java.util.Optional; import org.joda.time.LocalDate; import org.junit.Before; import org.junit.Test; @@ -43,7 +46,6 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class PublishSpec11ReportActionTest { - private final String spec11BodyTemplate = "{LIST_OF_THREATS}\n{REPLY_TO_EMAIL}"; private final LocalDate date = new LocalDate(2018, 6, 5); private Dataflow dataflow; @@ -69,41 +71,44 @@ public class PublishSpec11ReportActionTest { expectedJob = new Job(); when(get.execute()).thenReturn(expectedJob); emailUtils = mock(Spec11EmailUtils.class); + parser = mock(Spec11RegistrarThreatMatchesParser.class); response = new FakeResponse(); parser = mock(Spec11RegistrarThreatMatchesParser.class); - when(parser.getRegistrarThreatMatches()).thenReturn(sampleThreatMatches()); publishAction = new PublishSpec11ReportAction( "test-project", - spec11BodyTemplate, + "Super Cool Registry", "12345", emailUtils, - mock(Spec11RegistrarThreatMatchesParser.class), + parser, dataflow, response, date); } @Test - public void testJobDone_emailsResultsOnSecondOfMonth() throws Exception { + public void testJobDone_emailsOnlyMonthlyResultsOnSecondOfMonth() throws Exception { + LocalDate secondOfMonth = date.withDayOfMonth(2); + when(parser.getRegistrarThreatMatches(secondOfMonth)).thenReturn(sampleThreatMatches()); expectedJob.setCurrentState("JOB_STATE_DONE"); publishAction = new PublishSpec11ReportAction( "test-project", - spec11BodyTemplate, + "Super Cool Registry", "12345", emailUtils, parser, dataflow, response, - date.withDayOfMonth(2)); + secondOfMonth); publishAction.run(); assertThat(response.getStatus()).isEqualTo(SC_OK); verify(emailUtils) .emailSpec11Reports( - spec11BodyTemplate, - "Google Registry Monthly Threat Detector [2018-06-02]", + Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL, + "Super Cool Registry Monthly Threat Detector [2018-06-02]", sampleThreatMatches()); + verifyNoMoreInteractions(emailUtils); } @Test @@ -139,10 +144,31 @@ public class PublishSpec11ReportActionTest { } @Test - public void testJobDone_doesNotEmailResults() { + public void testJobDone_onlyDailyResults() throws Exception { + LocalDate yesterday = date.minusDays(1); + when(parser.getPreviousDateWithMatches(date)).thenReturn(Optional.of(yesterday)); + when(parser.getRegistrarThreatMatches(date)).thenReturn(sampleThreatMatches()); + when(parser.getRegistrarThreatMatches(yesterday)).thenReturn(ImmutableSet.of()); expectedJob.setCurrentState("JOB_STATE_DONE"); publishAction.run(); assertThat(response.getStatus()).isEqualTo(SC_OK); + verify(emailUtils) + .emailSpec11Reports( + Spec11EmailSoyInfo.DAILY_SPEC_11_EMAIL, + "Super Cool Registry Daily Threat Detector [2018-06-05]", + sampleThreatMatches()); verifyNoMoreInteractions(emailUtils); } + + @Test + public void testJobDone_failsDueToNoPreviousResults() { + when(parser.getPreviousDateWithMatches(date)).thenReturn(Optional.empty()); + expectedJob.setCurrentState("JOB_STATE_DONE"); + publishAction.run(); + assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT); + verify(emailUtils) + .sendAlertEmail( + String.format("Spec11 Diff Error %s", date), + String.format("Could not find a previous file within the past month of %s", date)); + } } diff --git a/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java b/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java index c170e0c54..2f841e361 100644 --- a/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java +++ b/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java @@ -25,6 +25,8 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.collect.ImmutableList; +import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo; import google.registry.testing.FakeClock; import google.registry.testing.FakeSleeper; import google.registry.util.Retrier; @@ -53,6 +55,7 @@ import org.mockito.stubbing.Answer; public class Spec11EmailUtilsTest { private static final int RETRY_COUNT = 2; + private static final ImmutableList FAKE_RESOURCES = ImmutableList.of("foo"); private SendEmailService emailService; private Spec11EmailUtils emailUtils; @@ -66,50 +69,75 @@ public class Spec11EmailUtilsTest { .thenAnswer((args) -> new MimeMessage(Session.getInstance(new Properties(), null))); parser = mock(Spec11RegistrarThreatMatchesParser.class); - when(parser.getRegistrarThreatMatches()).thenReturn(sampleThreatMatches()); + LocalDate date = new LocalDate(2018, 7, 15); + when(parser.getRegistrarThreatMatches(date)).thenReturn(sampleThreatMatches()); gotMessage = ArgumentCaptor.forClass(Message.class); emailUtils = new Spec11EmailUtils( emailService, - new LocalDate(2018, 7, 15), + date, "my-sender@test.com", "my-receiver@test.com", "my-reply-to@test.com", + FAKE_RESOURCES, + "Super Cool Registry", new Retrier(new FakeSleeper(new FakeClock()), RETRY_COUNT)); } @Test public void testSuccess_emailSpec11Reports() throws Exception { emailUtils.emailSpec11Reports( - "{LIST_OF_THREATS}\n{REPLY_TO_EMAIL}", - "Google Registry Monthly Threat Detector [2018-07-15]", + Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL, + "Super Cool Registry Monthly Threat Detector [2018-07-15]", sampleThreatMatches()); // We inspect individual parameters because Message doesn't implement equals(). verify(emailService, times(3)).sendMessage(gotMessage.capture()); List capturedMessages = gotMessage.getAllValues(); + String emailFormat = + "Dear registrar partner,

Super Cool Registry previously notified you when the following " + + "domains managed by your registrar were flagged for potential security concerns." + + "

The following domains that you manage continue to be flagged by our analysis " + + "for potential security concerns. This may be because the registrants have not " + + "completed the requisite steps to mitigate the potential security abuse and/or have " + + "it reviewed and delisted.

" + + "%s
Domain NameThreat Type

Please work with the registrant to mitigate any security issues " + + "and have the domains delisted.

Some helpful resources for getting off a blocked " + + "list include:
  • foo

You will continue to receive a monthly summary " + + "of all domains managed by your registrar that remain on our lists of potential " + + "security threats. You will additionally receive a daily notice when any new domains " + + "that are added to these lists. Once the registrant has resolved the security issues " + + "and followed the steps to have his or her domain reviewed and delisted it will " + + "automatically be removed from our monthly reporting.

If you have any q" + + "uestions regarding this notice, please contact my-reply-to@test.com.

"; + validateMessage( capturedMessages.get(0), "my-sender@test.com", "a@fake.com", "my-reply-to@test.com", - "Google Registry Monthly Threat Detector [2018-07-15]", - "a.com - MALWARE\n\nmy-reply-to@test.com"); + "Super Cool Registry Monthly Threat Detector [2018-07-15]", + String.format(emailFormat, "a.comMALWARE"), + "text/html"); validateMessage( capturedMessages.get(1), "my-sender@test.com", "b@fake.com", "my-reply-to@test.com", - "Google Registry Monthly Threat Detector [2018-07-15]", - "b.com - MALWARE\nc.com - MALWARE\n\nmy-reply-to@test.com"); + "Super Cool Registry Monthly Threat Detector [2018-07-15]", + String.format( + emailFormat, + "b.comMALWAREc.comMALWARE"), + "text/html"); validateMessage( capturedMessages.get(2), "my-sender@test.com", "my-receiver@test.com", null, "Spec11 Pipeline Success 2018-07-15", - "Spec11 reporting completed successfully."); + "Spec11 reporting completed successfully.", + "text/plain"); } @Test @@ -139,7 +167,9 @@ public class Spec11EmailUtilsTest { RuntimeException thrown = assertThrows( RuntimeException.class, - () -> emailUtils.emailSpec11Reports("foo", "bar", sampleThreatMatches())); + () -> + emailUtils.emailSpec11Reports( + Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL, "bar", sampleThreatMatches())); assertThat(thrown).hasMessageThat().isEqualTo("Emailing spec11 report failed"); assertThat(thrown) .hasCauseThat() @@ -155,7 +185,8 @@ public class Spec11EmailUtilsTest { "my-receiver@test.com", null, "Spec11 Emailing Failure 2018-07-15", - "Emailing spec11 reports failed due to expected"); + "Emailing spec11 reports failed due to expected", + "text/plain"); } @Test @@ -168,7 +199,8 @@ public class Spec11EmailUtilsTest { "my-receiver@test.com", null, "Spec11 Pipeline Alert: 2018-07", - "Alert!"); + "Alert!", + "text/plain"); } private void validateMessage( @@ -177,7 +209,8 @@ public class Spec11EmailUtilsTest { String recipient, @Nullable String replyTo, String subject, - String body) + String body, + String contentType) throws MessagingException, IOException { assertThat(message.getFrom()).asList().containsExactly(new InternetAddress(from)); assertThat(message.getRecipients(RecipientType.TO)) @@ -192,7 +225,7 @@ public class Spec11EmailUtilsTest { } assertThat(message.getRecipients(RecipientType.CC)).isNull(); assertThat(message.getSubject()).isEqualTo(subject); - assertThat(message.getContentType()).isEqualTo("text/plain"); - assertThat(message.getContent().toString()).isEqualTo(body); + assertThat(message.getContentType()).isEqualTo(contentType); + assertThat(message.getContent()).isEqualTo(body); } } diff --git a/javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java b/javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java index 3492492f1..c13d72ab8 100644 --- a/javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java +++ b/javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java @@ -15,20 +15,20 @@ package google.registry.reporting.spec11; import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import google.registry.beam.spec11.ThreatMatch; import google.registry.gcs.GcsUtils; import google.registry.testing.TestDataHelper; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; -import java.util.List; import org.joda.time.LocalDate; -import org.json.JSONException; import org.json.JSONObject; import org.junit.Before; import org.junit.Test; @@ -39,61 +39,96 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class Spec11RegistrarThreatMatchesParserTest { + private static final String TODAY = "2018-07-21"; + private static final String YESTERDAY = "2018-07-20"; + private final GcsUtils gcsUtils = mock(GcsUtils.class); private final Spec11RegistrarThreatMatchesParser parser = - new Spec11RegistrarThreatMatchesParser(new LocalDate(2018, 7, 21), gcsUtils, "test-bucket"); + new Spec11RegistrarThreatMatchesParser(gcsUtils, "test-bucket"); @Before public void setUp() { - when(gcsUtils.openInputStream( - new GcsFilename( - "test-bucket", "icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_2018-07-21"))) - .thenAnswer( - (args) -> - new ByteArrayInputStream( - loadFile("spec11_fake_report").getBytes(StandardCharsets.UTF_8))); + setupFile("spec11_fake_report", TODAY); } @Test public void testSuccess_retrievesReport() throws Exception { - List matches = parser.getRegistrarThreatMatches(); - assertThat(matches).isEqualTo(sampleThreatMatches()); + assertThat(parser.getRegistrarThreatMatches(LocalDate.parse(TODAY))) + .isEqualTo(sampleThreatMatches()); } - /** Returns a {@link String} from a file in the {@code spec11/testdata/} directory. */ - public static String loadFile(String filename) { - return TestDataHelper.loadFile(Spec11EmailUtils.class, filename); + @Test + public void testFindPrevious_exists() throws Exception { + setupFile("spec11_fake_report_previous_day", YESTERDAY); + assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY))) + .hasValue(LocalDate.parse(YESTERDAY)); + } + + @Test + public void testFindPrevious_notFound() { + assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY))).isEmpty(); + } + + @Test + public void testFindPrevious_olderThanYesterdayFound() throws Exception { + setupFile("spec11_fake_report_previous_day", "2018-07-14"); + + assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY))) + .hasValue(LocalDate.parse("2018-07-14")); } /** The expected contents of the sample spec11 report file */ - public static ImmutableList sampleThreatMatches() throws JSONException { - return ImmutableList.of( - RegistrarThreatMatches.create( - "a@fake.com", - ImmutableList.of( - ThreatMatch.fromJSON( - new JSONObject( - ImmutableMap.of( - "threatType", "MALWARE", - "platformType", "ANY_PLATFORM", - "threatEntryMetadata", "NONE", - "fullyQualifiedDomainName", "a.com"))))), - RegistrarThreatMatches.create( - "b@fake.com", - ImmutableList.of( - ThreatMatch.fromJSON( - new JSONObject( - ImmutableMap.of( - "threatType", "MALWARE", - "platformType", "ANY_PLATFORM", - "threatEntryMetadata", "NONE", - "fullyQualifiedDomainName", "b.com"))), - ThreatMatch.fromJSON( - new JSONObject( - ImmutableMap.of( - "threatType", "MALWARE", - "platformType", "ANY_PLATFORM", - "threatEntryMetadata", "NONE", - "fullyQualifiedDomainName", "c.com")))))); + public static ImmutableSet sampleThreatMatches() throws Exception { + return ImmutableSet.of(getMatchA(), getMatchB()); + } + + private void setupFile(String fileWithContent, String fileDate) { + GcsFilename gcsFilename = + new GcsFilename( + "test-bucket", + String.format("icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_%s", fileDate)); + when(gcsUtils.existsAndNotEmpty(gcsFilename)).thenReturn(true); + when(gcsUtils.openInputStream(gcsFilename)) + .thenAnswer( + (args) -> + new ByteArrayInputStream( + loadFile(fileWithContent).getBytes(StandardCharsets.UTF_8))); + } + + private static String loadFile(String filename) { + return TestDataHelper.loadFile(Spec11EmailUtils.class, filename); + } + + private static RegistrarThreatMatches getMatchA() throws Exception { + return RegistrarThreatMatches.create( + "a@fake.com", + ImmutableList.of( + ThreatMatch.fromJSON( + new JSONObject( + ImmutableMap.of( + "threatType", "MALWARE", + "platformType", "ANY_PLATFORM", + "threatEntryMetadata", "NONE", + "fullyQualifiedDomainName", "a.com"))))); + } + + private static RegistrarThreatMatches getMatchB() throws Exception { + return RegistrarThreatMatches.create( + "b@fake.com", + ImmutableList.of( + ThreatMatch.fromJSON( + new JSONObject( + ImmutableMap.of( + "threatType", "MALWARE", + "platformType", "ANY_PLATFORM", + "threatEntryMetadata", "NONE", + "fullyQualifiedDomainName", "b.com"))), + ThreatMatch.fromJSON( + new JSONObject( + ImmutableMap.of( + "threatType", "MALWARE", + "platformType", "ANY_PLATFORM", + "threatEntryMetadata", "NONE", + "fullyQualifiedDomainName", "c.com"))))); } } diff --git a/javatests/google/registry/reporting/spec11/testdata/spec11_fake_report_previous_day b/javatests/google/registry/reporting/spec11/testdata/spec11_fake_report_previous_day new file mode 100644 index 000000000..2dec89ae7 --- /dev/null +++ b/javatests/google/registry/reporting/spec11/testdata/spec11_fake_report_previous_day @@ -0,0 +1,3 @@ +Map from registrar email to detected subdomain threats: +{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"a.dev","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"a@fake.com"} +{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"b.com","platformType":"ANY_PLATFORM"},{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"c.com","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"b@fake.com"}