diff --git a/java/google/registry/beam/spec11/Spec11Pipeline.java b/java/google/registry/beam/spec11/Spec11Pipeline.java index 60ee35bb4..3b80b0ae6 100644 --- a/java/google/registry/beam/spec11/Spec11Pipeline.java +++ b/java/google/registry/beam/spec11/Spec11Pipeline.java @@ -55,6 +55,21 @@ import org.json.JSONObject; */ public class Spec11Pipeline implements Serializable { + /** + * Returns the subdirectory spec11 reports reside in for a given yearMonth in yyyy-MM format. + * + * @see google.registry.beam.spec11.Spec11Pipeline + * @see google.registry.reporting.spec11.Spec11EmailUtils + */ + public static String getSpec11Subdirectory(String yearMonth) { + return String.format("icann/spec11/%s/SPEC11_MONTHLY_REPORT", yearMonth); + } + + /** The JSON object field we put the registrar's e-mail address for Spec11 reports. */ + public static final String REGISTRAR_EMAIL_FIELD = "registrarEmailAddress"; + /** The JSON object field we put the threat match array for Spec11 reports. */ + public static final String THREAT_MATCHES_FIELD = "threatMatches"; + @Inject @Config("projectId") String projectId; @@ -68,8 +83,8 @@ public class Spec11Pipeline implements Serializable { String spec11TemplateUrl; @Inject - @Config("spec11BucketUrl") - String spec11BucketUrl; + @Config("reportingBucketUrl") + String reportingBucketUrl; @Inject Retrier retrier; @@ -165,12 +180,12 @@ public class Spec11Pipeline implements Serializable { (KV> kv) -> { JSONObject output = new JSONObject(); try { - output.put("registrarEmailAddress", kv.getKey()); + output.put(REGISTRAR_EMAIL_FIELD, kv.getKey()); JSONArray threatMatches = new JSONArray(); for (ThreatMatch match : kv.getValue()) { threatMatches.put(match.toJSON()); } - output.put("threatMatches", threatMatches); + output.put(THREAT_MATCHES_FIELD, threatMatches); return output.toString(); } catch (JSONException e) { throw new RuntimeException( @@ -187,8 +202,10 @@ public class Spec11Pipeline implements Serializable { yearMonthProvider, yearMonth -> String.format( - "%s/%s/%s-monthly-report", spec11BucketUrl, yearMonth, yearMonth))) + "%s/%s", + reportingBucketUrl, getSpec11Subdirectory(yearMonth)))) .withoutSharding() .withHeader("Map from registrar email to detected subdomain threats:")); } + } diff --git a/java/google/registry/beam/spec11/ThreatMatch.java b/java/google/registry/beam/spec11/ThreatMatch.java index 49c420089..c509a8a45 100644 --- a/java/google/registry/beam/spec11/ThreatMatch.java +++ b/java/google/registry/beam/spec11/ThreatMatch.java @@ -26,9 +26,10 @@ public abstract class ThreatMatch implements Serializable { private static final String THREAT_TYPE_FIELD = "threatType"; private static final String PLATFORM_TYPE_FIELD = "platformType"; private static final String METADATA_FIELD = "threatEntryMetadata"; + private static final String DOMAIN_NAME_FIELD = "fullyQualifiedDomainName"; /** Returns what kind of threat it is (malware, phishing etc.) */ - abstract String threatType(); + public abstract String threatType(); /** Returns what platforms it affects (Windows, Linux etc.) */ abstract String platformType(); /** @@ -40,7 +41,7 @@ public abstract class ThreatMatch implements Serializable { */ abstract String metadata(); /** Returns the fully qualified domain name [SLD].[TLD] of the matched threat. */ - abstract String fullyQualifiedDomainName(); + public abstract String fullyQualifiedDomainName(); /** * Constructs a {@link ThreatMatch} by parsing a {@code SafeBrowsing API} response {@link @@ -59,14 +60,21 @@ public abstract class ThreatMatch implements Serializable { fullyQualifiedDomainName); } - /** Returns a {@link String} containing the simplest details about this threat. */ - String getSimpleDetails() { - return String.format("%s;%s", this.fullyQualifiedDomainName(), this.threatType()); - } /** Returns a {@link JSONObject} representing a subset of this object's data. */ JSONObject toJSON() throws JSONException { return new JSONObject() - .put("fullyQualifiedDomainName", fullyQualifiedDomainName()) - .put("threatType", threatType()); + .put(THREAT_TYPE_FIELD, threatType()) + .put(PLATFORM_TYPE_FIELD, platformType()) + .put(METADATA_FIELD, metadata()) + .put(DOMAIN_NAME_FIELD, fullyQualifiedDomainName()); + } + + /** Parses a {@link JSONObject} and returns an equivalent {@link ThreatMatch}. */ + public static ThreatMatch fromJSON(JSONObject threatMatch) throws JSONException { + return new AutoValue_ThreatMatch( + threatMatch.getString(THREAT_TYPE_FIELD), + threatMatch.getString(PLATFORM_TYPE_FIELD), + threatMatch.getString(METADATA_FIELD), + threatMatch.getString(DOMAIN_NAME_FIELD)); } } diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index 55038911c..e6780253d 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -564,10 +564,24 @@ public final class RegistryConfig { */ @Provides @Config("reportingBucket") - public static String provideIcannReportingBucket(@Config("projectId") String projectId) { + public static String provideReportingBucket(@Config("projectId") String projectId) { return projectId + "-reporting"; } + /** + * Returns the Google Cloud Storage bucket URL for Spec11 and ICANN transaction and activity + * reports to be uploaded. + * + * @see google.registry.reporting.icann.IcannReportingUploadAction + * @see google.registry.reporting.spec11.PublishSpec11ReportAction + */ + @Provides + @Config("reportingBucketUrl") + public static String provideReportingBucketUrl( + @Config("reportingBucket") String reportingBucket) { + return "gs://" + reportingBucket; + } + /** * Returns the URL we send HTTP PUT requests for ICANN monthly transactions reports. * @@ -613,17 +627,6 @@ public final class RegistryConfig { return "gs://" + billingBucket; } - /** - * Returns the URL of the GCS subdirectory we store Spec11 reports in. - * - * @see google.registry.beam.spec11.Spec11Pipeline - */ - @Provides - @Config("spec11BucketUrl") - public static String provideSpec11BucketUrl(@Config("reportingBucket") String reportingBucket) { - return "gs://" + reportingBucket + "/icann/spec11"; - } - /** * Returns whether or not we should publish invoices to partners automatically by default. * diff --git a/java/google/registry/env/common/backend/WEB-INF/web.xml b/java/google/registry/env/common/backend/WEB-INF/web.xml index 63df151ea..17afdfc93 100644 --- a/java/google/registry/env/common/backend/WEB-INF/web.xml +++ b/java/google/registry/env/common/backend/WEB-INF/web.xml @@ -115,6 +115,15 @@ /_dr/task/generateSpec11 + + + backend-servlet + /_dr/task/publishSpec11 + + diff --git a/java/google/registry/env/common/default/WEB-INF/queue.xml b/java/google/registry/env/common/default/WEB-INF/queue.xml index 8719f4cf9..3e14770b7 100644 --- a/java/google/registry/env/common/default/WEB-INF/queue.xml +++ b/java/google/registry/env/common/default/WEB-INF/queue.xml @@ -126,9 +126,9 @@ - + - billing + beam-reporting 1/m 1 diff --git a/java/google/registry/module/backend/BackendRequestComponent.java b/java/google/registry/module/backend/BackendRequestComponent.java index c0dd3df2c..baba1f7fa 100644 --- a/java/google/registry/module/backend/BackendRequestComponent.java +++ b/java/google/registry/module/backend/BackendRequestComponent.java @@ -76,6 +76,8 @@ import google.registry.reporting.icann.IcannReportingModule; import google.registry.reporting.icann.IcannReportingStagingAction; import google.registry.reporting.icann.IcannReportingUploadAction; import google.registry.reporting.spec11.GenerateSpec11ReportAction; +import google.registry.reporting.spec11.PublishSpec11ReportAction; +import google.registry.reporting.spec11.Spec11Module; import google.registry.request.RequestComponentBuilder; import google.registry.request.RequestModule; import google.registry.request.RequestScope; @@ -108,6 +110,7 @@ import google.registry.tmch.TmchSmdrlAction; ReportingModule.class, RequestModule.class, SheetModule.class, + Spec11Module.class, TmchModule.class, VoidDnsWriterModule.class, WhiteboxModule.class, @@ -138,6 +141,7 @@ interface BackendRequestComponent { NordnUploadAction nordnUploadAction(); NordnVerifyAction nordnVerifyAction(); PublishDnsUpdatesAction publishDnsUpdatesAction(); + PublishSpec11ReportAction publishSpec11ReportAction(); ReadDnsQueueAction readDnsQueueAction(); RdeContactImportAction rdeContactImportAction(); RdeDomainImportAction rdeDomainImportAction(); diff --git a/java/google/registry/reporting/BUILD b/java/google/registry/reporting/BUILD index ca07813b7..e2bd95c53 100644 --- a/java/google/registry/reporting/BUILD +++ b/java/google/registry/reporting/BUILD @@ -8,9 +8,15 @@ java_library( name = "reporting", srcs = glob(["*.java"]), deps = [ + "//java/google/registry/config", "//java/google/registry/request", "//java/google/registry/util", + "@com_google_api_client_appengine", + "@com_google_apis_google_api_services_dataflow", + "@com_google_appengine_api_1_0_sdk", "@com_google_dagger", + "@com_google_guava", + "@com_google_http_client", "@javax_servlet_api", "@joda_time", ], diff --git a/java/google/registry/reporting/ReportingModule.java b/java/google/registry/reporting/ReportingModule.java index 67bfdd25c..6a8ca71eb 100644 --- a/java/google/registry/reporting/ReportingModule.java +++ b/java/google/registry/reporting/ReportingModule.java @@ -15,29 +15,49 @@ package google.registry.reporting; import static google.registry.request.RequestParameters.extractOptionalParameter; +import static google.registry.request.RequestParameters.extractRequiredParameter; +import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.services.dataflow.Dataflow; +import com.google.common.collect.ImmutableSet; import dagger.Module; import dagger.Provides; +import google.registry.config.RegistryConfig.Config; import google.registry.request.HttpException.BadRequestException; import google.registry.request.Parameter; import google.registry.util.Clock; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; import javax.servlet.http.HttpServletRequest; import org.joda.time.YearMonth; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; -/** - * Dagger module for injecting common settings for all Backend tasks. - */ +/** Dagger module for injecting common settings for all reporting tasks. */ @Module public class ReportingModule { + private static final String CLOUD_PLATFORM_SCOPE = + "https://www.googleapis.com/auth/cloud-platform"; + + public static final String BEAM_QUEUE = "beam-reporting"; /** * The request parameter name used by reporting actions that takes a year/month parameter, which * defaults to the last month. */ public static final String PARAM_YEAR_MONTH = "yearMonth"; + /** The request parameter specifying the jobId for a running Dataflow pipeline. */ + public static final String PARAM_JOB_ID = "jobId"; + + /** Provides the Cloud Dataflow jobId for a pipeline. */ + @Provides + @Parameter(PARAM_JOB_ID) + static String provideJobId(HttpServletRequest req) { + return extractRequiredParameter(req, PARAM_JOB_ID); + } /** Extracts an optional YearMonth in yyyy-MM format from the request. */ @Provides @@ -64,4 +84,21 @@ public class ReportingModule { @Parameter(PARAM_YEAR_MONTH) Optional yearMonthOptional, Clock clock) { return yearMonthOptional.orElseGet(() -> new YearMonth(clock.nowUtc().minusMonths(1))); } + + /** Constructs a {@link Dataflow} API client with default settings. */ + @Provides + static Dataflow provideDataflow( + @Config("projectId") String projectId, + HttpTransport transport, + JsonFactory jsonFactory, + Function, AppIdentityCredential> appIdentityCredentialFunc) { + + return new Dataflow.Builder( + transport, + jsonFactory, + appIdentityCredentialFunc.apply(ImmutableSet.of(CLOUD_PLATFORM_SCOPE))) + .setApplicationName(String.format("%s billing", projectId)) + .build(); + } + } diff --git a/java/google/registry/reporting/ReportingUtils.java b/java/google/registry/reporting/ReportingUtils.java new file mode 100644 index 000000000..279a3695b --- /dev/null +++ b/java/google/registry/reporting/ReportingUtils.java @@ -0,0 +1,39 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting; + +import com.google.appengine.api.taskqueue.QueueFactory; +import com.google.appengine.api.taskqueue.TaskOptions; +import org.joda.time.Duration; +import org.joda.time.YearMonth; + +/** Static methods common to various reporting tasks. */ +public class ReportingUtils { + + private static final int ENQUEUE_DELAY_MINUTES = 10; + + /** Enqueues a task that takes a Beam jobId and the {@link YearMonth} as parameters. */ + public static void enqueueBeamReportingTask(String path, String jobId, YearMonth yearMonth) { + TaskOptions publishTask = + TaskOptions.Builder.withUrl(path) + .method(TaskOptions.Method.POST) + // Dataflow jobs tend to take about 10 minutes to complete. + .countdownMillis(Duration.standardMinutes(ENQUEUE_DELAY_MINUTES).getMillis()) + .param(ReportingModule.PARAM_JOB_ID, jobId) + // Need to pass this through to ensure transitive yearMonth dependencies are satisfied. + .param(ReportingModule.PARAM_YEAR_MONTH, yearMonth.toString()); + QueueFactory.getQueue(ReportingModule.BEAM_QUEUE).add(publishTask); + } +} diff --git a/java/google/registry/reporting/billing/BillingModule.java b/java/google/registry/reporting/billing/BillingModule.java index d4a1889bf..b57c75f09 100644 --- a/java/google/registry/reporting/billing/BillingModule.java +++ b/java/google/registry/reporting/billing/BillingModule.java @@ -15,22 +15,14 @@ package google.registry.reporting.billing; import static google.registry.request.RequestParameters.extractOptionalBooleanParameter; -import static google.registry.request.RequestParameters.extractRequiredParameter; import static java.lang.annotation.RetentionPolicy.RUNTIME; -import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.services.dataflow.Dataflow; -import com.google.common.collect.ImmutableSet; import dagger.Module; import dagger.Provides; import google.registry.config.RegistryConfig.Config; import google.registry.request.Parameter; import java.lang.annotation.Documented; import java.lang.annotation.Retention; -import java.util.Set; -import java.util.function.Function; import javax.inject.Qualifier; import javax.servlet.http.HttpServletRequest; import org.joda.time.YearMonth; @@ -43,20 +35,9 @@ public final class BillingModule { public static final String OVERALL_INVOICE_PREFIX = "CRR-INV"; public static final String INVOICES_DIRECTORY = "invoices"; - static final String PARAM_JOB_ID = "jobId"; static final String PARAM_SHOULD_PUBLISH = "shouldPublish"; - static final String BILLING_QUEUE = "billing"; static final String CRON_QUEUE = "retryable-cron-tasks"; - private static final String CLOUD_PLATFORM_SCOPE = - "https://www.googleapis.com/auth/cloud-platform"; - - /** Provides the invoicing Dataflow jobId enqueued by {@link GenerateInvoicesAction}. */ - @Provides - @Parameter(PARAM_JOB_ID) - static String provideJobId(HttpServletRequest req) { - return extractRequiredParameter(req, PARAM_JOB_ID); - } @Provides @Parameter(PARAM_SHOULD_PUBLISH) @@ -73,23 +54,7 @@ public final class BillingModule { return String.format("%s/%s/", INVOICES_DIRECTORY, yearMonth.toString()); } - /** Constructs a {@link Dataflow} API client with default settings. */ - @Provides - static Dataflow provideDataflow( - @Config("projectId") String projectId, - HttpTransport transport, - JsonFactory jsonFactory, - Function, AppIdentityCredential> appIdentityCredentialFunc) { - - return new Dataflow.Builder( - transport, - jsonFactory, - appIdentityCredentialFunc.apply(ImmutableSet.of(CLOUD_PLATFORM_SCOPE))) - .setApplicationName(String.format("%s billing", projectId)) - .build(); - } - - /** Dagger qualifier for the subdirectory we stage to/upload from. */ + /** Dagger qualifier for the subdirectory we stage to/upload from for invoices. */ @Qualifier @Documented @Retention(RUNTIME) diff --git a/java/google/registry/reporting/billing/GenerateInvoicesAction.java b/java/google/registry/reporting/billing/GenerateInvoicesAction.java index e9f72bad1..ed325a6dc 100644 --- a/java/google/registry/reporting/billing/GenerateInvoicesAction.java +++ b/java/google/registry/reporting/billing/GenerateInvoicesAction.java @@ -14,7 +14,7 @@ package google.registry.reporting.billing; -import static google.registry.reporting.ReportingModule.PARAM_YEAR_MONTH; +import static google.registry.reporting.ReportingUtils.enqueueBeamReportingTask; import static google.registry.reporting.billing.BillingModule.PARAM_SHOULD_PUBLISH; import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; @@ -24,8 +24,6 @@ import com.google.api.services.dataflow.Dataflow; import com.google.api.services.dataflow.model.LaunchTemplateParameters; import com.google.api.services.dataflow.model.LaunchTemplateResponse; import com.google.api.services.dataflow.model.RuntimeEnvironment; -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TaskOptions; import com.google.common.collect.ImmutableMap; import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; @@ -36,7 +34,6 @@ import google.registry.request.Response; import google.registry.request.auth.Auth; import java.io.IOException; import javax.inject.Inject; -import org.joda.time.Duration; import org.joda.time.YearMonth; /** @@ -108,7 +105,7 @@ public class GenerateInvoicesAction implements Runnable { logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString()); String jobId = launchResponse.getJob().getId(); if (shouldPublish) { - enqueuePublishTask(jobId); + enqueueBeamReportingTask(PublishInvoicesAction.PATH, jobId, yearMonth); } } catch (IOException e) { logger.atWarning().withCause(e).log("Template Launch failed"); @@ -122,16 +119,4 @@ public class GenerateInvoicesAction implements Runnable { response.setContentType(MediaType.PLAIN_TEXT_UTF_8); response.setPayload("Launched dataflow template."); } - - private void enqueuePublishTask(String jobId) { - TaskOptions publishTask = - TaskOptions.Builder.withUrl(PublishInvoicesAction.PATH) - .method(TaskOptions.Method.POST) - // Dataflow jobs tend to take about 10 minutes to complete. - .countdownMillis(Duration.standardMinutes(10).getMillis()) - .param(BillingModule.PARAM_JOB_ID, jobId) - // Need to pass this through to ensure transitive yearMonth dependencies are satisfied. - .param(PARAM_YEAR_MONTH, yearMonth.toString()); - QueueFactory.getQueue(BillingModule.BILLING_QUEUE).add(publishTask); - } } diff --git a/java/google/registry/reporting/billing/PublishInvoicesAction.java b/java/google/registry/reporting/billing/PublishInvoicesAction.java index e170a9dd6..2661abd3c 100644 --- a/java/google/registry/reporting/billing/PublishInvoicesAction.java +++ b/java/google/registry/reporting/billing/PublishInvoicesAction.java @@ -28,6 +28,7 @@ import com.google.appengine.api.taskqueue.TaskOptions; import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; +import google.registry.reporting.ReportingModule; import google.registry.request.Action; import google.registry.request.Parameter; import google.registry.request.Response; @@ -62,7 +63,7 @@ public class PublishInvoicesAction implements Runnable { @Inject PublishInvoicesAction( @Config("projectId") String projectId, - @Parameter(BillingModule.PARAM_JOB_ID) String jobId, + @Parameter(ReportingModule.PARAM_JOB_ID) String jobId, BillingEmailUtils emailUtils, Dataflow dataflow, Response response, diff --git a/java/google/registry/reporting/spec11/BUILD b/java/google/registry/reporting/spec11/BUILD index 393e419b8..6317887a2 100644 --- a/java/google/registry/reporting/spec11/BUILD +++ b/java/google/registry/reporting/spec11/BUILD @@ -8,10 +8,14 @@ java_library( name = "spec11", srcs = glob(["*.java"]), deps = [ + "//java/google/registry/beam/spec11", "//java/google/registry/config", + "//java/google/registry/gcs", "//java/google/registry/keyring/api", + "//java/google/registry/reporting", "//java/google/registry/request", "//java/google/registry/request/auth", + "//java/google/registry/util", "@com_google_api_client_appengine", "@com_google_apis_google_api_services_dataflow", "@com_google_appengine_api_1_0_sdk", @@ -28,5 +32,6 @@ java_library( "@org_apache_beam_runners_google_cloud_dataflow_java", "@org_apache_beam_sdks_java_core", "@org_apache_beam_sdks_java_io_google_cloud_platform", + "@org_json", ], ) diff --git a/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java b/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java index b9230e659..2a41c3b67 100644 --- a/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java +++ b/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java @@ -14,6 +14,7 @@ package google.registry.reporting.spec11; +import static google.registry.reporting.ReportingUtils.enqueueBeamReportingTask; import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_OK; @@ -32,6 +33,7 @@ import google.registry.request.Response; import google.registry.request.auth.Auth; import java.io.IOException; import javax.inject.Inject; +import org.joda.time.YearMonth; /** * Invokes the {@code Spec11Pipeline} Beam template via the REST api. @@ -51,6 +53,7 @@ public class GenerateSpec11ReportAction implements Runnable { private final String spec11TemplateUrl; private final String jobZone; private final String apiKey; + private final YearMonth yearMonth; private final Response response; private final Dataflow dataflow; @@ -61,6 +64,7 @@ public class GenerateSpec11ReportAction implements Runnable { @Config("spec11TemplateUrl") String spec11TemplateUrl, @Config("defaultJobZone") String jobZone, @Key("safeBrowsingAPIKey") String apiKey, + YearMonth yearMonth, Response response, Dataflow dataflow) { this.projectId = projectId; @@ -68,6 +72,7 @@ public class GenerateSpec11ReportAction implements Runnable { this.spec11TemplateUrl = spec11TemplateUrl; this.jobZone = jobZone; this.apiKey = apiKey; + this.yearMonth = yearMonth; this.response = response; this.dataflow = dataflow; } @@ -77,12 +82,14 @@ public class GenerateSpec11ReportAction implements Runnable { try { LaunchTemplateParameters params = new LaunchTemplateParameters() - .setJobName("spec11_action") + .setJobName(String.format("spec11_%s", yearMonth.toString())) .setEnvironment( new RuntimeEnvironment() .setZone(jobZone) .setTempLocation(beamBucketUrl + "/temporary")) - .setParameters(ImmutableMap.of("safeBrowsingApiKey", apiKey)); + .setParameters( + ImmutableMap.of( + "safeBrowsingApiKey", apiKey, "yearMonth", yearMonth.toString("yyyy-MM"))); LaunchTemplateResponse launchResponse = dataflow .projects() @@ -90,7 +97,8 @@ public class GenerateSpec11ReportAction implements Runnable { .launch(projectId, params) .setGcsPath(spec11TemplateUrl) .execute(); - // TODO(b/111545355): Send an e-mail alert interpreting the results. + enqueueBeamReportingTask( + PublishSpec11ReportAction.PATH, launchResponse.getJob().getId(), yearMonth); logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString()); } catch (IOException e) { logger.atWarning().withCause(e).log("Template Launch failed"); diff --git a/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java b/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java new file mode 100644 index 000000000..cf851e1aa --- /dev/null +++ b/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java @@ -0,0 +1,110 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.spec11; + +import static google.registry.request.Action.Method.POST; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED; +import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; +import static javax.servlet.http.HttpServletResponse.SC_OK; + +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.model.Job; +import com.google.common.flogger.FluentLogger; +import com.google.common.net.MediaType; +import google.registry.config.RegistryConfig.Config; +import google.registry.reporting.ReportingModule; +import google.registry.request.Action; +import google.registry.request.Parameter; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import java.io.IOException; +import javax.inject.Inject; +import org.joda.time.YearMonth; + +/** + * Retries until a {@code Dataflow} job with a given {@code jobId} completes, continuing the Spec11 + * pipeline accordingly. + * + *

This calls {@link Spec11EmailUtils#emailSpec11Reports()} on success or {@link + * Spec11EmailUtils#sendFailureAlertEmail(String)} on failure. + */ +@Action(path = PublishSpec11ReportAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_OR_ADMIN) +public class PublishSpec11ReportAction implements Runnable { + + static final String PATH = "/_dr/task/publishSpec11"; + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final String JOB_DONE = "JOB_STATE_DONE"; + private static final String JOB_FAILED = "JOB_STATE_FAILED"; + + private final String projectId; + private final String jobId; + private final Spec11EmailUtils emailUtils; + private final Dataflow dataflow; + private final Response response; + private final YearMonth yearMonth; + + @Inject + PublishSpec11ReportAction( + @Config("projectId") String projectId, + @Parameter(ReportingModule.PARAM_JOB_ID) String jobId, + Spec11EmailUtils emailUtils, + Dataflow dataflow, + Response response, + YearMonth yearMonth) { + this.projectId = projectId; + this.jobId = jobId; + this.emailUtils = emailUtils; + this.dataflow = dataflow; + this.response = response; + this.yearMonth = yearMonth; + } + + @Override + public void run() { + try { + logger.atInfo().log("Starting publish job."); + Job job = dataflow.projects().jobs().get(projectId, jobId).execute(); + String state = job.getCurrentState(); + switch (state) { + case JOB_DONE: + logger.atInfo().log("Dataflow job %s finished successfully, publishing results.", jobId); + response.setStatus(SC_OK); + emailUtils.emailSpec11Reports(); + break; + case JOB_FAILED: + logger.atSevere().log("Dataflow job %s finished unsuccessfully.", jobId); + response.setStatus(SC_NO_CONTENT); + emailUtils.sendFailureAlertEmail( + String.format( + "Spec11 %s job %s ended in status failure.", yearMonth.toString(), jobId)); + break; + default: + logger.atInfo().log("Job in non-terminal state %s, retrying:", state); + response.setStatus(SC_NOT_MODIFIED); + break; + } + } catch (IOException e) { + logger.atSevere().withCause(e).log("Failed to publish Spec11 reports."); + emailUtils.sendFailureAlertEmail( + String.format( + "Spec11 %s publish action failed due to %s", yearMonth.toString(), e.getMessage())); + response.setStatus(SC_INTERNAL_SERVER_ERROR); + response.setContentType(MediaType.PLAIN_TEXT_UTF_8); + response.setPayload(String.format("Template launch failed: %s", e.getMessage())); + } + } +} diff --git a/java/google/registry/reporting/spec11/Spec11EmailUtils.java b/java/google/registry/reporting/spec11/Spec11EmailUtils.java new file mode 100644 index 000000000..a9abfbe5d --- /dev/null +++ b/java/google/registry/reporting/spec11/Spec11EmailUtils.java @@ -0,0 +1,153 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.spec11; + +import static com.google.common.base.Throwables.getRootCause; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.common.collect.ImmutableList; +import com.google.common.io.CharStreams; +import google.registry.beam.spec11.Spec11Pipeline; +import google.registry.beam.spec11.ThreatMatch; +import google.registry.config.RegistryConfig.Config; +import google.registry.gcs.GcsUtils; +import google.registry.reporting.spec11.Spec11Module.Spec11ReportDirectory; +import google.registry.util.Retrier; +import google.registry.util.SendEmailService; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import javax.inject.Inject; +import javax.mail.Message; +import javax.mail.Message.RecipientType; +import javax.mail.MessagingException; +import javax.mail.internet.InternetAddress; +import org.joda.time.YearMonth; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** Provides e-mail functionality for Spec11 tasks, such as sending Spec11 reports to registrars. */ +public class Spec11EmailUtils { + + private final SendEmailService emailService; + private final YearMonth yearMonth; + private final String alertSenderAddress; + private final String alertRecipientAddress; + private final String reportingBucket; + private final String spec11ReportDirectory; + private final GcsUtils gcsUtils; + private final Retrier retrier; + + @Inject + Spec11EmailUtils( + SendEmailService emailService, + YearMonth yearMonth, + @Config("alertSenderEmailAddress") String alertSenderAddress, + @Config("alertRecipientEmailAddress") String alertRecipientAddress, + @Config("reportingBucket") String reportingBucket, + @Spec11ReportDirectory String spec11ReportDirectory, + GcsUtils gcsUtils, + Retrier retrier) { + this.emailService = emailService; + this.yearMonth = yearMonth; + this.alertSenderAddress = alertSenderAddress; + this.alertRecipientAddress = alertRecipientAddress; + this.reportingBucket = reportingBucket; + this.spec11ReportDirectory = spec11ReportDirectory; + this.gcsUtils = gcsUtils; + this.retrier = retrier; + } + + /** + * Processes a Spec11 report on GCS for a given month and e-mails registrars based on the + * contents. + */ + void emailSpec11Reports() { + try { + retrier.callWithRetry( + () -> { + // Grab the file as an inputstream + GcsFilename spec11ReportFilename = + new GcsFilename(reportingBucket, spec11ReportDirectory); + try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) { + ImmutableList reportLines = + ImmutableList.copyOf( + CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n")); + // Iterate from 1 to size() to skip the header at line 0. + for (int i = 1; i < reportLines.size(); i++) { + emailRegistrar(reportLines.get(i)); + } + } + }, + IOException.class, + MessagingException.class); + } catch (Throwable e) { + // Send an alert with the root cause, unwrapping the retrier's RuntimeException + sendFailureAlertEmail( + String.format( + "Emailing spec11 reports failed due to %s", + getRootCause(e).getMessage())); + throw new RuntimeException("Emailing spec11 report failed", e); + } + } + + private void emailRegistrar(String line) throws MessagingException, JSONException { + // Parse the Spec11 report JSON + JSONObject reportJSON = new JSONObject(line); + String registrarEmail = reportJSON.getString(Spec11Pipeline.REGISTRAR_EMAIL_FIELD); + JSONArray threatMatches = reportJSON.getJSONArray(Spec11Pipeline.THREAT_MATCHES_FIELD); + // TODO(b/112354588): Reword this e-mail according to business team's opinions. + StringBuilder body = + new StringBuilder("Hello registrar partner,\n") + .append("The SafeBrowsing API has detected problems with the following domains:\n"); + for (int i = 0; i < threatMatches.length(); i++) { + ThreatMatch threatMatch = ThreatMatch.fromJSON(threatMatches.getJSONObject(i)); + body.append( + String.format( + "%s - %s\n", threatMatch.fullyQualifiedDomainName(), threatMatch.threatType())); + } + body.append("At the moment, no action is required. This is purely informatory.") + .append("Regards,\nGoogle Registry\n"); + Message msg = emailService.createMessage(); + msg.setSubject( + String.format("Spec11 Monthly Threat Detector [%s]", yearMonth.toString())); + msg.setText(body.toString()); + msg.setFrom(new InternetAddress(alertSenderAddress)); + msg.setRecipient(RecipientType.TO, new InternetAddress(registrarEmail)); + emailService.sendMessage(msg); + + } + + /** Sends an e-mail to the provided alert e-mail address indicating a spec11 failure. */ + void sendFailureAlertEmail(String body) { + try { + retrier.callWithRetry( + () -> { + Message msg = emailService.createMessage(); + msg.setFrom(new InternetAddress(alertSenderAddress)); + msg.addRecipient(RecipientType.TO, new InternetAddress(alertRecipientAddress)); + msg.setSubject(String.format("Spec11 Pipeline Alert: %s", yearMonth.toString())); + msg.setText(body); + emailService.sendMessage(msg); + return null; + }, + MessagingException.class); + } catch (Throwable e) { + throw new RuntimeException("The spec11 alert e-mail system failed.", e); + } + } +} diff --git a/java/google/registry/reporting/spec11/Spec11Module.java b/java/google/registry/reporting/spec11/Spec11Module.java new file mode 100644 index 000000000..0135459e6 --- /dev/null +++ b/java/google/registry/reporting/spec11/Spec11Module.java @@ -0,0 +1,42 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.spec11; + +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +import dagger.Module; +import dagger.Provides; +import google.registry.beam.spec11.Spec11Pipeline; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import javax.inject.Qualifier; +import org.joda.time.YearMonth; + +/** Module for dependencies required by Spec11 reporting. */ +@Module +public class Spec11Module { + + @Provides + @Spec11ReportDirectory + static String provideDirectoryPrefix(YearMonth yearMonth) { + return Spec11Pipeline.getSpec11Subdirectory(yearMonth.toString("yyyy-MM")); + } + + /** Dagger qualifier for the subdirectory we stage to/upload from for Spec11 reports. */ + @Qualifier + @Documented + @Retention(RUNTIME) + @interface Spec11ReportDirectory {} +} diff --git a/javatests/google/registry/beam/spec11/Spec11PipelineTest.java b/javatests/google/registry/beam/spec11/Spec11PipelineTest.java index 639f09b7b..b21d000ba 100644 --- a/javatests/google/registry/beam/spec11/Spec11PipelineTest.java +++ b/javatests/google/registry/beam/spec11/Spec11PipelineTest.java @@ -84,7 +84,7 @@ public class Spec11PipelineTest { public void initializePipeline() throws IOException { spec11Pipeline = new Spec11Pipeline(); spec11Pipeline.projectId = "test-project"; - spec11Pipeline.spec11BucketUrl = tempFolder.getRoot().getAbsolutePath(); + spec11Pipeline.reportingBucketUrl = tempFolder.getRoot().getAbsolutePath(); File beamTempFolder = tempFolder.newFolder(); spec11Pipeline.beamStagingUrl = beamTempFolder.getAbsolutePath() + "/staging"; spec11Pipeline.spec11TemplateUrl = beamTempFolder.getAbsolutePath() + "/templates/invoicing"; @@ -175,10 +175,14 @@ public class Spec11PipelineTest { new JSONObject() .put("fullyQualifiedDomainName", "111.com") .put("threatType", "MALWARE") + .put("threatEntryMetadata", "NONE") + .put("platformType", "WINDOWS") .toString(), new JSONObject() .put("fullyQualifiedDomainName", "222.com") .put("threatType", "MALWARE") + .put("threatEntryMetadata", "NONE") + .put("platformType", "WINDOWS") .toString()); } @@ -273,7 +277,8 @@ public class Spec11PipelineTest { File resultFile = new File( String.format( - "%s/2018-06/2018-06-monthly-report", tempFolder.getRoot().getAbsolutePath())); + "%s/icann/spec11/2018-06/SPEC11_MONTHLY_REPORT", + tempFolder.getRoot().getAbsolutePath())); return ImmutableList.copyOf( ResourceUtils.readResourceUtf8(resultFile.toURI().toURL()).split("\n")); } diff --git a/javatests/google/registry/module/backend/testdata/backend_routing.txt b/javatests/google/registry/module/backend/testdata/backend_routing.txt index 633d4c788..f4c5cba98 100644 --- a/javatests/google/registry/module/backend/testdata/backend_routing.txt +++ b/javatests/google/registry/module/backend/testdata/backend_routing.txt @@ -32,6 +32,7 @@ PATH CLASS METHOD /_dr/task/pollBigqueryJob BigqueryPollJobAction GET,POST y INTERNAL APP IGNORED /_dr/task/publishDnsUpdates PublishDnsUpdatesAction POST y INTERNAL APP IGNORED /_dr/task/publishInvoices PublishInvoicesAction POST n INTERNAL,API APP ADMIN +/_dr/task/publishSpec11 PublishSpec11ReportAction POST n INTERNAL,API APP ADMIN /_dr/task/rdeReport RdeReportAction POST n INTERNAL APP IGNORED /_dr/task/rdeStaging RdeStagingAction GET,POST n INTERNAL APP IGNORED /_dr/task/rdeUpload RdeUploadAction POST n INTERNAL APP IGNORED diff --git a/javatests/google/registry/reporting/billing/GenerateInvoicesActionTest.java b/javatests/google/registry/reporting/billing/GenerateInvoicesActionTest.java index ea4375d69..7d9ab3393 100644 --- a/javatests/google/registry/reporting/billing/GenerateInvoicesActionTest.java +++ b/javatests/google/registry/reporting/billing/GenerateInvoicesActionTest.java @@ -109,7 +109,7 @@ public class GenerateInvoicesActionTest { .method("POST") .param("jobId", "12345") .param("yearMonth", "2017-10"); - assertTasksEnqueued("billing", matcher); + assertTasksEnqueued("beam-reporting", matcher); } @Test diff --git a/javatests/google/registry/reporting/billing/PublishInvoicesActionTest.java b/javatests/google/registry/reporting/billing/PublishInvoicesActionTest.java index 912ec1fe1..8ccde1d33 100644 --- a/javatests/google/registry/reporting/billing/PublishInvoicesActionTest.java +++ b/javatests/google/registry/reporting/billing/PublishInvoicesActionTest.java @@ -41,6 +41,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +/** Unit tests for {@link PublishInvoicesAction}. */ @RunWith(JUnit4.class) public class PublishInvoicesActionTest { diff --git a/javatests/google/registry/reporting/spec11/BUILD b/javatests/google/registry/reporting/spec11/BUILD index 73a1be187..e7c9b00ba 100644 --- a/javatests/google/registry/reporting/spec11/BUILD +++ b/javatests/google/registry/reporting/spec11/BUILD @@ -10,8 +10,11 @@ load("//java/com/google/testing/builddefs:GenTestRules.bzl", "GenTestRules") java_library( name = "spec11", srcs = glob(["*.java"]), + resources = glob(["testdata/*"]), deps = [ + "//java/google/registry/gcs", "//java/google/registry/reporting/spec11", + "//java/google/registry/util", "//javatests/google/registry/testing", "@com_google_apis_google_api_services_dataflow", "@com_google_appengine_api_1_0_sdk", @@ -27,6 +30,7 @@ java_library( "@org_apache_beam_runners_google_cloud_dataflow_java", "@org_apache_beam_sdks_java_core", "@org_apache_beam_sdks_java_io_google_cloud_platform", + "@org_json", "@org_mockito_all", ], ) diff --git a/javatests/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java b/javatests/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java index 0f7864d08..017f51e9e 100644 --- a/javatests/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java +++ b/javatests/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java @@ -15,6 +15,7 @@ package google.registry.reporting.spec11; import static com.google.common.truth.Truth.assertThat; +import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -30,9 +31,13 @@ import com.google.api.services.dataflow.model.LaunchTemplateResponse; import com.google.api.services.dataflow.model.RuntimeEnvironment; import com.google.common.collect.ImmutableMap; import com.google.common.net.MediaType; +import google.registry.testing.AppEngineRule; import google.registry.testing.FakeResponse; +import google.registry.testing.TaskQueueHelper.TaskMatcher; import java.io.IOException; +import org.joda.time.YearMonth; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -41,6 +46,9 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class GenerateSpec11ReportActionTest { + @Rule + public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build(); + private FakeResponse response; private Dataflow dataflow; private Projects dataflowProjects; @@ -61,7 +69,7 @@ public class GenerateSpec11ReportActionTest { dataflowLaunch = mock(Launch.class); LaunchTemplateResponse launchTemplateResponse = new LaunchTemplateResponse(); // Ultimately we get back this job response with a given id. - launchTemplateResponse.setJob(new Job().setReplaceJobId("jobid")); + launchTemplateResponse.setJob(new Job().setId("jobid")); when(dataflow.projects()).thenReturn(dataflowProjects); when(dataflowProjects.templates()).thenReturn(dataflowTemplates); when(dataflowTemplates.launch(any(String.class), any(LaunchTemplateParameters.class))) @@ -79,22 +87,32 @@ public class GenerateSpec11ReportActionTest { "gs://template", "us-east1-c", "api_key/a", + YearMonth.parse("2018-06"), response, dataflow); action.run(); LaunchTemplateParameters expectedLaunchTemplateParameters = new LaunchTemplateParameters() - .setJobName("spec11_action") + .setJobName("spec11_2018-06") .setEnvironment( new RuntimeEnvironment() .setZone("us-east1-c") .setTempLocation("gs://my-bucket-beam/temporary")) - .setParameters(ImmutableMap.of("safeBrowsingApiKey", "api_key/a")); + .setParameters( + ImmutableMap.of("safeBrowsingApiKey", "api_key/a", "yearMonth", "2018-06")); verify(dataflowTemplates).launch("test", expectedLaunchTemplateParameters); verify(dataflowLaunch).setGcsPath("gs://template"); assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8); assertThat(response.getPayload()).isEqualTo("Launched Spec11 dataflow template."); + + TaskMatcher matcher = + new TaskMatcher() + .url("/_dr/task/publishSpec11") + .method("POST") + .param("jobId", "jobid") + .param("yearMonth", "2018-06"); + assertTasksEnqueued("beam-reporting", matcher); } } diff --git a/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java b/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java new file mode 100644 index 000000000..7f444136f --- /dev/null +++ b/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java @@ -0,0 +1,107 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.spec11; + +import static com.google.common.truth.Truth.assertThat; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED; +import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; +import static javax.servlet.http.HttpServletResponse.SC_OK; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.Dataflow.Projects; +import com.google.api.services.dataflow.Dataflow.Projects.Jobs; +import com.google.api.services.dataflow.Dataflow.Projects.Jobs.Get; +import com.google.api.services.dataflow.model.Job; +import com.google.common.net.MediaType; +import google.registry.testing.FakeResponse; +import java.io.IOException; +import org.joda.time.YearMonth; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link PublishSpec11ReportAction}. */ +@RunWith(JUnit4.class) +public class PublishSpec11ReportActionTest { + + private Dataflow dataflow; + private Projects projects; + private Jobs jobs; + private Get get; + private Spec11EmailUtils emailUtils; + + private Job expectedJob; + private FakeResponse response; + private PublishSpec11ReportAction publishAction; + + @Before + public void setUp() throws IOException { + dataflow = mock(Dataflow.class); + projects = mock(Projects.class); + jobs = mock(Jobs.class); + get = mock(Get.class); + when(dataflow.projects()).thenReturn(projects); + when(projects.jobs()).thenReturn(jobs); + when(jobs.get("test-project", "12345")).thenReturn(get); + expectedJob = new Job(); + when(get.execute()).thenReturn(expectedJob); + emailUtils = mock(Spec11EmailUtils.class); + response = new FakeResponse(); + publishAction = + new PublishSpec11ReportAction( + "test-project", "12345", emailUtils, dataflow, response, new YearMonth(2018, 6)); + } + + @Test + public void testJobDone_emailsResults() { + expectedJob.setCurrentState("JOB_STATE_DONE"); + publishAction.run(); + assertThat(response.getStatus()).isEqualTo(SC_OK); + verify(emailUtils).emailSpec11Reports(); + } + + @Test + public void testJobFailed_returnsNonRetriableResponse() { + expectedJob.setCurrentState("JOB_STATE_FAILED"); + publishAction.run(); + assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT); + verify(emailUtils).sendFailureAlertEmail("Spec11 2018-06 job 12345 ended in status failure."); + } + + @Test + public void testJobIndeterminate_returnsRetriableResponse() { + expectedJob.setCurrentState("JOB_STATE_RUNNING"); + publishAction.run(); + assertThat(response.getStatus()).isEqualTo(SC_NOT_MODIFIED); + verifyNoMoreInteractions(emailUtils); + } + + @Test + public void testIOException_returnsFailureMessage() throws IOException { + when(get.execute()).thenThrow(new IOException("expected")); + publishAction.run(); + assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR); + assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8); + assertThat(response.getPayload()).isEqualTo("Template launch failed: expected"); + verify(emailUtils) + .sendFailureAlertEmail("Spec11 2018-06 publish action failed due to expected"); + } +} diff --git a/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java b/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java new file mode 100644 index 000000000..99eddfb03 --- /dev/null +++ b/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java @@ -0,0 +1,192 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.spec11; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static google.registry.testing.JUnitBackports.assertThrows; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.appengine.tools.cloudstorage.GcsFilename; +import google.registry.gcs.GcsUtils; +import google.registry.testing.FakeClock; +import google.registry.testing.FakeSleeper; +import google.registry.testing.TestDataHelper; +import google.registry.util.Retrier; +import google.registry.util.SendEmailService; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Properties; +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.Session; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; +import org.joda.time.YearMonth; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** Unit tests for {@link Spec11EmailUtils}. */ +@RunWith(JUnit4.class) +public class Spec11EmailUtilsTest { + + private static final int RETRY_COUNT = 2; + + private SendEmailService emailService; + private Spec11EmailUtils emailUtils; + private GcsUtils gcsUtils; + private ArgumentCaptor gotMessage; + + @Before + public void setUp() { + emailService = mock(SendEmailService.class); + when(emailService.createMessage()) + .thenAnswer((args) -> new MimeMessage(Session.getInstance(new Properties(), null))); + + gcsUtils = mock(GcsUtils.class); + when(gcsUtils.openInputStream( + new GcsFilename("test-bucket", "icann/spec11/2018-06/SPEC11_MONTHLY_REPORT"))) + .thenAnswer( + (args) -> + new ByteArrayInputStream( + loadFile("spec11_fake_report").getBytes(StandardCharsets.UTF_8))); + + gotMessage = ArgumentCaptor.forClass(Message.class); + + emailUtils = + new Spec11EmailUtils( + emailService, + new YearMonth(2018, 6), + "my-sender@test.com", + "my-receiver@test.com", + "test-bucket", + "icann/spec11/2018-06/SPEC11_MONTHLY_REPORT", + gcsUtils, + new Retrier(new FakeSleeper(new FakeClock()), RETRY_COUNT)); + } + + @Test + public void testSuccess_emailSpec11Reports() throws MessagingException, IOException { + emailUtils.emailSpec11Reports(); + // We inspect individual parameters because Message doesn't implement equals(). + verify(emailService, times(2)).sendMessage(gotMessage.capture()); + List capturedMessages = gotMessage.getAllValues(); + validateMessage( + capturedMessages.get(0), + "my-sender@test.com", + "a@fake.com", + "Spec11 Monthly Threat Detector [2018-06]", + "Hello registrar partner,\n" + + "The SafeBrowsing API has detected problems with the following domains:\n" + + "a.com - MALWARE\n" + + "At the moment, no action is required. This is purely informatory." + + "Regards,\nGoogle Registry\n"); + validateMessage( + capturedMessages.get(1), + "my-sender@test.com", + "b@fake.com", + "Spec11 Monthly Threat Detector [2018-06]", + "Hello registrar partner,\n" + + "The SafeBrowsing API has detected problems with the following domains:\n" + + "b.com - MALWARE\n" + + "c.com - MALWARE\n" + + "At the moment, no action is required. This is purely informatory." + + "Regards,\nGoogle Registry\n"); + } + + @Test + public void testFailure_tooManyRetries_emailsAlert() throws MessagingException, IOException { + Message throwingMessage = mock(Message.class); + doThrow(new MessagingException("expected")).when(throwingMessage).setSubject(any(String.class)); + // Only return the throwingMessage enough times to force failure. The last invocation will + // be for the alert e-mail we're looking to verify. + when(emailService.createMessage()) + .thenAnswer( + new Answer() { + private int count = 0; + + @Override + public Message answer(InvocationOnMock invocation) { + if (count < RETRY_COUNT) { + count++; + return throwingMessage; + } else if (count == RETRY_COUNT) { + return new MimeMessage(Session.getDefaultInstance(new Properties(), null)); + } else { + assertWithMessage("Attempted to generate too many messages!").fail(); + return null; + } + } + }); + RuntimeException thrown = + assertThrows(RuntimeException.class, () -> emailUtils.emailSpec11Reports()); + assertThat(thrown).hasMessageThat().isEqualTo("Emailing spec11 report failed"); + assertThat(thrown) + .hasCauseThat() + .hasMessageThat() + .isEqualTo("javax.mail.MessagingException: expected"); + // We should have created RETRY_COUNT failing messages and one final alert message + verify(emailService, times(RETRY_COUNT + 1)).createMessage(); + // Verify we sent an e-mail alert + verify(emailService).sendMessage(gotMessage.capture()); + validateMessage( + gotMessage.getValue(), + "my-sender@test.com", + "my-receiver@test.com", + "Spec11 Pipeline Alert: 2018-06", + "Emailing spec11 reports failed due to expected"); + } + + @Test + public void testSuccess_sendAlertEmail() throws MessagingException, IOException { + emailUtils.sendFailureAlertEmail("Alert!"); + verify(emailService).sendMessage(gotMessage.capture()); + validateMessage( + gotMessage.getValue(), + "my-sender@test.com", + "my-receiver@test.com", + "Spec11 Pipeline Alert: 2018-06", + "Alert!"); + } + + private void validateMessage( + Message message, String from, String recipient, String subject, String body) + throws MessagingException, IOException { + assertThat(message.getFrom()).asList().containsExactly(new InternetAddress(from)); + assertThat(message.getAllRecipients()) + .asList() + .containsExactly(new InternetAddress(recipient)); + assertThat(message.getSubject()).isEqualTo(subject); + assertThat(message.getContentType()).isEqualTo("text/plain"); + assertThat(message.getContent().toString()).isEqualTo(body); + } + + /** Returns a {@link String} from a file in the {@code spec11/testdata/} directory. */ + public static String loadFile(String filename) { + return TestDataHelper.loadFile(Spec11EmailUtils.class, filename); + } +} diff --git a/javatests/google/registry/reporting/spec11/testdata/spec11_fake_report b/javatests/google/registry/reporting/spec11/testdata/spec11_fake_report new file mode 100644 index 000000000..ce9ae9cfe --- /dev/null +++ b/javatests/google/registry/reporting/spec11/testdata/spec11_fake_report @@ -0,0 +1,3 @@ +Map from registrar email to detected subdomain threats: +{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"a.com","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"a@fake.com"} +{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"b.com","platformType":"ANY_PLATFORM"},{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"c.com","platformType":"ANY_PLATFORM"}],"registrarEmailAddress":"b@fake.com"}