diff --git a/java/google/registry/beam/spec11/BUILD b/java/google/registry/beam/spec11/BUILD index 74696060c..2eb6dc942 100644 --- a/java/google/registry/beam/spec11/BUILD +++ b/java/google/registry/beam/spec11/BUILD @@ -9,6 +9,7 @@ java_library( srcs = glob(["*.java"]), resources = glob(["sql/*"]), deps = [ + "//java/com/google/common/time", "//java/google/registry/beam", "//java/google/registry/config", "//java/google/registry/util", diff --git a/java/google/registry/beam/spec11/Spec11Pipeline.java b/java/google/registry/beam/spec11/Spec11Pipeline.java index b96d8aef0..fb09616d1 100644 --- a/java/google/registry/beam/spec11/Spec11Pipeline.java +++ b/java/google/registry/beam/spec11/Spec11Pipeline.java @@ -15,17 +15,14 @@ package google.registry.beam.spec11; import static google.registry.beam.BeamUtils.getQueryFromFile; -import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.auth.oauth2.GoogleCredentials; import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn; -import google.registry.config.CredentialModule.LocalCredentialJson; import google.registry.config.RegistryConfig.Config; import google.registry.util.Retrier; import google.registry.util.SqlTemplate; -import java.io.ByteArrayInputStream; -import java.io.IOException; import java.io.Serializable; +import java.time.LocalDate; +import java.time.YearMonth; import javax.inject.Inject; import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; @@ -61,13 +58,14 @@ import org.json.JSONObject; public class Spec11Pipeline implements Serializable { /** - * Returns the subdirectory spec11 reports reside in for a given yearMonth in yyyy-MM format. + * Returns the subdirectory spec11 reports reside in for a given local date in yyyy-MM-dd format. * * @see google.registry.beam.spec11.Spec11Pipeline * @see google.registry.reporting.spec11.Spec11EmailUtils */ - public static String getSpec11Subdirectory(String yearMonth) { - return String.format("icann/spec11/%s/SPEC11_MONTHLY_REPORT", yearMonth); + public static String getSpec11ReportFilePath(LocalDate localDate) { + YearMonth yearMonth = YearMonth.of(localDate.getYear(), localDate.getMonth()); + return String.format("icann/spec11/%s/SPEC11_MONTHLY_REPORT_%s", yearMonth, localDate); } /** The JSON object field we put the registrar's e-mail address for Spec11 reports. */ @@ -91,27 +89,24 @@ public class Spec11Pipeline implements Serializable { @Config("reportingBucketUrl") String reportingBucketUrl; - @Inject - Retrier retrier; - - @Inject @LocalCredentialJson String credentialJson; + @Inject Retrier retrier; @Inject Spec11Pipeline() {} /** Custom options for running the spec11 pipeline. */ interface Spec11PipelineOptions extends DataflowPipelineOptions { - /** Returns the yearMonth we're generating the report for, in yyyy-MM format. */ - @Description("The yearMonth we generate the report for, in yyyy-MM format.") - ValueProvider getYearMonth(); + /** Returns the local date we're generating the report for, in yyyy-MM-dd format. */ + @Description("The local date we generate the report for, in yyyy-MM-dd format.") + ValueProvider getDate(); /** - * Sets the yearMonth we generate invoices for. + * Sets the local date we generate invoices for. * - *

This is implicitly set when executing the Dataflow template, by specifying the "yearMonth" + *

This is implicitly set when executing the Dataflow template, by specifying the "date" * parameter. */ - void setYearMonth(ValueProvider value); + void setDate(ValueProvider value); /** Returns the SafeBrowsing API key we use to evaluate subdomain health. */ @Description("The API key we use to access the SafeBrowsing API.") @@ -130,12 +125,6 @@ public class Spec11Pipeline implements Serializable { public void deploy() { // We can't store options as a member variable due to serialization concerns. Spec11PipelineOptions options = PipelineOptionsFactory.as(Spec11PipelineOptions.class); - try { - options.setGcpCredential( - GoogleCredentials.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8)))); - } catch (IOException e) { - throw new RuntimeException("Cannot obtain local credential to deploy the spec11 pipeline", e); - } options.setProject(projectId); options.setRunner(DataflowRunner.class); // This causes p.run() to stage the pipeline as a template on GCS, as opposed to running it. @@ -162,7 +151,7 @@ public class Spec11Pipeline implements Serializable { evaluateUrlHealth( domains, new EvaluateSafeBrowsingFn(options.getSafeBrowsingApiKey(), retrier), - options.getYearMonth()); + options.getDate()); p.run(); } @@ -174,7 +163,7 @@ public class Spec11Pipeline implements Serializable { void evaluateUrlHealth( PCollection domains, EvaluateSafeBrowsingFn evaluateSafeBrowsingFn, - ValueProvider yearMonthProvider) { + ValueProvider dateProvider) { domains .apply("Run through SafeBrowsingAPI", ParDo.of(evaluateSafeBrowsingFn)) .apply( @@ -212,13 +201,13 @@ public class Spec11Pipeline implements Serializable { TextIO.write() .to( NestedValueProvider.of( - yearMonthProvider, - yearMonth -> + dateProvider, + date -> String.format( "%s/%s", - reportingBucketUrl, getSpec11Subdirectory(yearMonth)))) + reportingBucketUrl, + getSpec11ReportFilePath(LocalDate.parse(date))))) .withoutSharding() .withHeader("Map from registrar email to detected subdomain threats:")); } - } diff --git a/java/google/registry/env/production/default/WEB-INF/cron.xml b/java/google/registry/env/production/default/WEB-INF/cron.xml index d4f27477f..c1ba7d62d 100644 --- a/java/google/registry/env/production/default/WEB-INF/cron.xml +++ b/java/google/registry/env/production/default/WEB-INF/cron.xml @@ -289,13 +289,12 @@ - Starts the beam/spec11/Spec11Pipeline Dataflow template, which creates last month's Spec11 - report. This report is stored in gs://[PROJECT-ID]-reporting/icann/spec11/yyyy-MM. - Upon Dataflow job completion, sends an e-mail to all registrars with domain registrations - flagged by the SafeBrowsing API. + Starts the beam/spec11/Spec11Pipeline Dataflow template, which creates today's Spec11 + report. This report is stored in gs://[PROJECT-ID]-reporting/icann/spec11/yyyy-MM/. + This job will only send email notifications on the second of every month. See GenerateSpec11ReportAction for more details. - 2 of month 15:00 + every day 15:00 backend diff --git a/java/google/registry/reporting/ReportingModule.java b/java/google/registry/reporting/ReportingModule.java index b02fd7028..a295636a9 100644 --- a/java/google/registry/reporting/ReportingModule.java +++ b/java/google/registry/reporting/ReportingModule.java @@ -26,8 +26,11 @@ import google.registry.config.RegistryConfig.Config; import google.registry.request.HttpException.BadRequestException; import google.registry.request.Parameter; import google.registry.util.Clock; +import java.time.LocalDate; +import java.time.format.DateTimeParseException; import java.util.Optional; import javax.servlet.http.HttpServletRequest; +import org.joda.time.DateTime; import org.joda.time.YearMonth; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -37,11 +40,20 @@ import org.joda.time.format.DateTimeFormatter; public class ReportingModule { public static final String BEAM_QUEUE = "beam-reporting"; + /** * The request parameter name used by reporting actions that takes a year/month parameter, which * defaults to the last month. */ + // TODO(b/120497263): remove this and replace with the date public static final String PARAM_YEAR_MONTH = "yearMonth"; + + /** + * The request parameter name used by reporting actions that take a local date as a parameter, + * which defaults to the current date. + */ + public static final String PARAM_DATE = "date"; + /** The request parameter specifying the jobId for a running Dataflow pipeline. */ public static final String PARAM_JOB_ID = "jobId"; @@ -74,18 +86,49 @@ public class ReportingModule { */ @Provides static YearMonth provideYearMonth( - @Parameter(PARAM_YEAR_MONTH) Optional yearMonthOptional, Clock clock) { - return yearMonthOptional.orElseGet(() -> new YearMonth(clock.nowUtc().minusMonths(1))); + @Parameter(PARAM_YEAR_MONTH) Optional yearMonthOptional, LocalDate date) { + return yearMonthOptional.orElseGet( + () -> new YearMonth(date.getYear(), date.getMonthValue() - 1)); + } + + /** Extracts an optional date in yyyy-MM-dd format from the request. */ + @Provides + @Parameter(PARAM_DATE) + static Optional provideDateOptional(HttpServletRequest req) { + java.time.format.DateTimeFormatter formatter = + java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; + Optional optionalDateString = extractOptionalParameter(req, PARAM_DATE); + try { + return optionalDateString.map(s -> LocalDate.parse(s, formatter)); + } catch (DateTimeParseException e) { + throw new BadRequestException( + String.format( + "date must be in yyyy-MM-dd format, got %s instead", + optionalDateString.orElse("UNSPECIFIED LOCAL DATE")), + e); + } + } + + /** + * Provides the local date in yyyy-MM-dd format, if not specified in the request, defaults to the + * current date. + */ + @Provides + static LocalDate provideDate( + @Parameter(PARAM_DATE) Optional dateOptional, Clock clock) { + return dateOptional.orElseGet( + () -> { + DateTime now = clock.nowUtc(); + return LocalDate.of(now.getYear(), now.getMonthOfYear(), now.getDayOfMonth()); + }); } /** Constructs a {@link Dataflow} API client with default settings. */ @Provides static Dataflow provideDataflow( @DefaultCredential GoogleCredential credential, @Config("projectId") String projectId) { - return new Dataflow.Builder(credential.getTransport(), credential.getJsonFactory(), credential) .setApplicationName(String.format("%s billing", projectId)) .build(); } - } diff --git a/java/google/registry/reporting/ReportingUtils.java b/java/google/registry/reporting/ReportingUtils.java index 279a3695b..233e18017 100644 --- a/java/google/registry/reporting/ReportingUtils.java +++ b/java/google/registry/reporting/ReportingUtils.java @@ -16,6 +16,7 @@ package google.registry.reporting; import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.api.taskqueue.TaskOptions; +import java.util.Map; import org.joda.time.Duration; import org.joda.time.YearMonth; @@ -25,15 +26,13 @@ public class ReportingUtils { private static final int ENQUEUE_DELAY_MINUTES = 10; /** Enqueues a task that takes a Beam jobId and the {@link YearMonth} as parameters. */ - public static void enqueueBeamReportingTask(String path, String jobId, YearMonth yearMonth) { + public static void enqueueBeamReportingTask(String path, Map parameters) { TaskOptions publishTask = TaskOptions.Builder.withUrl(path) .method(TaskOptions.Method.POST) // Dataflow jobs tend to take about 10 minutes to complete. - .countdownMillis(Duration.standardMinutes(ENQUEUE_DELAY_MINUTES).getMillis()) - .param(ReportingModule.PARAM_JOB_ID, jobId) - // Need to pass this through to ensure transitive yearMonth dependencies are satisfied. - .param(ReportingModule.PARAM_YEAR_MONTH, yearMonth.toString()); + .countdownMillis(Duration.standardMinutes(ENQUEUE_DELAY_MINUTES).getMillis()); + parameters.forEach(publishTask::param); QueueFactory.getQueue(ReportingModule.BEAM_QUEUE).add(publishTask); } } diff --git a/java/google/registry/reporting/billing/GenerateInvoicesAction.java b/java/google/registry/reporting/billing/GenerateInvoicesAction.java index ed325a6dc..8d573eaeb 100644 --- a/java/google/registry/reporting/billing/GenerateInvoicesAction.java +++ b/java/google/registry/reporting/billing/GenerateInvoicesAction.java @@ -28,11 +28,13 @@ import com.google.common.collect.ImmutableMap; import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; +import google.registry.reporting.ReportingModule; import google.registry.request.Action; import google.registry.request.Parameter; import google.registry.request.Response; import google.registry.request.auth.Auth; import java.io.IOException; +import java.util.Map; import javax.inject.Inject; import org.joda.time.YearMonth; @@ -105,7 +107,13 @@ public class GenerateInvoicesAction implements Runnable { logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString()); String jobId = launchResponse.getJob().getId(); if (shouldPublish) { - enqueueBeamReportingTask(PublishInvoicesAction.PATH, jobId, yearMonth); + Map beamTaskParameters = + ImmutableMap.of( + ReportingModule.PARAM_JOB_ID, + jobId, + ReportingModule.PARAM_YEAR_MONTH, + yearMonth.toString()); + enqueueBeamReportingTask(PublishInvoicesAction.PATH, beamTaskParameters); } } catch (IOException e) { logger.atWarning().withCause(e).log("Template Launch failed"); diff --git a/java/google/registry/reporting/spec11/BUILD b/java/google/registry/reporting/spec11/BUILD index 6317887a2..a7b4f296a 100644 --- a/java/google/registry/reporting/spec11/BUILD +++ b/java/google/registry/reporting/spec11/BUILD @@ -20,6 +20,7 @@ java_library( "@com_google_apis_google_api_services_dataflow", "@com_google_appengine_api_1_0_sdk", "@com_google_appengine_tools_appengine_gcs_client", + "@com_google_auto_value", "@com_google_dagger", "@com_google_flogger", "@com_google_flogger_system_backend", diff --git a/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java b/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java index 2a41c3b67..21b1021bf 100644 --- a/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java +++ b/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java @@ -28,12 +28,14 @@ import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.keyring.api.KeyModule.Key; +import google.registry.reporting.ReportingModule; import google.registry.request.Action; import google.registry.request.Response; import google.registry.request.auth.Auth; import java.io.IOException; +import java.time.LocalDate; +import java.util.Map; import javax.inject.Inject; -import org.joda.time.YearMonth; /** * Invokes the {@code Spec11Pipeline} Beam template via the REST api. @@ -53,7 +55,7 @@ public class GenerateSpec11ReportAction implements Runnable { private final String spec11TemplateUrl; private final String jobZone; private final String apiKey; - private final YearMonth yearMonth; + private final LocalDate date; private final Response response; private final Dataflow dataflow; @@ -64,7 +66,7 @@ public class GenerateSpec11ReportAction implements Runnable { @Config("spec11TemplateUrl") String spec11TemplateUrl, @Config("defaultJobZone") String jobZone, @Key("safeBrowsingAPIKey") String apiKey, - YearMonth yearMonth, + LocalDate date, Response response, Dataflow dataflow) { this.projectId = projectId; @@ -72,7 +74,7 @@ public class GenerateSpec11ReportAction implements Runnable { this.spec11TemplateUrl = spec11TemplateUrl; this.jobZone = jobZone; this.apiKey = apiKey; - this.yearMonth = yearMonth; + this.date = date; this.response = response; this.dataflow = dataflow; } @@ -82,14 +84,14 @@ public class GenerateSpec11ReportAction implements Runnable { try { LaunchTemplateParameters params = new LaunchTemplateParameters() - .setJobName(String.format("spec11_%s", yearMonth.toString())) + .setJobName(String.format("spec11_%s", date)) .setEnvironment( new RuntimeEnvironment() .setZone(jobZone) .setTempLocation(beamBucketUrl + "/temporary")) .setParameters( ImmutableMap.of( - "safeBrowsingApiKey", apiKey, "yearMonth", yearMonth.toString("yyyy-MM"))); + "safeBrowsingApiKey", apiKey, ReportingModule.PARAM_DATE, date.toString())); LaunchTemplateResponse launchResponse = dataflow .projects() @@ -97,8 +99,13 @@ public class GenerateSpec11ReportAction implements Runnable { .launch(projectId, params) .setGcsPath(spec11TemplateUrl) .execute(); - enqueueBeamReportingTask( - PublishSpec11ReportAction.PATH, launchResponse.getJob().getId(), yearMonth); + Map beamTaskParameters = + ImmutableMap.of( + ReportingModule.PARAM_JOB_ID, + launchResponse.getJob().getId(), + ReportingModule.PARAM_DATE, + date.toString()); + enqueueBeamReportingTask(PublishSpec11ReportAction.PATH, beamTaskParameters); logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString()); } catch (IOException e) { logger.atWarning().withCause(e).log("Template Launch failed"); diff --git a/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java b/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java index d8866617f..f58ff60a9 100644 --- a/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java +++ b/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java @@ -31,8 +31,8 @@ import google.registry.request.Parameter; import google.registry.request.Response; import google.registry.request.auth.Auth; import java.io.IOException; +import java.time.LocalDate; import javax.inject.Inject; -import org.joda.time.YearMonth; /** * Retries until a {@code Dataflow} job with a given {@code jobId} completes, continuing the Spec11 @@ -55,7 +55,7 @@ public class PublishSpec11ReportAction implements Runnable { private final Spec11EmailUtils emailUtils; private final Dataflow dataflow; private final Response response; - private final YearMonth yearMonth; + private final LocalDate date; @Inject PublishSpec11ReportAction( @@ -64,13 +64,13 @@ public class PublishSpec11ReportAction implements Runnable { Spec11EmailUtils emailUtils, Dataflow dataflow, Response response, - YearMonth yearMonth) { + LocalDate date) { this.projectId = projectId; this.jobId = jobId; this.emailUtils = emailUtils; this.dataflow = dataflow; this.response = response; - this.yearMonth = yearMonth; + this.date = date; } @Override @@ -81,17 +81,19 @@ public class PublishSpec11ReportAction implements Runnable { String state = job.getCurrentState(); switch (state) { case JOB_DONE: - logger.atInfo().log("Dataflow job %s finished successfully, publishing results.", jobId); + logger.atInfo().log( + "Dataflow job %s finished successfully, publishing results if appropriate.", jobId); response.setStatus(SC_OK); - emailUtils.emailSpec11Reports(); + if (shouldSendSpec11Email()) { + emailUtils.emailSpec11Reports(); + } break; case JOB_FAILED: logger.atSevere().log("Dataflow job %s finished unsuccessfully.", jobId); response.setStatus(SC_NO_CONTENT); emailUtils.sendAlertEmail( - String.format("Spec11 Dataflow Pipeline Failure %s", yearMonth.toString()), - String.format( - "Spec11 %s job %s ended in status failure.", yearMonth.toString(), jobId)); + String.format("Spec11 Dataflow Pipeline Failure %s", date), + String.format("Spec11 %s job %s ended in status failure.", date, jobId)); break; default: logger.atInfo().log("Job in non-terminal state %s, retrying:", state); @@ -101,12 +103,16 @@ public class PublishSpec11ReportAction implements Runnable { } catch (IOException e) { logger.atSevere().withCause(e).log("Failed to publish Spec11 reports."); emailUtils.sendAlertEmail( - String.format("Spec11 Publish Failure %s", yearMonth.toString()), - String.format( - "Spec11 %s publish action failed due to %s", yearMonth.toString(), e.getMessage())); + String.format("Spec11 Publish Failure %s", date), + String.format("Spec11 %s publish action failed due to %s", date, e.getMessage())); response.setStatus(SC_INTERNAL_SERVER_ERROR); response.setContentType(MediaType.PLAIN_TEXT_UTF_8); response.setPayload(String.format("Template launch failed: %s", e.getMessage())); } } + + private boolean shouldSendSpec11Email() { + // TODO(b/120496893): send emails every day with the diff content + return date.getDayOfMonth() == 2; + } } diff --git a/java/google/registry/reporting/spec11/RegistrarThreatMatches.java b/java/google/registry/reporting/spec11/RegistrarThreatMatches.java new file mode 100644 index 000000000..c1a2f68fa --- /dev/null +++ b/java/google/registry/reporting/spec11/RegistrarThreatMatches.java @@ -0,0 +1,33 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.spec11; + +import com.google.auto.value.AutoValue; +import google.registry.beam.spec11.ThreatMatch; +import java.util.List; + +/** Value class representing the registrar and list-of-threat-matches pair stored in GCS. */ +@AutoValue +public abstract class RegistrarThreatMatches { + + public abstract String registrarEmailAddress(); + + public abstract List threatMatches(); + + static RegistrarThreatMatches create( + String registrarEmailAddress, List threatMatches) { + return new AutoValue_RegistrarThreatMatches(registrarEmailAddress, threatMatches); + } +} diff --git a/java/google/registry/reporting/spec11/Spec11EmailUtils.java b/java/google/registry/reporting/spec11/Spec11EmailUtils.java index c29e2003b..1661efc93 100644 --- a/java/google/registry/reporting/spec11/Spec11EmailUtils.java +++ b/java/google/registry/reporting/spec11/Spec11EmailUtils.java @@ -15,30 +15,19 @@ package google.registry.reporting.spec11; import static com.google.common.base.Throwables.getRootCause; -import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.common.collect.ImmutableList; -import com.google.common.io.CharStreams; -import google.registry.beam.spec11.Spec11Pipeline; import google.registry.beam.spec11.ThreatMatch; import google.registry.config.RegistryConfig.Config; -import google.registry.gcs.GcsUtils; -import google.registry.reporting.spec11.Spec11Module.Spec11ReportDirectory; import google.registry.util.Retrier; import google.registry.util.SendEmailService; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import javax.inject.Inject; import javax.mail.Message; import javax.mail.Message.RecipientType; import javax.mail.MessagingException; import javax.mail.internet.InternetAddress; import org.joda.time.YearMonth; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; /** Provides e-mail functionality for Spec11 tasks, such as sending Spec11 reports to registrars. */ public class Spec11EmailUtils { @@ -48,10 +37,8 @@ public class Spec11EmailUtils { private final String outgoingEmailAddress; private final String alertRecipientAddress; private final String spec11ReplyToAddress; - private final String reportingBucket; - private final String spec11ReportDirectory; private final String spec11EmailBodyTemplate; - private final GcsUtils gcsUtils; + private final Spec11RegistrarThreatMatchesParser spec11RegistrarThreatMatchesParser; private final Retrier retrier; @Inject @@ -62,19 +49,15 @@ public class Spec11EmailUtils { @Config("alertRecipientEmailAddress") String alertRecipientAddress, @Config("spec11ReplyToEmailAddress") String spec11ReplyToAddress, @Config("spec11EmailBodyTemplate") String spec11EmailBodyTemplate, - @Config("reportingBucket") String reportingBucket, - @Spec11ReportDirectory String spec11ReportDirectory, - GcsUtils gcsUtils, + Spec11RegistrarThreatMatchesParser spec11RegistrarThreatMatchesParser, Retrier retrier) { this.emailService = emailService; this.yearMonth = yearMonth; this.outgoingEmailAddress = outgoingEmailAddress; this.alertRecipientAddress = alertRecipientAddress; this.spec11ReplyToAddress = spec11ReplyToAddress; - this.reportingBucket = reportingBucket; - this.spec11ReportDirectory = spec11ReportDirectory; + this.spec11RegistrarThreatMatchesParser = spec11RegistrarThreatMatchesParser; this.spec11EmailBodyTemplate = spec11EmailBodyTemplate; - this.gcsUtils = gcsUtils; this.retrier = retrier; } @@ -86,17 +69,10 @@ public class Spec11EmailUtils { try { retrier.callWithRetry( () -> { - // Grab the file as an inputstream - GcsFilename spec11ReportFilename = - new GcsFilename(reportingBucket, spec11ReportDirectory); - try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) { - ImmutableList reportLines = - ImmutableList.copyOf( - CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n")); - // Iterate from 1 to size() to skip the header at line 0. - for (int i = 1; i < reportLines.size(); i++) { - emailRegistrar(reportLines.get(i)); - } + ImmutableList registrarThreatMatchesList = + spec11RegistrarThreatMatchesParser.getRegistrarThreatMatches(); + for (RegistrarThreatMatches registrarThreatMatches : registrarThreatMatchesList) { + emailRegistrar(registrarThreatMatches); } }, IOException.class, @@ -105,9 +81,7 @@ public class Spec11EmailUtils { // Send an alert with the root cause, unwrapping the retrier's RuntimeException sendAlertEmail( String.format("Spec11 Emailing Failure %s", yearMonth.toString()), - String.format( - "Emailing spec11 reports failed due to %s", - getRootCause(e).getMessage())); + String.format("Emailing spec11 reports failed due to %s", getRootCause(e).getMessage())); throw new RuntimeException("Emailing spec11 report failed", e); } sendAlertEmail( @@ -115,14 +89,11 @@ public class Spec11EmailUtils { "Spec11 reporting completed successfully."); } - private void emailRegistrar(String line) throws MessagingException, JSONException { - // Parse the Spec11 report JSON - JSONObject reportJSON = new JSONObject(line); - String registrarEmail = reportJSON.getString(Spec11Pipeline.REGISTRAR_EMAIL_FIELD); - JSONArray threatMatches = reportJSON.getJSONArray(Spec11Pipeline.THREAT_MATCHES_FIELD); + private void emailRegistrar(RegistrarThreatMatches registrarThreatMatches) + throws MessagingException { + String registrarEmail = registrarThreatMatches.registrarEmailAddress(); StringBuilder threatList = new StringBuilder(); - for (int i = 0; i < threatMatches.length(); i++) { - ThreatMatch threatMatch = ThreatMatch.fromJSON(threatMatches.getJSONObject(i)); + for (ThreatMatch threatMatch : registrarThreatMatches.threatMatches()) { threatList.append( String.format( "%s - %s\n", threatMatch.fullyQualifiedDomainName(), threatMatch.threatType())); @@ -134,7 +105,7 @@ public class Spec11EmailUtils { Message msg = emailService.createMessage(); msg.setSubject( String.format("Google Registry Monthly Threat Detector [%s]", yearMonth.toString())); - msg.setText(body.toString()); + msg.setText(body); msg.setFrom(new InternetAddress(outgoingEmailAddress)); msg.addRecipient(RecipientType.TO, new InternetAddress(registrarEmail)); msg.addRecipient(RecipientType.BCC, new InternetAddress(spec11ReplyToAddress)); diff --git a/java/google/registry/reporting/spec11/Spec11Module.java b/java/google/registry/reporting/spec11/Spec11Module.java index 0135459e6..d9fd5a492 100644 --- a/java/google/registry/reporting/spec11/Spec11Module.java +++ b/java/google/registry/reporting/spec11/Spec11Module.java @@ -21,22 +21,22 @@ import dagger.Provides; import google.registry.beam.spec11.Spec11Pipeline; import java.lang.annotation.Documented; import java.lang.annotation.Retention; +import java.time.LocalDate; import javax.inject.Qualifier; -import org.joda.time.YearMonth; /** Module for dependencies required by Spec11 reporting. */ @Module public class Spec11Module { @Provides - @Spec11ReportDirectory - static String provideDirectoryPrefix(YearMonth yearMonth) { - return Spec11Pipeline.getSpec11Subdirectory(yearMonth.toString("yyyy-MM")); + @Spec11ReportFilePath + static String provideSpec11ReportFilePath(LocalDate localDate) { + return Spec11Pipeline.getSpec11ReportFilePath(localDate); } /** Dagger qualifier for the subdirectory we stage to/upload from for Spec11 reports. */ @Qualifier @Documented @Retention(RUNTIME) - @interface Spec11ReportDirectory {} + @interface Spec11ReportFilePath {} } diff --git a/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java b/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java new file mode 100644 index 000000000..6f09ddc9b --- /dev/null +++ b/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java @@ -0,0 +1,78 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.spec11; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.common.collect.ImmutableList; +import com.google.common.io.CharStreams; +import google.registry.beam.spec11.Spec11Pipeline; +import google.registry.beam.spec11.ThreatMatch; +import google.registry.config.RegistryConfig.Config; +import google.registry.gcs.GcsUtils; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.time.LocalDate; +import javax.inject.Inject; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** Parser to retrieve which registrar-threat matches we should notify via email */ +public class Spec11RegistrarThreatMatchesParser { + + private final LocalDate date; + private final GcsUtils gcsUtils; + private final String reportingBucket; + + @Inject + public Spec11RegistrarThreatMatchesParser( + LocalDate date, GcsUtils gcsUtils, @Config("reportingBucket") String reportingBucket) { + this.date = date; + this.gcsUtils = gcsUtils; + this.reportingBucket = reportingBucket; + } + + /** Gets the list of registrar:set-of-threat-match pairings from the file in GCS. */ + public ImmutableList getRegistrarThreatMatches() + throws IOException, JSONException { + // TODO(b/120078223): this should only be the diff of this run and the prior run. + GcsFilename spec11ReportFilename = + new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(date)); + ImmutableList.Builder builder = ImmutableList.builder(); + try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) { + ImmutableList reportLines = + ImmutableList.copyOf(CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n")); + // Iterate from 1 to size() to skip the header at line 0. + for (int i = 1; i < reportLines.size(); i++) { + builder.add(parseRegistrarThreatMatch(reportLines.get(i))); + } + return builder.build(); + } + } + + private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException { + JSONObject reportJSON = new JSONObject(line); + String registrarEmail = reportJSON.getString(Spec11Pipeline.REGISTRAR_EMAIL_FIELD); + JSONArray threatMatchesArray = reportJSON.getJSONArray(Spec11Pipeline.THREAT_MATCHES_FIELD); + ImmutableList.Builder threatMatches = ImmutableList.builder(); + for (int i = 0; i < threatMatchesArray.length(); i++) { + threatMatches.add(ThreatMatch.fromJSON(threatMatchesArray.getJSONObject(i))); + } + return RegistrarThreatMatches.create(registrarEmail, threatMatches.build()); + } +} diff --git a/javatests/google/registry/beam/spec11/Spec11PipelineTest.java b/javatests/google/registry/beam/spec11/Spec11PipelineTest.java index b21d000ba..0f1e9a5b4 100644 --- a/javatests/google/registry/beam/spec11/Spec11PipelineTest.java +++ b/javatests/google/registry/beam/spec11/Spec11PipelineTest.java @@ -132,7 +132,7 @@ public class Spec11PipelineTest { // Apply input and evaluation transforms PCollection input = p.apply(Create.of(inputRows)); - spec11Pipeline.evaluateUrlHealth(input, evalFn, StaticValueProvider.of("2018-06")); + spec11Pipeline.evaluateUrlHealth(input, evalFn, StaticValueProvider.of("2018-06-01")); p.run(); // Verify header and 3 threat matches for 2 registrars are found @@ -277,7 +277,7 @@ public class Spec11PipelineTest { File resultFile = new File( String.format( - "%s/icann/spec11/2018-06/SPEC11_MONTHLY_REPORT", + "%s/icann/spec11/2018-06/SPEC11_MONTHLY_REPORT_2018-06-01", tempFolder.getRoot().getAbsolutePath())); return ImmutableList.copyOf( ResourceUtils.readResourceUtf8(resultFile.toURI().toURL()).split("\n")); diff --git a/javatests/google/registry/reporting/ReportingModuleTest.java b/javatests/google/registry/reporting/ReportingModuleTest.java index 2ead79087..f532e1659 100644 --- a/javatests/google/registry/reporting/ReportingModuleTest.java +++ b/javatests/google/registry/reporting/ReportingModuleTest.java @@ -19,9 +19,11 @@ import static google.registry.testing.JUnitBackports.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.google.common.truth.Truth8; import google.registry.request.HttpException.BadRequestException; import google.registry.testing.FakeClock; import google.registry.util.Clock; +import java.time.LocalDate; import java.util.Optional; import javax.servlet.http.HttpServletRequest; import org.joda.time.DateTime; @@ -37,6 +39,7 @@ public class ReportingModuleTest { private HttpServletRequest req = mock(HttpServletRequest.class); private Clock clock; + @Before public void setUp() { clock = new FakeClock(DateTime.parse("2017-07-01TZ")); @@ -45,7 +48,14 @@ public class ReportingModuleTest { @Test public void testEmptyYearMonthParameter_returnsEmptyYearMonthOptional() { when(req.getParameter("yearMonth")).thenReturn(""); - assertThat(ReportingModule.provideYearMonthOptional(req)).isEqualTo(Optional.empty()); + Truth8.assertThat(ReportingModule.provideYearMonthOptional(req)).isEmpty(); + } + + @Test + public void testValidYearMonthParameter_returnsThatMonth() { + when(req.getParameter("yearMonth")).thenReturn("2017-05"); + Truth8.assertThat(ReportingModule.provideYearMonthOptional(req)) + .hasValue(new YearMonth(2017, 5)); } @Test @@ -61,14 +71,50 @@ public class ReportingModuleTest { @Test public void testEmptyYearMonth_returnsLastMonth() { - assertThat(ReportingModule.provideYearMonth(Optional.empty(), clock)) + assertThat(ReportingModule.provideYearMonth(Optional.empty(), LocalDate.of(2017, 7, 6))) .isEqualTo(new YearMonth(2017, 6)); } @Test public void testGivenYearMonth_returnsThatMonth() { - assertThat(ReportingModule.provideYearMonth(Optional.of(new YearMonth(2017, 5)), clock)) + assertThat( + ReportingModule.provideYearMonth( + Optional.of(new YearMonth(2017, 5)), LocalDate.of(2017, 7, 6))) .isEqualTo(new YearMonth(2017, 5)); } + @Test + public void testEmptyDateParameter_returnsEmptyDateOptional() { + when(req.getParameter("date")).thenReturn(""); + Truth8.assertThat(ReportingModule.provideDateOptional(req)).isEmpty(); + } + + @Test + public void testValidDateParameter_returnsThatDate() { + when(req.getParameter("date")).thenReturn("2017-05-13"); + Truth8.assertThat(ReportingModule.provideDateOptional(req)) + .hasValue(LocalDate.of(2017, 5, 13)); + } + + @Test + public void testInvalidDateParameter_throwsException() { + when(req.getParameter("date")).thenReturn("20170513"); + BadRequestException thrown = + assertThrows(BadRequestException.class, () -> ReportingModule.provideDateOptional(req)); + assertThat(thrown) + .hasMessageThat() + .contains("date must be in yyyy-MM-dd format, got 20170513 instead"); + } + + @Test + public void testEmptyDate_returnsToday() { + assertThat(ReportingModule.provideDate(Optional.empty(), clock)) + .isEqualTo(LocalDate.of(2017, 7, 1)); + } + + @Test + public void testGivenDate_returnsThatDate() { + assertThat(ReportingModule.provideDate(Optional.of(LocalDate.of(2017, 7, 2)), clock)) + .isEqualTo(LocalDate.of(2017, 7, 2)); + } } diff --git a/javatests/google/registry/reporting/spec11/BUILD b/javatests/google/registry/reporting/spec11/BUILD index fb60fa9eb..6eb863e8b 100644 --- a/javatests/google/registry/reporting/spec11/BUILD +++ b/javatests/google/registry/reporting/spec11/BUILD @@ -12,6 +12,7 @@ java_library( srcs = glob(["*.java"]), resources = glob(["testdata/*"]), deps = [ + "//java/google/registry/beam/spec11", "//java/google/registry/gcs", "//java/google/registry/reporting/spec11", "//java/google/registry/util", diff --git a/javatests/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java b/javatests/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java index 017f51e9e..64a70cc53 100644 --- a/javatests/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java +++ b/javatests/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java @@ -35,7 +35,7 @@ import google.registry.testing.AppEngineRule; import google.registry.testing.FakeResponse; import google.registry.testing.TaskQueueHelper.TaskMatcher; import java.io.IOException; -import org.joda.time.YearMonth; +import java.time.LocalDate; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -87,20 +87,20 @@ public class GenerateSpec11ReportActionTest { "gs://template", "us-east1-c", "api_key/a", - YearMonth.parse("2018-06"), + LocalDate.parse("2018-06-11"), response, dataflow); action.run(); LaunchTemplateParameters expectedLaunchTemplateParameters = new LaunchTemplateParameters() - .setJobName("spec11_2018-06") + .setJobName("spec11_2018-06-11") .setEnvironment( new RuntimeEnvironment() .setZone("us-east1-c") .setTempLocation("gs://my-bucket-beam/temporary")) .setParameters( - ImmutableMap.of("safeBrowsingApiKey", "api_key/a", "yearMonth", "2018-06")); + ImmutableMap.of("safeBrowsingApiKey", "api_key/a", "date", "2018-06-11")); verify(dataflowTemplates).launch("test", expectedLaunchTemplateParameters); verify(dataflowLaunch).setGcsPath("gs://template"); assertThat(response.getStatus()).isEqualTo(200); @@ -112,7 +112,7 @@ public class GenerateSpec11ReportActionTest { .url("/_dr/task/publishSpec11") .method("POST") .param("jobId", "jobid") - .param("yearMonth", "2018-06"); + .param("date", "2018-06-11"); assertTasksEnqueued("beam-reporting", matcher); } } diff --git a/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java b/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java index ce28bac42..4a86093c2 100644 --- a/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java +++ b/javatests/google/registry/reporting/spec11/PublishSpec11ReportActionTest.java @@ -32,7 +32,7 @@ import com.google.api.services.dataflow.model.Job; import com.google.common.net.MediaType; import google.registry.testing.FakeResponse; import java.io.IOException; -import org.joda.time.YearMonth; +import java.time.LocalDate; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -67,12 +67,15 @@ public class PublishSpec11ReportActionTest { response = new FakeResponse(); publishAction = new PublishSpec11ReportAction( - "test-project", "12345", emailUtils, dataflow, response, new YearMonth(2018, 6)); + "test-project", "12345", emailUtils, dataflow, response, LocalDate.of(2018, 6, 5)); } @Test - public void testJobDone_emailsResults() { + public void testJobDone_emailsResultsOnSecondOfMonth() { expectedJob.setCurrentState("JOB_STATE_DONE"); + publishAction = + new PublishSpec11ReportAction( + "test-project", "12345", emailUtils, dataflow, response, LocalDate.of(2018, 6, 2)); publishAction.run(); assertThat(response.getStatus()).isEqualTo(SC_OK); verify(emailUtils).emailSpec11Reports(); @@ -85,8 +88,8 @@ public class PublishSpec11ReportActionTest { assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT); verify(emailUtils) .sendAlertEmail( - "Spec11 Dataflow Pipeline Failure 2018-06", - "Spec11 2018-06 job 12345 ended in status failure."); + "Spec11 Dataflow Pipeline Failure 2018-06-05", + "Spec11 2018-06-05 job 12345 ended in status failure."); } @Test @@ -106,7 +109,15 @@ public class PublishSpec11ReportActionTest { assertThat(response.getPayload()).isEqualTo("Template launch failed: expected"); verify(emailUtils) .sendAlertEmail( - "Spec11 Publish Failure 2018-06", - "Spec11 2018-06 publish action failed due to expected"); + "Spec11 Publish Failure 2018-06-05", + "Spec11 2018-06-05 publish action failed due to expected"); + } + + @Test + public void testJobDone_doesNotEmailResults() { + expectedJob.setCurrentState("JOB_STATE_DONE"); + publishAction.run(); + assertThat(response.getStatus()).isEqualTo(SC_OK); + verifyNoMoreInteractions(emailUtils); } } diff --git a/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java b/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java index 8bf3c1386..01d09c5e8 100644 --- a/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java +++ b/javatests/google/registry/reporting/spec11/Spec11EmailUtilsTest.java @@ -16,6 +16,7 @@ package google.registry.reporting.spec11; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; +import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches; import static google.registry.testing.JUnitBackports.assertThrows; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doThrow; @@ -24,16 +25,11 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import google.registry.gcs.GcsUtils; import google.registry.testing.FakeClock; import google.registry.testing.FakeSleeper; -import google.registry.testing.TestDataHelper; import google.registry.util.Retrier; import google.registry.util.SendEmailService; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Properties; import javax.annotation.Nullable; @@ -60,36 +56,29 @@ public class Spec11EmailUtilsTest { private SendEmailService emailService; private Spec11EmailUtils emailUtils; - private GcsUtils gcsUtils; + private Spec11RegistrarThreatMatchesParser parser; private ArgumentCaptor gotMessage; @Before - public void setUp() { + public void setUp() throws Exception { emailService = mock(SendEmailService.class); when(emailService.createMessage()) .thenAnswer((args) -> new MimeMessage(Session.getInstance(new Properties(), null))); - gcsUtils = mock(GcsUtils.class); - when(gcsUtils.openInputStream( - new GcsFilename("test-bucket", "icann/spec11/2018-06/SPEC11_MONTHLY_REPORT"))) - .thenAnswer( - (args) -> - new ByteArrayInputStream( - loadFile("spec11_fake_report").getBytes(StandardCharsets.UTF_8))); + parser = mock(Spec11RegistrarThreatMatchesParser.class); + when(parser.getRegistrarThreatMatches()).thenReturn(sampleThreatMatches()); gotMessage = ArgumentCaptor.forClass(Message.class); emailUtils = new Spec11EmailUtils( emailService, - new YearMonth(2018, 6), + new YearMonth(2018, 7), "my-sender@test.com", "my-receiver@test.com", "my-reply-to@test.com", "{LIST_OF_THREATS}\n{REPLY_TO_EMAIL}", - "test-bucket", - "icann/spec11/2018-06/SPEC11_MONTHLY_REPORT", - gcsUtils, + parser, new Retrier(new FakeSleeper(new FakeClock()), RETRY_COUNT)); } @@ -104,21 +93,21 @@ public class Spec11EmailUtilsTest { "my-sender@test.com", "a@fake.com", "my-reply-to@test.com", - "Google Registry Monthly Threat Detector [2018-06]", + "Google Registry Monthly Threat Detector [2018-07]", "a.com - MALWARE\n\nmy-reply-to@test.com"); validateMessage( capturedMessages.get(1), "my-sender@test.com", "b@fake.com", "my-reply-to@test.com", - "Google Registry Monthly Threat Detector [2018-06]", + "Google Registry Monthly Threat Detector [2018-07]", "b.com - MALWARE\nc.com - MALWARE\n\nmy-reply-to@test.com"); validateMessage( capturedMessages.get(2), "my-sender@test.com", "my-receiver@test.com", null, - "Spec11 Pipeline Success 2018-06", + "Spec11 Pipeline Success 2018-07", "Spec11 reporting completed successfully."); } @@ -162,20 +151,20 @@ public class Spec11EmailUtilsTest { "my-sender@test.com", "my-receiver@test.com", null, - "Spec11 Emailing Failure 2018-06", + "Spec11 Emailing Failure 2018-07", "Emailing spec11 reports failed due to expected"); } @Test public void testSuccess_sendAlertEmail() throws MessagingException, IOException { - emailUtils.sendAlertEmail("Spec11 Pipeline Alert: 2018-06", "Alert!"); + emailUtils.sendAlertEmail("Spec11 Pipeline Alert: 2018-07", "Alert!"); verify(emailService).sendMessage(gotMessage.capture()); validateMessage( gotMessage.getValue(), "my-sender@test.com", "my-receiver@test.com", null, - "Spec11 Pipeline Alert: 2018-06", + "Spec11 Pipeline Alert: 2018-07", "Alert!"); } @@ -203,9 +192,4 @@ public class Spec11EmailUtilsTest { assertThat(message.getContentType()).isEqualTo("text/plain"); assertThat(message.getContent().toString()).isEqualTo(body); } - - /** Returns a {@link String} from a file in the {@code spec11/testdata/} directory. */ - public static String loadFile(String filename) { - return TestDataHelper.loadFile(Spec11EmailUtils.class, filename); - } } diff --git a/javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java b/javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java new file mode 100644 index 000000000..6e2b8f506 --- /dev/null +++ b/javatests/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java @@ -0,0 +1,99 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.spec11; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import google.registry.beam.spec11.ThreatMatch; +import google.registry.gcs.GcsUtils; +import google.registry.testing.TestDataHelper; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.util.List; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link Spec11RegistrarThreatMatchesParser}. */ +@RunWith(JUnit4.class) +public class Spec11RegistrarThreatMatchesParserTest { + + private final GcsUtils gcsUtils = mock(GcsUtils.class); + private final Spec11RegistrarThreatMatchesParser parser = + new Spec11RegistrarThreatMatchesParser(LocalDate.of(2018, 7, 21), gcsUtils, "test-bucket"); + + @Before + public void setUp() { + when(gcsUtils.openInputStream( + new GcsFilename( + "test-bucket", "icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_2018-07-21"))) + .thenAnswer( + (args) -> + new ByteArrayInputStream( + loadFile("spec11_fake_report").getBytes(StandardCharsets.UTF_8))); + } + + @Test + public void testSuccess_retrievesReport() throws Exception { + List matches = parser.getRegistrarThreatMatches(); + assertThat(matches).isEqualTo(sampleThreatMatches()); + } + + /** Returns a {@link String} from a file in the {@code spec11/testdata/} directory. */ + public static String loadFile(String filename) { + return TestDataHelper.loadFile(Spec11EmailUtils.class, filename); + } + + /** The expected contents of the sample spec11 report file */ + public static ImmutableList sampleThreatMatches() throws JSONException { + return ImmutableList.of( + RegistrarThreatMatches.create( + "a@fake.com", + ImmutableList.of( + ThreatMatch.fromJSON( + new JSONObject( + ImmutableMap.of( + "threatType", "MALWARE", + "platformType", "ANY_PLATFORM", + "threatEntryMetadata", "NONE", + "fullyQualifiedDomainName", "a.com"))))), + RegistrarThreatMatches.create( + "b@fake.com", + ImmutableList.of( + ThreatMatch.fromJSON( + new JSONObject( + ImmutableMap.of( + "threatType", "MALWARE", + "platformType", "ANY_PLATFORM", + "threatEntryMetadata", "NONE", + "fullyQualifiedDomainName", "b.com"))), + ThreatMatch.fromJSON( + new JSONObject( + ImmutableMap.of( + "threatType", "MALWARE", + "platformType", "ANY_PLATFORM", + "threatEntryMetadata", "NONE", + "fullyQualifiedDomainName", "c.com")))))); + } +}