mirror of
https://github.com/google/nomulus.git
synced 2025-05-31 09:44:03 +02:00
Fix reporting module parameters and yearMonth usage
This is a final refactor to address Nick's comments in [] where YearMonth really should be injected as a Joda type instead of a raw string, and the HTTP parameters should be separate from the default-provided dependencies. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=173539965
This commit is contained in:
parent
d22986a0a3
commit
4a9b8b918a
14 changed files with 109 additions and 87 deletions
|
@ -20,13 +20,13 @@ import static google.registry.reporting.IcannReportingModule.ICANN_REPORTING_DAT
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import com.google.common.io.Resources;
|
import com.google.common.io.Resources;
|
||||||
import google.registry.config.RegistryConfig.Config;
|
import google.registry.config.RegistryConfig.Config;
|
||||||
import google.registry.request.Parameter;
|
|
||||||
import google.registry.util.ResourceUtils;
|
import google.registry.util.ResourceUtils;
|
||||||
import google.registry.util.SqlTemplate;
|
import google.registry.util.SqlTemplate;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import javax.inject.Inject;
|
import javax.inject.Inject;
|
||||||
import org.joda.time.LocalDate;
|
import org.joda.time.LocalDate;
|
||||||
|
import org.joda.time.YearMonth;
|
||||||
import org.joda.time.format.DateTimeFormat;
|
import org.joda.time.format.DateTimeFormat;
|
||||||
import org.joda.time.format.DateTimeFormatter;
|
import org.joda.time.format.DateTimeFormatter;
|
||||||
|
|
||||||
|
@ -44,7 +44,9 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
|
||||||
static final String ACTIVITY_REPORT_AGGREGATION = "activity_report_aggregation";
|
static final String ACTIVITY_REPORT_AGGREGATION = "activity_report_aggregation";
|
||||||
|
|
||||||
@Inject @Config("projectId") String projectId;
|
@Inject @Config("projectId") String projectId;
|
||||||
@Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
|
|
||||||
|
@Inject YearMonth yearMonth;
|
||||||
|
|
||||||
@Inject ActivityReportingQueryBuilder() {}
|
@Inject ActivityReportingQueryBuilder() {}
|
||||||
|
|
||||||
/** Returns the aggregate query which generates the activity report from the saved view. */
|
/** Returns the aggregate query which generates the activity report from the saved view. */
|
||||||
|
@ -60,11 +62,9 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
|
||||||
/** Sets the month we're doing activity reporting for, and returns the view query map. */
|
/** Sets the month we're doing activity reporting for, and returns the view query map. */
|
||||||
@Override
|
@Override
|
||||||
public ImmutableMap<String, String> getViewQueryMap() throws IOException {
|
public ImmutableMap<String, String> getViewQueryMap() throws IOException {
|
||||||
LocalDate reportDate =
|
LocalDate firstDayOfMonth = yearMonth.toLocalDate(1);
|
||||||
DateTimeFormat.forPattern("yyyy-MM").parseLocalDate(yearMonth).withDayOfMonth(1);
|
|
||||||
LocalDate firstDayOfMonth = reportDate;
|
|
||||||
// The pattern-matching is inclusive, so we subtract 1 day to only report that month's data.
|
// The pattern-matching is inclusive, so we subtract 1 day to only report that month's data.
|
||||||
LocalDate lastDayOfMonth = reportDate.plusMonths(1).minusDays(1);
|
LocalDate lastDayOfMonth = yearMonth.toLocalDate(1).plusMonths(1).minusDays(1);
|
||||||
return createQueryMap(firstDayOfMonth, lastDayOfMonth);
|
return createQueryMap(firstDayOfMonth, lastDayOfMonth);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -135,9 +135,9 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/** Returns the table name of the query, suffixed with the yearMonth in _YYYYMM format. */
|
/** Returns the table name of the query, suffixed with the yearMonth in _yyyyMM format. */
|
||||||
private String getTableName(String queryName) {
|
private String getTableName(String queryName) {
|
||||||
return String.format("%s_%s", queryName, yearMonth.replace("-", ""));
|
return String.format("%s_%s", queryName, DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Returns {@link String} for file in {@code reporting/sql/} directory. */
|
/** Returns {@link String} for file in {@code reporting/sql/} directory. */
|
||||||
|
|
|
@ -16,6 +16,7 @@ package google.registry.reporting;
|
||||||
|
|
||||||
import static google.registry.request.RequestParameters.extractOptionalEnumParameter;
|
import static google.registry.request.RequestParameters.extractOptionalEnumParameter;
|
||||||
import static google.registry.request.RequestParameters.extractOptionalParameter;
|
import static google.registry.request.RequestParameters.extractOptionalParameter;
|
||||||
|
import static java.lang.annotation.RetentionPolicy.RUNTIME;
|
||||||
|
|
||||||
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
||||||
import com.google.api.client.http.HttpTransport;
|
import com.google.api.client.http.HttpTransport;
|
||||||
|
@ -29,10 +30,15 @@ import google.registry.request.HttpException.BadRequestException;
|
||||||
import google.registry.request.Parameter;
|
import google.registry.request.Parameter;
|
||||||
import google.registry.util.Clock;
|
import google.registry.util.Clock;
|
||||||
import google.registry.util.SendEmailService;
|
import google.registry.util.SendEmailService;
|
||||||
|
import java.lang.annotation.Documented;
|
||||||
|
import java.lang.annotation.Retention;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
import javax.inject.Qualifier;
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
import org.joda.time.Duration;
|
import org.joda.time.Duration;
|
||||||
|
import org.joda.time.YearMonth;
|
||||||
import org.joda.time.format.DateTimeFormat;
|
import org.joda.time.format.DateTimeFormat;
|
||||||
|
import org.joda.time.format.DateTimeFormatter;
|
||||||
|
|
||||||
/** Module for dependencies required by ICANN monthly transactions/activity reporting. */
|
/** Module for dependencies required by ICANN monthly transactions/activity reporting. */
|
||||||
@Module
|
@Module
|
||||||
|
@ -44,9 +50,7 @@ public final class IcannReportingModule {
|
||||||
ACTIVITY
|
ACTIVITY
|
||||||
}
|
}
|
||||||
|
|
||||||
static final String PARAM_OPTIONAL_YEAR_MONTH = "yearMonthOptional";
|
|
||||||
static final String PARAM_YEAR_MONTH = "yearMonth";
|
static final String PARAM_YEAR_MONTH = "yearMonth";
|
||||||
static final String PARAM_OPTIONAL_SUBDIR = "subdirOptional";
|
|
||||||
static final String PARAM_SUBDIR = "subdir";
|
static final String PARAM_SUBDIR = "subdir";
|
||||||
static final String PARAM_REPORT_TYPE = "reportType";
|
static final String PARAM_REPORT_TYPE = "reportType";
|
||||||
static final String ICANN_REPORTING_DATA_SET = "icann_reporting";
|
static final String ICANN_REPORTING_DATA_SET = "icann_reporting";
|
||||||
|
@ -55,42 +59,45 @@ public final class IcannReportingModule {
|
||||||
private static final String DEFAULT_SUBDIR = "icann/monthly";
|
private static final String DEFAULT_SUBDIR = "icann/monthly";
|
||||||
private static final String BIGQUERY_SCOPE = "https://www.googleapis.com/auth/cloud-platform";
|
private static final String BIGQUERY_SCOPE = "https://www.googleapis.com/auth/cloud-platform";
|
||||||
|
|
||||||
/** Extracts an optional yearMonth in yyyy-MM format from the request. */
|
/** Extracts an optional YearMonth in yyyy-MM format from the request. */
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter(PARAM_OPTIONAL_YEAR_MONTH)
|
@Parameter(PARAM_YEAR_MONTH)
|
||||||
static Optional<String> provideYearMonthOptional(HttpServletRequest req) {
|
static Optional<YearMonth> provideYearMonthOptional(HttpServletRequest req) {
|
||||||
return extractOptionalParameter(req, PARAM_YEAR_MONTH);
|
DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM");
|
||||||
|
Optional<String> optionalYearMonthStr = extractOptionalParameter(req, PARAM_YEAR_MONTH);
|
||||||
|
try {
|
||||||
|
return optionalYearMonthStr.map(s -> YearMonth.parse(s, formatter));
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
throw new BadRequestException(
|
||||||
|
String.format(
|
||||||
|
"yearMonth must be in yyyy-MM format, got %s instead",
|
||||||
|
optionalYearMonthStr.orElse("UNSPECIFIED YEARMONTH")));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Provides the yearMonth in yyyy-MM format, defaults to one month prior to run time. */
|
/** Provides the yearMonth in yyyy-MM format, defaults to one month prior to run time. */
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter(PARAM_YEAR_MONTH)
|
static YearMonth provideYearMonth(
|
||||||
static String provideYearMonth(
|
@Parameter(PARAM_YEAR_MONTH) Optional<YearMonth> yearMonthOptional, Clock clock) {
|
||||||
@Parameter(PARAM_OPTIONAL_YEAR_MONTH) Optional<String> yearMonthOptional, Clock clock) {
|
return yearMonthOptional.orElseGet(() -> new YearMonth(clock.nowUtc().minusMonths(1)));
|
||||||
String yearMonth =
|
|
||||||
yearMonthOptional.orElse(
|
|
||||||
DateTimeFormat.forPattern("yyyy-MM").print(clock.nowUtc().minusMonths(1)));
|
|
||||||
if (!yearMonth.matches("[0-9]{4}-[0-9]{2}")) {
|
|
||||||
throw new BadRequestException(
|
|
||||||
String.format("yearMonth must be in yyyy-MM format, got %s instead", yearMonth));
|
|
||||||
}
|
|
||||||
return yearMonth;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Provides an optional subdirectory to store/upload reports to, extracted from the request. */
|
/** Provides an optional subdirectory to store/upload reports to, extracted from the request. */
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter(PARAM_OPTIONAL_SUBDIR)
|
@Parameter(PARAM_SUBDIR)
|
||||||
static Optional<String> provideSubdirOptional(HttpServletRequest req) {
|
static Optional<String> provideSubdirOptional(HttpServletRequest req) {
|
||||||
return extractOptionalParameter(req, PARAM_SUBDIR);
|
return extractOptionalParameter(req, PARAM_SUBDIR);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Provides the subdirectory to store/upload reports to, defaults to icann/monthly/yearMonth. */
|
/** Provides the subdirectory to store/upload reports to, defaults to icann/monthly/yearMonth. */
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter(PARAM_SUBDIR)
|
@ReportingSubdir
|
||||||
static String provideSubdir(
|
static String provideSubdir(
|
||||||
@Parameter(PARAM_OPTIONAL_SUBDIR) Optional<String> subdirOptional,
|
@Parameter(PARAM_SUBDIR) Optional<String> subdirOptional, YearMonth yearMonth) {
|
||||||
@Parameter(PARAM_YEAR_MONTH) String yearMonth) {
|
String subdir =
|
||||||
String subdir = subdirOptional.orElse(String.format("%s/%s", DEFAULT_SUBDIR, yearMonth));
|
subdirOptional.orElse(
|
||||||
|
String.format(
|
||||||
|
"%s/%s", DEFAULT_SUBDIR, DateTimeFormat.forPattern("yyyy-MM").print(yearMonth)));
|
||||||
if (subdir.startsWith("/") || subdir.endsWith("/")) {
|
if (subdir.startsWith("/") || subdir.endsWith("/")) {
|
||||||
throw new BadRequestException(
|
throw new BadRequestException(
|
||||||
String.format("subdir must not start or end with a \"/\", got %s instead.", subdir));
|
String.format("subdir must not start or end with a \"/\", got %s instead.", subdir));
|
||||||
|
@ -100,14 +107,15 @@ public final class IcannReportingModule {
|
||||||
|
|
||||||
/** Provides an optional reportType to store/upload reports to, extracted from the request. */
|
/** Provides an optional reportType to store/upload reports to, extracted from the request. */
|
||||||
@Provides
|
@Provides
|
||||||
|
@Parameter(PARAM_REPORT_TYPE)
|
||||||
static Optional<ReportType> provideReportTypeOptional(HttpServletRequest req) {
|
static Optional<ReportType> provideReportTypeOptional(HttpServletRequest req) {
|
||||||
return extractOptionalEnumParameter(req, ReportType.class, PARAM_REPORT_TYPE);
|
return extractOptionalEnumParameter(req, ReportType.class, PARAM_REPORT_TYPE);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Provides a list of reportTypes specified. If absent, we default to both report types. */
|
/** Provides a list of reportTypes specified. If absent, we default to both report types. */
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter(PARAM_REPORT_TYPE)
|
static ImmutableList<ReportType> provideReportTypes(
|
||||||
static ImmutableList<ReportType> provideReportTypes(Optional<ReportType> reportTypeOptional) {
|
@Parameter(PARAM_REPORT_TYPE) Optional<ReportType> reportTypeOptional) {
|
||||||
return reportTypeOptional.map(ImmutableList::of)
|
return reportTypeOptional.map(ImmutableList::of)
|
||||||
.orElseGet(() -> ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS));
|
.orElseGet(() -> ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS));
|
||||||
}
|
}
|
||||||
|
@ -144,5 +152,11 @@ public final class IcannReportingModule {
|
||||||
static SendEmailService provideSendEmailService() {
|
static SendEmailService provideSendEmailService() {
|
||||||
return new SendEmailService();
|
return new SendEmailService();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Dagger qualifier for the subdirectory we stage to/upload from. */
|
||||||
|
@Qualifier
|
||||||
|
@Documented
|
||||||
|
@Retention(RUNTIME)
|
||||||
|
public @interface ReportingSubdir {}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -34,7 +34,7 @@ import google.registry.bigquery.BigqueryUtils.TableType;
|
||||||
import google.registry.config.RegistryConfig.Config;
|
import google.registry.config.RegistryConfig.Config;
|
||||||
import google.registry.gcs.GcsUtils;
|
import google.registry.gcs.GcsUtils;
|
||||||
import google.registry.reporting.IcannReportingModule.ReportType;
|
import google.registry.reporting.IcannReportingModule.ReportType;
|
||||||
import google.registry.request.Parameter;
|
import google.registry.reporting.IcannReportingModule.ReportingSubdir;
|
||||||
import google.registry.util.FormattingLogger;
|
import google.registry.util.FormattingLogger;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
@ -47,6 +47,8 @@ import java.util.Map.Entry;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.concurrent.ExecutionException;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import javax.inject.Inject;
|
import javax.inject.Inject;
|
||||||
|
import org.joda.time.YearMonth;
|
||||||
|
import org.joda.time.format.DateTimeFormat;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Class containing methods for staging ICANN monthly reports on GCS.
|
* Class containing methods for staging ICANN monthly reports on GCS.
|
||||||
|
@ -58,10 +60,9 @@ public class IcannReportingStager {
|
||||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||||
|
|
||||||
@Inject @Config("icannReportingBucket") String reportingBucket;
|
@Inject @Config("icannReportingBucket") String reportingBucket;
|
||||||
@Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
|
|
||||||
|
|
||||||
@Inject
|
@Inject YearMonth yearMonth;
|
||||||
@Parameter(IcannReportingModule.PARAM_SUBDIR)
|
@Inject @ReportingSubdir
|
||||||
String subdir;
|
String subdir;
|
||||||
|
|
||||||
@Inject ActivityReportingQueryBuilder activityQueryBuilder;
|
@Inject ActivityReportingQueryBuilder activityQueryBuilder;
|
||||||
|
@ -242,7 +243,9 @@ public class IcannReportingStager {
|
||||||
String reportFilename =
|
String reportFilename =
|
||||||
String.format(
|
String.format(
|
||||||
"%s-%s-%s.csv",
|
"%s-%s-%s.csv",
|
||||||
tld, Ascii.toLowerCase(reportType.toString()), yearMonth.replace("-", ""));
|
tld,
|
||||||
|
Ascii.toLowerCase(reportType.toString()),
|
||||||
|
DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
|
||||||
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
|
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
|
||||||
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
|
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
|
||||||
gcsUtils.createFromBytes(gcsFilename, reportBytes);
|
gcsUtils.createFromBytes(gcsFilename, reportBytes);
|
||||||
|
|
|
@ -24,7 +24,6 @@ import com.google.common.net.MediaType;
|
||||||
import google.registry.bigquery.BigqueryJobFailureException;
|
import google.registry.bigquery.BigqueryJobFailureException;
|
||||||
import google.registry.reporting.IcannReportingModule.ReportType;
|
import google.registry.reporting.IcannReportingModule.ReportType;
|
||||||
import google.registry.request.Action;
|
import google.registry.request.Action;
|
||||||
import google.registry.request.Parameter;
|
|
||||||
import google.registry.request.Response;
|
import google.registry.request.Response;
|
||||||
import google.registry.request.auth.Auth;
|
import google.registry.request.auth.Auth;
|
||||||
import google.registry.util.FormattingLogger;
|
import google.registry.util.FormattingLogger;
|
||||||
|
@ -56,10 +55,7 @@ public final class IcannReportingStagingAction implements Runnable {
|
||||||
|
|
||||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||||
|
|
||||||
@Inject
|
@Inject ImmutableList<ReportType> reportTypes;
|
||||||
@Parameter(IcannReportingModule.PARAM_REPORT_TYPE)
|
|
||||||
ImmutableList<ReportType> reportTypes;
|
|
||||||
|
|
||||||
@Inject IcannReportingStager stager;
|
@Inject IcannReportingStager stager;
|
||||||
@Inject Retrier retrier;
|
@Inject Retrier retrier;
|
||||||
@Inject Response response;
|
@Inject Response response;
|
||||||
|
|
|
@ -27,8 +27,8 @@ import com.google.common.collect.ImmutableMap;
|
||||||
import com.google.common.io.ByteStreams;
|
import com.google.common.io.ByteStreams;
|
||||||
import google.registry.config.RegistryConfig.Config;
|
import google.registry.config.RegistryConfig.Config;
|
||||||
import google.registry.gcs.GcsUtils;
|
import google.registry.gcs.GcsUtils;
|
||||||
|
import google.registry.reporting.IcannReportingModule.ReportingSubdir;
|
||||||
import google.registry.request.Action;
|
import google.registry.request.Action;
|
||||||
import google.registry.request.Parameter;
|
|
||||||
import google.registry.request.Response;
|
import google.registry.request.Response;
|
||||||
import google.registry.request.auth.Auth;
|
import google.registry.request.auth.Auth;
|
||||||
import google.registry.util.FormattingLogger;
|
import google.registry.util.FormattingLogger;
|
||||||
|
@ -62,8 +62,7 @@ public final class IcannReportingUploadAction implements Runnable {
|
||||||
@Config("icannReportingBucket")
|
@Config("icannReportingBucket")
|
||||||
String reportingBucket;
|
String reportingBucket;
|
||||||
|
|
||||||
@Inject
|
@Inject @ReportingSubdir
|
||||||
@Parameter(IcannReportingModule.PARAM_SUBDIR)
|
|
||||||
String subdir;
|
String subdir;
|
||||||
|
|
||||||
@Inject GcsUtils gcsUtils;
|
@Inject GcsUtils gcsUtils;
|
||||||
|
|
|
@ -20,13 +20,14 @@ import static google.registry.reporting.IcannReportingModule.ICANN_REPORTING_DAT
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import com.google.common.io.Resources;
|
import com.google.common.io.Resources;
|
||||||
import google.registry.config.RegistryConfig.Config;
|
import google.registry.config.RegistryConfig.Config;
|
||||||
import google.registry.request.Parameter;
|
|
||||||
import google.registry.util.ResourceUtils;
|
import google.registry.util.ResourceUtils;
|
||||||
import google.registry.util.SqlTemplate;
|
import google.registry.util.SqlTemplate;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import javax.inject.Inject;
|
import javax.inject.Inject;
|
||||||
import org.joda.time.DateTime;
|
import org.joda.time.DateTime;
|
||||||
|
import org.joda.time.LocalTime;
|
||||||
|
import org.joda.time.YearMonth;
|
||||||
import org.joda.time.format.DateTimeFormat;
|
import org.joda.time.format.DateTimeFormat;
|
||||||
import org.joda.time.format.DateTimeFormatter;
|
import org.joda.time.format.DateTimeFormatter;
|
||||||
|
|
||||||
|
@ -36,7 +37,9 @@ import org.joda.time.format.DateTimeFormatter;
|
||||||
public final class TransactionsReportingQueryBuilder implements QueryBuilder {
|
public final class TransactionsReportingQueryBuilder implements QueryBuilder {
|
||||||
|
|
||||||
@Inject @Config("projectId") String projectId;
|
@Inject @Config("projectId") String projectId;
|
||||||
@Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
|
|
||||||
|
@Inject YearMonth yearMonth;
|
||||||
|
|
||||||
@Inject TransactionsReportingQueryBuilder() {}
|
@Inject TransactionsReportingQueryBuilder() {}
|
||||||
|
|
||||||
static final String TRANSACTIONS_REPORT_AGGREGATION = "transactions_report_aggregation";
|
static final String TRANSACTIONS_REPORT_AGGREGATION = "transactions_report_aggregation";
|
||||||
|
@ -61,15 +64,9 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
|
||||||
@Override
|
@Override
|
||||||
public ImmutableMap<String, String> getViewQueryMap() throws IOException {
|
public ImmutableMap<String, String> getViewQueryMap() throws IOException {
|
||||||
// Set the earliest date to to yearMonth on day 1 at 00:00:00
|
// Set the earliest date to to yearMonth on day 1 at 00:00:00
|
||||||
DateTime earliestReportTime =
|
DateTime earliestReportTime = yearMonth.toLocalDate(1).toDateTime(new LocalTime(0, 0, 0));
|
||||||
DateTimeFormat.forPattern("yyyy-MM")
|
// Set the latest date to yearMonth on the last day at 23:59:59.999
|
||||||
.parseDateTime(yearMonth)
|
DateTime latestReportTime = earliestReportTime.plusMonths(1).minusMillis(1);
|
||||||
.withDayOfMonth(1)
|
|
||||||
.withHourOfDay(0)
|
|
||||||
.withMinuteOfHour(0)
|
|
||||||
.withSecondOfMinute(0);
|
|
||||||
// Set the latest date to yearMonth on the last day at 23:59:59
|
|
||||||
DateTime latestReportTime = earliestReportTime.plusMonths(1).minusSeconds(1);
|
|
||||||
return createQueryMap(earliestReportTime, latestReportTime);
|
return createQueryMap(earliestReportTime, latestReportTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -95,7 +92,7 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
|
||||||
.build();
|
.build();
|
||||||
queriesBuilder.put(getTableName(TOTAL_DOMAINS), totalDomainsQuery);
|
queriesBuilder.put(getTableName(TOTAL_DOMAINS), totalDomainsQuery);
|
||||||
|
|
||||||
DateTimeFormatter timestampFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
|
DateTimeFormatter timestampFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
|
||||||
String totalNameserversQuery =
|
String totalNameserversQuery =
|
||||||
SqlTemplate.create(getQueryFromFile("total_nameservers.sql"))
|
SqlTemplate.create(getQueryFromFile("total_nameservers.sql"))
|
||||||
.put("PROJECT_ID", projectId)
|
.put("PROJECT_ID", projectId)
|
||||||
|
@ -173,9 +170,9 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
|
||||||
return queriesBuilder.build();
|
return queriesBuilder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Returns the table name of the query, suffixed with the yearMonth in _YYYYMM format. */
|
/** Returns the table name of the query, suffixed with the yearMonth in _yyyyMM format. */
|
||||||
private String getTableName(String queryName) {
|
private String getTableName(String queryName) {
|
||||||
return String.format("%s_%s", queryName, yearMonth.replace("-", ""));
|
return String.format("%s_%s", queryName, DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Returns {@link String} for file in {@code reporting/sql/} directory. */
|
/** Returns {@link String} for file in {@code reporting/sql/} directory. */
|
||||||
|
|
|
@ -19,6 +19,7 @@ import static com.google.common.truth.Truth.assertThat;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import org.joda.time.YearMonth;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.runner.RunWith;
|
import org.junit.runner.RunWith;
|
||||||
import org.junit.runners.JUnit4;
|
import org.junit.runners.JUnit4;
|
||||||
|
@ -29,7 +30,7 @@ public class ActivityReportingQueryBuilderTest {
|
||||||
|
|
||||||
private ActivityReportingQueryBuilder getQueryBuilder() {
|
private ActivityReportingQueryBuilder getQueryBuilder() {
|
||||||
ActivityReportingQueryBuilder queryBuilder = new ActivityReportingQueryBuilder();
|
ActivityReportingQueryBuilder queryBuilder = new ActivityReportingQueryBuilder();
|
||||||
queryBuilder.yearMonth = "2017-09";
|
queryBuilder.yearMonth = new YearMonth(2017, 9);
|
||||||
queryBuilder.projectId = "domain-registry-alpha";
|
queryBuilder.projectId = "domain-registry-alpha";
|
||||||
return queryBuilder;
|
return queryBuilder;
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,6 +16,7 @@ package google.registry.reporting;
|
||||||
|
|
||||||
import static com.google.common.truth.Truth.assertThat;
|
import static com.google.common.truth.Truth.assertThat;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
import google.registry.reporting.IcannReportingModule.ReportType;
|
import google.registry.reporting.IcannReportingModule.ReportType;
|
||||||
import google.registry.request.HttpException.BadRequestException;
|
import google.registry.request.HttpException.BadRequestException;
|
||||||
|
@ -25,6 +26,7 @@ import google.registry.util.Clock;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
import org.joda.time.DateTime;
|
import org.joda.time.DateTime;
|
||||||
|
import org.joda.time.YearMonth;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Rule;
|
import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
@ -46,32 +48,41 @@ public class IcannReportingModuleTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testEmptyYearMonth_returnsCurrentDate() {
|
public void testEmptyYearMonthParameter_returnsEmptyYearMonthOptional() {
|
||||||
assertThat(IcannReportingModule.provideYearMonth(Optional.empty(), clock)).isEqualTo("2017-06");
|
when(req.getParameter("yearMonth")).thenReturn("");
|
||||||
|
assertThat(IcannReportingModule.provideYearMonthOptional(req)).isEqualTo(Optional.empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testInvalidYearMonthParameter_throwsException() {
|
||||||
|
when(req.getParameter("yearMonth")).thenReturn("201705");
|
||||||
|
thrown.expect(
|
||||||
|
BadRequestException.class, "yearMonth must be in yyyy-MM format, got 201705 instead");
|
||||||
|
IcannReportingModule.provideYearMonthOptional(req);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testEmptyYearMonth_returnsLastMonth() {
|
||||||
|
assertThat(IcannReportingModule.provideYearMonth(Optional.empty(), clock))
|
||||||
|
.isEqualTo(new YearMonth(2017, 6));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testGivenYearMonth_returnsThatMonth() {
|
public void testGivenYearMonth_returnsThatMonth() {
|
||||||
assertThat(IcannReportingModule.provideYearMonth(Optional.of("2017-05"), clock))
|
assertThat(IcannReportingModule.provideYearMonth(Optional.of(new YearMonth(2017, 5)), clock))
|
||||||
.isEqualTo("2017-05");
|
.isEqualTo(new YearMonth(2017, 5));
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testInvalidYearMonth_throwsException() {
|
|
||||||
thrown.expect(
|
|
||||||
BadRequestException.class, "yearMonth must be in yyyy-MM format, got 201705 instead");
|
|
||||||
IcannReportingModule.provideYearMonth(Optional.of("201705"), clock);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testEmptySubDir_returnsDefaultSubdir() {
|
public void testEmptySubDir_returnsDefaultSubdir() {
|
||||||
assertThat(IcannReportingModule.provideSubdir(Optional.empty(), "2017-06"))
|
assertThat(IcannReportingModule.provideSubdir(Optional.empty(), new YearMonth(2017, 6)))
|
||||||
.isEqualTo("icann/monthly/2017-06");
|
.isEqualTo("icann/monthly/2017-06");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testGivenSubdir_returnsManualSubdir() {
|
public void testGivenSubdir_returnsManualSubdir() {
|
||||||
assertThat(IcannReportingModule.provideSubdir(Optional.of("manual/dir"), "2017-06"))
|
assertThat(
|
||||||
|
IcannReportingModule.provideSubdir(Optional.of("manual/dir"), new YearMonth(2017, 6)))
|
||||||
.isEqualTo("manual/dir");
|
.isEqualTo("manual/dir");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -80,7 +91,7 @@ public class IcannReportingModuleTest {
|
||||||
thrown.expect(
|
thrown.expect(
|
||||||
BadRequestException.class,
|
BadRequestException.class,
|
||||||
"subdir must not start or end with a \"/\", got /whoops instead.");
|
"subdir must not start or end with a \"/\", got /whoops instead.");
|
||||||
IcannReportingModule.provideSubdir(Optional.of("/whoops"), "2017-06");
|
IcannReportingModule.provideSubdir(Optional.of("/whoops"), new YearMonth(2017, 6));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -39,6 +39,7 @@ import java.util.concurrent.ExecutionException;
|
||||||
import java.util.concurrent.Executor;
|
import java.util.concurrent.Executor;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.concurrent.TimeoutException;
|
import java.util.concurrent.TimeoutException;
|
||||||
|
import org.joda.time.YearMonth;
|
||||||
import org.junit.Rule;
|
import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.runner.RunWith;
|
import org.junit.runner.RunWith;
|
||||||
|
@ -62,14 +63,14 @@ public class IcannReportingStagerTest {
|
||||||
IcannReportingStager action = new IcannReportingStager();
|
IcannReportingStager action = new IcannReportingStager();
|
||||||
ActivityReportingQueryBuilder activityBuilder = new ActivityReportingQueryBuilder();
|
ActivityReportingQueryBuilder activityBuilder = new ActivityReportingQueryBuilder();
|
||||||
activityBuilder.projectId = "test-project";
|
activityBuilder.projectId = "test-project";
|
||||||
activityBuilder.yearMonth = "2017-06";
|
activityBuilder.yearMonth = new YearMonth(2017, 6);
|
||||||
action.activityQueryBuilder = activityBuilder;
|
action.activityQueryBuilder = activityBuilder;
|
||||||
TransactionsReportingQueryBuilder transactionsBuilder = new TransactionsReportingQueryBuilder();
|
TransactionsReportingQueryBuilder transactionsBuilder = new TransactionsReportingQueryBuilder();
|
||||||
transactionsBuilder.projectId = "test-project";
|
transactionsBuilder.projectId = "test-project";
|
||||||
transactionsBuilder.yearMonth = "2017-06";
|
transactionsBuilder.yearMonth = new YearMonth(2017, 6);
|
||||||
action.transactionsQueryBuilder = transactionsBuilder;
|
action.transactionsQueryBuilder = transactionsBuilder;
|
||||||
action.reportingBucket = "test-bucket";
|
action.reportingBucket = "test-bucket";
|
||||||
action.yearMonth = "2017-06";
|
action.yearMonth = new YearMonth(2017, 6);
|
||||||
action.subdir = "icann/monthly/2017-06";
|
action.subdir = "icann/monthly/2017-06";
|
||||||
action.bigquery = bigquery;
|
action.bigquery = bigquery;
|
||||||
action.gcsUtils = new GcsUtils(gcsService, 1024);
|
action.gcsUtils = new GcsUtils(gcsService, 1024);
|
||||||
|
|
|
@ -131,4 +131,3 @@ public class IcannReportingStagingActionTest {
|
||||||
+ " check logs for more details.");
|
+ " check logs for more details.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -19,6 +19,7 @@ import static com.google.common.truth.Truth.assertThat;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import org.joda.time.YearMonth;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.runner.RunWith;
|
import org.junit.runner.RunWith;
|
||||||
import org.junit.runners.JUnit4;
|
import org.junit.runners.JUnit4;
|
||||||
|
@ -29,7 +30,7 @@ public class TransactionsReportingQueryBuilderTest {
|
||||||
|
|
||||||
private TransactionsReportingQueryBuilder getQueryBuilder() {
|
private TransactionsReportingQueryBuilder getQueryBuilder() {
|
||||||
TransactionsReportingQueryBuilder queryBuilder = new TransactionsReportingQueryBuilder();
|
TransactionsReportingQueryBuilder queryBuilder = new TransactionsReportingQueryBuilder();
|
||||||
queryBuilder.yearMonth = "2017-09";
|
queryBuilder.yearMonth = new YearMonth(2017, 9);
|
||||||
queryBuilder.projectId = "domain-registry-alpha";
|
queryBuilder.projectId = "domain-registry-alpha";
|
||||||
return queryBuilder;
|
return queryBuilder;
|
||||||
}
|
}
|
||||||
|
|
|
@ -45,12 +45,12 @@ JOIN (
|
||||||
`domain-registry-alpha.latest_datastore_export.DomainBase`,
|
`domain-registry-alpha.latest_datastore_export.DomainBase`,
|
||||||
UNNEST(nsHosts) AS hosts
|
UNNEST(nsHosts) AS hosts
|
||||||
WHERE _d = 'DomainResource'
|
WHERE _d = 'DomainResource'
|
||||||
AND creationTime <= TIMESTAMP("2017-09-30 23:59:59")
|
AND creationTime <= TIMESTAMP("2017-09-30 23:59:59.999")
|
||||||
AND deletionTime > TIMESTAMP("2017-09-30 23:59:59") ) AS domain_table
|
AND deletionTime > TIMESTAMP("2017-09-30 23:59:59.999") ) AS domain_table
|
||||||
ON
|
ON
|
||||||
host_table.__key__.name = domain_table.referencedHostName
|
host_table.__key__.name = domain_table.referencedHostName
|
||||||
WHERE creationTime <= TIMESTAMP("2017-09-30 23:59:59")
|
WHERE creationTime <= TIMESTAMP("2017-09-30 23:59:59.999")
|
||||||
AND deletionTime > TIMESTAMP("2017-09-30 23:59:59")
|
AND deletionTime > TIMESTAMP("2017-09-30 23:59:59.999")
|
||||||
GROUP BY tld, registrarName
|
GROUP BY tld, registrarName
|
||||||
ORDER BY tld, registrarName
|
ORDER BY tld, registrarName
|
||||||
|
|
||||||
|
|
|
@ -63,8 +63,8 @@ FROM (
|
||||||
WHERE entries.domainTransactionRecords IS NOT NULL )
|
WHERE entries.domainTransactionRecords IS NOT NULL )
|
||||||
-- Only look at this month's data
|
-- Only look at this month's data
|
||||||
WHERE reportingTime
|
WHERE reportingTime
|
||||||
BETWEEN TIMESTAMP('2017-09-01 00:00:00')
|
BETWEEN TIMESTAMP('2017-09-01 00:00:00.000')
|
||||||
AND TIMESTAMP('2017-09-30 23:59:59')
|
AND TIMESTAMP('2017-09-30 23:59:59.999')
|
||||||
GROUP BY
|
GROUP BY
|
||||||
tld,
|
tld,
|
||||||
clientId,
|
clientId,
|
||||||
|
|
|
@ -63,8 +63,8 @@ FROM (
|
||||||
WHERE entries.domainTransactionRecords IS NOT NULL )
|
WHERE entries.domainTransactionRecords IS NOT NULL )
|
||||||
-- Only look at this month's data
|
-- Only look at this month's data
|
||||||
WHERE reportingTime
|
WHERE reportingTime
|
||||||
BETWEEN TIMESTAMP('2017-09-01 00:00:00')
|
BETWEEN TIMESTAMP('2017-09-01 00:00:00.000')
|
||||||
AND TIMESTAMP('2017-09-30 23:59:59')
|
AND TIMESTAMP('2017-09-30 23:59:59.999')
|
||||||
GROUP BY
|
GROUP BY
|
||||||
tld,
|
tld,
|
||||||
clientId,
|
clientId,
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue