zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-15 08:57:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

Only inject @Parameter-created variables in the Action itself

Browse source 
Icann reports have 3 parameter-provided injections:

- yearMonth
- subdir
- reportType

We move all of them away from the "inner classes" and only @Inject them in the Actions themselves.

This has 2 benefits:
- it's much clearer what all the parameter inputs of the Actions are
- the "inner injected classes" don't assume anything about the Action that uses them - they will work just as well for JSON actions as for "regular" actions.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=233625765
This commit is contained in:
guyben 2019-02-12 09:56:26 -08:00 • committed by jianglai
parent e6c46cab58
commit 4097dae3b2
20 changed files with 364 additions and 270 deletions
Download patch file Download diff file Expand all files Collapse all files

67
java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java
View file

@ -16,13 +16,12 @@ package google.registry.reporting.icann;
import static google.registry.reporting.icann.IcannReportingModule.DATASTORE_EXPORT_DATA_SET; import static google.registry.reporting.icann.IcannReportingModule.DATASTORE_EXPORT_DATA_SET;
import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET; import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET;
import static google.registry.reporting.icann.QueryBuilderUtils.getQueryFromFile;
import static google.registry.reporting.icann.QueryBuilderUtils.getTableName;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.io.Resources;
import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryConfig.Config;
import google.registry.util.ResourceUtils;
import google.registry.util.SqlTemplate; import google.registry.util.SqlTemplate;
import java.net.URL;
import javax.inject.Inject; import javax.inject.Inject;
import org.joda.time.LocalDate; import org.joda.time.LocalDate;
import org.joda.time.YearMonth; import org.joda.time.YearMonth;
@ -44,8 +43,6 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
@Config("projectId") @Config("projectId")
String projectId; String projectId;
@Inject YearMonth yearMonth;
@Inject DnsCountQueryCoordinator dnsCountQueryCoordinator; @Inject DnsCountQueryCoordinator dnsCountQueryCoordinator;
@Inject @Inject
@ -53,28 +50,19 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
/** Returns the aggregate query which generates the activity report from the saved view. */ /** Returns the aggregate query which generates the activity report from the saved view. */
@Override @Override
public String getReportQuery() { public String getReportQuery(YearMonth yearMonth) {
return String.format( return String.format(
"#standardSQL\nSELECT * FROM `%s.%s.%s`", "#standardSQL\nSELECT * FROM `%s.%s.%s`",
projectId, ICANN_REPORTING_DATA_SET, getTableName(ACTIVITY_REPORT_AGGREGATION)); projectId, ICANN_REPORTING_DATA_SET, getTableName(ACTIVITY_REPORT_AGGREGATION, yearMonth));
} }
/** Sets the month we're doing activity reporting for, and returns the view query map. */ /** Sets the month we're doing activity reporting for, and returns the view query map. */
@Override @Override
public ImmutableMap<String, String> getViewQueryMap() { public ImmutableMap<String, String> getViewQueryMap(YearMonth yearMonth) {
LocalDate firstDayOfMonth = yearMonth.toLocalDate(1); LocalDate firstDayOfMonth = yearMonth.toLocalDate(1);
// The pattern-matching is inclusive, so we subtract 1 day to only report that month's data. // The pattern-matching is inclusive, so we subtract 1 day to only report that month's data.
LocalDate lastDayOfMonth = yearMonth.toLocalDate(1).plusMonths(1).minusDays(1); LocalDate lastDayOfMonth = yearMonth.toLocalDate(1).plusMonths(1).minusDays(1);
return createQueryMap(firstDayOfMonth, lastDayOfMonth);
}
public void prepareForQuery() throws Exception {
dnsCountQueryCoordinator.prepareForQuery();
}
/** Returns a map from view name to its associated SQL query. */
private ImmutableMap<String, String> createQueryMap(
LocalDate firstDayOfMonth, LocalDate lastDayOfMonth) {
ImmutableMap.Builder<String, String> queriesBuilder = ImmutableMap.builder(); ImmutableMap.Builder<String, String> queriesBuilder = ImmutableMap.builder();
String operationalRegistrarsQuery = String operationalRegistrarsQuery =
@ -83,10 +71,11 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET) .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
.put("REGISTRAR_TABLE", "Registrar") .put("REGISTRAR_TABLE", "Registrar")
.build(); .build();
queriesBuilder.put(getTableName(REGISTRAR_OPERATING_STATUS), operationalRegistrarsQuery); queriesBuilder.put(
getTableName(REGISTRAR_OPERATING_STATUS, yearMonth), operationalRegistrarsQuery);
String dnsCountsQuery = dnsCountQueryCoordinator.createQuery(); String dnsCountsQuery = dnsCountQueryCoordinator.createQuery(yearMonth);
queriesBuilder.put(getTableName(DNS_COUNTS), dnsCountsQuery); queriesBuilder.put(getTableName(DNS_COUNTS, yearMonth), dnsCountsQuery);
// Convert reportingMonth into YYYYMMDD format for Bigquery table partition pattern-matching. // Convert reportingMonth into YYYYMMDD format for Bigquery table partition pattern-matching.
DateTimeFormatter logTableFormatter = DateTimeFormat.forPattern("yyyyMMdd"); DateTimeFormatter logTableFormatter = DateTimeFormat.forPattern("yyyyMMdd");
@ -99,55 +88,47 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
.put("FIRST_DAY_OF_MONTH", logTableFormatter.print(firstDayOfMonth)) .put("FIRST_DAY_OF_MONTH", logTableFormatter.print(firstDayOfMonth))
.put("LAST_DAY_OF_MONTH", logTableFormatter.print(lastDayOfMonth)) .put("LAST_DAY_OF_MONTH", logTableFormatter.print(lastDayOfMonth))
.build(); .build();
queriesBuilder.put(getTableName(MONTHLY_LOGS), monthlyLogsQuery); queriesBuilder.put(getTableName(MONTHLY_LOGS, yearMonth), monthlyLogsQuery);
String eppQuery = String eppQuery =
SqlTemplate.create(getQueryFromFile("epp_metrics.sql")) SqlTemplate.create(getQueryFromFile("epp_metrics.sql"))
.put("PROJECT_ID", projectId) .put("PROJECT_ID", projectId)
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET) .put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
.put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS)) .put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS, yearMonth))
// All metadata logs for reporting come from google.registry.flows.FlowReporter. // All metadata logs for reporting come from google.registry.flows.FlowReporter.
.put( .put(
"METADATA_LOG_PREFIX", "METADATA_LOG_PREFIX",
"google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA") "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
.build(); .build();
queriesBuilder.put(getTableName(EPP_METRICS), eppQuery); queriesBuilder.put(getTableName(EPP_METRICS, yearMonth), eppQuery);
String whoisQuery = String whoisQuery =
SqlTemplate.create(getQueryFromFile("whois_counts.sql")) SqlTemplate.create(getQueryFromFile("whois_counts.sql"))
.put("PROJECT_ID", projectId) .put("PROJECT_ID", projectId)
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET) .put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
.put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS)) .put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS, yearMonth))
.build(); .build();
queriesBuilder.put(getTableName(WHOIS_COUNTS), whoisQuery); queriesBuilder.put(getTableName(WHOIS_COUNTS, yearMonth), whoisQuery);
String aggregateQuery = String aggregateQuery =
SqlTemplate.create(getQueryFromFile("activity_report_aggregation.sql")) SqlTemplate.create(getQueryFromFile("activity_report_aggregation.sql"))
.put("PROJECT_ID", projectId) .put("PROJECT_ID", projectId)
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET) .put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
.put("REGISTRAR_OPERATING_STATUS_TABLE", getTableName(REGISTRAR_OPERATING_STATUS)) .put(
.put("DNS_COUNTS_TABLE", getTableName(DNS_COUNTS)) "REGISTRAR_OPERATING_STATUS_TABLE",
.put("EPP_METRICS_TABLE", getTableName(EPP_METRICS)) getTableName(REGISTRAR_OPERATING_STATUS, yearMonth))
.put("WHOIS_COUNTS_TABLE", getTableName(WHOIS_COUNTS)) .put("DNS_COUNTS_TABLE", getTableName(DNS_COUNTS, yearMonth))
.put("EPP_METRICS_TABLE", getTableName(EPP_METRICS, yearMonth))
.put("WHOIS_COUNTS_TABLE", getTableName(WHOIS_COUNTS, yearMonth))
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET) .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
.put("REGISTRY_TABLE", "Registry") .put("REGISTRY_TABLE", "Registry")
.build(); .build();
queriesBuilder.put(getTableName(ACTIVITY_REPORT_AGGREGATION), aggregateQuery); queriesBuilder.put(getTableName(ACTIVITY_REPORT_AGGREGATION, yearMonth), aggregateQuery);
return queriesBuilder.build(); return queriesBuilder.build();
} }
/** Returns the table name of the query, suffixed with the yearMonth in _yyyyMM format. */ public void prepareForQuery(YearMonth yearMonth) throws Exception {
private String getTableName(String queryName) { dnsCountQueryCoordinator.prepareForQuery(yearMonth);
return String.format("%s_%s", queryName, DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
}
/** Returns {@link String} for file in {@code reporting/sql/} directory. */
private static String getQueryFromFile(String filename) {
return ResourceUtils.readResourceUtf8(getUrl(filename));
}
private static URL getUrl(String filename) {
return Resources.getResource(ActivityReportingQueryBuilder.class, "sql/" + filename);
} }
} }

1
java/google/registry/reporting/icann/BUILD
View file

@ -14,6 +14,7 @@ java_library(
"//java/google/registry/gcs", "//java/google/registry/gcs",
"//java/google/registry/keyring/api", "//java/google/registry/keyring/api",
"//java/google/registry/model", "//java/google/registry/model",
"//java/google/registry/reporting",
"//java/google/registry/request", "//java/google/registry/request",
"//java/google/registry/request/auth", "//java/google/registry/request/auth",
"//java/google/registry/util", "//java/google/registry/util",

5
java/google/registry/reporting/icann/BasicDnsCountQueryCoordinator.java
View file

@ -17,6 +17,7 @@ package google.registry.reporting.icann;
import com.google.common.io.Resources; import com.google.common.io.Resources;
import google.registry.util.ResourceUtils; import google.registry.util.ResourceUtils;
import google.registry.util.SqlTemplate; import google.registry.util.SqlTemplate;
import org.joda.time.YearMonth;
/** /**
* DNS Count query for the basic case. * DNS Count query for the basic case.
@ -26,7 +27,7 @@ public class BasicDnsCountQueryCoordinator implements DnsCountQueryCoordinator {
BasicDnsCountQueryCoordinator(DnsCountQueryCoordinator.Params params) {} BasicDnsCountQueryCoordinator(DnsCountQueryCoordinator.Params params) {}
@Override @Override
public String createQuery() { public String createQuery(YearMonth yearMonth) {
return SqlTemplate.create( return SqlTemplate.create(
ResourceUtils.readResourceUtf8( ResourceUtils.readResourceUtf8(
Resources.getResource(this.getClass(), "sql/" + "dns_counts.sql"))) Resources.getResource(this.getClass(), "sql/" + "dns_counts.sql")))
@ -34,5 +35,5 @@ public class BasicDnsCountQueryCoordinator implements DnsCountQueryCoordinator {
} }
@Override @Override
public void prepareForQuery() throws Exception {} public void prepareForQuery(YearMonth yearMonth) throws Exception {}
} }

10
java/google/registry/reporting/icann/DnsCountQueryCoordinator.java
View file

@ -37,22 +37,18 @@ public interface DnsCountQueryCoordinator {
public class Params { public class Params {
public BigqueryConnection bigquery; public BigqueryConnection bigquery;
/** The year and month of the report. */
public YearMonth yearMonth;
/** The Google Cloud project id. */ /** The Google Cloud project id. */
public String projectId; public String projectId;
public Params(BigqueryConnection bigquery, YearMonth yearMonth, String projectId) { public Params(BigqueryConnection bigquery, String projectId) {
this.bigquery = bigquery; this.bigquery = bigquery;
this.yearMonth = yearMonth;
this.projectId = projectId; this.projectId = projectId;
} }
} }
/** Creates the string used to query bigtable for DNS count information. */ /** Creates the string used to query bigtable for DNS count information. */
String createQuery(); String createQuery(YearMonth yearMonth);
/** Do any necessry preparation for the DNS query. */ /** Do any necessry preparation for the DNS query. */
void prepareForQuery() throws Exception; void prepareForQuery(YearMonth yearMonth) throws Exception;
} }

4
java/google/registry/reporting/icann/DnsCountQueryCoordinatorModule.java
View file

@ -21,7 +21,6 @@ import dagger.Module;
import dagger.Provides; import dagger.Provides;
import google.registry.bigquery.BigqueryConnection; import google.registry.bigquery.BigqueryConnection;
import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryConfig.Config;
import org.joda.time.YearMonth;
/** Dagger module to provide the DnsCountQueryCoordinator. */ /** Dagger module to provide the DnsCountQueryCoordinator. */
@Module @Module
@ -31,10 +30,9 @@ public class DnsCountQueryCoordinatorModule {
static DnsCountQueryCoordinator provideDnsCountQueryCoordinator( static DnsCountQueryCoordinator provideDnsCountQueryCoordinator(
@Config("dnsCountQueryCoordinatorClass") String customClass, @Config("dnsCountQueryCoordinatorClass") String customClass,
BigqueryConnection bigquery, BigqueryConnection bigquery,
YearMonth yearMonth,
@Config("projectId") String projectId) { @Config("projectId") String projectId) {
DnsCountQueryCoordinator.Params params = DnsCountQueryCoordinator.Params params =
new DnsCountQueryCoordinator.Params(bigquery, yearMonth, projectId); new DnsCountQueryCoordinator.Params(bigquery, projectId);
DnsCountQueryCoordinator result = DnsCountQueryCoordinator result =
instantiate(getClassFromString(customClass, DnsCountQueryCoordinator.class), params); instantiate(getClassFromString(customClass, DnsCountQueryCoordinator.class), params);
return result; return result;

54
java/google/registry/reporting/icann/IcannReportingModule.java
View file

@ -14,25 +14,20 @@
package google.registry.reporting.icann; package google.registry.reporting.icann;
import static google.registry.request.RequestParameters.extractOptionalEnumParameter;
import static google.registry.request.RequestParameters.extractOptionalParameter; import static google.registry.request.RequestParameters.extractOptionalParameter;
import static java.lang.annotation.RetentionPolicy.RUNTIME; import static google.registry.request.RequestParameters.extractRequiredParameter;
import static google.registry.request.RequestParameters.extractSetOfEnumParameters;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import dagger.Module; import dagger.Module;
import dagger.Provides; import dagger.Provides;
import google.registry.bigquery.BigqueryConnection; import google.registry.bigquery.BigqueryConnection;
import google.registry.request.HttpException.BadRequestException; import google.registry.request.HttpException.BadRequestException;
import google.registry.request.Parameter; import google.registry.request.Parameter;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.util.Optional; import java.util.Optional;
import javax.inject.Qualifier;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import org.joda.time.Duration; import org.joda.time.Duration;
import org.joda.time.YearMonth;
import org.joda.time.format.DateTimeFormat;
/** Module for dependencies required by ICANN monthly transactions/activity reporting. */ /** Module for dependencies required by ICANN monthly transactions/activity reporting. */
@Module @Module
@ -45,28 +40,26 @@ public final class IcannReportingModule {
} }
static final String PARAM_SUBDIR = "subdir"; static final String PARAM_SUBDIR = "subdir";
static final String PARAM_REPORT_TYPE = "reportType"; static final String PARAM_REPORT_TYPES = "reportTypes";
static final String ICANN_REPORTING_DATA_SET = "icann_reporting"; static final String ICANN_REPORTING_DATA_SET = "icann_reporting";
static final String DATASTORE_EXPORT_DATA_SET = "latest_datastore_export"; static final String DATASTORE_EXPORT_DATA_SET = "latest_datastore_export";
static final String MANIFEST_FILE_NAME = "MANIFEST.txt"; static final String MANIFEST_FILE_NAME = "MANIFEST.txt";
private static final String DEFAULT_SUBDIR = "icann/monthly";
/** Provides an optional subdirectory to store/upload reports to, extracted from the request. */ /** Provides an optional subdirectory to store/upload reports to, extracted from the request. */
@Provides @Provides
@Parameter(PARAM_SUBDIR) @Parameter(PARAM_SUBDIR)
static Optional<String> provideSubdirOptional(HttpServletRequest req) { static Optional<String> provideSubdirOptional(HttpServletRequest req) {
return extractOptionalParameter(req, PARAM_SUBDIR); return extractOptionalParameter(req, PARAM_SUBDIR).map(IcannReportingModule::checkSubdirValid);
} }
/** Provides the subdirectory to store/upload reports to, defaults to icann/monthly/yearMonth. */ /** Provides the subdirectory to store/upload reports to, extracted from the request. */
@Provides @Provides
@ReportingSubdir @Parameter(PARAM_SUBDIR)
static String provideSubdir( static String provideSubdir(HttpServletRequest req) {
@Parameter(PARAM_SUBDIR) Optional<String> subdirOptional, YearMonth yearMonth) { return checkSubdirValid(extractRequiredParameter(req, PARAM_SUBDIR));
String subdir = }
subdirOptional.orElse(
String.format( static String checkSubdirValid(String subdir) {
"%s/%s", DEFAULT_SUBDIR, DateTimeFormat.forPattern("yyyy-MM").print(yearMonth)));
if (subdir.startsWith("/") || subdir.endsWith("/")) { if (subdir.startsWith("/") || subdir.endsWith("/")) {
throw new BadRequestException( throw new BadRequestException(
String.format("subdir must not start or end with a \"/\", got %s instead.", subdir)); String.format("subdir must not start or end with a \"/\", got %s instead.", subdir));
@ -76,17 +69,11 @@ public final class IcannReportingModule {
/** Provides an optional reportType to store/upload reports to, extracted from the request. */ /** Provides an optional reportType to store/upload reports to, extracted from the request. */
@Provides @Provides
@Parameter(PARAM_REPORT_TYPE) @Parameter(PARAM_REPORT_TYPES)
static Optional<ReportType> provideReportTypeOptional(HttpServletRequest req) { static ImmutableSet<ReportType> provideReportTypes(HttpServletRequest req) {
return extractOptionalEnumParameter(req, ReportType.class, PARAM_REPORT_TYPE); ImmutableSet<ReportType> reportTypes =
} extractSetOfEnumParameters(req, ReportType.class, PARAM_REPORT_TYPES);
return reportTypes.isEmpty() ? ImmutableSet.copyOf(ReportType.values()) : reportTypes;
/** Provides a list of reportTypes specified. If absent, we default to both report types. */
@Provides
static ImmutableList<ReportType> provideReportTypes(
@Parameter(PARAM_REPORT_TYPE) Optional<ReportType> reportTypeOptional) {
return reportTypeOptional.map(ImmutableList::of)
.orElseGet(() -> ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS));
} }
/** /**
@ -113,11 +100,4 @@ public final class IcannReportingModule {
throw new RuntimeException("Could not initialize BigqueryConnection!", e); throw new RuntimeException("Could not initialize BigqueryConnection!", e);
} }
} }
/** Dagger qualifier for the subdirectory we stage to/upload from. */
@Qualifier
@Documented
@Retention(RUNTIME)
public @interface ReportingSubdir {}
} }

42
java/google/registry/reporting/icann/IcannReportingStager.java
View file

@ -36,7 +36,6 @@ import google.registry.bigquery.BigqueryUtils.TableType;
import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryConfig.Config;
import google.registry.gcs.GcsUtils; import google.registry.gcs.GcsUtils;
import google.registry.reporting.icann.IcannReportingModule.ReportType; import google.registry.reporting.icann.IcannReportingModule.ReportType;
import google.registry.reporting.icann.IcannReportingModule.ReportingSubdir;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
@ -62,10 +61,6 @@ public class IcannReportingStager {
@Inject @Config("reportingBucket") String reportingBucket; @Inject @Config("reportingBucket") String reportingBucket;
@Inject YearMonth yearMonth;
@Inject @ReportingSubdir
String subdir;
@Inject ActivityReportingQueryBuilder activityQueryBuilder; @Inject ActivityReportingQueryBuilder activityQueryBuilder;
@Inject TransactionsReportingQueryBuilder transactionsQueryBuilder; @Inject TransactionsReportingQueryBuilder transactionsQueryBuilder;
@Inject GcsUtils gcsUtils; @Inject GcsUtils gcsUtils;
@ -79,16 +74,17 @@ public class IcannReportingStager {
* *
* <p>This is factored out to facilitate choosing which reports to upload, * <p>This is factored out to facilitate choosing which reports to upload,
*/ */
ImmutableList<String> stageReports(ReportType reportType) throws Exception { ImmutableList<String> stageReports(YearMonth yearMonth, String subdir, ReportType reportType)
throws Exception {
QueryBuilder queryBuilder = QueryBuilder queryBuilder =
(reportType == ReportType.ACTIVITY) ? activityQueryBuilder : transactionsQueryBuilder; (reportType == ReportType.ACTIVITY) ? activityQueryBuilder : transactionsQueryBuilder;
if (reportType == ReportType.ACTIVITY) { if (reportType == ReportType.ACTIVITY) {
// Prepare for the DNS count query, which may have special needs. // Prepare for the DNS count query, which may have special needs.
activityQueryBuilder.prepareForQuery(); activityQueryBuilder.prepareForQuery(yearMonth);
} }
ImmutableMap<String, String> viewQueryMap = queryBuilder.getViewQueryMap(); ImmutableMap<String, String> viewQueryMap = queryBuilder.getViewQueryMap(yearMonth);
// Generate intermediary views // Generate intermediary views
for (Entry<String, String> entry : viewQueryMap.entrySet()) { for (Entry<String, String> entry : viewQueryMap.entrySet()) {
createIntermediaryTableView(entry.getKey(), entry.getValue(), reportType); createIntermediaryTableView(entry.getKey(), entry.getValue(), reportType);
@ -96,14 +92,14 @@ public class IcannReportingStager {
// Get an in-memory table of the aggregate query's result // Get an in-memory table of the aggregate query's result
ImmutableTable<Integer, TableFieldSchema, Object> reportTable = ImmutableTable<Integer, TableFieldSchema, Object> reportTable =
bigquery.queryToLocalTableSync(queryBuilder.getReportQuery()); bigquery.queryToLocalTableSync(queryBuilder.getReportQuery(yearMonth));
// Get report headers from the table schema and convert into CSV format // Get report headers from the table schema and convert into CSV format
String headerRow = constructRow(getHeaders(reportTable.columnKeySet())); String headerRow = constructRow(getHeaders(reportTable.columnKeySet()));
return (reportType == ReportType.ACTIVITY) return (reportType == ReportType.ACTIVITY)
? stageActivityReports(headerRow, reportTable.rowMap().values()) ? stageActivityReports(yearMonth, subdir, headerRow, reportTable.rowMap().values())
: stageTransactionsReports(headerRow, reportTable.rowMap().values()); : stageTransactionsReports(yearMonth, subdir, headerRow, reportTable.rowMap().values());
} }
private void createIntermediaryTableView(String queryName, String query, ReportType reportType) private void createIntermediaryTableView(String queryName, String query, ReportType reportType)
@ -129,7 +125,10 @@ public class IcannReportingStager {
/** Creates and stores activity reports on GCS, returns a list of files stored. */ /** Creates and stores activity reports on GCS, returns a list of files stored. */
private ImmutableList<String> stageActivityReports( private ImmutableList<String> stageActivityReports(
String headerRow, ImmutableCollection<Map<TableFieldSchema, Object>> rows) YearMonth yearMonth,
String subdir,
String headerRow,
ImmutableCollection<Map<TableFieldSchema, Object>> rows)
throws IOException { throws IOException {
ImmutableList.Builder<String> manifestBuilder = new ImmutableList.Builder<>(); ImmutableList.Builder<String> manifestBuilder = new ImmutableList.Builder<>();
// Create a report csv for each tld from query table, and upload to GCS // Create a report csv for each tld from query table, and upload to GCS
@ -142,14 +141,18 @@ public class IcannReportingStager {
ImmutableList<String> rowStrings = ImmutableList.of(constructRow(row.values())); ImmutableList<String> rowStrings = ImmutableList.of(constructRow(row.values()));
// Create and upload the activity report with a single row // Create and upload the activity report with a single row
manifestBuilder.add( manifestBuilder.add(
saveReportToGcs(tld, createReport(headerRow, rowStrings), ReportType.ACTIVITY)); saveReportToGcs(
tld, yearMonth, subdir, createReport(headerRow, rowStrings), ReportType.ACTIVITY));
} }
return manifestBuilder.build(); return manifestBuilder.build();
} }
/** Creates and stores transactions reports on GCS, returns a list of files stored. */ /** Creates and stores transactions reports on GCS, returns a list of files stored. */
private ImmutableList<String> stageTransactionsReports( private ImmutableList<String> stageTransactionsReports(
String headerRow, ImmutableCollection<Map<TableFieldSchema, Object>> rows) YearMonth yearMonth,
String subdir,
String headerRow,
ImmutableCollection<Map<TableFieldSchema, Object>> rows)
throws IOException { throws IOException {
// Map from tld to rows // Map from tld to rows
ListMultimap<String, String> tldToRows = ArrayListMultimap.create(); ListMultimap<String, String> tldToRows = ArrayListMultimap.create();
@ -175,7 +178,11 @@ public class IcannReportingStager {
tldToRows.put(tld, constructTotalRow(tldToTotals.get(tld))); tldToRows.put(tld, constructTotalRow(tldToTotals.get(tld)));
manifestBuilder.add( manifestBuilder.add(
saveReportToGcs( saveReportToGcs(
tld, createReport(headerRow, tldToRows.get(tld)), ReportType.TRANSACTIONS)); tld,
yearMonth,
subdir,
createReport(headerRow, tldToRows.get(tld)),
ReportType.TRANSACTIONS));
} }
return manifestBuilder.build(); return manifestBuilder.build();
} }
@ -238,7 +245,8 @@ public class IcannReportingStager {
} }
/** Stores a report on GCS, returning the name of the file stored. */ /** Stores a report on GCS, returning the name of the file stored. */
private String saveReportToGcs(String tld, String reportCsv, ReportType reportType) private String saveReportToGcs(
String tld, YearMonth yearMonth, String subdir, String reportCsv, ReportType reportType)
throws IOException { throws IOException {
// Upload resulting CSV file to GCS // Upload resulting CSV file to GCS
byte[] reportBytes = reportCsv.getBytes(UTF_8); byte[] reportBytes = reportCsv.getBytes(UTF_8);
@ -256,7 +264,7 @@ public class IcannReportingStager {
} }
/** Creates and stores a manifest file on GCS, indicating which reports were generated. */ /** Creates and stores a manifest file on GCS, indicating which reports were generated. */
void createAndUploadManifest(ImmutableList<String> filenames) throws IOException { void createAndUploadManifest(String subdir, ImmutableList<String> filenames) throws IOException {
String reportBucketname = String.format("%s/%s", reportingBucket, subdir); String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME); final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
StringBuilder manifestString = new StringBuilder(); StringBuilder manifestString = new StringBuilder();

28
java/google/registry/reporting/icann/IcannReportingStagingAction.java
View file

@ -15,6 +15,8 @@
package google.registry.reporting.icann; package google.registry.reporting.icann;
import static com.google.common.base.Throwables.getRootCause; import static com.google.common.base.Throwables.getRootCause;
import static google.registry.reporting.icann.IcannReportingModule.PARAM_REPORT_TYPES;
import static google.registry.reporting.icann.IcannReportingModule.PARAM_SUBDIR;
import static google.registry.request.Action.Method.POST; import static google.registry.request.Action.Method.POST;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_OK; import static javax.servlet.http.HttpServletResponse.SC_OK;
@ -24,18 +26,21 @@ import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method; import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.flogger.FluentLogger; import com.google.common.flogger.FluentLogger;
import com.google.common.net.MediaType; import com.google.common.net.MediaType;
import google.registry.bigquery.BigqueryJobFailureException; import google.registry.bigquery.BigqueryJobFailureException;
import google.registry.reporting.icann.IcannReportingModule.ReportType; import google.registry.reporting.icann.IcannReportingModule.ReportType;
import google.registry.reporting.icann.IcannReportingModule.ReportingSubdir;
import google.registry.request.Action; import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response; import google.registry.request.Response;
import google.registry.request.auth.Auth; import google.registry.request.auth.Auth;
import google.registry.util.Retrier; import google.registry.util.Retrier;
import java.util.Optional;
import javax.inject.Inject; import javax.inject.Inject;
import org.joda.time.Duration; import org.joda.time.Duration;
import org.joda.time.YearMonth; import org.joda.time.YearMonth;
import org.joda.time.format.DateTimeFormat;
/** /**
* Action that generates monthly ICANN activity and transactions reports. * Action that generates monthly ICANN activity and transactions reports.
@ -53,7 +58,7 @@ import org.joda.time.YearMonth;
* to "icann/monthly/[yearMonth]". * to "icann/monthly/[yearMonth]".
* *
* <p>reportTypes: the type of reports to generate. You can specify either 'activity' or * <p>reportTypes: the type of reports to generate. You can specify either 'activity' or
* 'transactions'. Defaults to generating both. * 'transactions'. If none specified - defaults to generating both.
*/ */
@Action( @Action(
service = Action.Service.BACKEND, service = Action.Service.BACKEND,
@ -66,10 +71,11 @@ public final class IcannReportingStagingAction implements Runnable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final String CRON_QUEUE = "retryable-cron-tasks"; private static final String CRON_QUEUE = "retryable-cron-tasks";
private static final String DEFAULT_SUBDIR = "icann/monthly";
@Inject YearMonth yearMonth; @Inject YearMonth yearMonth;
@Inject @ReportingSubdir String subdir; @Inject @Parameter(PARAM_SUBDIR) Optional<String> overrideSubdir;
@Inject ImmutableList<ReportType> reportTypes; @Inject @Parameter(PARAM_REPORT_TYPES) ImmutableSet<ReportType> reportTypes;
@Inject IcannReportingStager stager; @Inject IcannReportingStager stager;
@Inject Retrier retrier; @Inject Retrier retrier;
@Inject Response response; @Inject Response response;
@ -79,14 +85,15 @@ public final class IcannReportingStagingAction implements Runnable {
@Override @Override
public void run() { public void run() {
try { try {
String subdir = getSubdir(yearMonth);
retrier.callWithRetry( retrier.callWithRetry(
() -> { () -> {
ImmutableList.Builder<String> manifestedFilesBuilder = new ImmutableList.Builder<>(); ImmutableList.Builder<String> manifestedFilesBuilder = new ImmutableList.Builder<>();
for (ReportType reportType : reportTypes) { for (ReportType reportType : reportTypes) {
manifestedFilesBuilder.addAll(stager.stageReports(reportType)); manifestedFilesBuilder.addAll(stager.stageReports(yearMonth, subdir, reportType));
} }
ImmutableList<String> manifestedFiles = manifestedFilesBuilder.build(); ImmutableList<String> manifestedFiles = manifestedFilesBuilder.build();
stager.createAndUploadManifest(manifestedFiles); stager.createAndUploadManifest(subdir, manifestedFiles);
logger.atInfo().log("Completed staging %d report files.", manifestedFiles.size()); logger.atInfo().log("Completed staging %d report files.", manifestedFiles.size());
emailUtils.emailResults( emailUtils.emailResults(
@ -104,7 +111,7 @@ public final class IcannReportingStagingAction implements Runnable {
TaskOptions.Builder.withUrl(IcannReportingUploadAction.PATH) TaskOptions.Builder.withUrl(IcannReportingUploadAction.PATH)
.method(Method.POST) .method(Method.POST)
.countdownMillis(Duration.standardMinutes(2).getMillis()) .countdownMillis(Duration.standardMinutes(2).getMillis())
.param(IcannReportingModule.PARAM_SUBDIR, subdir); .param(PARAM_SUBDIR, subdir);
QueueFactory.getQueue(CRON_QUEUE).add(uploadTask); QueueFactory.getQueue(CRON_QUEUE).add(uploadTask);
return null; return null;
}, },
@ -124,4 +131,11 @@ public final class IcannReportingStagingAction implements Runnable {
throw new RuntimeException("Staging action failed.", e); throw new RuntimeException("Staging action failed.", e);
} }
} }
String getSubdir(YearMonth yearMonth) {
return IcannReportingModule.checkSubdirValid(
overrideSubdir.orElse(
String.format(
"%s/%s", DEFAULT_SUBDIR, DateTimeFormat.forPattern("yyyy-MM").print(yearMonth))));
}
} }

6
java/google/registry/reporting/icann/IcannReportingUploadAction.java
View file

@ -17,6 +17,7 @@ package google.registry.reporting.icann;
import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static google.registry.reporting.icann.IcannReportingModule.MANIFEST_FILE_NAME; import static google.registry.reporting.icann.IcannReportingModule.MANIFEST_FILE_NAME;
import static google.registry.reporting.icann.IcannReportingModule.PARAM_SUBDIR;
import static google.registry.request.Action.Method.POST; import static google.registry.request.Action.Method.POST;
import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_OK; import static javax.servlet.http.HttpServletResponse.SC_OK;
@ -29,8 +30,8 @@ import com.google.common.flogger.FluentLogger;
import com.google.common.io.ByteStreams; import com.google.common.io.ByteStreams;
import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryConfig.Config;
import google.registry.gcs.GcsUtils; import google.registry.gcs.GcsUtils;
import google.registry.reporting.icann.IcannReportingModule.ReportingSubdir;
import google.registry.request.Action; import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response; import google.registry.request.Response;
import google.registry.request.auth.Auth; import google.registry.request.auth.Auth;
import google.registry.util.Retrier; import google.registry.util.Retrier;
@ -67,8 +68,7 @@ public final class IcannReportingUploadAction implements Runnable {
@Config("reportingBucket") @Config("reportingBucket")
String reportingBucket; String reportingBucket;
@Inject @ReportingSubdir @Inject @Parameter(PARAM_SUBDIR) String subdir;
String subdir;
@Inject GcsUtils gcsUtils; @Inject GcsUtils gcsUtils;
@Inject IcannHttpReporter icannReporter; @Inject IcannHttpReporter icannReporter;

5
java/google/registry/reporting/icann/QueryBuilder.java
View file

@ -15,13 +15,14 @@
package google.registry.reporting.icann; package google.registry.reporting.icann;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import org.joda.time.YearMonth;
/** Interface defining the necessary methods to construct ICANN reporting SQL queries. */ /** Interface defining the necessary methods to construct ICANN reporting SQL queries. */
public interface QueryBuilder { public interface QueryBuilder {
/** Returns a map from an intermediary view's table name to the query that generates it. */ /** Returns a map from an intermediary view's table name to the query that generates it. */
ImmutableMap<String, String> getViewQueryMap(); ImmutableMap<String, String> getViewQueryMap(YearMonth yearMonth);
/** Returns a query that retrieves the overall report from the previously generated view. */ /** Returns a query that retrieves the overall report from the previously generated view. */
String getReportQuery(); String getReportQuery(YearMonth yearMonth);
} }

40
java/google/registry/reporting/icann/QueryBuilderUtils.java Normal file
View file

@ -0,0 +1,40 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.reporting.icann;
import com.google.common.io.Resources;
import google.registry.util.ResourceUtils;
import java.net.URL;
import org.joda.time.YearMonth;
import org.joda.time.format.DateTimeFormat;
final class QueryBuilderUtils {
private QueryBuilderUtils() {}
/** Returns the table name of the query, suffixed with the yearMonth in _yyyyMM format. */
static String getTableName(String queryName, YearMonth yearMonth) {
return String.format("%s_%s", queryName, DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
}
/** Returns {@link String} for file in {@code reporting/sql/} directory. */
static String getQueryFromFile(String filename) {
return ResourceUtils.readResourceUtf8(getUrl(filename));
}
private static URL getUrl(String filename) {
return Resources.getResource(QueryBuilderUtils.class, "sql/" + filename);
}
}

64
java/google/registry/reporting/icann/TransactionsReportingQueryBuilder.java
View file

@ -16,13 +16,12 @@ package google.registry.reporting.icann;
import static google.registry.reporting.icann.IcannReportingModule.DATASTORE_EXPORT_DATA_SET; import static google.registry.reporting.icann.IcannReportingModule.DATASTORE_EXPORT_DATA_SET;
import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET; import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET;
import static google.registry.reporting.icann.QueryBuilderUtils.getQueryFromFile;
import static google.registry.reporting.icann.QueryBuilderUtils.getTableName;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.io.Resources;
import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryConfig.Config;
import google.registry.util.ResourceUtils;
import google.registry.util.SqlTemplate; import google.registry.util.SqlTemplate;
import java.net.URL;
import javax.inject.Inject; import javax.inject.Inject;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.joda.time.LocalTime; import org.joda.time.LocalTime;
@ -37,8 +36,6 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
@Inject @Config("projectId") String projectId; @Inject @Config("projectId") String projectId;
@Inject YearMonth yearMonth;
@Inject TransactionsReportingQueryBuilder() {} @Inject TransactionsReportingQueryBuilder() {}
static final String TRANSACTIONS_REPORT_AGGREGATION = "transactions_report_aggregation"; static final String TRANSACTIONS_REPORT_AGGREGATION = "transactions_report_aggregation";
@ -51,27 +48,21 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
/** Returns the aggregate query which generates the transactions report from the saved view. */ /** Returns the aggregate query which generates the transactions report from the saved view. */
@Override @Override
public String getReportQuery() { public String getReportQuery(YearMonth yearMonth) {
return String.format( return String.format(
"#standardSQL\nSELECT * FROM `%s.%s.%s`", "#standardSQL\nSELECT * FROM `%s.%s.%s`",
projectId, projectId,
ICANN_REPORTING_DATA_SET, ICANN_REPORTING_DATA_SET,
getTableName(TRANSACTIONS_REPORT_AGGREGATION)); getTableName(TRANSACTIONS_REPORT_AGGREGATION, yearMonth));
} }
/** Sets the month we're doing transactions reporting for, and returns the view query map. */ /** Sets the month we're doing transactions reporting for, and returns the view query map. */
@Override @Override
public ImmutableMap<String, String> getViewQueryMap() { public ImmutableMap<String, String> getViewQueryMap(YearMonth yearMonth) {
// Set the earliest date to to yearMonth on day 1 at 00:00:00 // Set the earliest date to to yearMonth on day 1 at 00:00:00
DateTime earliestReportTime = yearMonth.toLocalDate(1).toDateTime(new LocalTime(0, 0, 0)); DateTime earliestReportTime = yearMonth.toLocalDate(1).toDateTime(new LocalTime(0, 0, 0));
// Set the latest date to yearMonth on the last day at 23:59:59.999 // Set the latest date to yearMonth on the last day at 23:59:59.999
DateTime latestReportTime = earliestReportTime.plusMonths(1).minusMillis(1); DateTime latestReportTime = earliestReportTime.plusMonths(1).minusMillis(1);
return createQueryMap(earliestReportTime, latestReportTime);
}
/** Returns a map from view name to its associated SQL query. */
private ImmutableMap<String, String> createQueryMap(
DateTime earliestReportTime, DateTime latestReportTime) {
ImmutableMap.Builder<String, String> queriesBuilder = ImmutableMap.builder(); ImmutableMap.Builder<String, String> queriesBuilder = ImmutableMap.builder();
String registrarIanaIdQuery = String registrarIanaIdQuery =
@ -80,7 +71,7 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET) .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
.put("REGISTRAR_TABLE", "Registrar") .put("REGISTRAR_TABLE", "Registrar")
.build(); .build();
queriesBuilder.put(getTableName(REGISTRAR_IANA_ID), registrarIanaIdQuery); queriesBuilder.put(getTableName(REGISTRAR_IANA_ID, yearMonth), registrarIanaIdQuery);
String totalDomainsQuery = String totalDomainsQuery =
SqlTemplate.create(getQueryFromFile("total_domains.sql")) SqlTemplate.create(getQueryFromFile("total_domains.sql"))
@ -89,7 +80,7 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
.put("DOMAINBASE_TABLE", "DomainBase") .put("DOMAINBASE_TABLE", "DomainBase")
.put("REGISTRAR_TABLE", "Registrar") .put("REGISTRAR_TABLE", "Registrar")
.build(); .build();
queriesBuilder.put(getTableName(TOTAL_DOMAINS), totalDomainsQuery); queriesBuilder.put(getTableName(TOTAL_DOMAINS, yearMonth), totalDomainsQuery);
DateTimeFormatter timestampFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS"); DateTimeFormatter timestampFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
String totalNameserversQuery = String totalNameserversQuery =
@ -101,7 +92,7 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
.put("REGISTRAR_TABLE", "Registrar") .put("REGISTRAR_TABLE", "Registrar")
.put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime)) .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
.build(); .build();
queriesBuilder.put(getTableName(TOTAL_NAMESERVERS), totalNameserversQuery); queriesBuilder.put(getTableName(TOTAL_NAMESERVERS, yearMonth), totalNameserversQuery);
String transactionCountsQuery = String transactionCountsQuery =
SqlTemplate.create(getQueryFromFile("transaction_counts.sql")) SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
@ -117,7 +108,7 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
.put("TRANSFER_NACKED_FIELD", "TRANSFER_GAINING_NACKED") .put("TRANSFER_NACKED_FIELD", "TRANSFER_GAINING_NACKED")
.put("DEFAULT_FIELD", "field") .put("DEFAULT_FIELD", "field")
.build(); .build();
queriesBuilder.put(getTableName(TRANSACTION_COUNTS), transactionCountsQuery); queriesBuilder.put(getTableName(TRANSACTION_COUNTS, yearMonth), transactionCountsQuery);
String transactionTransferLosingQuery = String transactionTransferLosingQuery =
SqlTemplate.create(getQueryFromFile("transaction_counts.sql")) SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
@ -134,7 +125,8 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
.put("TRANSFER_NACKED_FIELD", "TRANSFER_LOSING_NACKED") .put("TRANSFER_NACKED_FIELD", "TRANSFER_LOSING_NACKED")
.put("DEFAULT_FIELD", "NULL") .put("DEFAULT_FIELD", "NULL")
.build(); .build();
queriesBuilder.put(getTableName(TRANSACTION_TRANSFER_LOSING), transactionTransferLosingQuery); queriesBuilder.put(
getTableName(TRANSACTION_TRANSFER_LOSING, yearMonth), transactionTransferLosingQuery);
// App Engine log table suffixes use YYYYMMDD format // App Engine log table suffixes use YYYYMMDD format
DateTimeFormatter logTableFormatter = DateTimeFormat.forPattern("yyyyMMdd"); DateTimeFormatter logTableFormatter = DateTimeFormat.forPattern("yyyyMMdd");
@ -152,7 +144,7 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
"METADATA_LOG_PREFIX", "METADATA_LOG_PREFIX",
"google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA") "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
.build(); .build();
queriesBuilder.put(getTableName(ATTEMPTED_ADDS), attemptedAddsQuery); queriesBuilder.put(getTableName(ATTEMPTED_ADDS, yearMonth), attemptedAddsQuery);
String aggregateQuery = String aggregateQuery =
SqlTemplate.create(getQueryFromFile("transactions_report_aggregation.sql")) SqlTemplate.create(getQueryFromFile("transactions_report_aggregation.sql"))
@ -160,31 +152,17 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET) .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
.put("REGISTRY_TABLE", "Registry") .put("REGISTRY_TABLE", "Registry")
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET) .put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
.put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID)) .put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID, yearMonth))
.put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS)) .put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS, yearMonth))
.put("TOTAL_NAMESERVERS_TABLE", getTableName(TOTAL_NAMESERVERS)) .put("TOTAL_NAMESERVERS_TABLE", getTableName(TOTAL_NAMESERVERS, yearMonth))
.put("TRANSACTION_COUNTS_TABLE", getTableName(TRANSACTION_COUNTS)) .put("TRANSACTION_COUNTS_TABLE", getTableName(TRANSACTION_COUNTS, yearMonth))
.put("TRANSACTION_TRANSFER_LOSING_TABLE", getTableName(TRANSACTION_TRANSFER_LOSING)) .put(
.put("ATTEMPTED_ADDS_TABLE", getTableName(ATTEMPTED_ADDS)) "TRANSACTION_TRANSFER_LOSING_TABLE",
getTableName(TRANSACTION_TRANSFER_LOSING, yearMonth))
.put("ATTEMPTED_ADDS_TABLE", getTableName(ATTEMPTED_ADDS, yearMonth))
.build(); .build();
queriesBuilder.put(getTableName(TRANSACTIONS_REPORT_AGGREGATION), aggregateQuery); queriesBuilder.put(getTableName(TRANSACTIONS_REPORT_AGGREGATION, yearMonth), aggregateQuery);
return queriesBuilder.build(); return queriesBuilder.build();
} }
/** Returns the table name of the query, suffixed with the yearMonth in _yyyyMM format. */
private String getTableName(String queryName) {
return String.format("%s_%s", queryName, DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
}
/** Returns {@link String} for file in {@code reporting/sql/} directory. */
private static String getQueryFromFile(String filename) {
return ResourceUtils.readResourceUtf8(getUrl(filename));
}
private static URL getUrl(String filename) {
return Resources.getResource(
ActivityReportingQueryBuilder.class, "sql/" + filename);
}
} }

64
java/google/registry/request/RequestParameters.java
View file

@ -96,7 +96,7 @@ public final class RequestParameters {
} }
/** /**
* Returns all GET or POST parameters associated with {@code name} (or {@code nameLegacy}). * Returns all GET or POST parameters associated with {@code name}.
* *
* <p>The parameter value is assumed to be a comma-delimited set of values - so tlds=com,net would * <p>The parameter value is assumed to be a comma-delimited set of values - so tlds=com,net would
* result in ImmutableSet.of("com", "net"). * result in ImmutableSet.of("com", "net").
@ -109,36 +109,49 @@ public final class RequestParameters {
* @param name the name of the parameter, should be in plural form (e.g. tlds=, not tld=) * @param name the name of the parameter, should be in plural form (e.g. tlds=, not tld=)
*/ */
public static ImmutableSet<String> extractSetOfParameters(HttpServletRequest req, String name) { public static ImmutableSet<String> extractSetOfParameters(HttpServletRequest req, String name) {
// First we make sure the user didn't accidentally try to pass the "set of parameters" as
// multiple tld=a&tld=b parameters instead of tld=a,b
String[] parameters = req.getParameterValues(name); String[] parameters = req.getParameterValues(name);
if (parameters == null || parameters.length == 0) { if (parameters != null && parameters.length > 1) {
return ImmutableSet.of();
}
if (parameters.length > 1) {
throw new BadRequestException( throw new BadRequestException(
String.format( String.format(
"Bad 'set of parameters' input! Received multiple values instead of single " "Bad 'set of parameters' input! Received multiple values instead of single "
+ "comma-delimited value for parameter %s", + "comma-delimited value for parameter %s",
name)); name));
} }
// Now we parse the single parameter.
// We use the req.getParameter(name) instead of parameters[0] to make tests more consistent (all
// extractXxx read the data from req.getParameter, so mocking the parameter is consistent)
String parameter = req.getParameter(name);
if (parameter == null || parameter.isEmpty()) {
return ImmutableSet.of();
}
return Splitter.on(',') return Splitter.on(',')
.splitToList(parameters[0]) .splitToList(parameter)
.stream() .stream()
.filter(s -> !s.isEmpty()) .filter(s -> !s.isEmpty())
.collect(toImmutableSet()); .collect(toImmutableSet());
} }
/** /**
* Returns the first GET or POST parameter associated with {@code name}, absent otherwise. * Returns all GET or POST parameters associated with {@code name}.
* *
* @throws BadRequestException if request parameter named {@code name} is not equal to any of the * <p>The parameter value is assumed to be a comma-delimited set of values - so tlds=com,net would
* values in {@code enumClass} * result in ImmutableSet.of("com", "net").
*
* <p>Empty strings are not supported, and are automatically removed from the result.
*
* <p>Both missing parameter and parameter with empty value result in an empty set.
*
* @param req the request that has the parameter
* @param enumClass the Class of the expected Enum type
* @param name the name of the parameter, should be in plural form (e.g. tlds=, not tld=)
* @throws BadRequestException if any of the comma-delimited values of the request parameter named
* {@code name} aren't equal to any of the values in {@code enumClass}
*/ */
public static <C extends Enum<C>> Optional<C> extractOptionalEnumParameter( public static <C extends Enum<C>> Optional<C> extractOptionalEnumParameter(
HttpServletRequest req, Class<C> enumClass, String name) { HttpServletRequest req, Class<C> enumClass, String name) {
String stringParam = req.getParameter(name); return extractOptionalParameter(req, name).map(value -> getEnumValue(enumClass, value, name));
return isNullOrEmpty(stringParam)
? Optional.empty()
: Optional.of(extractEnumParameter(req, enumClass, name));
} }
/** /**
@ -149,11 +162,32 @@ public final class RequestParameters {
*/ */
public static <C extends Enum<C>> public static <C extends Enum<C>>
C extractEnumParameter(HttpServletRequest req, Class<C> enumClass, String name) { C extractEnumParameter(HttpServletRequest req, Class<C> enumClass, String name) {
return getEnumValue(enumClass, extractRequiredParameter(req, name), name);
}
/**
* Returns the first GET or POST parameter associated with {@code name}.
*
* @throws BadRequestException if request parameter named {@code name} is absent, empty, or not
* equal to any of the values in {@code enumClass}
*/
public static <C extends Enum<C>> ImmutableSet<C> extractSetOfEnumParameters(
HttpServletRequest req, Class<C> enumClass, String name) {
return extractSetOfParameters(req, name).stream()
.map(value -> getEnumValue(enumClass, value, name))
.collect(toImmutableSet());
}
/** Translates a string name into the enum value, or throws a BadRequestException. */
private static <C extends Enum<C>> C getEnumValue(
Class<C> enumClass, String value, String parameterName) {
try { try {
return Enum.valueOf(enumClass, Ascii.toUpperCase(extractRequiredParameter(req, name))); return Enum.valueOf(enumClass, Ascii.toUpperCase(value));
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
throw new BadRequestException( throw new BadRequestException(
String.format("Invalid %s parameter: %s", enumClass.getSimpleName(), name)); String.format(
"Invalid parameter %s: expected enum of type %s, but got '%s'",
parameterName, enumClass.getSimpleName(), value));
} }
} }

10
javatests/google/registry/reporting/icann/ActivityReportingQueryBuilderTest.java
View file

@ -27,21 +27,21 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class) @RunWith(JUnit4.class)
public class ActivityReportingQueryBuilderTest { public class ActivityReportingQueryBuilderTest {
private final YearMonth yearMonth = new YearMonth(2017, 9);
private ActivityReportingQueryBuilder getQueryBuilder() { private ActivityReportingQueryBuilder getQueryBuilder() {
ActivityReportingQueryBuilder queryBuilder = new ActivityReportingQueryBuilder(); ActivityReportingQueryBuilder queryBuilder = new ActivityReportingQueryBuilder();
queryBuilder.yearMonth = new YearMonth(2017, 9);
queryBuilder.projectId = "domain-registry-alpha"; queryBuilder.projectId = "domain-registry-alpha";
queryBuilder.dnsCountQueryCoordinator = queryBuilder.dnsCountQueryCoordinator =
new BasicDnsCountQueryCoordinator( new BasicDnsCountQueryCoordinator(
new BasicDnsCountQueryCoordinator.Params(null, queryBuilder.yearMonth, new BasicDnsCountQueryCoordinator.Params(null, queryBuilder.projectId));
queryBuilder.projectId));
return queryBuilder; return queryBuilder;
} }
@Test @Test
public void testAggregateQueryMatch() { public void testAggregateQueryMatch() {
ActivityReportingQueryBuilder queryBuilder = getQueryBuilder(); ActivityReportingQueryBuilder queryBuilder = getQueryBuilder();
assertThat(queryBuilder.getReportQuery()) assertThat(queryBuilder.getReportQuery(yearMonth))
.isEqualTo( .isEqualTo(
"#standardSQL\nSELECT * FROM " "#standardSQL\nSELECT * FROM "
+ "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201709`"); + "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201709`");
@ -59,7 +59,7 @@ public class ActivityReportingQueryBuilderTest {
ActivityReportingQueryBuilder.ACTIVITY_REPORT_AGGREGATION); ActivityReportingQueryBuilder.ACTIVITY_REPORT_AGGREGATION);
ActivityReportingQueryBuilder queryBuilder = getQueryBuilder(); ActivityReportingQueryBuilder queryBuilder = getQueryBuilder();
ImmutableMap<String, String> actualQueries = queryBuilder.getViewQueryMap(); ImmutableMap<String, String> actualQueries = queryBuilder.getViewQueryMap(yearMonth);
for (String queryName : expectedQueryNames) { for (String queryName : expectedQueryNames) {
String actualTableName = String.format("%s_201709", queryName); String actualTableName = String.format("%s_201709", queryName);
String testFilename = String.format("%s_test.sql", queryName); String testFilename = String.format("%s_test.sql", queryName);

1
javatests/google/registry/reporting/icann/BUILD
View file

@ -26,6 +26,7 @@ java_library(
"@com_google_http_client", "@com_google_http_client",
"@com_google_truth", "@com_google_truth",
"@com_google_truth_extensions_truth_java8_extension", "@com_google_truth_extensions_truth_java8_extension",
"@javax_servlet_api",
"@joda_time", "@joda_time",
"@junit", "@junit",
"@org_mockito_all", "@org_mockito_all",

59
javatests/google/registry/reporting/icann/IcannReportingModuleTest.java
View file

@ -15,12 +15,10 @@
package google.registry.reporting.icann; package google.registry.reporting.icann;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static google.registry.testing.JUnitBackports.assertThrows; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import google.registry.reporting.icann.IcannReportingModule.ReportType; import javax.servlet.http.HttpServletRequest;
import google.registry.request.HttpException.BadRequestException;
import java.util.Optional;
import org.joda.time.YearMonth;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.junit.runners.JUnit4; import org.junit.runners.JUnit4;
@ -30,39 +28,30 @@ import org.junit.runners.JUnit4;
public class IcannReportingModuleTest { public class IcannReportingModuleTest {
@Test @Test
public void testEmptySubDir_returnsDefaultSubdir() { public void testProvideReportTypes() {
assertThat(IcannReportingModule.provideSubdir(Optional.empty(), new YearMonth(2017, 6))) HttpServletRequest req = mock(HttpServletRequest.class);
.isEqualTo("icann/monthly/2017-06");
}
@Test when(req.getParameter("reportTypes")).thenReturn(null);
public void testGivenSubdir_returnsManualSubdir() { assertThat(IcannReportingModule.provideReportTypes(req))
assertThat( .containsExactly(
IcannReportingModule.provideSubdir(Optional.of("manual/dir"), new YearMonth(2017, 6))) IcannReportingModule.ReportType.ACTIVITY, IcannReportingModule.ReportType.TRANSACTIONS);
.isEqualTo("manual/dir");
}
@Test when(req.getParameter("reportTypes")).thenReturn("");
public void testInvalidSubdir_throwsException() { assertThat(IcannReportingModule.provideReportTypes(req))
BadRequestException thrown = .containsExactly(
assertThrows( IcannReportingModule.ReportType.ACTIVITY, IcannReportingModule.ReportType.TRANSACTIONS);
BadRequestException.class,
() ->
IcannReportingModule.provideSubdir(Optional.of("/whoops"), new YearMonth(2017, 6)));
assertThat(thrown)
.hasMessageThat()
.contains("subdir must not start or end with a \"/\", got /whoops instead.");
}
@Test when(req.getParameter("reportTypes")).thenReturn("activity");
public void testGivenReportType_returnsReportType() { assertThat(IcannReportingModule.provideReportTypes(req))
assertThat(IcannReportingModule.provideReportTypes(Optional.of(ReportType.ACTIVITY))) .containsExactly(IcannReportingModule.ReportType.ACTIVITY);
.containsExactly(ReportType.ACTIVITY);
}
@Test when(req.getParameter("reportTypes")).thenReturn("transactions");
public void testNoReportType_returnsBothReportTypes() { assertThat(IcannReportingModule.provideReportTypes(req))
assertThat(IcannReportingModule.provideReportTypes(Optional.empty())) .containsExactly(IcannReportingModule.ReportType.TRANSACTIONS);
.containsExactly(ReportType.ACTIVITY, ReportType.TRANSACTIONS);
when(req.getParameter("reportTypes")).thenReturn("activity,transactions");
assertThat(IcannReportingModule.provideReportTypes(req))
.containsExactly(
IcannReportingModule.ReportType.ACTIVITY, IcannReportingModule.ReportType.TRANSACTIONS);
} }
} }

12
javatests/google/registry/reporting/icann/IcannReportingStagerTest.java
View file

@ -50,6 +50,8 @@ public class IcannReportingStagerTest {
BigqueryConnection bigquery = mock(BigqueryConnection.class); BigqueryConnection bigquery = mock(BigqueryConnection.class);
FakeResponse response = new FakeResponse(); FakeResponse response = new FakeResponse();
GcsService gcsService = GcsServiceFactory.createGcsService(); GcsService gcsService = GcsServiceFactory.createGcsService();
YearMonth yearMonth = new YearMonth(2017, 6);
String subdir = "icann/monthly/2017-06";
@Rule @Rule
public final AppEngineRule appEngine = AppEngineRule.builder() public final AppEngineRule appEngine = AppEngineRule.builder()
@ -61,16 +63,12 @@ public class IcannReportingStagerTest {
IcannReportingStager action = new IcannReportingStager(); IcannReportingStager action = new IcannReportingStager();
ActivityReportingQueryBuilder activityBuilder = new ActivityReportingQueryBuilder(); ActivityReportingQueryBuilder activityBuilder = new ActivityReportingQueryBuilder();
activityBuilder.projectId = "test-project"; activityBuilder.projectId = "test-project";
activityBuilder.yearMonth = new YearMonth(2017, 6);
activityBuilder.dnsCountQueryCoordinator = new BasicDnsCountQueryCoordinator(null); activityBuilder.dnsCountQueryCoordinator = new BasicDnsCountQueryCoordinator(null);
action.activityQueryBuilder = activityBuilder; action.activityQueryBuilder = activityBuilder;
TransactionsReportingQueryBuilder transactionsBuilder = new TransactionsReportingQueryBuilder(); TransactionsReportingQueryBuilder transactionsBuilder = new TransactionsReportingQueryBuilder();
transactionsBuilder.projectId = "test-project"; transactionsBuilder.projectId = "test-project";
transactionsBuilder.yearMonth = new YearMonth(2017, 6);
action.transactionsQueryBuilder = transactionsBuilder; action.transactionsQueryBuilder = transactionsBuilder;
action.reportingBucket = "test-bucket"; action.reportingBucket = "test-bucket";
action.yearMonth = new YearMonth(2017, 6);
action.subdir = "icann/monthly/2017-06";
action.bigquery = bigquery; action.bigquery = bigquery;
action.gcsUtils = new GcsUtils(gcsService, 1024); action.gcsUtils = new GcsUtils(gcsService, 1024);
return action; return action;
@ -100,7 +98,7 @@ public class IcannReportingStagerTest {
.build(); .build();
when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(activityReportTable); when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(activityReportTable);
IcannReportingStager stager = createStager(); IcannReportingStager stager = createStager();
stager.stageReports(ReportType.ACTIVITY); stager.stageReports(yearMonth, subdir, ReportType.ACTIVITY);
String expectedReport1 = "fooField,barField\r\n12,34"; String expectedReport1 = "fooField,barField\r\n12,34";
String expectedReport2 = "fooField,barField\r\n56,78"; String expectedReport2 = "fooField,barField\r\n56,78";
@ -143,7 +141,7 @@ public class IcannReportingStagerTest {
.build(); .build();
when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(transactionReportTable); when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(transactionReportTable);
IcannReportingStager stager = createStager(); IcannReportingStager stager = createStager();
stager.stageReports(ReportType.TRANSACTIONS); stager.stageReports(yearMonth, subdir, ReportType.TRANSACTIONS);
String expectedReport1 = String expectedReport1 =
"registrar,iana,field\r\n\"reg1\",123,10\r\n\"reg2\",456,20\r\nTotals,,30"; "registrar,iana,field\r\n\"reg1\",123,10\r\n\"reg2\",456,20\r\nTotals,,30";
@ -165,7 +163,7 @@ public class IcannReportingStagerTest {
IcannReportingStager stager = createStager(); IcannReportingStager stager = createStager();
ImmutableList<String> filenames = ImmutableList<String> filenames =
ImmutableList.of("fooTld-transactions-201706.csv", "barTld-activity-201706.csv"); ImmutableList.of("fooTld-transactions-201706.csv", "barTld-activity-201706.csv");
stager.createAndUploadManifest(filenames); stager.createAndUploadManifest(subdir, filenames);
String expectedManifest = "fooTld-transactions-201706.csv\nbarTld-activity-201706.csv\n"; String expectedManifest = "fooTld-transactions-201706.csv\nbarTld-activity-201706.csv\n";
byte[] generatedManifest = byte[] generatedManifest =

99
javatests/google/registry/reporting/icann/IcannReportingStagingActionTest.java
View file

@ -24,14 +24,17 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import google.registry.bigquery.BigqueryJobFailureException; import google.registry.bigquery.BigqueryJobFailureException;
import google.registry.reporting.icann.IcannReportingModule.ReportType; import google.registry.reporting.icann.IcannReportingModule.ReportType;
import google.registry.request.HttpException.BadRequestException;
import google.registry.testing.AppEngineRule; import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock; import google.registry.testing.FakeClock;
import google.registry.testing.FakeResponse; import google.registry.testing.FakeResponse;
import google.registry.testing.FakeSleeper; import google.registry.testing.FakeSleeper;
import google.registry.testing.TaskQueueHelper.TaskMatcher; import google.registry.testing.TaskQueueHelper.TaskMatcher;
import google.registry.util.Retrier; import google.registry.util.Retrier;
import java.util.Optional;
import org.joda.time.YearMonth; import org.joda.time.YearMonth;
import org.junit.Before; import org.junit.Before;
import org.junit.Rule; import org.junit.Rule;
@ -46,6 +49,9 @@ public class IcannReportingStagingActionTest {
FakeResponse response = new FakeResponse(); FakeResponse response = new FakeResponse();
IcannReportingStager stager = mock(IcannReportingStager.class); IcannReportingStager stager = mock(IcannReportingStager.class);
ReportingEmailUtils emailUtils = mock(ReportingEmailUtils.class); ReportingEmailUtils emailUtils = mock(ReportingEmailUtils.class);
YearMonth yearMonth = new YearMonth(2017, 6);
String subdir = "default/dir";
IcannReportingStagingAction action;
@Rule @Rule
public final AppEngineRule appEngine = AppEngineRule.builder() public final AppEngineRule appEngine = AppEngineRule.builder()
@ -56,37 +62,36 @@ public class IcannReportingStagingActionTest {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
when(stager.stageReports(ReportType.ACTIVITY)).thenReturn(ImmutableList.of("a", "b")); action = new IcannReportingStagingAction();
when(stager.stageReports(ReportType.TRANSACTIONS)).thenReturn(ImmutableList.of("c", "d")); action.yearMonth = yearMonth;
} action.overrideSubdir = Optional.of(subdir);
action.reportTypes = ImmutableSet.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS);
private static void assertUploadTaskEnqueued(String subDir) {
TaskMatcher matcher =
new TaskMatcher()
.url("/_dr/task/icannReportingUpload")
.method("POST")
.param("subdir", subDir);
assertTasksEnqueued("retryable-cron-tasks", matcher);
}
private IcannReportingStagingAction createAction(ImmutableList<ReportType> reportingMode) {
IcannReportingStagingAction action = new IcannReportingStagingAction();
action.yearMonth = new YearMonth(2017, 6);
action.subdir = "default/dir";
action.reportTypes = reportingMode;
action.response = response; action.response = response;
action.stager = stager; action.stager = stager;
action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
action.emailUtils = emailUtils; action.emailUtils = emailUtils;
return action;
when(stager.stageReports(yearMonth, subdir, ReportType.ACTIVITY))
.thenReturn(ImmutableList.of("a", "b"));
when(stager.stageReports(yearMonth, subdir, ReportType.TRANSACTIONS))
.thenReturn(ImmutableList.of("c", "d"));
}
private static void assertUploadTaskEnqueued(String subdir) {
TaskMatcher matcher =
new TaskMatcher()
.url("/_dr/task/icannReportingUpload")
.method("POST")
.param("subdir", subdir);
assertTasksEnqueued("retryable-cron-tasks", matcher);
} }
@Test @Test
public void testActivityReportingMode_onlyStagesActivityReports() throws Exception { public void testActivityReportingMode_onlyStagesActivityReports() throws Exception {
IcannReportingStagingAction action = createAction(ImmutableList.of(ReportType.ACTIVITY)); action.reportTypes = ImmutableSet.of(ReportType.ACTIVITY);
action.run(); action.run();
verify(stager).stageReports(ReportType.ACTIVITY); verify(stager).stageReports(yearMonth, subdir, ReportType.ACTIVITY);
verify(stager).createAndUploadManifest(ImmutableList.of("a", "b")); verify(stager).createAndUploadManifest(subdir, ImmutableList.of("a", "b"));
verify(emailUtils) verify(emailUtils)
.emailResults( .emailResults(
"ICANN Monthly report staging summary [SUCCESS]", "ICANN Monthly report staging summary [SUCCESS]",
@ -96,12 +101,10 @@ public class IcannReportingStagingActionTest {
@Test @Test
public void testAbsentReportingMode_stagesBothReports() throws Exception { public void testAbsentReportingMode_stagesBothReports() throws Exception {
IcannReportingStagingAction action =
createAction(ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS));
action.run(); action.run();
verify(stager).stageReports(ReportType.ACTIVITY); verify(stager).stageReports(yearMonth, subdir, ReportType.ACTIVITY);
verify(stager).stageReports(ReportType.TRANSACTIONS); verify(stager).stageReports(yearMonth, subdir, ReportType.TRANSACTIONS);
verify(stager).createAndUploadManifest(ImmutableList.of("a", "b", "c", "d")); verify(stager).createAndUploadManifest(subdir, ImmutableList.of("a", "b", "c", "d"));
verify(emailUtils) verify(emailUtils)
.emailResults( .emailResults(
"ICANN Monthly report staging summary [SUCCESS]", "ICANN Monthly report staging summary [SUCCESS]",
@ -111,15 +114,13 @@ public class IcannReportingStagingActionTest {
@Test @Test
public void testRetryOnBigqueryException() throws Exception { public void testRetryOnBigqueryException() throws Exception {
IcannReportingStagingAction action = when(stager.stageReports(yearMonth, subdir, ReportType.TRANSACTIONS))
createAction(ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS));
when(stager.stageReports(ReportType.TRANSACTIONS))
.thenThrow(new BigqueryJobFailureException("Expected failure", null, null, null)) .thenThrow(new BigqueryJobFailureException("Expected failure", null, null, null))
.thenReturn(ImmutableList.of("c", "d")); .thenReturn(ImmutableList.of("c", "d"));
action.run(); action.run();
verify(stager, times(2)).stageReports(ReportType.ACTIVITY); verify(stager, times(2)).stageReports(yearMonth, subdir, ReportType.ACTIVITY);
verify(stager, times(2)).stageReports(ReportType.TRANSACTIONS); verify(stager, times(2)).stageReports(yearMonth, subdir, ReportType.TRANSACTIONS);
verify(stager).createAndUploadManifest(ImmutableList.of("a", "b", "c", "d")); verify(stager).createAndUploadManifest(subdir, ImmutableList.of("a", "b", "c", "d"));
verify(emailUtils) verify(emailUtils)
.emailResults( .emailResults(
"ICANN Monthly report staging summary [SUCCESS]", "ICANN Monthly report staging summary [SUCCESS]",
@ -129,15 +130,14 @@ public class IcannReportingStagingActionTest {
@Test @Test
public void testEmailEng_onMoreThanRetriableFailure() throws Exception { public void testEmailEng_onMoreThanRetriableFailure() throws Exception {
IcannReportingStagingAction action = action.reportTypes = ImmutableSet.of(ReportType.ACTIVITY);
createAction(ImmutableList.of(ReportType.ACTIVITY)); when(stager.stageReports(yearMonth, subdir, ReportType.ACTIVITY))
when(stager.stageReports(ReportType.ACTIVITY))
.thenThrow(new BigqueryJobFailureException("Expected failure", null, null, null)); .thenThrow(new BigqueryJobFailureException("Expected failure", null, null, null));
RuntimeException thrown = assertThrows(RuntimeException.class, action::run); RuntimeException thrown = assertThrows(RuntimeException.class, action::run);
assertThat(thrown).hasCauseThat().isInstanceOf(BigqueryJobFailureException.class); assertThat(thrown).hasCauseThat().isInstanceOf(BigqueryJobFailureException.class);
assertThat(thrown).hasMessageThat().isEqualTo("Staging action failed."); assertThat(thrown).hasMessageThat().isEqualTo("Staging action failed.");
assertThat(thrown).hasCauseThat().hasMessageThat().isEqualTo("Expected failure"); assertThat(thrown).hasCauseThat().hasMessageThat().isEqualTo("Expected failure");
verify(stager, times(3)).stageReports(ReportType.ACTIVITY); verify(stager, times(3)).stageReports(yearMonth, subdir, ReportType.ACTIVITY);
verify(emailUtils) verify(emailUtils)
.emailResults( .emailResults(
"ICANN Monthly report staging summary [FAILURE]", "ICANN Monthly report staging summary [FAILURE]",
@ -146,4 +146,29 @@ public class IcannReportingStagingActionTest {
// Assert no upload task enqueued // Assert no upload task enqueued
assertNoTasksEnqueued("retryable-cron-tasks"); assertNoTasksEnqueued("retryable-cron-tasks");
} }
@Test
public void testEmptySubDir_returnsDefaultSubdir() {
action.overrideSubdir = Optional.empty();
assertThat(action.getSubdir(new YearMonth(2017, 6))).isEqualTo("icann/monthly/2017-06");
}
@Test
public void testGivenSubdir_returnsManualSubdir() {
action.overrideSubdir = Optional.of("manual/dir");
assertThat(action.getSubdir(new YearMonth(2017, 6))).isEqualTo("manual/dir");
}
@Test
public void testInvalidSubdir_throwsException() {
action.overrideSubdir = Optional.of("/whoops");
BadRequestException thrown =
assertThrows(
BadRequestException.class,
() ->
action.getSubdir(new YearMonth(2017, 6)));
assertThat(thrown)
.hasMessageThat()
.contains("subdir must not start or end with a \"/\", got /whoops instead.");
}
} }

7
javatests/google/registry/reporting/icann/TransactionsReportingQueryBuilderTest.java
View file

@ -27,9 +27,10 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class) @RunWith(JUnit4.class)
public class TransactionsReportingQueryBuilderTest { public class TransactionsReportingQueryBuilderTest {
private final YearMonth yearMonth = new YearMonth(2017, 9);
private TransactionsReportingQueryBuilder getQueryBuilder() { private TransactionsReportingQueryBuilder getQueryBuilder() {
TransactionsReportingQueryBuilder queryBuilder = new TransactionsReportingQueryBuilder(); TransactionsReportingQueryBuilder queryBuilder = new TransactionsReportingQueryBuilder();
queryBuilder.yearMonth = new YearMonth(2017, 9);
queryBuilder.projectId = "domain-registry-alpha"; queryBuilder.projectId = "domain-registry-alpha";
return queryBuilder; return queryBuilder;
} }
@ -37,7 +38,7 @@ public class TransactionsReportingQueryBuilderTest {
@Test @Test
public void testAggregateQueryMatch() { public void testAggregateQueryMatch() {
TransactionsReportingQueryBuilder queryBuilder = getQueryBuilder(); TransactionsReportingQueryBuilder queryBuilder = getQueryBuilder();
assertThat(queryBuilder.getReportQuery()) assertThat(queryBuilder.getReportQuery(yearMonth))
.isEqualTo( .isEqualTo(
"#standardSQL\nSELECT * FROM " "#standardSQL\nSELECT * FROM "
+ "`domain-registry-alpha.icann_reporting.transactions_report_aggregation_201709`"); + "`domain-registry-alpha.icann_reporting.transactions_report_aggregation_201709`");
@ -56,7 +57,7 @@ public class TransactionsReportingQueryBuilderTest {
TransactionsReportingQueryBuilder.ATTEMPTED_ADDS); TransactionsReportingQueryBuilder.ATTEMPTED_ADDS);
TransactionsReportingQueryBuilder queryBuilder = getQueryBuilder(); TransactionsReportingQueryBuilder queryBuilder = getQueryBuilder();
ImmutableMap<String, String> actualQueries = queryBuilder.getViewQueryMap(); ImmutableMap<String, String> actualQueries = queryBuilder.getViewQueryMap(yearMonth);
for (String queryName : expectedQueryNames) { for (String queryName : expectedQueryNames) {
String actualTableName = String.format("%s_201709", queryName); String actualTableName = String.format("%s_201709", queryName);
String testFilename = String.format("%s_test.sql", queryName); String testFilename = String.format("%s_test.sql", queryName);

56
javatests/google/registry/request/RequestParametersTest.java
View file

@ -23,6 +23,7 @@ import static google.registry.request.RequestParameters.extractOptionalEnumParam
import static google.registry.request.RequestParameters.extractOptionalParameter; import static google.registry.request.RequestParameters.extractOptionalParameter;
import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter; import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
import static google.registry.request.RequestParameters.extractRequiredParameter; import static google.registry.request.RequestParameters.extractRequiredParameter;
import static google.registry.request.RequestParameters.extractSetOfEnumParameters;
import static google.registry.request.RequestParameters.extractSetOfParameters; import static google.registry.request.RequestParameters.extractSetOfParameters;
import static google.registry.testing.JUnitBackports.assertThrows; import static google.registry.testing.JUnitBackports.assertThrows;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
@ -86,25 +87,25 @@ public class RequestParametersTest {
@Test @Test
public void testExtractSetOfParameters_empty_returnsEmpty() { public void testExtractSetOfParameters_empty_returnsEmpty() {
when(req.getParameterValues("spin")).thenReturn(new String[]{""}); when(req.getParameter("spin")).thenReturn("");
assertThat(extractSetOfParameters(req, "spin")).isEmpty(); assertThat(extractSetOfParameters(req, "spin")).isEmpty();
} }
@Test @Test
public void testExtractSetOfParameters_oneValue_returnsValue() { public void testExtractSetOfParameters_oneValue_returnsValue() {
when(req.getParameterValues("spin")).thenReturn(new String[]{"bog"}); when(req.getParameter("spin")).thenReturn("bog");
assertThat(extractSetOfParameters(req, "spin")).containsExactly("bog"); assertThat(extractSetOfParameters(req, "spin")).containsExactly("bog");
} }
@Test @Test
public void testExtractSetOfParameters_multipleValues_returnsAll() { public void testExtractSetOfParameters_multipleValues_returnsAll() {
when(req.getParameterValues("spin")).thenReturn(new String[]{"bog,gob"}); when(req.getParameter("spin")).thenReturn("bog,gob");
assertThat(extractSetOfParameters(req, "spin")).containsExactly("bog", "gob"); assertThat(extractSetOfParameters(req, "spin")).containsExactly("bog", "gob");
} }
@Test @Test
public void testExtractSetOfParameters_multipleValuesWithEmpty_removesEmpty() { public void testExtractSetOfParameters_multipleValuesWithEmpty_removesEmpty() {
when(req.getParameterValues("spin")).thenReturn(new String[]{",bog,,gob,"}); when(req.getParameter("spin")).thenReturn(",bog,,gob,");
assertThat(extractSetOfParameters(req, "spin")).containsExactly("bog", "gob"); assertThat(extractSetOfParameters(req, "spin")).containsExactly("bog", "gob");
} }
@ -116,6 +117,53 @@ public class RequestParametersTest {
assertThat(thrown).hasMessageThat().contains("spin"); assertThat(thrown).hasMessageThat().contains("spin");
} }
@Test
public void testExtractSetOfEnumParameters_notPresent_returnsEmpty() {
assertThat(extractSetOfEnumParameters(req, Club.class, "spin")).isEmpty();
}
@Test
public void testExtractSetOfEnumParameters_empty_returnsEmpty() {
when(req.getParameter("spin")).thenReturn("");
assertThat(extractSetOfEnumParameters(req, Club.class, "spin")).isEmpty();
}
@Test
public void testExtractSetOfEnumParameters_oneValue_returnsValue() {
when(req.getParameter("spin")).thenReturn("DANCE");
assertThat(extractSetOfEnumParameters(req, Club.class, "spin")).containsExactly(Club.DANCE);
}
@Test
public void testExtractSetOfEnumParameters_multipleValues_returnsAll() {
when(req.getParameter("spin")).thenReturn("DANCE,FLOOR");
assertThat(extractSetOfEnumParameters(req, Club.class, "spin"))
.containsExactly(Club.DANCE, Club.FLOOR);
}
@Test
public void testExtractSetOfEnumParameters_multipleValuesWithEmpty_removesEmpty() {
when(req.getParameter("spin")).thenReturn(",DANCE,,FLOOR,");
assertThat(extractSetOfEnumParameters(req, Club.class, "spin"))
.containsExactly(Club.DANCE, Club.FLOOR);
}
@Test
public void testExtractSetOfEnumParameters_multipleValues_caseInsensitive() {
when(req.getParameter("spin")).thenReturn("danCE,FlooR");
assertThat(extractSetOfEnumParameters(req, Club.class, "spin"))
.containsExactly(Club.DANCE, Club.FLOOR);
}
@Test
public void testExtractSetOfEnumParameters_multipleParameters_error() {
when(req.getParameterValues("spin")).thenReturn(new String[]{"DANCE", "FLOOR"});
BadRequestException thrown =
assertThrows(
BadRequestException.class, () -> extractSetOfEnumParameters(req, Club.class, "spin"));
assertThat(thrown).hasMessageThat().contains("spin");
}
@Test @Test
public void testExtractBooleanParameter_notPresent_returnsFalse() { public void testExtractBooleanParameter_notPresent_returnsFalse() {
assertThat(extractBooleanParameter(req, "love")).isFalse(); assertThat(extractBooleanParameter(req, "love")).isFalse();