mirror of
https://github.com/google/nomulus.git
synced 2025-05-13 07:57:13 +02:00
Add activity report generation code
This adds Bigquery API client code to generate the activity reports from our now standardSQL queries. The naming mirrors that of RDE (Staging generates the reports and uploads them to GCS). ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=164656344
This commit is contained in:
larryruili
Ben McIlwain
parent
2547313ef9
commit
477617eec9
25 changed files with 612 additions and 117 deletions
|
|
@ -16,10 +16,13 @@ package google.registry.reporting;
|
|||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.Resources;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.util.ResourceUtils;
|
||||
import google.registry.util.SqlTemplate;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.LocalDate;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
import org.joda.time.format.DateTimeFormatter;
|
||||
|
|
@ -31,24 +34,44 @@ public final class ActivityReportingQueryBuilder {
|
|||
|
||||
// Names for intermediary tables for overall activity reporting query.
|
||||
static final String ICANN_REPORTING_DATA_SET = "icann_reporting";
|
||||
static final String MONTHLY_LOGS_TABLE = "monthly_logs";
|
||||
static final String ACTIVITY_REPORT_AGGREGATION = "activity_report_aggregation";
|
||||
static final String MONTHLY_LOGS = "monthly_logs";
|
||||
static final String REGISTRAR_OPERATING_STATUS = "registrar_operating_status";
|
||||
static final String DNS_COUNTS = "dns_counts";
|
||||
static final String EPP_METRICS = "epp_metrics";
|
||||
static final String WHOIS_COUNTS = "whois_counts";
|
||||
|
||||
/** Sets the month we're doing activity reporting for, and initializes the query map. */
|
||||
static ImmutableMap<String, String> getQueryMap(
|
||||
LocalDate reportingMonth, String projectId) throws IOException {
|
||||
// Convert reportingMonth into YYYYMM01 format for Bigquery table partition pattern-matching.
|
||||
DateTimeFormatter formatter = DateTimeFormat.forPattern("YYYYMM01");
|
||||
String startOfMonth = formatter.print(reportingMonth);
|
||||
String endOfMonth = formatter.print(reportingMonth.plusMonths(1));
|
||||
return createQueryMap(startOfMonth, endOfMonth, projectId);
|
||||
@Inject @Config("projectId") String projectId;
|
||||
@Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
|
||||
@Inject ActivityReportingQueryBuilder() {}
|
||||
|
||||
/** Returns the aggregate query which generates the activity report from the saved view. */
|
||||
String getActivityReportQuery() throws IOException {
|
||||
return String.format(
|
||||
"#standardSQL\nSELECT * FROM `%s.%s.%s`",
|
||||
projectId,
|
||||
ICANN_REPORTING_DATA_SET,
|
||||
getTableName(ACTIVITY_REPORT_AGGREGATION));
|
||||
}
|
||||
|
||||
private static ImmutableMap<String, String> createQueryMap(
|
||||
String startOfMonth, String endOfMonth, String projectId) throws IOException {
|
||||
/** Returns the table name of the query, suffixed with the yearMonth in _YYYYMM format. */
|
||||
private String getTableName(String queryName) {
|
||||
return String.format("%s_%s", queryName, yearMonth.replace("-", ""));
|
||||
}
|
||||
|
||||
/** Sets the month we're doing activity reporting for, and returns the view query map. */
|
||||
ImmutableMap<String, String> getViewQueryMap() throws IOException {
|
||||
LocalDate reportDate = DateTimeFormat.forPattern("yyyy-MM").parseLocalDate(yearMonth);
|
||||
// Convert reportingMonth into YYYYMM01 format for Bigquery table partition pattern-matching.
|
||||
DateTimeFormatter formatter = DateTimeFormat.forPattern("YYYYMM01");
|
||||
String startOfMonth = formatter.print(reportDate);
|
||||
String endOfMonth = formatter.print(reportDate.plusMonths(1));
|
||||
return createQueryMap(startOfMonth, endOfMonth);
|
||||
}
|
||||
|
||||
/** Returns a map from view name to its associated SQL query. */
|
||||
private ImmutableMap<String, String> createQueryMap(
|
||||
String startOfMonth, String endOfMonth) throws IOException {
|
||||
|
||||
ImmutableMap.Builder<String, String> queriesBuilder = ImmutableMap.builder();
|
||||
String operationalRegistrarsQuery =
|
||||
|
|
@ -57,12 +80,12 @@ public final class ActivityReportingQueryBuilder {
|
|||
.put("REGISTRAR_DATA_SET", "registrar_data")
|
||||
.put("REGISTRAR_STATUS_TABLE", "registrar_status")
|
||||
.build();
|
||||
queriesBuilder.put(REGISTRAR_OPERATING_STATUS, operationalRegistrarsQuery);
|
||||
queriesBuilder.put(getTableName(REGISTRAR_OPERATING_STATUS), operationalRegistrarsQuery);
|
||||
|
||||
// TODO(b/62626209): Make this use the CloudDNS counts instead.
|
||||
String dnsCountsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("dns_counts.sql")).build();
|
||||
queriesBuilder.put(DNS_COUNTS, dnsCountsQuery);
|
||||
queriesBuilder.put(getTableName(DNS_COUNTS), dnsCountsQuery);
|
||||
|
||||
// The monthly logs query is a shared dependency for epp counts and whois metrics
|
||||
String monthlyLogsQuery =
|
||||
|
|
@ -73,35 +96,37 @@ public final class ActivityReportingQueryBuilder {
|
|||
.put("START_OF_MONTH", startOfMonth)
|
||||
.put("END_OF_MONTH", endOfMonth)
|
||||
.build();
|
||||
queriesBuilder.put("monthly_logs", monthlyLogsQuery);
|
||||
queriesBuilder.put(getTableName(MONTHLY_LOGS), monthlyLogsQuery);
|
||||
|
||||
String eppQuery =
|
||||
SqlTemplate.create(getQueryFromFile("epp_metrics.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
|
||||
.put("MONTHLY_LOGS_TABLE", MONTHLY_LOGS_TABLE)
|
||||
.put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS))
|
||||
.build();
|
||||
queriesBuilder.put(EPP_METRICS, eppQuery);
|
||||
queriesBuilder.put(getTableName(EPP_METRICS), eppQuery);
|
||||
|
||||
String whoisQuery =
|
||||
SqlTemplate.create(getQueryFromFile("whois_counts.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("ICANN_REPORTING_DATA_SET", MONTHLY_LOGS_TABLE)
|
||||
.put("MONTHLY_LOGS_TABLE", MONTHLY_LOGS_TABLE)
|
||||
.build();
|
||||
queriesBuilder.put(WHOIS_COUNTS, whoisQuery);
|
||||
|
||||
String activityQuery =
|
||||
SqlTemplate.create(getQueryFromFile("activity_report_aggregation.sql"))
|
||||
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
|
||||
.put("REGISTRAR_OPERATING_STATUS_TABLE", REGISTRAR_OPERATING_STATUS)
|
||||
.put("DNS_COUNTS_TABLE", DNS_COUNTS)
|
||||
.put("EPP_METRICS_TABLE", EPP_METRICS)
|
||||
.put("WHOIS_COUNTS_TABLE", WHOIS_COUNTS)
|
||||
.put("LATEST_SNAPSHOT_DATA_SET", "latest_snapshot")
|
||||
.put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS))
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(WHOIS_COUNTS), whoisQuery);
|
||||
|
||||
String aggregateQuery =
|
||||
SqlTemplate.create(getQueryFromFile("activity_report_aggregation.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
|
||||
.put("REGISTRAR_OPERATING_STATUS_TABLE", getTableName(REGISTRAR_OPERATING_STATUS))
|
||||
.put("DNS_COUNTS_TABLE", getTableName(DNS_COUNTS))
|
||||
.put("EPP_METRICS_TABLE", getTableName(EPP_METRICS))
|
||||
.put("WHOIS_COUNTS_TABLE", getTableName(WHOIS_COUNTS))
|
||||
// TODO(larryruili): Change to "latest_datastore_export" when cl/163124895 in prod.
|
||||
.put("LATEST_DATASTORE_EXPORT", "latest_datastore_views")
|
||||
.put("REGISTRY_TABLE", "Registry")
|
||||
.build();
|
||||
queriesBuilder.put("activity_report_aggregation", activityQuery);
|
||||
queriesBuilder.put(getTableName(ACTIVITY_REPORT_AGGREGATION), aggregateQuery);
|
||||
|
||||
return queriesBuilder.build();
|
||||
}
|
||||
|
|
@ -115,3 +140,4 @@ public final class ActivityReportingQueryBuilder {
|
|||
return Resources.getResource(ActivityReportingQueryBuilder.class, "sql/" + filename);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue