mirror of
https://github.com/google/nomulus.git
synced 2025-05-13 16:07:15 +02:00
Add transaction report generation code
------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=167311547
This commit is contained in:
larryruili
jianglai
parent
f26bfbf632
commit
06f3215659
25 changed files with 1284 additions and 75 deletions
|
|
@ -0,0 +1,189 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.reporting;
|
||||
|
||||
import static google.registry.reporting.IcannReportingModule.DATASTORE_EXPORT_DATA_SET;
|
||||
import static google.registry.reporting.IcannReportingModule.ICANN_REPORTING_DATA_SET;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.Resources;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.util.ResourceUtils;
|
||||
import google.registry.util.SqlTemplate;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
import org.joda.time.format.DateTimeFormatter;
|
||||
|
||||
/**
|
||||
* Utility class that produces SQL queries used to generate activity reports from Bigquery.
|
||||
*/
|
||||
public final class TransactionsReportingQueryBuilder implements QueryBuilder {
|
||||
|
||||
@Inject @Config("projectId") String projectId;
|
||||
@Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
|
||||
@Inject TransactionsReportingQueryBuilder() {}
|
||||
|
||||
static final String TRANSACTIONS_REPORT_AGGREGATION = "transactions_report_aggregation";
|
||||
static final String REGISTRAR_IANA_ID = "registrar_iana_id";
|
||||
static final String TOTAL_DOMAINS = "total_domains";
|
||||
static final String TOTAL_NAMESERVERS = "total_nameservers";
|
||||
static final String TRANSACTION_COUNTS = "transaction_counts";
|
||||
static final String TRANSACTION_TRANSFER_LOSING = "transaction_transfer_losing";
|
||||
static final String ATTEMPTED_ADDS = "attempted_adds";
|
||||
|
||||
/** Returns the aggregate query which generates the transactions report from the saved view. */
|
||||
@Override
|
||||
public String getReportQuery() throws IOException {
|
||||
return String.format(
|
||||
"#standardSQL\nSELECT * FROM `%s.%s.%s`",
|
||||
projectId,
|
||||
ICANN_REPORTING_DATA_SET,
|
||||
getTableName(TRANSACTIONS_REPORT_AGGREGATION));
|
||||
}
|
||||
|
||||
/** Sets the month we're doing transactions reporting for, and returns the view query map. */
|
||||
@Override
|
||||
public ImmutableMap<String, String> getViewQueryMap() throws IOException {
|
||||
// Set the earliest date to to yearMonth on day 1 at 00:00:00
|
||||
DateTime earliestReportTime =
|
||||
DateTimeFormat.forPattern("yyyy-MM")
|
||||
.parseDateTime(yearMonth)
|
||||
.withDayOfMonth(1)
|
||||
.withHourOfDay(0)
|
||||
.withMinuteOfHour(0)
|
||||
.withSecondOfMinute(0);
|
||||
// Set the latest date to yearMonth on the last day at 23:59:59
|
||||
DateTime latestReportTime = earliestReportTime.plusMonths(1).minusSeconds(1);
|
||||
return createQueryMap(earliestReportTime, latestReportTime);
|
||||
}
|
||||
|
||||
/** Returns a map from view name to its associated SQL query. */
|
||||
private ImmutableMap<String, String> createQueryMap(
|
||||
DateTime earliestReportTime, DateTime latestReportTime) throws IOException {
|
||||
|
||||
ImmutableMap.Builder<String, String> queriesBuilder = ImmutableMap.builder();
|
||||
String registrarIanaIdQuery =
|
||||
SqlTemplate.create(getQueryFromFile("registrar_iana_id.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(REGISTRAR_IANA_ID), registrarIanaIdQuery);
|
||||
|
||||
String totalDomainsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("total_domains.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("DOMAINBASE_TABLE", "DomainBase")
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(TOTAL_DOMAINS), totalDomainsQuery);
|
||||
|
||||
DateTimeFormatter timestampFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
|
||||
String totalNameserversQuery =
|
||||
SqlTemplate.create(getQueryFromFile("total_nameservers.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("HOSTRESOURCE_TABLE", "HostResource")
|
||||
.put("DOMAINBASE_TABLE", "DomainBase")
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(TOTAL_NAMESERVERS), totalNameserversQuery);
|
||||
|
||||
String transactionCountsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("HISTORYENTRY_TABLE", "HistoryEntry")
|
||||
.put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
|
||||
.put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
|
||||
.put("CLIENT_ID", "clientId")
|
||||
.put("TRANSFER_SUCCESS_FIELD", "TRANSFER_GAINING_SUCCESSFUL")
|
||||
.put("TRANSFER_NACKED_FIELD", "TRANSFER_GAINING_NACKED")
|
||||
.put("DEFAULT_FIELD", "field")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(TRANSACTION_COUNTS), transactionCountsQuery);
|
||||
|
||||
String transactionTransferLosingQuery =
|
||||
SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("HISTORYENTRY_TABLE", "HistoryEntry")
|
||||
.put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
|
||||
.put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
|
||||
.put("CLIENT_ID", "otherClientId")
|
||||
.put("TRANSFER_SUCCESS_FIELD", "TRANSFER_LOSING_SUCCESSFUL")
|
||||
.put("TRANSFER_NACKED_FIELD", "TRANSFER_LOSING_NACKED")
|
||||
.put("DEFAULT_FIELD", "NULL")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(TRANSACTION_TRANSFER_LOSING), transactionTransferLosingQuery);
|
||||
|
||||
// App Engine log table suffixes use YYYYMMDD format
|
||||
DateTimeFormatter logTableFormatter = DateTimeFormat.forPattern("yyyyMMdd");
|
||||
String attemptedAddsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("attempted_adds.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("APPENGINE_LOGS_DATA_SET", "appengine_logs")
|
||||
.put("REQUEST_TABLE", "appengine_googleapis_com_request_log_")
|
||||
.put("FIRST_DAY_OF_MONTH", logTableFormatter.print(earliestReportTime))
|
||||
.put("LAST_DAY_OF_MONTH", logTableFormatter.print(latestReportTime))
|
||||
// All metadata logs for reporting come from google.registry.flows.FlowReporter.
|
||||
.put(
|
||||
"METADATA_LOG_PREFIX",
|
||||
"google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(ATTEMPTED_ADDS), attemptedAddsQuery);
|
||||
|
||||
String aggregateQuery =
|
||||
SqlTemplate.create(getQueryFromFile("transactions_report_aggregation.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
|
||||
.put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID))
|
||||
.put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS))
|
||||
.put("TOTAL_NAMESERVERS_TABLE", getTableName(TOTAL_NAMESERVERS))
|
||||
.put("TRANSACTION_COUNTS_TABLE", getTableName(TRANSACTION_COUNTS))
|
||||
.put("TRANSACTION_TRANSFER_LOSING_TABLE", getTableName(TRANSACTION_TRANSFER_LOSING))
|
||||
.put("ATTEMPTED_ADDS_TABLE", getTableName(ATTEMPTED_ADDS))
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(TRANSACTIONS_REPORT_AGGREGATION), aggregateQuery);
|
||||
|
||||
return queriesBuilder.build();
|
||||
}
|
||||
|
||||
/** Returns the table name of the query, suffixed with the yearMonth in _YYYYMM format. */
|
||||
private String getTableName(String queryName) {
|
||||
return String.format("%s_%s", queryName, yearMonth.replace("-", ""));
|
||||
}
|
||||
|
||||
/** Returns {@link String} for file in {@code reporting/sql/} directory. */
|
||||
private static String getQueryFromFile(String filename) throws IOException {
|
||||
return ResourceUtils.readResourceUtf8(getUrl(filename));
|
||||
}
|
||||
|
||||
private static URL getUrl(String filename) {
|
||||
return Resources.getResource(
|
||||
ActivityReportingQueryBuilder.class, "sql/" + filename);
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue