mirror of
https://github.com/google/nomulus.git
synced 2025-07-23 19:20:44 +02:00
Refactor ICANN reporting and billing into common package
This moves the default yearMonth logic into a common ReportingModule, rather than the coarse-scoped BackendModule, which may not want the default parameter extraction logic, as well as moving the 'yearMonth' parameter constant to the common package it's used in. This also provides a basis for future consolidation of the ReportingEmailUtils and BillingEmailUtils classes, which have modest overlap. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=183130311
This commit is contained in:
parent
9d532cb507
commit
74ced1e907
71 changed files with 233 additions and 142 deletions
|
@ -0,0 +1,151 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import static google.registry.reporting.icann.IcannReportingModule.DATASTORE_EXPORT_DATA_SET;
|
||||
import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.Resources;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.util.ResourceUtils;
|
||||
import google.registry.util.SqlTemplate;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.LocalDate;
|
||||
import org.joda.time.YearMonth;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
import org.joda.time.format.DateTimeFormatter;
|
||||
|
||||
/**
|
||||
* Utility class that produces SQL queries used to generate activity reports from Bigquery.
|
||||
*/
|
||||
public final class ActivityReportingQueryBuilder implements QueryBuilder {
|
||||
|
||||
// Names for intermediary tables for overall activity reporting query.
|
||||
static final String REGISTRAR_OPERATING_STATUS = "registrar_operating_status";
|
||||
static final String DNS_COUNTS = "dns_counts";
|
||||
static final String MONTHLY_LOGS = "monthly_logs";
|
||||
static final String EPP_METRICS = "epp_metrics";
|
||||
static final String WHOIS_COUNTS = "whois_counts";
|
||||
static final String ACTIVITY_REPORT_AGGREGATION = "activity_report_aggregation";
|
||||
|
||||
@Inject @Config("projectId") String projectId;
|
||||
|
||||
@Inject YearMonth yearMonth;
|
||||
|
||||
@Inject ActivityReportingQueryBuilder() {}
|
||||
|
||||
/** Returns the aggregate query which generates the activity report from the saved view. */
|
||||
@Override
|
||||
public String getReportQuery() throws IOException {
|
||||
return String.format(
|
||||
"#standardSQL\nSELECT * FROM `%s.%s.%s`",
|
||||
projectId,
|
||||
ICANN_REPORTING_DATA_SET,
|
||||
getTableName(ACTIVITY_REPORT_AGGREGATION));
|
||||
}
|
||||
|
||||
/** Sets the month we're doing activity reporting for, and returns the view query map. */
|
||||
@Override
|
||||
public ImmutableMap<String, String> getViewQueryMap() throws IOException {
|
||||
LocalDate firstDayOfMonth = yearMonth.toLocalDate(1);
|
||||
// The pattern-matching is inclusive, so we subtract 1 day to only report that month's data.
|
||||
LocalDate lastDayOfMonth = yearMonth.toLocalDate(1).plusMonths(1).minusDays(1);
|
||||
return createQueryMap(firstDayOfMonth, lastDayOfMonth);
|
||||
}
|
||||
|
||||
/** Returns a map from view name to its associated SQL query. */
|
||||
private ImmutableMap<String, String> createQueryMap(
|
||||
LocalDate firstDayOfMonth, LocalDate lastDayOfMonth) throws IOException {
|
||||
|
||||
ImmutableMap.Builder<String, String> queriesBuilder = ImmutableMap.builder();
|
||||
String operationalRegistrarsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("registrar_operating_status.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(REGISTRAR_OPERATING_STATUS), operationalRegistrarsQuery);
|
||||
|
||||
String dnsCountsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("dns_counts.sql")).build();
|
||||
queriesBuilder.put(getTableName(DNS_COUNTS), dnsCountsQuery);
|
||||
|
||||
// Convert reportingMonth into YYYYMMDD format for Bigquery table partition pattern-matching.
|
||||
DateTimeFormatter logTableFormatter = DateTimeFormat.forPattern("yyyyMMdd");
|
||||
// The monthly logs are a shared dependency for epp counts and whois metrics
|
||||
String monthlyLogsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("monthly_logs.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("APPENGINE_LOGS_DATA_SET", "appengine_logs")
|
||||
.put("REQUEST_TABLE", "appengine_googleapis_com_request_log_")
|
||||
.put("FIRST_DAY_OF_MONTH", logTableFormatter.print(firstDayOfMonth))
|
||||
.put("LAST_DAY_OF_MONTH", logTableFormatter.print(lastDayOfMonth))
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(MONTHLY_LOGS), monthlyLogsQuery);
|
||||
|
||||
String eppQuery =
|
||||
SqlTemplate.create(getQueryFromFile("epp_metrics.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
|
||||
.put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS))
|
||||
// All metadata logs for reporting come from google.registry.flows.FlowReporter.
|
||||
.put(
|
||||
"METADATA_LOG_PREFIX",
|
||||
"google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(EPP_METRICS), eppQuery);
|
||||
|
||||
String whoisQuery =
|
||||
SqlTemplate.create(getQueryFromFile("whois_counts.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
|
||||
.put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS))
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(WHOIS_COUNTS), whoisQuery);
|
||||
|
||||
String aggregateQuery =
|
||||
SqlTemplate.create(getQueryFromFile("activity_report_aggregation.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
|
||||
.put("REGISTRAR_OPERATING_STATUS_TABLE", getTableName(REGISTRAR_OPERATING_STATUS))
|
||||
.put("DNS_COUNTS_TABLE", getTableName(DNS_COUNTS))
|
||||
.put("EPP_METRICS_TABLE", getTableName(EPP_METRICS))
|
||||
.put("WHOIS_COUNTS_TABLE", getTableName(WHOIS_COUNTS))
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRY_TABLE", "Registry")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(ACTIVITY_REPORT_AGGREGATION), aggregateQuery);
|
||||
|
||||
return queriesBuilder.build();
|
||||
}
|
||||
|
||||
|
||||
/** Returns the table name of the query, suffixed with the yearMonth in _yyyyMM format. */
|
||||
private String getTableName(String queryName) {
|
||||
return String.format("%s_%s", queryName, DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
|
||||
}
|
||||
|
||||
/** Returns {@link String} for file in {@code reporting/sql/} directory. */
|
||||
private static String getQueryFromFile(String filename) throws IOException {
|
||||
return ResourceUtils.readResourceUtf8(getUrl(filename));
|
||||
}
|
||||
|
||||
private static URL getUrl(String filename) {
|
||||
return Resources.getResource(ActivityReportingQueryBuilder.class, "sql/" + filename);
|
||||
}
|
||||
}
|
34
java/google/registry/reporting/icann/BUILD
Normal file
34
java/google/registry/reporting/icann/BUILD
Normal file
|
@ -0,0 +1,34 @@
|
|||
package(
|
||||
default_visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
java_library(
|
||||
name = "icann",
|
||||
srcs = glob(["*.java"]),
|
||||
resources = glob(["sql/*"]),
|
||||
deps = [
|
||||
"//java/google/registry/bigquery",
|
||||
"//java/google/registry/config",
|
||||
"//java/google/registry/gcs",
|
||||
"//java/google/registry/keyring/api",
|
||||
"//java/google/registry/model",
|
||||
"//java/google/registry/request",
|
||||
"//java/google/registry/request/auth",
|
||||
"//java/google/registry/util",
|
||||
"//java/google/registry/xjc",
|
||||
"//java/google/registry/xml",
|
||||
"@com_google_api_client",
|
||||
"@com_google_apis_google_api_services_bigquery",
|
||||
"@com_google_appengine_api_1_0_sdk",
|
||||
"@com_google_appengine_tools_appengine_gcs_client",
|
||||
"@com_google_code_findbugs_jsr305",
|
||||
"@com_google_dagger",
|
||||
"@com_google_guava",
|
||||
"@com_google_http_client",
|
||||
"@com_google_http_client_jackson2",
|
||||
"@javax_servlet_api",
|
||||
"@joda_time",
|
||||
],
|
||||
)
|
167
java/google/registry/reporting/icann/IcannHttpReporter.java
Normal file
167
java/google/registry/reporting/icann/IcannHttpReporter.java
Normal file
|
@ -0,0 +1,167 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.net.MediaType.CSV_UTF_8;
|
||||
import static google.registry.model.registry.Registries.assertTldExists;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.api.client.http.ByteArrayContent;
|
||||
import com.google.api.client.http.GenericUrl;
|
||||
import com.google.api.client.http.HttpHeaders;
|
||||
import com.google.api.client.http.HttpRequest;
|
||||
import com.google.api.client.http.HttpResponse;
|
||||
import com.google.api.client.http.HttpTransport;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.keyring.api.KeyModule.Key;
|
||||
import google.registry.reporting.icann.IcannReportingModule.ReportType;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import google.registry.xjc.XjcXmlTransformer;
|
||||
import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
|
||||
import google.registry.xjc.iirdea.XjcIirdeaResult;
|
||||
import google.registry.xml.XmlException;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.YearMonth;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
|
||||
/**
|
||||
* Class that uploads a CSV file to ICANN's endpoint via an HTTP PUT call.
|
||||
*
|
||||
* <p>It uses basic authorization credentials as specified in the "Registry Interfaces" draft.
|
||||
*
|
||||
* <p>Note that there's a lot of hard-coded logic extracting parameters from the report filenames.
|
||||
* These are safe, as long as they follow the tld-reportType-yearMonth.csv filename format.
|
||||
*
|
||||
* @see IcannReportingUploadAction
|
||||
* @see <a href=https://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-07#page-9>ICANN
|
||||
* Reporting Specification</a>
|
||||
*/
|
||||
public class IcannHttpReporter {
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@Inject HttpTransport httpTransport;
|
||||
@Inject @Key("icannReportingPassword") String password;
|
||||
@Inject @Config("icannTransactionsReportingUploadUrl") String icannTransactionsUrl;
|
||||
@Inject @Config("icannActivityReportingUploadUrl") String icannActivityUrl;
|
||||
@Inject IcannHttpReporter() {}
|
||||
|
||||
/** Uploads {@code reportBytes} to ICANN, returning whether or not it succeeded. */
|
||||
public boolean send(byte[] reportBytes, String reportFilename) throws XmlException, IOException {
|
||||
validateReportFilename(reportFilename);
|
||||
GenericUrl uploadUrl = new GenericUrl(makeUrl(reportFilename));
|
||||
HttpRequest request =
|
||||
httpTransport
|
||||
.createRequestFactory()
|
||||
.buildPutRequest(uploadUrl, new ByteArrayContent(CSV_UTF_8.toString(), reportBytes));
|
||||
|
||||
HttpHeaders headers = request.getHeaders();
|
||||
headers.setBasicAuthentication(getTld(reportFilename) + "_ry", password);
|
||||
headers.setContentType(CSV_UTF_8.toString());
|
||||
request.setHeaders(headers);
|
||||
request.setFollowRedirects(false);
|
||||
|
||||
HttpResponse response = null;
|
||||
logger.infofmt(
|
||||
"Sending report to %s with content length %s",
|
||||
uploadUrl.toString(), request.getContent().getLength());
|
||||
boolean success = true;
|
||||
try {
|
||||
response = request.execute();
|
||||
byte[] content;
|
||||
try {
|
||||
content = ByteStreams.toByteArray(response.getContent());
|
||||
} finally {
|
||||
response.getContent().close();
|
||||
}
|
||||
logger.infofmt(
|
||||
"Received response code %s with content %s",
|
||||
response.getStatusCode(), new String(content, UTF_8));
|
||||
XjcIirdeaResult result = parseResult(content);
|
||||
if (result.getCode().getValue() != 1000) {
|
||||
success = false;
|
||||
logger.warningfmt(
|
||||
"PUT rejected, status code %s:\n%s\n%s",
|
||||
result.getCode(),
|
||||
result.getMsg(),
|
||||
result.getDescription());
|
||||
}
|
||||
} finally {
|
||||
if (response != null) {
|
||||
response.disconnect();
|
||||
} else {
|
||||
success = false;
|
||||
logger.warningfmt(
|
||||
"Received null response from ICANN server at %s", uploadUrl.toString());
|
||||
}
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
private XjcIirdeaResult parseResult(byte[] content) throws XmlException, IOException {
|
||||
XjcIirdeaResponseElement response =
|
||||
XjcXmlTransformer.unmarshal(
|
||||
XjcIirdeaResponseElement.class, new ByteArrayInputStream(content));
|
||||
return response.getResult();
|
||||
}
|
||||
|
||||
/** Verifies a given report filename matches the pattern tld-reportType-yyyyMM.csv. */
|
||||
private void validateReportFilename(String filename) {
|
||||
checkArgument(
|
||||
filename.matches("[a-z0-9.\\-]+-((activity)|(transactions))-[0-9]{6}\\.csv"),
|
||||
"Expected file format: tld-reportType-yyyyMM.csv, got %s instead",
|
||||
filename);
|
||||
assertTldExists(getTld(filename));
|
||||
}
|
||||
|
||||
private String getTld(String filename) {
|
||||
// Extract the TLD, up to second-to-last hyphen in the filename (works with international TLDs)
|
||||
return filename.substring(0, filename.lastIndexOf('-', filename.lastIndexOf('-') - 1));
|
||||
}
|
||||
|
||||
private String makeUrl(String filename) {
|
||||
// Filename is in the format tld-reportType-yearMonth.csv
|
||||
String tld = getTld(filename);
|
||||
// Remove the tld- prefix and csv suffix
|
||||
String remainder = filename.substring(tld.length() + 1, filename.length() - 4);
|
||||
List<String> elements = Splitter.on('-').splitToList(remainder);
|
||||
ReportType reportType = ReportType.valueOf(Ascii.toUpperCase(elements.get(0)));
|
||||
// Re-add hyphen between year and month, because ICANN is inconsistent between filename and URL
|
||||
String yearMonth =
|
||||
YearMonth.parse(elements.get(1), DateTimeFormat.forPattern("yyyyMM")).toString("yyyy-MM");
|
||||
return String.format("%s/%s/%s", getUrlPrefix(reportType), tld, yearMonth);
|
||||
}
|
||||
|
||||
private String getUrlPrefix(ReportType reportType) {
|
||||
switch (reportType) {
|
||||
case TRANSACTIONS:
|
||||
return icannTransactionsUrl;
|
||||
case ACTIVITY:
|
||||
return icannActivityUrl;
|
||||
default:
|
||||
throw new IllegalStateException(
|
||||
String.format(
|
||||
"Received invalid reportTypes! Expected ACTIVITY or TRANSACTIONS, got %s.",
|
||||
reportType));
|
||||
}
|
||||
}
|
||||
}
|
136
java/google/registry/reporting/icann/IcannReportingModule.java
Normal file
136
java/google/registry/reporting/icann/IcannReportingModule.java
Normal file
|
@ -0,0 +1,136 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import static google.registry.request.RequestParameters.extractOptionalEnumParameter;
|
||||
import static google.registry.request.RequestParameters.extractOptionalParameter;
|
||||
import static java.lang.annotation.RetentionPolicy.RUNTIME;
|
||||
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
||||
import com.google.api.client.http.HttpTransport;
|
||||
import com.google.api.client.json.jackson2.JacksonFactory;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import google.registry.bigquery.BigqueryConnection;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.util.SendEmailService;
|
||||
import java.lang.annotation.Documented;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.util.Optional;
|
||||
import javax.inject.Qualifier;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import org.joda.time.Duration;
|
||||
import org.joda.time.YearMonth;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
|
||||
/** Module for dependencies required by ICANN monthly transactions/activity reporting. */
|
||||
@Module
|
||||
public final class IcannReportingModule {
|
||||
|
||||
/** Enum determining the type of report to generate or upload. */
|
||||
public enum ReportType {
|
||||
TRANSACTIONS,
|
||||
ACTIVITY
|
||||
}
|
||||
|
||||
static final String PARAM_SUBDIR = "subdir";
|
||||
static final String PARAM_REPORT_TYPE = "reportType";
|
||||
static final String ICANN_REPORTING_DATA_SET = "icann_reporting";
|
||||
static final String DATASTORE_EXPORT_DATA_SET = "latest_datastore_export";
|
||||
static final String MANIFEST_FILE_NAME = "MANIFEST.txt";
|
||||
private static final String DEFAULT_SUBDIR = "icann/monthly";
|
||||
private static final String BIGQUERY_SCOPE = "https://www.googleapis.com/auth/cloud-platform";
|
||||
|
||||
/** Provides an optional subdirectory to store/upload reports to, extracted from the request. */
|
||||
@Provides
|
||||
@Parameter(PARAM_SUBDIR)
|
||||
static Optional<String> provideSubdirOptional(HttpServletRequest req) {
|
||||
return extractOptionalParameter(req, PARAM_SUBDIR);
|
||||
}
|
||||
|
||||
/** Provides the subdirectory to store/upload reports to, defaults to icann/monthly/yearMonth. */
|
||||
@Provides
|
||||
@ReportingSubdir
|
||||
static String provideSubdir(
|
||||
@Parameter(PARAM_SUBDIR) Optional<String> subdirOptional, YearMonth yearMonth) {
|
||||
String subdir =
|
||||
subdirOptional.orElse(
|
||||
String.format(
|
||||
"%s/%s", DEFAULT_SUBDIR, DateTimeFormat.forPattern("yyyy-MM").print(yearMonth)));
|
||||
if (subdir.startsWith("/") || subdir.endsWith("/")) {
|
||||
throw new BadRequestException(
|
||||
String.format("subdir must not start or end with a \"/\", got %s instead.", subdir));
|
||||
}
|
||||
return subdir;
|
||||
}
|
||||
|
||||
/** Provides an optional reportType to store/upload reports to, extracted from the request. */
|
||||
@Provides
|
||||
@Parameter(PARAM_REPORT_TYPE)
|
||||
static Optional<ReportType> provideReportTypeOptional(HttpServletRequest req) {
|
||||
return extractOptionalEnumParameter(req, ReportType.class, PARAM_REPORT_TYPE);
|
||||
}
|
||||
|
||||
/** Provides a list of reportTypes specified. If absent, we default to both report types. */
|
||||
@Provides
|
||||
static ImmutableList<ReportType> provideReportTypes(
|
||||
@Parameter(PARAM_REPORT_TYPE) Optional<ReportType> reportTypeOptional) {
|
||||
return reportTypeOptional.map(ImmutableList::of)
|
||||
.orElseGet(() -> ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS));
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a BigqueryConnection with default settings.
|
||||
*
|
||||
* <p>We use Bigquery to generate ICANN monthly reports via large aggregate SQL queries.
|
||||
*
|
||||
* @see ActivityReportingQueryBuilder
|
||||
* @see google.registry.tools.BigqueryParameters for justifications of defaults.
|
||||
*/
|
||||
@Provides
|
||||
static BigqueryConnection provideBigqueryConnection(HttpTransport transport) {
|
||||
try {
|
||||
GoogleCredential credential = GoogleCredential
|
||||
.getApplicationDefault(transport, new JacksonFactory());
|
||||
BigqueryConnection connection =
|
||||
new BigqueryConnection.Builder()
|
||||
.setExecutorService(MoreExecutors.newDirectExecutorService())
|
||||
.setCredential(credential.createScoped(ImmutableList.of(BIGQUERY_SCOPE)))
|
||||
.setDatasetId(ICANN_REPORTING_DATA_SET)
|
||||
.setOverwrite(true)
|
||||
.setPollInterval(Duration.standardSeconds(1))
|
||||
.build();
|
||||
connection.initialize();
|
||||
return connection;
|
||||
} catch (Throwable e) {
|
||||
throw new RuntimeException("Could not initialize BigqueryConnection!", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Provides
|
||||
static SendEmailService provideSendEmailService() {
|
||||
return new SendEmailService();
|
||||
}
|
||||
|
||||
/** Dagger qualifier for the subdirectory we stage to/upload from. */
|
||||
@Qualifier
|
||||
@Documented
|
||||
@Retention(RUNTIME)
|
||||
public @interface ReportingSubdir {}
|
||||
}
|
||||
|
267
java/google/registry/reporting/icann/IcannReportingStager.java
Normal file
267
java/google/registry/reporting/icann/IcannReportingStager.java
Normal file
|
@ -0,0 +1,267 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.reporting.icann.IcannReportingModule.MANIFEST_FILE_NAME;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.collect.ArrayListMultimap;
|
||||
import com.google.common.collect.ImmutableCollection;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableTable;
|
||||
import com.google.common.collect.ListMultimap;
|
||||
import google.registry.bigquery.BigqueryConnection;
|
||||
import google.registry.bigquery.BigqueryUtils.TableType;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.reporting.icann.IcannReportingModule.ReportType;
|
||||
import google.registry.reporting.icann.IcannReportingModule.ReportingSubdir;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.YearMonth;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
|
||||
/**
|
||||
* Class containing methods for staging ICANN monthly reports on GCS.
|
||||
*
|
||||
* <p>The main entrypoint is stageReports, which generates a given type of reports.
|
||||
*/
|
||||
public class IcannReportingStager {
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@Inject @Config("icannReportingBucket") String reportingBucket;
|
||||
|
||||
@Inject YearMonth yearMonth;
|
||||
@Inject @ReportingSubdir
|
||||
String subdir;
|
||||
|
||||
@Inject ActivityReportingQueryBuilder activityQueryBuilder;
|
||||
@Inject TransactionsReportingQueryBuilder transactionsQueryBuilder;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
@Inject BigqueryConnection bigquery;
|
||||
|
||||
@Inject
|
||||
IcannReportingStager() {}
|
||||
|
||||
/**
|
||||
* Creates and stores reports of a given type on GCS.
|
||||
*
|
||||
* <p>This is factored out to facilitate choosing which reports to upload,
|
||||
*/
|
||||
ImmutableList<String> stageReports(ReportType reportType) throws Exception {
|
||||
QueryBuilder queryBuilder =
|
||||
(reportType == ReportType.ACTIVITY) ? activityQueryBuilder : transactionsQueryBuilder;
|
||||
|
||||
|
||||
ImmutableMap<String, String> viewQueryMap = queryBuilder.getViewQueryMap();
|
||||
// Generate intermediary views
|
||||
for (Entry<String, String> entry : viewQueryMap.entrySet()) {
|
||||
createIntermediaryTableView(entry.getKey(), entry.getValue(), reportType);
|
||||
}
|
||||
|
||||
// Get an in-memory table of the aggregate query's result
|
||||
ImmutableTable<Integer, TableFieldSchema, Object> reportTable =
|
||||
bigquery.queryToLocalTableSync(queryBuilder.getReportQuery());
|
||||
|
||||
// Get report headers from the table schema and convert into CSV format
|
||||
String headerRow = constructRow(getHeaders(reportTable.columnKeySet()));
|
||||
|
||||
return (reportType == ReportType.ACTIVITY)
|
||||
? stageActivityReports(headerRow, reportTable.rowMap().values())
|
||||
: stageTransactionsReports(headerRow, reportTable.rowMap().values());
|
||||
}
|
||||
|
||||
private void createIntermediaryTableView(String queryName, String query, ReportType reportType)
|
||||
throws ExecutionException, InterruptedException {
|
||||
// Later views depend on the results of earlier ones, so query everything synchronously
|
||||
logger.infofmt("Generating intermediary view %s", queryName);
|
||||
bigquery.query(
|
||||
query,
|
||||
bigquery.buildDestinationTable(queryName)
|
||||
.description(String.format(
|
||||
"An intermediary view to generate %s reports for this month.", reportType))
|
||||
.type(TableType.VIEW)
|
||||
.build()
|
||||
).get();
|
||||
}
|
||||
|
||||
private Iterable<String> getHeaders(ImmutableSet<TableFieldSchema> fields) {
|
||||
return fields
|
||||
.stream()
|
||||
.map((schema) -> schema.getName().replace('_', '-'))
|
||||
.collect(toImmutableList());
|
||||
}
|
||||
|
||||
/** Creates and stores activity reports on GCS, returns a list of files stored. */
|
||||
private ImmutableList<String> stageActivityReports(
|
||||
String headerRow, ImmutableCollection<Map<TableFieldSchema, Object>> rows)
|
||||
throws IOException {
|
||||
ImmutableList.Builder<String> manifestBuilder = new ImmutableList.Builder<>();
|
||||
// Create a report csv for each tld from query table, and upload to GCS
|
||||
for (Map<TableFieldSchema, Object> row : rows) {
|
||||
// Get the tld (first cell in each row)
|
||||
String tld = row.values().iterator().next().toString();
|
||||
if (isNullOrEmpty(tld)) {
|
||||
throw new RuntimeException("Found an empty row in the activity report table!");
|
||||
}
|
||||
ImmutableList<String> rowStrings = ImmutableList.of(constructRow(row.values()));
|
||||
// Create and upload the activity report with a single row
|
||||
manifestBuilder.add(
|
||||
saveReportToGcs(tld, createReport(headerRow, rowStrings), ReportType.ACTIVITY));
|
||||
}
|
||||
return manifestBuilder.build();
|
||||
}
|
||||
|
||||
/** Creates and stores transactions reports on GCS, returns a list of files stored. */
|
||||
private ImmutableList<String> stageTransactionsReports(
|
||||
String headerRow, ImmutableCollection<Map<TableFieldSchema, Object>> rows)
|
||||
throws IOException {
|
||||
// Map from tld to rows
|
||||
ListMultimap<String, String> tldToRows = ArrayListMultimap.create();
|
||||
// Map from tld to totals
|
||||
HashMap<String, List<Integer>> tldToTotals = new HashMap<>();
|
||||
for (Map<TableFieldSchema, Object> row : rows) {
|
||||
// Get the tld (first cell in each row)
|
||||
String tld = row.values().iterator().next().toString();
|
||||
if (isNullOrEmpty(tld)) {
|
||||
throw new RuntimeException("Found an empty row in the transactions report table!");
|
||||
}
|
||||
tldToRows.put(tld, constructRow(row.values()));
|
||||
// Construct totals for each tld, skipping non-summable columns (TLD, registrar name, iana-id)
|
||||
if (!tldToTotals.containsKey(tld)) {
|
||||
tldToTotals.put(tld, new ArrayList<>(Collections.nCopies(row.values().size() - 3, 0)));
|
||||
}
|
||||
addToTotal(tldToTotals.get(tld), row);
|
||||
}
|
||||
ImmutableList.Builder<String> manifestBuilder = new ImmutableList.Builder<>();
|
||||
// Create and upload a transactions report for each tld via its rows
|
||||
for (String tld : tldToRows.keySet()) {
|
||||
// Append the totals row
|
||||
tldToRows.put(tld, constructTotalRow(tldToTotals.get(tld)));
|
||||
manifestBuilder.add(
|
||||
saveReportToGcs(
|
||||
tld, createReport(headerRow, tldToRows.get(tld)), ReportType.TRANSACTIONS));
|
||||
}
|
||||
return manifestBuilder.build();
|
||||
}
|
||||
|
||||
/** Adds a row's values to an existing list of integers (totals). */
|
||||
private void addToTotal(List<Integer> totals, Map<TableFieldSchema, Object> row) {
|
||||
List<Integer> rowVals =
|
||||
row.values()
|
||||
.stream()
|
||||
// Ignore TLD, Registrar name and IANA id
|
||||
.skip(3)
|
||||
.map((Object o) -> Integer.parseInt(o.toString()))
|
||||
.collect(toImmutableList());
|
||||
checkState(
|
||||
rowVals.size() == totals.size(),
|
||||
"Number of elements in totals not equal to number of elements in row!");
|
||||
for (int i = 0; i < rowVals.size(); i++) {
|
||||
totals.set(i, totals.get(i) + rowVals.get(i));
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns a list of integers (totals) as a comma separated string. */
|
||||
private String constructTotalRow(List<Integer> totals) {
|
||||
return "Totals,," + totals.stream().map(Object::toString).collect(Collectors.joining(","));
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes a row of the report by appending the string representation of all objects in an iterable
|
||||
* with commas separating individual fields.
|
||||
*
|
||||
* <p>This discards the first object, which is assumed to be the TLD field.
|
||||
* */
|
||||
private String constructRow(Iterable<?> iterable) {
|
||||
Iterator<?> rowIter = iterable.iterator();
|
||||
StringBuilder rowString = new StringBuilder();
|
||||
// Skip the TLD column
|
||||
rowIter.next();
|
||||
while (rowIter.hasNext()) {
|
||||
rowString.append(String.format("%s,", rowIter.next().toString()));
|
||||
}
|
||||
// Remove trailing comma
|
||||
rowString.deleteCharAt(rowString.length() - 1);
|
||||
return rowString.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a report given its headers and rows as a string.
|
||||
*
|
||||
* <p>Note that activity reports will only have one row, while transactions reports may have
|
||||
* multiple rows.
|
||||
*/
|
||||
private String createReport(String headers, List<String> rows) {
|
||||
StringBuilder reportCsv = new StringBuilder(headers);
|
||||
for (String row : rows) {
|
||||
// Add CRLF between rows per ICANN specification
|
||||
reportCsv.append("\r\n");
|
||||
reportCsv.append(row);
|
||||
}
|
||||
return reportCsv.toString();
|
||||
}
|
||||
|
||||
/** Stores a report on GCS, returning the name of the file stored. */
|
||||
private String saveReportToGcs(String tld, String reportCsv, ReportType reportType)
|
||||
throws IOException {
|
||||
// Upload resulting CSV file to GCS
|
||||
byte[] reportBytes = reportCsv.getBytes(UTF_8);
|
||||
String reportFilename =
|
||||
String.format(
|
||||
"%s-%s-%s.csv",
|
||||
tld,
|
||||
Ascii.toLowerCase(reportType.toString()),
|
||||
DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
|
||||
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
|
||||
gcsUtils.createFromBytes(gcsFilename, reportBytes);
|
||||
logger.infofmt(
|
||||
"Wrote %d bytes to file location %s",
|
||||
reportBytes.length,
|
||||
gcsFilename.toString());
|
||||
return reportFilename;
|
||||
}
|
||||
|
||||
/** Creates and stores a manifest file on GCS, indicating which reports were generated. */
|
||||
void createAndUploadManifest(ImmutableList<String> filenames) throws IOException {
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
|
||||
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
|
||||
StringBuilder manifestString = new StringBuilder();
|
||||
filenames.forEach((filename) -> manifestString.append(filename).append("\n"));
|
||||
gcsUtils.createFromBytes(gcsFilename, manifestString.toString().getBytes(UTF_8));
|
||||
logger.infofmt(
|
||||
"Wrote %d filenames to manifest at %s", filenames.size(), gcsFilename.toString());
|
||||
}
|
||||
}
|
|
@ -0,0 +1,122 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.bigquery.BigqueryJobFailureException;
|
||||
import google.registry.reporting.icann.IcannReportingModule.ReportType;
|
||||
import google.registry.reporting.icann.IcannReportingModule.ReportingSubdir;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import google.registry.util.Retrier;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.Duration;
|
||||
import org.joda.time.YearMonth;
|
||||
|
||||
/**
|
||||
* Action that generates monthly ICANN activity and transactions reports.
|
||||
*
|
||||
* <p>The reports are stored in GCS under gs://[project-id]-reporting/[subdir]. We also store a
|
||||
* MANIFEST.txt file that contains a list of filenames generated, to facilitate subsequent uploads.
|
||||
*
|
||||
* <p>Parameters:
|
||||
*
|
||||
* <p>yearMonth: the reporting month in yyyy-MM format. Defaults to the previous month at runtime
|
||||
* (i.e. a run on 2017-09-01 defaults to 2017-08's reports).
|
||||
*
|
||||
* <p>subdir: the subdirectory of gs://[project-id]-reporting/ to upload to. For example:
|
||||
* "manual/dir" means reports will be stored under gs://[project-id]-reporting/manual/dir. Defaults
|
||||
* to "icann/monthly/[yearMonth]".
|
||||
*
|
||||
* <p>reportTypes: the type of reports to generate. You can specify either 'activity' or
|
||||
* 'transactions'. Defaults to generating both.
|
||||
*/
|
||||
@Action(path = IcannReportingStagingAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_ONLY)
|
||||
public final class IcannReportingStagingAction implements Runnable {
|
||||
|
||||
static final String PATH = "/_dr/task/icannReportingStaging";
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
private static final String CRON_QUEUE = "retryable-cron-tasks";
|
||||
|
||||
@Inject YearMonth yearMonth;
|
||||
@Inject @ReportingSubdir String subdir;
|
||||
@Inject ImmutableList<ReportType> reportTypes;
|
||||
@Inject IcannReportingStager stager;
|
||||
@Inject Retrier retrier;
|
||||
@Inject Response response;
|
||||
@Inject ReportingEmailUtils emailUtils;
|
||||
@Inject IcannReportingStagingAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
retrier.callWithRetry(
|
||||
() -> {
|
||||
ImmutableList.Builder<String> manifestedFilesBuilder = new ImmutableList.Builder<>();
|
||||
for (ReportType reportType : reportTypes) {
|
||||
manifestedFilesBuilder.addAll(stager.stageReports(reportType));
|
||||
}
|
||||
ImmutableList<String> manifestedFiles = manifestedFilesBuilder.build();
|
||||
stager.createAndUploadManifest(manifestedFiles);
|
||||
|
||||
logger.infofmt("Completed staging %d report files.", manifestedFiles.size());
|
||||
emailUtils.emailResults(
|
||||
"ICANN Monthly report staging summary [SUCCESS]",
|
||||
String.format(
|
||||
"Completed staging the following %d ICANN reports:\n%s",
|
||||
manifestedFiles.size(), Joiner.on('\n').join(manifestedFiles)));
|
||||
|
||||
response.setStatus(SC_OK);
|
||||
response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
|
||||
response.setPayload("Completed staging action.");
|
||||
|
||||
logger.infofmt("Enqueueing report upload :");
|
||||
TaskOptions uploadTask = TaskOptions.Builder.withUrl(IcannReportingUploadAction.PATH)
|
||||
.method(Method.POST)
|
||||
.countdownMillis(Duration.standardMinutes(2).getMillis())
|
||||
.param(IcannReportingModule.PARAM_SUBDIR, subdir);
|
||||
QueueFactory.getQueue(CRON_QUEUE).add(uploadTask);
|
||||
return null;
|
||||
},
|
||||
new Retrier.FailureReporter() {
|
||||
@Override
|
||||
public void beforeRetry(Throwable thrown, int failures, int maxAttempts) {}
|
||||
|
||||
@Override
|
||||
public void afterFinalFailure(Throwable thrown, int failures) {
|
||||
emailUtils.emailResults(
|
||||
"ICANN Monthly report staging summary [FAILURE]",
|
||||
String.format(
|
||||
"Staging failed due to %s, check logs for more details.", thrown.toString()));
|
||||
logger.severe(thrown, "Staging action failed.");
|
||||
response.setStatus(SC_INTERNAL_SERVER_ERROR);
|
||||
response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
|
||||
response.setPayload(String.format("Staging failed due to %s", thrown.toString()));
|
||||
}
|
||||
},
|
||||
BigqueryJobFailureException.class);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,150 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
|
||||
import static google.registry.reporting.icann.IcannReportingModule.MANIFEST_FILE_NAME;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.reporting.icann.IcannReportingModule.ReportingSubdir;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import google.registry.util.Retrier;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.stream.Collectors;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* Action that uploads the monthly activity/transactions reports from GCS to ICANN via an HTTP PUT.
|
||||
*
|
||||
* <p>This should be run after {@link IcannReportingStagingAction}, which writes out the month's
|
||||
* reports and a MANIFEST.txt file. This action reads the filenames from the MANIFEST.txt, and
|
||||
* attempts to upload every file in the manifest to ICANN's endpoint.
|
||||
*
|
||||
* <p>Parameters:
|
||||
*
|
||||
* <p>subdir: the subdirectory of gs://[project-id]-reporting/ to retrieve reports from. For
|
||||
* example: "manual/dir" means reports will be stored under gs://[project-id]-reporting/manual/dir.
|
||||
* Defaults to "icann/monthly/[last month in yyyy-MM format]".
|
||||
*/
|
||||
@Action(path = IcannReportingUploadAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
public final class IcannReportingUploadAction implements Runnable {
|
||||
|
||||
static final String PATH = "/_dr/task/icannReportingUpload";
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@Inject
|
||||
@Config("icannReportingBucket")
|
||||
String reportingBucket;
|
||||
|
||||
@Inject @ReportingSubdir
|
||||
String subdir;
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
@Inject IcannHttpReporter icannReporter;
|
||||
@Inject Retrier retrier;
|
||||
@Inject Response response;
|
||||
@Inject ReportingEmailUtils emailUtils;
|
||||
|
||||
@Inject
|
||||
IcannReportingUploadAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
|
||||
ImmutableList<String> manifestedFiles = getManifestedFiles(reportBucketname);
|
||||
ImmutableMap.Builder<String, Boolean> reportSummaryBuilder = new ImmutableMap.Builder<>();
|
||||
// Report on all manifested files
|
||||
for (String reportFilename : manifestedFiles) {
|
||||
logger.infofmt("Reading ICANN report %s from bucket %s", reportFilename, reportBucketname);
|
||||
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
|
||||
verifyFileExists(gcsFilename);
|
||||
boolean success = false;
|
||||
try {
|
||||
success =
|
||||
retrier.callWithRetry(
|
||||
() -> {
|
||||
final byte[] payload = readBytesFromGcs(gcsFilename);
|
||||
return icannReporter.send(payload, reportFilename);
|
||||
},
|
||||
IOException.class);
|
||||
} catch (RuntimeException e) {
|
||||
logger.warningfmt(e, "Upload to %s failed.", gcsFilename.toString());
|
||||
}
|
||||
reportSummaryBuilder.put(reportFilename, success);
|
||||
}
|
||||
emailUploadResults(reportSummaryBuilder.build());
|
||||
response.setStatus(SC_OK);
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
response.setPayload(
|
||||
String.format("OK, attempted uploading %d reports", manifestedFiles.size()));
|
||||
}
|
||||
|
||||
private void emailUploadResults(ImmutableMap<String, Boolean> reportSummary) {
|
||||
emailUtils.emailResults(
|
||||
String.format(
|
||||
"ICANN Monthly report upload summary: %d/%d succeeded",
|
||||
reportSummary.values().stream().filter((b) -> b).count(), reportSummary.size()),
|
||||
String.format(
|
||||
"Report Filename - Upload status:\n%s",
|
||||
reportSummary
|
||||
.entrySet()
|
||||
.stream()
|
||||
.map(
|
||||
(e) ->
|
||||
String.format("%s - %s", e.getKey(), e.getValue() ? "SUCCESS" : "FAILURE"))
|
||||
.collect(Collectors.joining("\n"))));
|
||||
}
|
||||
|
||||
private ImmutableList<String> getManifestedFiles(String reportBucketname) {
|
||||
GcsFilename manifestFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
|
||||
verifyFileExists(manifestFilename);
|
||||
return retrier.callWithRetry(
|
||||
() ->
|
||||
ImmutableList.copyOf(
|
||||
Splitter.on('\n')
|
||||
.omitEmptyStrings()
|
||||
.split(new String(readBytesFromGcs(manifestFilename), UTF_8))),
|
||||
IOException.class);
|
||||
}
|
||||
|
||||
private byte[] readBytesFromGcs(GcsFilename reportFilename) throws IOException {
|
||||
try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) {
|
||||
return ByteStreams.toByteArray(gcsInput);
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyFileExists(GcsFilename gcsFilename) {
|
||||
checkArgument(
|
||||
gcsUtils.existsAndNotEmpty(gcsFilename),
|
||||
"Object %s in bucket %s not found",
|
||||
gcsFilename.getObjectName(),
|
||||
gcsFilename.getBucketName());
|
||||
}
|
||||
}
|
28
java/google/registry/reporting/icann/QueryBuilder.java
Normal file
28
java/google/registry/reporting/icann/QueryBuilder.java
Normal file
|
@ -0,0 +1,28 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import java.io.IOException;
|
||||
|
||||
/** Interface defining the necessary methods to construct ICANN reporting SQL queries. */
|
||||
public interface QueryBuilder {
|
||||
|
||||
/** Returns a map from an intermediary view's table name to the query that generates it. */
|
||||
ImmutableMap<String, String> getViewQueryMap() throws IOException;
|
||||
|
||||
/** Returns a query that retrieves the overall report from the previously generated view. */
|
||||
String getReportQuery() throws IOException;
|
||||
}
|
|
@ -0,0 +1,48 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import google.registry.util.SendEmailService;
|
||||
import javax.inject.Inject;
|
||||
import javax.mail.Message;
|
||||
import javax.mail.Message.RecipientType;
|
||||
import javax.mail.internet.InternetAddress;
|
||||
|
||||
/** Static utils for emailing reporting results. */
|
||||
public class ReportingEmailUtils {
|
||||
|
||||
@Inject @Config("alertSenderEmailAddress") String sender;
|
||||
@Inject @Config("alertRecipientEmailAddress") String recipient;
|
||||
@Inject SendEmailService emailService;
|
||||
@Inject ReportingEmailUtils() {}
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
void emailResults(String subject, String body) {
|
||||
try {
|
||||
Message msg = emailService.createMessage();
|
||||
logger.infofmt("Emailing %s", recipient);
|
||||
msg.setFrom(new InternetAddress(sender));
|
||||
msg.addRecipient(RecipientType.TO, new InternetAddress(recipient));
|
||||
msg.setSubject(subject);
|
||||
msg.setText(body);
|
||||
emailService.sendMessage(msg);
|
||||
} catch (Exception e) {
|
||||
logger.warning(e, "E-mail service failed.");
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,188 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import static google.registry.reporting.icann.IcannReportingModule.DATASTORE_EXPORT_DATA_SET;
|
||||
import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.Resources;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.util.ResourceUtils;
|
||||
import google.registry.util.SqlTemplate;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.LocalTime;
|
||||
import org.joda.time.YearMonth;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
import org.joda.time.format.DateTimeFormatter;
|
||||
|
||||
/**
|
||||
* Utility class that produces SQL queries used to generate activity reports from Bigquery.
|
||||
*/
|
||||
public final class TransactionsReportingQueryBuilder implements QueryBuilder {
|
||||
|
||||
@Inject @Config("projectId") String projectId;
|
||||
|
||||
@Inject YearMonth yearMonth;
|
||||
|
||||
@Inject TransactionsReportingQueryBuilder() {}
|
||||
|
||||
static final String TRANSACTIONS_REPORT_AGGREGATION = "transactions_report_aggregation";
|
||||
static final String REGISTRAR_IANA_ID = "registrar_iana_id";
|
||||
static final String TOTAL_DOMAINS = "total_domains";
|
||||
static final String TOTAL_NAMESERVERS = "total_nameservers";
|
||||
static final String TRANSACTION_COUNTS = "transaction_counts";
|
||||
static final String TRANSACTION_TRANSFER_LOSING = "transaction_transfer_losing";
|
||||
static final String ATTEMPTED_ADDS = "attempted_adds";
|
||||
|
||||
/** Returns the aggregate query which generates the transactions report from the saved view. */
|
||||
@Override
|
||||
public String getReportQuery() throws IOException {
|
||||
return String.format(
|
||||
"#standardSQL\nSELECT * FROM `%s.%s.%s`",
|
||||
projectId,
|
||||
ICANN_REPORTING_DATA_SET,
|
||||
getTableName(TRANSACTIONS_REPORT_AGGREGATION));
|
||||
}
|
||||
|
||||
/** Sets the month we're doing transactions reporting for, and returns the view query map. */
|
||||
@Override
|
||||
public ImmutableMap<String, String> getViewQueryMap() throws IOException {
|
||||
// Set the earliest date to to yearMonth on day 1 at 00:00:00
|
||||
DateTime earliestReportTime = yearMonth.toLocalDate(1).toDateTime(new LocalTime(0, 0, 0));
|
||||
// Set the latest date to yearMonth on the last day at 23:59:59.999
|
||||
DateTime latestReportTime = earliestReportTime.plusMonths(1).minusMillis(1);
|
||||
return createQueryMap(earliestReportTime, latestReportTime);
|
||||
}
|
||||
|
||||
/** Returns a map from view name to its associated SQL query. */
|
||||
private ImmutableMap<String, String> createQueryMap(
|
||||
DateTime earliestReportTime, DateTime latestReportTime) throws IOException {
|
||||
|
||||
ImmutableMap.Builder<String, String> queriesBuilder = ImmutableMap.builder();
|
||||
String registrarIanaIdQuery =
|
||||
SqlTemplate.create(getQueryFromFile("registrar_iana_id.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(REGISTRAR_IANA_ID), registrarIanaIdQuery);
|
||||
|
||||
String totalDomainsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("total_domains.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("DOMAINBASE_TABLE", "DomainBase")
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(TOTAL_DOMAINS), totalDomainsQuery);
|
||||
|
||||
DateTimeFormatter timestampFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
|
||||
String totalNameserversQuery =
|
||||
SqlTemplate.create(getQueryFromFile("total_nameservers.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("HOSTRESOURCE_TABLE", "HostResource")
|
||||
.put("DOMAINBASE_TABLE", "DomainBase")
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(TOTAL_NAMESERVERS), totalNameserversQuery);
|
||||
|
||||
String transactionCountsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("HISTORYENTRY_TABLE", "HistoryEntry")
|
||||
.put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
|
||||
.put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
|
||||
.put("CLIENT_ID", "clientId")
|
||||
.put("TRANSFER_SUCCESS_FIELD", "TRANSFER_GAINING_SUCCESSFUL")
|
||||
.put("TRANSFER_NACKED_FIELD", "TRANSFER_GAINING_NACKED")
|
||||
.put("DEFAULT_FIELD", "field")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(TRANSACTION_COUNTS), transactionCountsQuery);
|
||||
|
||||
String transactionTransferLosingQuery =
|
||||
SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("HISTORYENTRY_TABLE", "HistoryEntry")
|
||||
.put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
|
||||
.put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
|
||||
.put("CLIENT_ID", "otherClientId")
|
||||
.put("TRANSFER_SUCCESS_FIELD", "TRANSFER_LOSING_SUCCESSFUL")
|
||||
.put("TRANSFER_NACKED_FIELD", "TRANSFER_LOSING_NACKED")
|
||||
.put("DEFAULT_FIELD", "NULL")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(TRANSACTION_TRANSFER_LOSING), transactionTransferLosingQuery);
|
||||
|
||||
// App Engine log table suffixes use YYYYMMDD format
|
||||
DateTimeFormatter logTableFormatter = DateTimeFormat.forPattern("yyyyMMdd");
|
||||
String attemptedAddsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("attempted_adds.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("APPENGINE_LOGS_DATA_SET", "appengine_logs")
|
||||
.put("REQUEST_TABLE", "appengine_googleapis_com_request_log_")
|
||||
.put("FIRST_DAY_OF_MONTH", logTableFormatter.print(earliestReportTime))
|
||||
.put("LAST_DAY_OF_MONTH", logTableFormatter.print(latestReportTime))
|
||||
// All metadata logs for reporting come from google.registry.flows.FlowReporter.
|
||||
.put(
|
||||
"METADATA_LOG_PREFIX",
|
||||
"google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(ATTEMPTED_ADDS), attemptedAddsQuery);
|
||||
|
||||
String aggregateQuery =
|
||||
SqlTemplate.create(getQueryFromFile("transactions_report_aggregation.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRY_TABLE", "Registry")
|
||||
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
|
||||
.put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID))
|
||||
.put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS))
|
||||
.put("TOTAL_NAMESERVERS_TABLE", getTableName(TOTAL_NAMESERVERS))
|
||||
.put("TRANSACTION_COUNTS_TABLE", getTableName(TRANSACTION_COUNTS))
|
||||
.put("TRANSACTION_TRANSFER_LOSING_TABLE", getTableName(TRANSACTION_TRANSFER_LOSING))
|
||||
.put("ATTEMPTED_ADDS_TABLE", getTableName(ATTEMPTED_ADDS))
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(TRANSACTIONS_REPORT_AGGREGATION), aggregateQuery);
|
||||
|
||||
return queriesBuilder.build();
|
||||
}
|
||||
|
||||
/** Returns the table name of the query, suffixed with the yearMonth in _yyyyMM format. */
|
||||
private String getTableName(String queryName) {
|
||||
return String.format("%s_%s", queryName, DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
|
||||
}
|
||||
|
||||
/** Returns {@link String} for file in {@code reporting/sql/} directory. */
|
||||
private static String getQueryFromFile(String filename) throws IOException {
|
||||
return ResourceUtils.readResourceUtf8(getUrl(filename));
|
||||
}
|
||||
|
||||
private static URL getUrl(String filename) {
|
||||
return Resources.getResource(
|
||||
ActivityReportingQueryBuilder.class, "sql/" + filename);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- This query pulls from all intermediary tables to create the activity
|
||||
-- report csv, via a table transpose and sum over all activity report fields.
|
||||
|
||||
SELECT
|
||||
RealTlds.tld AS tld,
|
||||
SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars,
|
||||
-- We use the Centralized Zone Data Service.
|
||||
"CZDS" AS zfa_passwords,
|
||||
SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries,
|
||||
SUM(IF(metricName = 'web-whois-queries', count, 0)) AS web_whois_queries,
|
||||
-- We don't support searchable WHOIS.
|
||||
0 AS searchable_whois_queries,
|
||||
-- DNS queries for UDP/TCP are all assumed to be received/responded.
|
||||
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_received,
|
||||
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_responded,
|
||||
SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_received,
|
||||
SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_responded,
|
||||
-- SRS metrics.
|
||||
SUM(IF(metricName = 'srs-dom-check', count, 0)) AS srs_dom_check,
|
||||
SUM(IF(metricName = 'srs-dom-create', count, 0)) AS srs_dom_create,
|
||||
SUM(IF(metricName = 'srs-dom-delete', count, 0)) AS srs_dom_delete,
|
||||
SUM(IF(metricName = 'srs-dom-info', count, 0)) AS srs_dom_info,
|
||||
SUM(IF(metricName = 'srs-dom-renew', count, 0)) AS srs_dom_renew,
|
||||
SUM(IF(metricName = 'srs-dom-rgp-restore-report', count, 0)) AS srs_dom_rgp_restore_report,
|
||||
SUM(IF(metricName = 'srs-dom-rgp-restore-request', count, 0)) AS srs_dom_rgp_restore_request,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-approve', count, 0)) AS srs_dom_transfer_approve,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-cancel', count, 0)) AS srs_dom_transfer_cancel,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-query', count, 0)) AS srs_dom_transfer_query,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-reject', count, 0)) AS srs_dom_transfer_reject,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-request', count, 0)) AS srs_dom_transfer_request,
|
||||
SUM(IF(metricName = 'srs-dom-update', count, 0)) AS srs_dom_update,
|
||||
SUM(IF(metricName = 'srs-host-check', count, 0)) AS srs_host_check,
|
||||
SUM(IF(metricName = 'srs-host-create', count, 0)) AS srs_host_create,
|
||||
SUM(IF(metricName = 'srs-host-delete', count, 0)) AS srs_host_delete,
|
||||
SUM(IF(metricName = 'srs-host-info', count, 0)) AS srs_host_info,
|
||||
SUM(IF(metricName = 'srs-host-update', count, 0)) AS srs_host_update,
|
||||
SUM(IF(metricName = 'srs-cont-check', count, 0)) AS srs_cont_check,
|
||||
SUM(IF(metricName = 'srs-cont-create', count, 0)) AS srs_cont_create,
|
||||
SUM(IF(metricName = 'srs-cont-delete', count, 0)) AS srs_cont_delete,
|
||||
SUM(IF(metricName = 'srs-cont-info', count, 0)) AS srs_cont_info,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-approve', count, 0)) AS srs_cont_transfer_approve,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-cancel', count, 0)) AS srs_cont_transfer_cancel,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request,
|
||||
SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update
|
||||
-- Cross join a list of all TLDs against TLD-specific metrics and then
|
||||
-- filter so that only metrics with that TLD or a NULL TLD are counted
|
||||
-- towards a given TLD.
|
||||
FROM (
|
||||
SELECT tldStr AS tld
|
||||
FROM `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
|
||||
WHERE tldType = 'REAL'
|
||||
) as RealTlds
|
||||
CROSS JOIN(
|
||||
SELECT
|
||||
tld,
|
||||
metricName,
|
||||
count
|
||||
FROM
|
||||
(
|
||||
-- BEGIN INTERMEDIARY DATA SOURCES --
|
||||
-- Dummy data source to ensure all TLDs appear in report, even if
|
||||
-- they have no recorded metrics for the month.
|
||||
SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 as count
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%REGISTRAR_OPERATING_STATUS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%DNS_COUNTS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%EPP_METRICS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%WHOIS_COUNTS_TABLE%`
|
||||
-- END INTERMEDIARY DATA SOURCES --
|
||||
)) AS TldMetrics
|
||||
WHERE RealTlds.tld = TldMetrics.tld OR TldMetrics.tld IS NULL
|
||||
GROUP BY tld
|
||||
ORDER BY tld
|
73
java/google/registry/reporting/icann/sql/attempted_adds.sql
Normal file
73
java/google/registry/reporting/icann/sql/attempted_adds.sql
Normal file
|
@ -0,0 +1,73 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Determine the number of attempted adds each registrar made.
|
||||
|
||||
-- Since the specification requests all 'attempted' adds, we regex the
|
||||
-- monthly App Engine logs, searching for all create commands and associating
|
||||
-- them with their corresponding registrars.
|
||||
|
||||
-- Example log generated by FlowReporter in App Engine logs:
|
||||
--google.registry.flows.FlowReporter
|
||||
-- recordToLogs: FLOW-LOG-SIGNATURE-METADATA:
|
||||
--{"serverTrid":"oNwL2J2eRya7bh7c9oHIzg==-2360a","clientId":"ipmirror"
|
||||
-- ,"commandType":"hello", "resourceType":"","flowClassName":"HelloFlow"
|
||||
-- ,"targetId":"","targetIds":[],"tld":"",
|
||||
-- "tlds":[],"icannActivityReportField":""}
|
||||
|
||||
-- This outer select just converts the registrar's clientId to their name.
|
||||
SELECT
|
||||
tld,
|
||||
registrar_table.registrarName AS registrar_name,
|
||||
'ATTEMPTED_ADDS' AS metricName,
|
||||
count AS metricValue
|
||||
FROM (
|
||||
SELECT
|
||||
JSON_EXTRACT_SCALAR(json, '$.tld') AS tld,
|
||||
JSON_EXTRACT_SCALAR(json, '$.clientId') AS clientId,
|
||||
COUNT(json) AS count
|
||||
FROM (
|
||||
-- Extract JSON metadata package from monthly logs
|
||||
SELECT
|
||||
REGEXP_EXTRACT(logMessages, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
|
||||
AS json
|
||||
FROM (
|
||||
SELECT
|
||||
protoPayload.resource AS requestPath,
|
||||
ARRAY(
|
||||
SELECT logMessage
|
||||
FROM UNNEST(protoPayload.line)) AS logMessage
|
||||
FROM
|
||||
`%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%REQUEST_TABLE%*`
|
||||
WHERE _TABLE_SUFFIX
|
||||
BETWEEN '%FIRST_DAY_OF_MONTH%'
|
||||
AND '%LAST_DAY_OF_MONTH%')
|
||||
JOIN UNNEST(logMessage) AS logMessages
|
||||
-- Look for metadata logs from epp and registrar console requests
|
||||
WHERE requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr')
|
||||
AND STARTS_WITH(logMessages, "%METADATA_LOG_PREFIX%")
|
||||
-- Look for domain creates
|
||||
AND REGEXP_CONTAINS(
|
||||
logMessages, r'"commandType":"create","resourceType":"domain"')
|
||||
-- Filter prober data
|
||||
AND NOT REGEXP_CONTAINS(
|
||||
logMessages, r'"prober-[a-z]{2}-((any)|(canary))"') )
|
||||
GROUP BY tld, clientId ) AS logs_table
|
||||
JOIN
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
|
||||
AS registrar_table
|
||||
ON logs_table.clientId = registrar_table.__key__.name
|
||||
ORDER BY tld, registrar_name
|
||||
|
29
java/google/registry/reporting/icann/sql/dns_counts.sql
Normal file
29
java/google/registry/reporting/icann/sql/dns_counts.sql
Normal file
|
@ -0,0 +1,29 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Query for DNS metrics.
|
||||
|
||||
-- You must configure this yourself to enable activity reporting, according
|
||||
-- to whatever metrics your DNS provider makes available. We hope to make
|
||||
-- this available in the open-source build in the near future.
|
||||
|
||||
SELECT
|
||||
STRING(NULL) AS tld,
|
||||
metricName,
|
||||
-1 AS count
|
||||
FROM ((
|
||||
SELECT 'dns-udp-queries' AS metricName)
|
||||
UNION ALL
|
||||
(SELECT 'dns-tcp-queries' AS metricName))
|
58
java/google/registry/reporting/icann/sql/epp_metrics.sql
Normal file
58
java/google/registry/reporting/icann/sql/epp_metrics.sql
Normal file
|
@ -0,0 +1,58 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Query FlowReporter JSON log messages and calculate SRS metrics.
|
||||
|
||||
-- We use ugly regex's over the monthly appengine logs to determine how many
|
||||
-- EPP requests we received for each command. For example:
|
||||
-- {"commandType":"check"...,"targetIds":["ais.a.how"],
|
||||
-- "tld":"","tlds":["a.how"],"icannActivityReportField":"srs-dom-check"}
|
||||
|
||||
SELECT
|
||||
-- Remove quotation marks from tld fields.
|
||||
REGEXP_EXTRACT(tld, '^"(.*)"$') AS tld,
|
||||
activityReportField AS metricName,
|
||||
COUNT(*) AS count
|
||||
FROM (
|
||||
SELECT
|
||||
-- TODO(b/32486667): Replace with JSON.parse() UDF when available for views
|
||||
SPLIT(
|
||||
REGEXP_EXTRACT(JSON_EXTRACT(json, '$.tlds'), r'^\[(.*)\]$')) AS tlds,
|
||||
JSON_EXTRACT_SCALAR(json,
|
||||
'$.resourceType') AS resourceType,
|
||||
JSON_EXTRACT_SCALAR(json,
|
||||
'$.icannActivityReportField') AS activityReportField
|
||||
FROM (
|
||||
SELECT
|
||||
-- Extract the logged JSON payload.
|
||||
REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
|
||||
AS json
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%MONTHLY_LOGS_TABLE%` AS logs
|
||||
JOIN
|
||||
UNNEST(logs.logMessage) AS logMessage
|
||||
WHERE
|
||||
STARTS_WITH(logMessage, "%METADATA_LOG_PREFIX%"))) AS regexes
|
||||
JOIN
|
||||
-- Unnest the JSON-parsed tlds.
|
||||
UNNEST(regexes.tlds) AS tld
|
||||
-- Exclude cases that can't be tabulated correctly, where activityReportField
|
||||
-- is null/empty, or TLD is null/empty despite being a domain flow.
|
||||
WHERE
|
||||
activityReportField != ''
|
||||
AND (tld != '' OR resourceType != 'domain')
|
||||
GROUP BY
|
||||
tld, metricName
|
||||
ORDER BY
|
||||
tld, metricName
|
30
java/google/registry/reporting/icann/sql/monthly_logs.sql
Normal file
30
java/google/registry/reporting/icann/sql/monthly_logs.sql
Normal file
|
@ -0,0 +1,30 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Query to fetch AppEngine request logs for the report month.
|
||||
|
||||
-- START_OF_MONTH and END_OF_MONTH should be in YYYYMM01 format.
|
||||
|
||||
SELECT
|
||||
protoPayload.resource AS requestPath,
|
||||
ARRAY(
|
||||
SELECT
|
||||
logMessage
|
||||
FROM
|
||||
UNNEST(protoPayload.line)) AS logMessage
|
||||
FROM
|
||||
`%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%REQUEST_TABLE%*`
|
||||
WHERE
|
||||
_TABLE_SUFFIX BETWEEN '%FIRST_DAY_OF_MONTH%' AND '%LAST_DAY_OF_MONTH%'
|
|
@ -0,0 +1,30 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Gather a list of all tld-registrar pairs, with their IANA IDs.
|
||||
|
||||
-- This establishes which registrars will appear in the reports.
|
||||
|
||||
SELECT
|
||||
allowed_tlds AS tld,
|
||||
registrarName AS registrar_name,
|
||||
ianaIdentifier AS iana_id
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`,
|
||||
UNNEST(allowedTlds) as allowed_tlds
|
||||
WHERE (type = 'REAL' OR type = 'INTERNAL')
|
||||
-- Filter out prober data
|
||||
AND NOT ENDS_WITH(allowed_tlds, ".test")
|
||||
ORDER BY tld, registrarName
|
|
@ -0,0 +1,27 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Query that counts the number of real registrars in system.
|
||||
|
||||
SELECT
|
||||
-- Applies to all TLDs, hence the 'null' magic value.
|
||||
STRING(NULL) AS tld,
|
||||
'operational-registrars' AS metricName,
|
||||
COUNT(registrarName) AS count
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
|
||||
WHERE
|
||||
(type = 'REAL' OR type = 'INTERNAL')
|
||||
GROUP BY metricName
|
38
java/google/registry/reporting/icann/sql/total_domains.sql
Normal file
38
java/google/registry/reporting/icann/sql/total_domains.sql
Normal file
|
@ -0,0 +1,38 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Determine the number of domains each registrar sponsors per tld.
|
||||
|
||||
-- This is just the number of fullyQualifiedDomainNames under each
|
||||
-- tld-registrar pair.
|
||||
|
||||
SELECT
|
||||
tld,
|
||||
registrarName as registrar_name,
|
||||
'TOTAL_DOMAINS' as metricName,
|
||||
COUNT(fullyQualifiedDomainName) as metricValue
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%DOMAINBASE_TABLE%`
|
||||
AS domain_table
|
||||
JOIN
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
|
||||
AS registrar_table
|
||||
ON
|
||||
currentSponsorClientId = registrar_table.__key__.name
|
||||
WHERE
|
||||
domain_table._d = 'DomainResource'
|
||||
AND (registrar_table.type = 'REAL' OR registrar_table.type = 'INTERNAL')
|
||||
GROUP BY tld, registrarName
|
||||
ORDER BY tld, registrarName
|
|
@ -0,0 +1,56 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Determine the number of referenced nameservers for a registrar's domains.
|
||||
|
||||
-- We count the number of unique hosts under each tld-registrar combo by
|
||||
-- collecting all domains' listed hosts that were still valid at the
|
||||
-- end of the reporting month.
|
||||
|
||||
SELECT
|
||||
tld,
|
||||
registrarName AS registrar_name,
|
||||
'TOTAL_NAMESERVERS' AS metricName,
|
||||
COUNT(fullyQualifiedHostName) AS metricValue
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%HOSTRESOURCE_TABLE%` AS host_table
|
||||
JOIN (
|
||||
SELECT
|
||||
__key__.name AS clientId,
|
||||
registrarName
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
|
||||
WHERE
|
||||
type = 'REAL'
|
||||
OR type = 'INTERNAL') AS registrar_table
|
||||
ON
|
||||
currentSponsorClientId = registrar_table.clientId
|
||||
JOIN (
|
||||
SELECT
|
||||
tld,
|
||||
hosts.name AS referencedHostName
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%DOMAINBASE_TABLE%`,
|
||||
UNNEST(nsHosts) AS hosts
|
||||
WHERE _d = 'DomainResource'
|
||||
AND creationTime <= TIMESTAMP("%LATEST_REPORT_TIME%")
|
||||
AND deletionTime > TIMESTAMP("%LATEST_REPORT_TIME%") ) AS domain_table
|
||||
ON
|
||||
host_table.__key__.name = domain_table.referencedHostName
|
||||
WHERE creationTime <= TIMESTAMP("%LATEST_REPORT_TIME%")
|
||||
AND deletionTime > TIMESTAMP("%LATEST_REPORT_TIME%")
|
||||
GROUP BY tld, registrarName
|
||||
ORDER BY tld, registrarName
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Counts the number of mutating transactions each registrar made.
|
||||
|
||||
-- We populate the fields through explicit logging of
|
||||
-- DomainTransactionRecords, which contain all necessary information for
|
||||
-- reporting (such as reporting time, report field, report amount, etc.
|
||||
|
||||
-- A special note on transfers: we only record 'TRANSFER_SUCCESSFUL' or
|
||||
-- 'TRANSFER_NACKED', and we can infer the gaining and losing parties
|
||||
-- from the enclosing HistoryEntry's clientId and otherClientId
|
||||
-- respectively. This query templates the client ID, field for transfer
|
||||
-- success, field for transfer nacks and default field. This allows us to
|
||||
-- create one query for TRANSFER_GAINING and the other report fields,
|
||||
-- and one query for TRANSFER_LOSING fields from the same template.
|
||||
|
||||
-- This outer select just converts the registrar's clientId to their name.
|
||||
SELECT
|
||||
tld,
|
||||
registrar_table.registrarName AS registrar_name,
|
||||
metricName,
|
||||
metricValue
|
||||
FROM (
|
||||
SELECT
|
||||
tld,
|
||||
clientId,
|
||||
CASE
|
||||
WHEN field = 'TRANSFER_SUCCESSFUL' THEN '%TRANSFER_SUCCESS_FIELD%'
|
||||
WHEN field = 'TRANSFER_NACKED' THEN '%TRANSFER_NACKED_FIELD%'
|
||||
ELSE %DEFAULT_FIELD%
|
||||
END AS metricName,
|
||||
SUM(amount) AS metricValue
|
||||
FROM (
|
||||
SELECT
|
||||
entries.%CLIENT_ID% AS clientId,
|
||||
entries.domainTransactionRecords.tld[SAFE_OFFSET(index)] AS tld,
|
||||
entries.domainTransactionRecords.reportingTime[SAFE_OFFSET(index)]
|
||||
AS reportingTime,
|
||||
entries.domainTransactionRecords.reportField[SAFE_OFFSET(index)]
|
||||
AS field,
|
||||
entries.domainTransactionRecords.reportAmount[SAFE_OFFSET(index)]
|
||||
AS amount
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%HISTORYENTRY_TABLE%`
|
||||
AS entries,
|
||||
-- This allows us to 'loop' through the arrays in parallel by index
|
||||
UNNEST(GENERATE_ARRAY(0, ARRAY_LENGTH(
|
||||
entries.domainTransactionRecords.tld) - 1)) AS index
|
||||
-- Ignore null entries
|
||||
WHERE entries.domainTransactionRecords IS NOT NULL )
|
||||
-- Only look at this month's data
|
||||
WHERE reportingTime
|
||||
BETWEEN TIMESTAMP('%EARLIEST_REPORT_TIME%')
|
||||
AND TIMESTAMP('%LATEST_REPORT_TIME%')
|
||||
GROUP BY
|
||||
tld,
|
||||
clientId,
|
||||
field ) AS counts_table
|
||||
JOIN
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
|
||||
AS registrar_table
|
||||
ON
|
||||
counts_table.clientId = registrar_table.__key__.name
|
|
@ -0,0 +1,100 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Construct the transaction reports' rows from the intermediary data views.
|
||||
|
||||
-- This query pulls from all intermediary tables to create the activity
|
||||
-- report csv, via a table transpose and sum over all activity report fields.
|
||||
|
||||
SELECT
|
||||
registrars.tld as tld,
|
||||
-- Surround registrar names with quotes to handle names containing a comma.
|
||||
FORMAT("\"%s\"", registrars.registrar_name) as registrar_name,
|
||||
registrars.iana_id as iana_id,
|
||||
SUM(IF(metrics.metricName = 'TOTAL_DOMAINS', metrics.metricValue, 0)) AS total_domains,
|
||||
SUM(IF(metrics.metricName = 'TOTAL_NAMESERVERS', metrics.metricValue, 0)) AS total_nameservers,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_1_YR', metrics.metricValue, 0)) AS net_adds_1_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_2_YR', metrics.metricValue, 0)) AS net_adds_2_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_3_YR', metrics.metricValue, 0)) AS net_adds_3_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_4_YR', metrics.metricValue, 0)) AS net_adds_4_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_5_YR', metrics.metricValue, 0)) AS net_adds_5_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_6_YR', metrics.metricValue, 0)) AS net_adds_6_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_7_YR', metrics.metricValue, 0)) AS net_adds_7_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_8_YR', metrics.metricValue, 0)) AS net_adds_8_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_9_YR', metrics.metricValue, 0)) AS net_adds_9_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_10_Yr', metrics.metricValue, 0)) AS net_adds_10_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_1_YR', metrics.metricValue, 0)) AS net_renews_1_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_2_YR', metrics.metricValue, 0)) AS net_renews_2_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_3_YR', metrics.metricValue, 0)) AS net_renews_3_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_4_YR', metrics.metricValue, 0)) AS net_renews_4_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_5_YR', metrics.metricValue, 0)) AS net_renews_5_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_6_YR', metrics.metricValue, 0)) AS net_renews_6_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_7_YR', metrics.metricValue, 0)) AS net_renews_7_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_8_YR', metrics.metricValue, 0)) AS net_renews_8_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_9_YR', metrics.metricValue, 0)) AS net_renews_9_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_10_YR', metrics.metricValue, 0)) AS net_renews_10_yr,
|
||||
SUM(IF(metrics.metricName = 'TRANSFER_GAINING_SUCCESSFUL', metrics.metricValue, 0)) AS transfer_gaining_successful,
|
||||
SUM(IF(metrics.metricName = 'TRANSFER_GAINING_NACKED', metrics.metricValue, 0)) AS transfer_gaining_nacked,
|
||||
SUM(IF(metrics.metricName = 'TRANSFER_LOSING_SUCCESSFUL', metrics.metricValue, 0)) AS transfer_losing_successful,
|
||||
SUM(IF(metrics.metricName = 'TRANSFER_LOSING_NACKED', metrics.metricValue, 0)) AS transfer_losing_nacked,
|
||||
-- We don't interact with transfer disputes
|
||||
0 AS transfer_disputed_won,
|
||||
0 AS transfer_disputed_lost,
|
||||
0 AS transfer_disputed_nodecision,
|
||||
SUM(IF(metrics.metricName = 'DELETED_DOMAINS_GRACE', metrics.metricValue, 0)) AS deleted_domains_grace,
|
||||
SUM(IF(metrics.metricName = 'DELETED_DOMAINS_NOGRACE', metrics.metricValue, 0)) AS deleted_domains_nograce,
|
||||
SUM(IF(metrics.metricName = 'RESTORED_DOMAINS', metrics.metricValue, 0)) AS restored_domains,
|
||||
-- We don't require restore reports
|
||||
0 AS restored_noreport,
|
||||
-- We don't enforce AGP limits right now
|
||||
0 AS agp_exemption_requests,
|
||||
0 AS agp_exemptions_granted,
|
||||
0 AS agp_exempted_domains,
|
||||
SUM(IF(metrics.metricName = 'ATTEMPTED_ADDS', metrics.metricValue, 0)) AS attempted_adds
|
||||
FROM
|
||||
-- Only produce reports for real TLDs
|
||||
(SELECT tldStr AS tld
|
||||
FROM `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
|
||||
WHERE tldType = 'REAL') AS registries
|
||||
JOIN
|
||||
(SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%REGISTRAR_IANA_ID_TABLE%`)
|
||||
AS registrars
|
||||
ON registries.tld = registrars.tld
|
||||
-- We LEFT JOIN to produce reports even if the registrar made no transactions
|
||||
LEFT OUTER JOIN (
|
||||
-- Gather all intermediary data views
|
||||
SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TOTAL_DOMAINS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TOTAL_NAMESERVERS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TRANSACTION_COUNTS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TRANSACTION_TRANSFER_LOSING_TABLE%`
|
||||
UNION ALL
|
||||
SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%ATTEMPTED_ADDS_TABLE%` ) AS metrics
|
||||
-- Join on tld and registrar name
|
||||
ON registrars.tld = metrics.tld
|
||||
AND registrars.registrar_name = metrics.registrar_name
|
||||
GROUP BY
|
||||
tld, registrar_name, iana_id
|
||||
ORDER BY
|
||||
tld, registrar_name
|
||||
|
33
java/google/registry/reporting/icann/sql/whois_counts.sql
Normal file
33
java/google/registry/reporting/icann/sql/whois_counts.sql
Normal file
|
@ -0,0 +1,33 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Query for WHOIS metrics.
|
||||
|
||||
-- This searches the monthly appengine logs for Whois requests, and
|
||||
-- counts the number of hits via both endpoints (port 43 and the web).
|
||||
|
||||
SELECT
|
||||
STRING(NULL) AS tld,
|
||||
CASE
|
||||
WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries'
|
||||
WHEN SUBSTR(requestPath, 0, 7) = '/whois/' THEN 'web-whois-queries'
|
||||
END AS metricName,
|
||||
COUNT(requestPath) AS count
|
||||
FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%MONTHLY_LOGS_TABLE%`
|
||||
GROUP BY
|
||||
metricName
|
||||
HAVING
|
||||
metricName IS NOT NULL
|
Loading…
Add table
Add a link
Reference in a new issue