Add transaction report generation code

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=167311547
This commit is contained in:
larryruili 2017-09-01 12:45:06 -07:00 committed by jianglai
parent f26bfbf632
commit 06f3215659
25 changed files with 1284 additions and 75 deletions

View file

@ -37,7 +37,7 @@ public class ActivityReportingQueryBuilderTest {
@Test
public void testAggregateQueryMatch() throws IOException {
ActivityReportingQueryBuilder queryBuilder = getQueryBuilder();
assertThat(queryBuilder.getActivityReportQuery())
assertThat(queryBuilder.getReportQuery())
.isEqualTo(
"#standardSQL\nSELECT * FROM "
+ "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201706`");
@ -55,15 +55,13 @@ public class ActivityReportingQueryBuilderTest {
ActivityReportingQueryBuilder.WHOIS_COUNTS,
ActivityReportingQueryBuilder.ACTIVITY_REPORT_AGGREGATION);
ImmutableMap.Builder<String, String> testQueryBuilder = ImmutableMap.builder();
for (String queryName : queryNames) {
String testFilename = String.format("%s_test.sql", queryName);
testQueryBuilder.put(
String.format("%s_201706", queryName), ReportingTestData.getString(testFilename));
}
ImmutableMap<String, String> expectedQueries = testQueryBuilder.build();
ImmutableMap<String, String> actualQueries = queryBuilder.getViewQueryMap();
assertThat(actualQueries).isEqualTo(expectedQueries);
for (String queryName : queryNames) {
String actualTableName = String.format("%s_201706", queryName);
String testFilename = String.format("%s_test.sql", queryName);
assertThat(actualQueries.get(actualTableName))
.isEqualTo(ReportingTestData.getString(testFilename));
}
}
}

View file

@ -32,6 +32,7 @@ import google.registry.bigquery.BigqueryConnection;
import google.registry.bigquery.BigqueryConnection.DestinationTable;
import google.registry.bigquery.BigqueryUtils.TableType;
import google.registry.gcs.GcsUtils;
import google.registry.reporting.IcannReportingModule.ReportType;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeResponse;
import java.util.concurrent.ExecutionException;
@ -51,7 +52,6 @@ public class IcannReportingStagingActionTest {
BigqueryConnection bigquery = mock(BigqueryConnection.class);
FakeResponse response = new FakeResponse();
ActivityReportingQueryBuilder queryBuilder;
GcsService gcsService = GcsServiceFactory.createGcsService();
@Rule
@ -60,23 +60,31 @@ public class IcannReportingStagingActionTest {
.withLocalModules()
.build();
private IcannReportingStagingAction createAction() {
private IcannReportingStagingAction createAction(ReportType reportType) {
IcannReportingStagingAction action = new IcannReportingStagingAction();
queryBuilder = new ActivityReportingQueryBuilder();
queryBuilder.projectId = "test-project";
queryBuilder.yearMonth = "2017-06";
if (reportType == ReportType.ACTIVITY) {
ActivityReportingQueryBuilder activityBuilder = new ActivityReportingQueryBuilder();
activityBuilder.projectId = "test-project";
activityBuilder.yearMonth = "2017-06";
action.queryBuilder = activityBuilder;
} else {
TransactionsReportingQueryBuilder transactionsBuilder =
new TransactionsReportingQueryBuilder();
transactionsBuilder.projectId = "test-project";
transactionsBuilder.yearMonth = "2017-06";
action.queryBuilder = transactionsBuilder;
}
action.reportType = reportType;
action.reportingBucket = "test-bucket";
action.yearMonth = "2017-06";
action.subdir = Optional.absent();
action.queryBuilder = queryBuilder;
action.bigquery = bigquery;
action.gcsUtils = new GcsUtils(gcsService, 1024);
action.response = response;
return action;
}
@Test
public void testRunSuccess() throws Exception {
private void setUpBigquery() {
when(bigquery.query(any(String.class), any(DestinationTable.class))).thenReturn(fakeFuture());
DestinationTable.Builder tableBuilder = new DestinationTable.Builder()
.datasetId("testdataset")
@ -84,8 +92,12 @@ public class IcannReportingStagingActionTest {
.name("tablename")
.overwrite(true);
when(bigquery.buildDestinationTable(any(String.class))).thenReturn(tableBuilder);
}
ImmutableTable<Integer, TableFieldSchema, Object> reportTable =
@Test
public void testRunSuccess_activityReport() throws Exception {
setUpBigquery();
ImmutableTable<Integer, TableFieldSchema, Object> activityReportTable =
new ImmutableTable.Builder<Integer, TableFieldSchema, Object>()
.put(1, new TableFieldSchema().setName("tld"), "fooTld")
.put(1, new TableFieldSchema().setName("fooField"), "12")
@ -94,8 +106,8 @@ public class IcannReportingStagingActionTest {
.put(2, new TableFieldSchema().setName("fooField"), "56")
.put(2, new TableFieldSchema().setName("barField"), "78")
.build();
when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(reportTable);
IcannReportingStagingAction action = createAction();
when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(activityReportTable);
IcannReportingStagingAction action = createAction(ReportType.ACTIVITY);
action.run();
String expectedReport1 = "fooField,barField\r\n12,34";
@ -112,6 +124,47 @@ public class IcannReportingStagingActionTest {
assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
}
@Test
public void testRunSuccess_transactionsReport() throws Exception {
setUpBigquery();
/*
The fake table result looks like:
tld registrar field
1 fooTld reg1 10
2 fooTld reg2 20
3 barTld reg1 30
*/
ImmutableTable<Integer, TableFieldSchema, Object> transactionReportTable =
new ImmutableTable.Builder<Integer, TableFieldSchema, Object>()
.put(1, new TableFieldSchema().setName("tld"), "fooTld")
.put(1, new TableFieldSchema().setName("registrar"), "reg1")
.put(1, new TableFieldSchema().setName("field"), "10")
.put(2, new TableFieldSchema().setName("tld"), "fooTld")
.put(2, new TableFieldSchema().setName("registrar"), "reg2")
.put(2, new TableFieldSchema().setName("field"), "20")
.put(3, new TableFieldSchema().setName("tld"), "barTld")
.put(3, new TableFieldSchema().setName("registrar"), "reg1")
.put(3, new TableFieldSchema().setName("field"), "30")
.build();
when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(transactionReportTable);
IcannReportingStagingAction action = createAction(ReportType.TRANSACTIONS);
action.reportType = ReportType.TRANSACTIONS;
action.run();
String expectedReport1 = "registrar,field\r\nreg1,10\r\nreg2,20";
String expectedReport2 = "registrar,field\r\nreg1,30";
byte[] generatedFile1 =
readGcsFile(
gcsService,
new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv"));
assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
byte[] generatedFile2 =
readGcsFile(
gcsService,
new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv"));
assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
}
private ListenableFuture<DestinationTable> fakeFuture() {
return new ListenableFuture<DestinationTable>() {
@Override

View file

@ -0,0 +1,69 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.reporting;
import static com.google.common.truth.Truth.assertThat;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link ActivityReportingQueryBuilder}. */
@RunWith(JUnit4.class)
public class TransactionsReportingQueryBuilderTest {
private TransactionsReportingQueryBuilder getQueryBuilder() {
TransactionsReportingQueryBuilder queryBuilder = new TransactionsReportingQueryBuilder();
queryBuilder.yearMonth = "2017-06";
queryBuilder.projectId = "domain-registry-alpha";
return queryBuilder;
}
@Test
public void testAggregateQueryMatch() throws IOException {
TransactionsReportingQueryBuilder queryBuilder = getQueryBuilder();
assertThat(queryBuilder.getReportQuery())
.isEqualTo(
"#standardSQL\nSELECT * FROM "
+ "`domain-registry-alpha.icann_reporting.transactions_report_aggregation_201706`");
}
@Test
public void testIntermediaryQueryMatch() throws IOException {
TransactionsReportingQueryBuilder queryBuilder = getQueryBuilder();
ImmutableList<String> queryNames =
ImmutableList.of(
TransactionsReportingQueryBuilder.TRANSACTIONS_REPORT_AGGREGATION,
TransactionsReportingQueryBuilder.REGISTRAR_IANA_ID,
TransactionsReportingQueryBuilder.TOTAL_DOMAINS,
TransactionsReportingQueryBuilder.TOTAL_NAMESERVERS,
TransactionsReportingQueryBuilder.TRANSACTION_COUNTS,
TransactionsReportingQueryBuilder.TRANSACTION_TRANSFER_LOSING,
TransactionsReportingQueryBuilder.ATTEMPTED_ADDS);
ImmutableMap<String, String> actualQueries = queryBuilder.getViewQueryMap();
for (String queryName : queryNames) {
String actualTableName = String.format("%s_201706", queryName);
String testFilename = String.format("%s_test.sql", queryName);
assertThat(actualQueries.get(actualTableName))
.isEqualTo(ReportingTestData.getString(testFilename));
}
}
}

View file

@ -27,7 +27,7 @@ SELECT
SUM(IF(metricName = 'web-whois-queries', count, 0)) AS web_whois_queries,
-- We don't support searchable WHOIS.
0 AS searchable_whois_queries,
-- DNS queries for UDP/TCP are all assumed to be recevied/responded.
-- DNS queries for UDP/TCP are all assumed to be received/responded.
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_received,
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_responded,
SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_received,
@ -66,7 +66,7 @@ SELECT
-- towards a given TLD.
FROM (
SELECT tldStr as tld
FROM `domain-registry-alpha.latest_datastore_views.Registry`
FROM `domain-registry-alpha.latest_datastore_export.Registry`
WHERE tldType = 'REAL'
) as RealTlds
CROSS JOIN(
@ -97,4 +97,3 @@ CROSS JOIN(
WHERE RealTlds.tld = TldMetrics.tld OR TldMetrics.tld IS NULL
GROUP BY tld
ORDER BY tld

View file

@ -0,0 +1,73 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- Determine the number of attempted adds each registrar made.
-- Since the specification requests all 'attempted' adds, we regex the
-- monthly App Engine logs, searching for all create commands and associating
-- them with their corresponding registrars.
-- Example log generated by FlowReporter in App Engine logs:
--google.registry.flows.FlowReporter
-- recordToLogs: FLOW-LOG-SIGNATURE-METADATA:
--{"serverTrid":"oNwL2J2eRya7bh7c9oHIzg==-2360a","clientId":"ipmirror"
-- ,"commandType":"hello", "resourceType":"","flowClassName":"HelloFlow"
-- ,"targetId":"","targetIds":[],"tld":"",
-- "tlds":[],"icannActivityReportField":""}
-- This outer select just converts the registrar's clientId to their name.
SELECT
tld,
registrar_table.registrarName AS registrar_name,
'ATTEMPTED_ADDS' AS metricName,
count AS metricValue
FROM (
SELECT
JSON_EXTRACT_SCALAR(json, '$.tld') AS tld,
JSON_EXTRACT_SCALAR(json, '$.clientId') AS clientId,
COUNT(json) AS count
FROM (
-- Extract JSON metadata package from monthly logs
SELECT
REGEXP_EXTRACT(logMessages, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
AS json
FROM (
SELECT
protoPayload.resource AS requestPath,
ARRAY(
SELECT logMessage
FROM UNNEST(protoPayload.line)) AS logMessage
FROM
`domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
WHERE _TABLE_SUFFIX
BETWEEN '20170601'
AND '20170630')
JOIN UNNEST(logMessage) AS logMessages
-- Look for metadata logs from epp and registrar console requests
WHERE requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr')
AND STARTS_WITH(logMessages, "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
-- Look for domain creates
AND REGEXP_CONTAINS(
logMessages, r'"commandType":"create","resourceType":"domain"')
-- Filter prober data
AND NOT REGEXP_CONTAINS(
logMessages, r'"prober-[a-z]{2}-((any)|(canary))"') )
GROUP BY tld, clientId ) AS logs_table
JOIN
`domain-registry-alpha.latest_datastore_export.Registrar`
AS registrar_table
ON logs_table.clientId = registrar_table.__key__.name
ORDER BY tld, registrar_name

View file

@ -27,4 +27,4 @@ SELECT
FROM
`domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
WHERE
_TABLE_SUFFIX BETWEEN '20170601' AND '20170701'
_TABLE_SUFFIX BETWEEN '20170601' AND '20170630'

View file

@ -0,0 +1,30 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- Gather a list of all tld-registrar pairs, with their IANA IDs.
-- This establishes which registrars will appear in the reports.
SELECT
allowed_tlds AS tld,
registrarName AS registrar_name,
ianaIdentifier AS iana_id
FROM
`domain-registry-alpha.latest_datastore_export.Registrar`,
UNNEST(allowedTlds) as allowed_tlds
WHERE (type = 'REAL' OR type = 'INTERNAL')
-- Filter out prober data
AND NOT ENDS_WITH(allowed_tlds, "test")
ORDER BY tld, registrarName

View file

@ -0,0 +1,38 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- Determine the number of domains each registrar sponsors per tld.
-- This is just the number of fullyQualifiedDomainNames under each
-- tld-registrar pair.
SELECT
tld,
registrarName as registrar_name,
'TOTAL_DOMAINS' as metricName,
COUNT(fullyQualifiedDomainName) as metricValue
FROM
`domain-registry-alpha.latest_datastore_export.DomainBase`
AS domain_table
JOIN
`domain-registry-alpha.latest_datastore_export.Registrar`
AS registrar_table
ON
currentSponsorClientId = registrar_table.__key__.name
WHERE
domain_table._d = "DomainResource"
AND (registrar_table.type = "REAL" OR registrar_table.type = "INTERNAL")
GROUP BY tld, registrarName
ORDER BY tld, registrarName

View file

@ -0,0 +1,56 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- Determine the number of referenced nameservers for a registrar's domains.
-- We count the number of unique hosts under each tld-registrar combo by
-- collecting all domains' listed hosts that were still valid at the
-- end of the reporting month.
SELECT
tld,
registrarName AS registrar_name,
'TOTAL_NAMESERVERS' AS metricName,
COUNT(fullyQualifiedHostName) AS metricValue
FROM
`domain-registry-alpha.latest_datastore_export.HostResource` AS host_table
JOIN (
SELECT
__key__.name AS clientId,
registrarName
FROM
`domain-registry-alpha.latest_datastore_export.Registrar`
WHERE
type = 'REAL'
OR type = 'INTERNAL') AS registrar_table
ON
currentSponsorClientId = registrar_table.clientId
JOIN (
SELECT
tld,
hosts.name AS referencedHostName
FROM
`domain-registry-alpha.latest_datastore_export.DomainBase`,
UNNEST(nsHosts) AS hosts
WHERE _d = 'DomainResource'
AND creationTime <= TIMESTAMP("2017-06-30 23:59:59")
AND deletionTime > TIMESTAMP("2017-06-30 23:59:59") ) AS domain_table
ON
host_table.__key__.name = domain_table.referencedHostName
WHERE creationTime <= TIMESTAMP("2017-06-30 23:59:59")
AND deletionTime > TIMESTAMP("2017-06-30 23:59:59")
GROUP BY tld, registrarName
ORDER BY tld, registrarName

View file

@ -0,0 +1,78 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- Counts the number of mutating transactions each registrar made.
-- We populate the fields through explicit logging of
-- DomainTransactionRecords, which contain all necessary information for
-- reporting (such as reporting time, report field, report amount, etc.
-- A special note on transfers: we only record 'TRANSFER_SUCCESSFUL' or
-- 'TRANSFER_NACKED', and we can infer the gaining and losing parties
-- from the enclosing HistoryEntry's clientId and otherClientId
-- respectively. This query templates the client ID, field for transfer
-- success, field for transfer nacks and default field. This allows us to
-- create one query for TRANSFER_GAINING and the other report fields,
-- and one query for TRANSFER_LOSING fields from the same template.
-- This outer select just converts the registrar's clientId to their name.
SELECT
tld,
registrar_table.registrarName AS registrar_name,
metricName,
metricValue
FROM (
SELECT
tld,
clientId,
CASE
WHEN field = 'TRANSFER_SUCCESSFUL' THEN 'TRANSFER_GAINING_SUCCESSFUL'
WHEN field = 'TRANSFER_NACKED' THEN 'TRANSFER_GAINING_NACKED'
ELSE field
END AS metricName,
SUM(amount) AS metricValue
FROM (
SELECT
entries.clientId AS clientId,
entries.domainTransactionRecords.tld[SAFE_OFFSET(index)] AS tld,
entries.domainTransactionRecords.reportingTime[SAFE_OFFSET(index)]
AS reportingTime,
entries.domainTransactionRecords.reportField[SAFE_OFFSET(index)]
AS field,
entries.domainTransactionRecords.reportAmount[SAFE_OFFSET(index)]
AS amount
FROM
`domain-registry-alpha.latest_datastore_export.HistoryEntry`
AS entries,
-- This allows us to 'loop' through the arrays in parallel by index
UNNEST(GENERATE_ARRAY(0, ARRAY_LENGTH(
entries.domainTransactionRecords.tld) - 1)) AS index
-- Ignore null entries
WHERE entries.domainTransactionRecords IS NOT NULL )
-- Only look at this month's data
WHERE reportingTime
BETWEEN TIMESTAMP('2017-06-01 00:00:00')
AND TIMESTAMP('2017-06-30 23:59:59')
-- Ignore prober data
AND NOT ENDS_WITH(tld, "test")
GROUP BY
tld,
clientId,
field ) AS counts_table
JOIN
`domain-registry-alpha.latest_datastore_export.Registrar`
AS registrar_table
ON
counts_table.clientId = registrar_table.__key__.name

View file

@ -0,0 +1,78 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- Counts the number of mutating transactions each registrar made.
-- We populate the fields through explicit logging of
-- DomainTransactionRecords, which contain all necessary information for
-- reporting (such as reporting time, report field, report amount, etc.
-- A special note on transfers: we only record 'TRANSFER_SUCCESSFUL' or
-- 'TRANSFER_NACKED', and we can infer the gaining and losing parties
-- from the enclosing HistoryEntry's clientId and otherClientId
-- respectively. This query templates the client ID, field for transfer
-- success, field for transfer nacks and default field. This allows us to
-- create one query for TRANSFER_GAINING and the other report fields,
-- and one query for TRANSFER_LOSING fields from the same template.
-- This outer select just converts the registrar's clientId to their name.
SELECT
tld,
registrar_table.registrarName AS registrar_name,
metricName,
metricValue
FROM (
SELECT
tld,
clientId,
CASE
WHEN field = 'TRANSFER_SUCCESSFUL' THEN 'TRANSFER_LOSING_SUCCESSFUL'
WHEN field = 'TRANSFER_NACKED' THEN 'TRANSFER_LOSING_NACKED'
ELSE NULL
END AS metricName,
SUM(amount) AS metricValue
FROM (
SELECT
entries.otherClientId AS clientId,
entries.domainTransactionRecords.tld[SAFE_OFFSET(index)] AS tld,
entries.domainTransactionRecords.reportingTime[SAFE_OFFSET(index)]
AS reportingTime,
entries.domainTransactionRecords.reportField[SAFE_OFFSET(index)]
AS field,
entries.domainTransactionRecords.reportAmount[SAFE_OFFSET(index)]
AS amount
FROM
`domain-registry-alpha.latest_datastore_export.HistoryEntry`
AS entries,
-- This allows us to 'loop' through the arrays in parallel by index
UNNEST(GENERATE_ARRAY(0, ARRAY_LENGTH(
entries.domainTransactionRecords.tld) - 1)) AS index
-- Ignore null entries
WHERE entries.domainTransactionRecords IS NOT NULL )
-- Only look at this month's data
WHERE reportingTime
BETWEEN TIMESTAMP('2017-06-01 00:00:00')
AND TIMESTAMP('2017-06-30 23:59:59')
-- Ignore prober data
AND NOT ENDS_WITH(tld, "test")
GROUP BY
tld,
clientId,
field ) AS counts_table
JOIN
`domain-registry-alpha.latest_datastore_export.Registrar`
AS registrar_table
ON
counts_table.clientId = registrar_table.__key__.name

View file

@ -0,0 +1,92 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- Construct the transaction reports' rows from the intermediary data views.
-- This query pulls from all intermediary tables to create the activity
-- report csv, via a table transpose and sum over all activity report fields.
SELECT
registrars.tld as tld,
registrars.registrar_name as registrar_name,
registrars.iana_id as iana_id,
SUM(IF(metrics.metricName = 'TOTAL_DOMAINS', metrics.metricValue, 0)) AS total_domains,
SUM(IF(metrics.metricName = 'TOTAL_NAMESERVERS', metrics.metricValue, 0)) AS total_nameservers,
SUM(IF(metrics.metricName = 'NET_ADDS_1_YR', metrics.metricValue, 0)) AS net_adds_1_yr,
SUM(IF(metrics.metricName = 'NET_ADDS_2_YR', metrics.metricValue, 0)) AS net_adds_2_yr,
SUM(IF(metrics.metricName = 'NET_ADDS_3_YR', metrics.metricValue, 0)) AS net_adds_3_yr,
SUM(IF(metrics.metricName = 'NET_ADDS_4_YR', metrics.metricValue, 0)) AS net_adds_4_yr,
SUM(IF(metrics.metricName = 'NET_ADDS_5_YR', metrics.metricValue, 0)) AS net_adds_5_yr,
SUM(IF(metrics.metricName = 'NET_ADDS_6_YR', metrics.metricValue, 0)) AS net_adds_6_yr,
SUM(IF(metrics.metricName = 'NET_ADDS_7_YR', metrics.metricValue, 0)) AS net_adds_7_yr,
SUM(IF(metrics.metricName = 'NET_ADDS_8_YR', metrics.metricValue, 0)) AS net_adds_8_yr,
SUM(IF(metrics.metricName = 'NET_ADDS_9_YR', metrics.metricValue, 0)) AS net_adds_9_yr,
SUM(IF(metrics.metricName = 'NET_ADDS_10_Yr', metrics.metricValue, 0)) AS net_adds_10_yr,
SUM(IF(metrics.metricName = 'NET_RENEWS_1_YR', metrics.metricValue, 0)) AS net_renews_1_yr,
SUM(IF(metrics.metricName = 'NET_RENEWS_2_YR', metrics.metricValue, 0)) AS net_renews_2_yr,
SUM(IF(metrics.metricName = 'NET_RENEWS_3_YR', metrics.metricValue, 0)) AS net_renews_3_yr,
SUM(IF(metrics.metricName = 'NET_RENEWS_4_YR', metrics.metricValue, 0)) AS net_renews_4_yr,
SUM(IF(metrics.metricName = 'NET_RENEWS_5_YR', metrics.metricValue, 0)) AS net_renews_5_yr,
SUM(IF(metrics.metricName = 'NET_RENEWS_6_YR', metrics.metricValue, 0)) AS net_renews_6_yr,
SUM(IF(metrics.metricName = 'NET_RENEWS_7_YR', metrics.metricValue, 0)) AS net_renews_7_yr,
SUM(IF(metrics.metricName = 'NET_RENEWS_8_YR', metrics.metricValue, 0)) AS net_renews_8_yr,
SUM(IF(metrics.metricName = 'NET_RENEWS_9_YR', metrics.metricValue, 0)) AS net_renews_9_yr,
SUM(IF(metrics.metricName = 'NET_RENEWS_10_YR', metrics.metricValue, 0)) AS net_renews_10_yr,
SUM(IF(metrics.metricName = 'TRANSFER_GAINING_SUCCESSFUL', metrics.metricValue, 0)) AS transfer_gaining_successful,
SUM(IF(metrics.metricName = 'TRANSFER_GAINING_NACKED', metrics.metricValue, 0)) AS transfer_gaining_nacked,
SUM(IF(metrics.metricName = 'TRANSFER_LOSING_SUCCESSFUL', metrics.metricValue, 0)) AS transfer_losing_successful,
SUM(IF(metrics.metricName = 'TRANSFER_LOSING_NACKED', metrics.metricValue, 0)) AS transfer_losing_nacked,
-- We don't interact with transfer disputes
0 AS transfer_disputed_won,
0 AS transfer_disputed_lost,
0 AS transfer_disputed_nodecision,
SUM(IF(metrics.metricName = 'DELETED_DOMAINS_GRACE', metrics.metricValue, 0)) AS deleted_domains_grace,
SUM(IF(metrics.metricName = 'DELETED_DOMAINS_NOGRACE', metrics.metricValue, 0)) AS deleted_domains_nograce,
SUM(IF(metrics.metricName = 'RESTORED_DOMAINS', metrics.metricValue, 0)) AS restored_domains,
-- We don't require restore reports
0 AS restored_noreport,
-- We don't enforce AGP limits right now
0 AS agp_exemption_requests,
0 AS agp_exemptions_granted,
0 AS agp_exempted_domains,
SUM(IF(metrics.metricName = 'ATTEMPTED_ADDS', metrics.metricValue, 0)) AS attempted_adds
FROM (
SELECT *
FROM `domain-registry-alpha.icann_reporting.registrar_iana_id_201706`) AS registrars
-- We LEFT JOIN to produce reports even if the registrar made no transactions
LEFT OUTER JOIN (
-- Gather all intermediary data views
SELECT *
FROM `domain-registry-alpha.icann_reporting.total_domains_201706`
UNION ALL
SELECT *
FROM `domain-registry-alpha.icann_reporting.total_nameservers_201706`
UNION ALL
SELECT *
FROM `domain-registry-alpha.icann_reporting.transaction_counts_201706`
UNION ALL
SELECT *
FROM `domain-registry-alpha.icann_reporting.transaction_transfer_losing_201706`
UNION ALL
SELECT *
FROM `domain-registry-alpha.icann_reporting.attempted_adds_201706` ) AS metrics
-- Join on tld and registrar name
ON registrars.tld = metrics.tld
AND registrars.registrar_name = metrics.registrar_name
GROUP BY
tld, registrar_name, iana_id
ORDER BY
tld, registrar_name