mirror of
https://github.com/google/nomulus.git
synced 2025-06-27 06:44:51 +02:00
Refactor ICANN reporting and billing into common package
This moves the default yearMonth logic into a common ReportingModule, rather than the coarse-scoped BackendModule, which may not want the default parameter extraction logic, as well as moving the 'yearMonth' parameter constant to the common package it's used in. This also provides a basis for future consolidation of the ReportingEmailUtils and BillingEmailUtils classes, which have modest overlap. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=183130311
This commit is contained in:
larryruili
Ben McIlwain
parent
9d532cb507
commit
74ced1e907
71 changed files with 233 additions and 142 deletions
|
|
@ -0,0 +1,97 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- This query pulls from all intermediary tables to create the activity
|
||||
-- report csv, via a table transpose and sum over all activity report fields.
|
||||
|
||||
SELECT
|
||||
RealTlds.tld AS tld,
|
||||
SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars,
|
||||
-- We use the Centralized Zone Data Service.
|
||||
"CZDS" AS zfa_passwords,
|
||||
SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries,
|
||||
SUM(IF(metricName = 'web-whois-queries', count, 0)) AS web_whois_queries,
|
||||
-- We don't support searchable WHOIS.
|
||||
0 AS searchable_whois_queries,
|
||||
-- DNS queries for UDP/TCP are all assumed to be received/responded.
|
||||
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_received,
|
||||
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_responded,
|
||||
SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_received,
|
||||
SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_responded,
|
||||
-- SRS metrics.
|
||||
SUM(IF(metricName = 'srs-dom-check', count, 0)) AS srs_dom_check,
|
||||
SUM(IF(metricName = 'srs-dom-create', count, 0)) AS srs_dom_create,
|
||||
SUM(IF(metricName = 'srs-dom-delete', count, 0)) AS srs_dom_delete,
|
||||
SUM(IF(metricName = 'srs-dom-info', count, 0)) AS srs_dom_info,
|
||||
SUM(IF(metricName = 'srs-dom-renew', count, 0)) AS srs_dom_renew,
|
||||
SUM(IF(metricName = 'srs-dom-rgp-restore-report', count, 0)) AS srs_dom_rgp_restore_report,
|
||||
SUM(IF(metricName = 'srs-dom-rgp-restore-request', count, 0)) AS srs_dom_rgp_restore_request,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-approve', count, 0)) AS srs_dom_transfer_approve,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-cancel', count, 0)) AS srs_dom_transfer_cancel,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-query', count, 0)) AS srs_dom_transfer_query,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-reject', count, 0)) AS srs_dom_transfer_reject,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-request', count, 0)) AS srs_dom_transfer_request,
|
||||
SUM(IF(metricName = 'srs-dom-update', count, 0)) AS srs_dom_update,
|
||||
SUM(IF(metricName = 'srs-host-check', count, 0)) AS srs_host_check,
|
||||
SUM(IF(metricName = 'srs-host-create', count, 0)) AS srs_host_create,
|
||||
SUM(IF(metricName = 'srs-host-delete', count, 0)) AS srs_host_delete,
|
||||
SUM(IF(metricName = 'srs-host-info', count, 0)) AS srs_host_info,
|
||||
SUM(IF(metricName = 'srs-host-update', count, 0)) AS srs_host_update,
|
||||
SUM(IF(metricName = 'srs-cont-check', count, 0)) AS srs_cont_check,
|
||||
SUM(IF(metricName = 'srs-cont-create', count, 0)) AS srs_cont_create,
|
||||
SUM(IF(metricName = 'srs-cont-delete', count, 0)) AS srs_cont_delete,
|
||||
SUM(IF(metricName = 'srs-cont-info', count, 0)) AS srs_cont_info,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-approve', count, 0)) AS srs_cont_transfer_approve,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-cancel', count, 0)) AS srs_cont_transfer_cancel,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request,
|
||||
SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update
|
||||
-- Cross join a list of all TLDs against TLD-specific metrics and then
|
||||
-- filter so that only metrics with that TLD or a NULL TLD are counted
|
||||
-- towards a given TLD.
|
||||
FROM (
|
||||
SELECT tldStr AS tld
|
||||
FROM `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
|
||||
WHERE tldType = 'REAL'
|
||||
) as RealTlds
|
||||
CROSS JOIN(
|
||||
SELECT
|
||||
tld,
|
||||
metricName,
|
||||
count
|
||||
FROM
|
||||
(
|
||||
-- BEGIN INTERMEDIARY DATA SOURCES --
|
||||
-- Dummy data source to ensure all TLDs appear in report, even if
|
||||
-- they have no recorded metrics for the month.
|
||||
SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 as count
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%REGISTRAR_OPERATING_STATUS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%DNS_COUNTS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%EPP_METRICS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%WHOIS_COUNTS_TABLE%`
|
||||
-- END INTERMEDIARY DATA SOURCES --
|
||||
)) AS TldMetrics
|
||||
WHERE RealTlds.tld = TldMetrics.tld OR TldMetrics.tld IS NULL
|
||||
GROUP BY tld
|
||||
ORDER BY tld
|
||||
73
java/google/registry/reporting/icann/sql/attempted_adds.sql
Normal file
73
java/google/registry/reporting/icann/sql/attempted_adds.sql
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Determine the number of attempted adds each registrar made.
|
||||
|
||||
-- Since the specification requests all 'attempted' adds, we regex the
|
||||
-- monthly App Engine logs, searching for all create commands and associating
|
||||
-- them with their corresponding registrars.
|
||||
|
||||
-- Example log generated by FlowReporter in App Engine logs:
|
||||
--google.registry.flows.FlowReporter
|
||||
-- recordToLogs: FLOW-LOG-SIGNATURE-METADATA:
|
||||
--{"serverTrid":"oNwL2J2eRya7bh7c9oHIzg==-2360a","clientId":"ipmirror"
|
||||
-- ,"commandType":"hello", "resourceType":"","flowClassName":"HelloFlow"
|
||||
-- ,"targetId":"","targetIds":[],"tld":"",
|
||||
-- "tlds":[],"icannActivityReportField":""}
|
||||
|
||||
-- This outer select just converts the registrar's clientId to their name.
|
||||
SELECT
|
||||
tld,
|
||||
registrar_table.registrarName AS registrar_name,
|
||||
'ATTEMPTED_ADDS' AS metricName,
|
||||
count AS metricValue
|
||||
FROM (
|
||||
SELECT
|
||||
JSON_EXTRACT_SCALAR(json, '$.tld') AS tld,
|
||||
JSON_EXTRACT_SCALAR(json, '$.clientId') AS clientId,
|
||||
COUNT(json) AS count
|
||||
FROM (
|
||||
-- Extract JSON metadata package from monthly logs
|
||||
SELECT
|
||||
REGEXP_EXTRACT(logMessages, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
|
||||
AS json
|
||||
FROM (
|
||||
SELECT
|
||||
protoPayload.resource AS requestPath,
|
||||
ARRAY(
|
||||
SELECT logMessage
|
||||
FROM UNNEST(protoPayload.line)) AS logMessage
|
||||
FROM
|
||||
`%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%REQUEST_TABLE%*`
|
||||
WHERE _TABLE_SUFFIX
|
||||
BETWEEN '%FIRST_DAY_OF_MONTH%'
|
||||
AND '%LAST_DAY_OF_MONTH%')
|
||||
JOIN UNNEST(logMessage) AS logMessages
|
||||
-- Look for metadata logs from epp and registrar console requests
|
||||
WHERE requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr')
|
||||
AND STARTS_WITH(logMessages, "%METADATA_LOG_PREFIX%")
|
||||
-- Look for domain creates
|
||||
AND REGEXP_CONTAINS(
|
||||
logMessages, r'"commandType":"create","resourceType":"domain"')
|
||||
-- Filter prober data
|
||||
AND NOT REGEXP_CONTAINS(
|
||||
logMessages, r'"prober-[a-z]{2}-((any)|(canary))"') )
|
||||
GROUP BY tld, clientId ) AS logs_table
|
||||
JOIN
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
|
||||
AS registrar_table
|
||||
ON logs_table.clientId = registrar_table.__key__.name
|
||||
ORDER BY tld, registrar_name
|
||||
|
||||
29
java/google/registry/reporting/icann/sql/dns_counts.sql
Normal file
29
java/google/registry/reporting/icann/sql/dns_counts.sql
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Query for DNS metrics.
|
||||
|
||||
-- You must configure this yourself to enable activity reporting, according
|
||||
-- to whatever metrics your DNS provider makes available. We hope to make
|
||||
-- this available in the open-source build in the near future.
|
||||
|
||||
SELECT
|
||||
STRING(NULL) AS tld,
|
||||
metricName,
|
||||
-1 AS count
|
||||
FROM ((
|
||||
SELECT 'dns-udp-queries' AS metricName)
|
||||
UNION ALL
|
||||
(SELECT 'dns-tcp-queries' AS metricName))
|
||||
58
java/google/registry/reporting/icann/sql/epp_metrics.sql
Normal file
58
java/google/registry/reporting/icann/sql/epp_metrics.sql
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Query FlowReporter JSON log messages and calculate SRS metrics.
|
||||
|
||||
-- We use ugly regex's over the monthly appengine logs to determine how many
|
||||
-- EPP requests we received for each command. For example:
|
||||
-- {"commandType":"check"...,"targetIds":["ais.a.how"],
|
||||
-- "tld":"","tlds":["a.how"],"icannActivityReportField":"srs-dom-check"}
|
||||
|
||||
SELECT
|
||||
-- Remove quotation marks from tld fields.
|
||||
REGEXP_EXTRACT(tld, '^"(.*)"$') AS tld,
|
||||
activityReportField AS metricName,
|
||||
COUNT(*) AS count
|
||||
FROM (
|
||||
SELECT
|
||||
-- TODO(b/32486667): Replace with JSON.parse() UDF when available for views
|
||||
SPLIT(
|
||||
REGEXP_EXTRACT(JSON_EXTRACT(json, '$.tlds'), r'^\[(.*)\]$')) AS tlds,
|
||||
JSON_EXTRACT_SCALAR(json,
|
||||
'$.resourceType') AS resourceType,
|
||||
JSON_EXTRACT_SCALAR(json,
|
||||
'$.icannActivityReportField') AS activityReportField
|
||||
FROM (
|
||||
SELECT
|
||||
-- Extract the logged JSON payload.
|
||||
REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
|
||||
AS json
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%MONTHLY_LOGS_TABLE%` AS logs
|
||||
JOIN
|
||||
UNNEST(logs.logMessage) AS logMessage
|
||||
WHERE
|
||||
STARTS_WITH(logMessage, "%METADATA_LOG_PREFIX%"))) AS regexes
|
||||
JOIN
|
||||
-- Unnest the JSON-parsed tlds.
|
||||
UNNEST(regexes.tlds) AS tld
|
||||
-- Exclude cases that can't be tabulated correctly, where activityReportField
|
||||
-- is null/empty, or TLD is null/empty despite being a domain flow.
|
||||
WHERE
|
||||
activityReportField != ''
|
||||
AND (tld != '' OR resourceType != 'domain')
|
||||
GROUP BY
|
||||
tld, metricName
|
||||
ORDER BY
|
||||
tld, metricName
|
||||
30
java/google/registry/reporting/icann/sql/monthly_logs.sql
Normal file
30
java/google/registry/reporting/icann/sql/monthly_logs.sql
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Query to fetch AppEngine request logs for the report month.
|
||||
|
||||
-- START_OF_MONTH and END_OF_MONTH should be in YYYYMM01 format.
|
||||
|
||||
SELECT
|
||||
protoPayload.resource AS requestPath,
|
||||
ARRAY(
|
||||
SELECT
|
||||
logMessage
|
||||
FROM
|
||||
UNNEST(protoPayload.line)) AS logMessage
|
||||
FROM
|
||||
`%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%REQUEST_TABLE%*`
|
||||
WHERE
|
||||
_TABLE_SUFFIX BETWEEN '%FIRST_DAY_OF_MONTH%' AND '%LAST_DAY_OF_MONTH%'
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Gather a list of all tld-registrar pairs, with their IANA IDs.
|
||||
|
||||
-- This establishes which registrars will appear in the reports.
|
||||
|
||||
SELECT
|
||||
allowed_tlds AS tld,
|
||||
registrarName AS registrar_name,
|
||||
ianaIdentifier AS iana_id
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`,
|
||||
UNNEST(allowedTlds) as allowed_tlds
|
||||
WHERE (type = 'REAL' OR type = 'INTERNAL')
|
||||
-- Filter out prober data
|
||||
AND NOT ENDS_WITH(allowed_tlds, ".test")
|
||||
ORDER BY tld, registrarName
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Query that counts the number of real registrars in system.
|
||||
|
||||
SELECT
|
||||
-- Applies to all TLDs, hence the 'null' magic value.
|
||||
STRING(NULL) AS tld,
|
||||
'operational-registrars' AS metricName,
|
||||
COUNT(registrarName) AS count
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
|
||||
WHERE
|
||||
(type = 'REAL' OR type = 'INTERNAL')
|
||||
GROUP BY metricName
|
||||
38
java/google/registry/reporting/icann/sql/total_domains.sql
Normal file
38
java/google/registry/reporting/icann/sql/total_domains.sql
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Determine the number of domains each registrar sponsors per tld.
|
||||
|
||||
-- This is just the number of fullyQualifiedDomainNames under each
|
||||
-- tld-registrar pair.
|
||||
|
||||
SELECT
|
||||
tld,
|
||||
registrarName as registrar_name,
|
||||
'TOTAL_DOMAINS' as metricName,
|
||||
COUNT(fullyQualifiedDomainName) as metricValue
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%DOMAINBASE_TABLE%`
|
||||
AS domain_table
|
||||
JOIN
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
|
||||
AS registrar_table
|
||||
ON
|
||||
currentSponsorClientId = registrar_table.__key__.name
|
||||
WHERE
|
||||
domain_table._d = 'DomainResource'
|
||||
AND (registrar_table.type = 'REAL' OR registrar_table.type = 'INTERNAL')
|
||||
GROUP BY tld, registrarName
|
||||
ORDER BY tld, registrarName
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Determine the number of referenced nameservers for a registrar's domains.
|
||||
|
||||
-- We count the number of unique hosts under each tld-registrar combo by
|
||||
-- collecting all domains' listed hosts that were still valid at the
|
||||
-- end of the reporting month.
|
||||
|
||||
SELECT
|
||||
tld,
|
||||
registrarName AS registrar_name,
|
||||
'TOTAL_NAMESERVERS' AS metricName,
|
||||
COUNT(fullyQualifiedHostName) AS metricValue
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%HOSTRESOURCE_TABLE%` AS host_table
|
||||
JOIN (
|
||||
SELECT
|
||||
__key__.name AS clientId,
|
||||
registrarName
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
|
||||
WHERE
|
||||
type = 'REAL'
|
||||
OR type = 'INTERNAL') AS registrar_table
|
||||
ON
|
||||
currentSponsorClientId = registrar_table.clientId
|
||||
JOIN (
|
||||
SELECT
|
||||
tld,
|
||||
hosts.name AS referencedHostName
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%DOMAINBASE_TABLE%`,
|
||||
UNNEST(nsHosts) AS hosts
|
||||
WHERE _d = 'DomainResource'
|
||||
AND creationTime <= TIMESTAMP("%LATEST_REPORT_TIME%")
|
||||
AND deletionTime > TIMESTAMP("%LATEST_REPORT_TIME%") ) AS domain_table
|
||||
ON
|
||||
host_table.__key__.name = domain_table.referencedHostName
|
||||
WHERE creationTime <= TIMESTAMP("%LATEST_REPORT_TIME%")
|
||||
AND deletionTime > TIMESTAMP("%LATEST_REPORT_TIME%")
|
||||
GROUP BY tld, registrarName
|
||||
ORDER BY tld, registrarName
|
||||
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Counts the number of mutating transactions each registrar made.
|
||||
|
||||
-- We populate the fields through explicit logging of
|
||||
-- DomainTransactionRecords, which contain all necessary information for
|
||||
-- reporting (such as reporting time, report field, report amount, etc.
|
||||
|
||||
-- A special note on transfers: we only record 'TRANSFER_SUCCESSFUL' or
|
||||
-- 'TRANSFER_NACKED', and we can infer the gaining and losing parties
|
||||
-- from the enclosing HistoryEntry's clientId and otherClientId
|
||||
-- respectively. This query templates the client ID, field for transfer
|
||||
-- success, field for transfer nacks and default field. This allows us to
|
||||
-- create one query for TRANSFER_GAINING and the other report fields,
|
||||
-- and one query for TRANSFER_LOSING fields from the same template.
|
||||
|
||||
-- This outer select just converts the registrar's clientId to their name.
|
||||
SELECT
|
||||
tld,
|
||||
registrar_table.registrarName AS registrar_name,
|
||||
metricName,
|
||||
metricValue
|
||||
FROM (
|
||||
SELECT
|
||||
tld,
|
||||
clientId,
|
||||
CASE
|
||||
WHEN field = 'TRANSFER_SUCCESSFUL' THEN '%TRANSFER_SUCCESS_FIELD%'
|
||||
WHEN field = 'TRANSFER_NACKED' THEN '%TRANSFER_NACKED_FIELD%'
|
||||
ELSE %DEFAULT_FIELD%
|
||||
END AS metricName,
|
||||
SUM(amount) AS metricValue
|
||||
FROM (
|
||||
SELECT
|
||||
entries.%CLIENT_ID% AS clientId,
|
||||
entries.domainTransactionRecords.tld[SAFE_OFFSET(index)] AS tld,
|
||||
entries.domainTransactionRecords.reportingTime[SAFE_OFFSET(index)]
|
||||
AS reportingTime,
|
||||
entries.domainTransactionRecords.reportField[SAFE_OFFSET(index)]
|
||||
AS field,
|
||||
entries.domainTransactionRecords.reportAmount[SAFE_OFFSET(index)]
|
||||
AS amount
|
||||
FROM
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%HISTORYENTRY_TABLE%`
|
||||
AS entries,
|
||||
-- This allows us to 'loop' through the arrays in parallel by index
|
||||
UNNEST(GENERATE_ARRAY(0, ARRAY_LENGTH(
|
||||
entries.domainTransactionRecords.tld) - 1)) AS index
|
||||
-- Ignore null entries
|
||||
WHERE entries.domainTransactionRecords IS NOT NULL )
|
||||
-- Only look at this month's data
|
||||
WHERE reportingTime
|
||||
BETWEEN TIMESTAMP('%EARLIEST_REPORT_TIME%')
|
||||
AND TIMESTAMP('%LATEST_REPORT_TIME%')
|
||||
GROUP BY
|
||||
tld,
|
||||
clientId,
|
||||
field ) AS counts_table
|
||||
JOIN
|
||||
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
|
||||
AS registrar_table
|
||||
ON
|
||||
counts_table.clientId = registrar_table.__key__.name
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Construct the transaction reports' rows from the intermediary data views.
|
||||
|
||||
-- This query pulls from all intermediary tables to create the activity
|
||||
-- report csv, via a table transpose and sum over all activity report fields.
|
||||
|
||||
SELECT
|
||||
registrars.tld as tld,
|
||||
-- Surround registrar names with quotes to handle names containing a comma.
|
||||
FORMAT("\"%s\"", registrars.registrar_name) as registrar_name,
|
||||
registrars.iana_id as iana_id,
|
||||
SUM(IF(metrics.metricName = 'TOTAL_DOMAINS', metrics.metricValue, 0)) AS total_domains,
|
||||
SUM(IF(metrics.metricName = 'TOTAL_NAMESERVERS', metrics.metricValue, 0)) AS total_nameservers,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_1_YR', metrics.metricValue, 0)) AS net_adds_1_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_2_YR', metrics.metricValue, 0)) AS net_adds_2_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_3_YR', metrics.metricValue, 0)) AS net_adds_3_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_4_YR', metrics.metricValue, 0)) AS net_adds_4_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_5_YR', metrics.metricValue, 0)) AS net_adds_5_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_6_YR', metrics.metricValue, 0)) AS net_adds_6_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_7_YR', metrics.metricValue, 0)) AS net_adds_7_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_8_YR', metrics.metricValue, 0)) AS net_adds_8_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_9_YR', metrics.metricValue, 0)) AS net_adds_9_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_ADDS_10_Yr', metrics.metricValue, 0)) AS net_adds_10_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_1_YR', metrics.metricValue, 0)) AS net_renews_1_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_2_YR', metrics.metricValue, 0)) AS net_renews_2_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_3_YR', metrics.metricValue, 0)) AS net_renews_3_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_4_YR', metrics.metricValue, 0)) AS net_renews_4_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_5_YR', metrics.metricValue, 0)) AS net_renews_5_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_6_YR', metrics.metricValue, 0)) AS net_renews_6_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_7_YR', metrics.metricValue, 0)) AS net_renews_7_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_8_YR', metrics.metricValue, 0)) AS net_renews_8_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_9_YR', metrics.metricValue, 0)) AS net_renews_9_yr,
|
||||
SUM(IF(metrics.metricName = 'NET_RENEWS_10_YR', metrics.metricValue, 0)) AS net_renews_10_yr,
|
||||
SUM(IF(metrics.metricName = 'TRANSFER_GAINING_SUCCESSFUL', metrics.metricValue, 0)) AS transfer_gaining_successful,
|
||||
SUM(IF(metrics.metricName = 'TRANSFER_GAINING_NACKED', metrics.metricValue, 0)) AS transfer_gaining_nacked,
|
||||
SUM(IF(metrics.metricName = 'TRANSFER_LOSING_SUCCESSFUL', metrics.metricValue, 0)) AS transfer_losing_successful,
|
||||
SUM(IF(metrics.metricName = 'TRANSFER_LOSING_NACKED', metrics.metricValue, 0)) AS transfer_losing_nacked,
|
||||
-- We don't interact with transfer disputes
|
||||
0 AS transfer_disputed_won,
|
||||
0 AS transfer_disputed_lost,
|
||||
0 AS transfer_disputed_nodecision,
|
||||
SUM(IF(metrics.metricName = 'DELETED_DOMAINS_GRACE', metrics.metricValue, 0)) AS deleted_domains_grace,
|
||||
SUM(IF(metrics.metricName = 'DELETED_DOMAINS_NOGRACE', metrics.metricValue, 0)) AS deleted_domains_nograce,
|
||||
SUM(IF(metrics.metricName = 'RESTORED_DOMAINS', metrics.metricValue, 0)) AS restored_domains,
|
||||
-- We don't require restore reports
|
||||
0 AS restored_noreport,
|
||||
-- We don't enforce AGP limits right now
|
||||
0 AS agp_exemption_requests,
|
||||
0 AS agp_exemptions_granted,
|
||||
0 AS agp_exempted_domains,
|
||||
SUM(IF(metrics.metricName = 'ATTEMPTED_ADDS', metrics.metricValue, 0)) AS attempted_adds
|
||||
FROM
|
||||
-- Only produce reports for real TLDs
|
||||
(SELECT tldStr AS tld
|
||||
FROM `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
|
||||
WHERE tldType = 'REAL') AS registries
|
||||
JOIN
|
||||
(SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%REGISTRAR_IANA_ID_TABLE%`)
|
||||
AS registrars
|
||||
ON registries.tld = registrars.tld
|
||||
-- We LEFT JOIN to produce reports even if the registrar made no transactions
|
||||
LEFT OUTER JOIN (
|
||||
-- Gather all intermediary data views
|
||||
SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TOTAL_DOMAINS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TOTAL_NAMESERVERS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TRANSACTION_COUNTS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TRANSACTION_TRANSFER_LOSING_TABLE%`
|
||||
UNION ALL
|
||||
SELECT *
|
||||
FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%ATTEMPTED_ADDS_TABLE%` ) AS metrics
|
||||
-- Join on tld and registrar name
|
||||
ON registrars.tld = metrics.tld
|
||||
AND registrars.registrar_name = metrics.registrar_name
|
||||
GROUP BY
|
||||
tld, registrar_name, iana_id
|
||||
ORDER BY
|
||||
tld, registrar_name
|
||||
|
||||
33
java/google/registry/reporting/icann/sql/whois_counts.sql
Normal file
33
java/google/registry/reporting/icann/sql/whois_counts.sql
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#standardSQL
|
||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Query for WHOIS metrics.
|
||||
|
||||
-- This searches the monthly appengine logs for Whois requests, and
|
||||
-- counts the number of hits via both endpoints (port 43 and the web).
|
||||
|
||||
SELECT
|
||||
STRING(NULL) AS tld,
|
||||
CASE
|
||||
WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries'
|
||||
WHEN SUBSTR(requestPath, 0, 7) = '/whois/' THEN 'web-whois-queries'
|
||||
END AS metricName,
|
||||
COUNT(requestPath) AS count
|
||||
FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%MONTHLY_LOGS_TABLE%`
|
||||
GROUP BY
|
||||
metricName
|
||||
HAVING
|
||||
metricName IS NOT NULL
|
||||
Loading…
Add table
Add a link
Reference in a new issue