From 311d5ac9b66c0a646fef009b35fb9dcef5546d4c Mon Sep 17 00:00:00 2001 From: gbrodman Date: Fri, 21 Jul 2023 14:28:14 -0400 Subject: [PATCH] Fix ICANN reporting and add rdap-queries field (#2081) This includes two changes, the second necessary for testing the first. 1. We add the rdap-queries field as mandated by the amendment to the registry agreement, https://itp.cdn.icann.org/en/files/registry-agreement/proposed-global-amendment-base-gtld-registry-agreement-12-04-2023-en.pdf. This is fairly similar to the whois-queries field where we just query the logs, but instead of searching for "whois" we search for "rdap". 2. BigQuery doesn't use MAX to refer to the bigger of two fields; MAX accepts an array as an argument. In order to do what we want (and to have the BigQuery statements succeed), we need to use GREATEST. Tested both versions in alpha and production BigQuery instances. --- .../icann/sql/cloud_sql_activity_report_aggregation.sql | 3 ++- .../reporting/icann/sql/cloud_sql_transaction_counts.sql | 2 +- .../google/registry/reporting/icann/sql/whois_counts.sql | 1 + .../icann/activity_report_aggregation_test_cloud_sql.sql | 3 ++- .../reporting/icann/transaction_counts_test_cloud_sql.sql | 2 +- .../registry/reporting/icann/whois_counts_test_cloud_sql.sql | 1 + 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_activity_report_aggregation.sql b/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_activity_report_aggregation.sql index ca1d59787..0c304b9de 100644 --- a/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_activity_report_aggregation.sql +++ b/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_activity_report_aggregation.sql @@ -58,7 +58,8 @@ SELECT SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query, SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject, SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request, - SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update + SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update, + SUM(IF(metricName = 'rdap-queries', count, 0)) AS rdap_queries -- Cross join a list of all TLDs against TLD-specific metrics and then -- filter so that only metrics with that TLD or a NULL TLD are counted -- towards a given TLD. diff --git a/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_transaction_counts.sql b/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_transaction_counts.sql index 006398a77..763958f7b 100644 --- a/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_transaction_counts.sql +++ b/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_transaction_counts.sql @@ -56,7 +56,7 @@ FROM ( -- should have negligible impact as the edge cage happens very rarely, more specifically -- when a cancellation happens during grace period by a registrar other than the the -- owning one. All the numbers here should be positive to pass ICANN validation. - MAX(report_amount, 0) AS amount, + GREATEST(report_amount, 0) AS amount, reporting_time AS reportingTime FROM EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql", ''' SELECT history_type, history_other_registrar_id, history_registrar_id, domain_repo_id, history_revision_id FROM "DomainHistory";''') AS dh diff --git a/core/src/main/resources/google/registry/reporting/icann/sql/whois_counts.sql b/core/src/main/resources/google/registry/reporting/icann/sql/whois_counts.sql index d118d1c33..07fa4d107 100644 --- a/core/src/main/resources/google/registry/reporting/icann/sql/whois_counts.sql +++ b/core/src/main/resources/google/registry/reporting/icann/sql/whois_counts.sql @@ -23,6 +23,7 @@ SELECT CASE WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries' WHEN SUBSTR(requestPath, 0, 7) = '/whois/' THEN 'web-whois-queries' + WHEN SUBSTR(requestPath, 0, 6) = '/rdap/' THEN 'rdap-queries' END AS metricName, COUNT(requestPath) AS count FROM diff --git a/core/src/test/resources/google/registry/reporting/icann/activity_report_aggregation_test_cloud_sql.sql b/core/src/test/resources/google/registry/reporting/icann/activity_report_aggregation_test_cloud_sql.sql index f1340963c..d933d6746 100644 --- a/core/src/test/resources/google/registry/reporting/icann/activity_report_aggregation_test_cloud_sql.sql +++ b/core/src/test/resources/google/registry/reporting/icann/activity_report_aggregation_test_cloud_sql.sql @@ -58,7 +58,8 @@ SELECT SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query, SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject, SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request, - SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update + SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update, + SUM(IF(metricName = 'rdap-queries', count, 0)) AS rdap_queries -- Cross join a list of all TLDs against TLD-specific metrics and then -- filter so that only metrics with that TLD or a NULL TLD are counted -- towards a given TLD. diff --git a/core/src/test/resources/google/registry/reporting/icann/transaction_counts_test_cloud_sql.sql b/core/src/test/resources/google/registry/reporting/icann/transaction_counts_test_cloud_sql.sql index cf0db80e4..2e52d80d3 100644 --- a/core/src/test/resources/google/registry/reporting/icann/transaction_counts_test_cloud_sql.sql +++ b/core/src/test/resources/google/registry/reporting/icann/transaction_counts_test_cloud_sql.sql @@ -56,7 +56,7 @@ FROM ( -- should have negligible impact as the edge cage happens very rarely, more specifically -- when a cancellation happens during grace period by a registrar other than the the -- owning one. All the numbers here should be positive to pass ICANN validation. - MAX(report_amount, 0) AS amount, + GREATEST(report_amount, 0) AS amount, reporting_time AS reportingTime FROM EXTERNAL_QUERY("projects/domain-registry-alpha/locations/us/connections/domain-registry-alpha-sql", ''' SELECT history_type, history_other_registrar_id, history_registrar_id, domain_repo_id, history_revision_id FROM "DomainHistory";''') AS dh diff --git a/core/src/test/resources/google/registry/reporting/icann/whois_counts_test_cloud_sql.sql b/core/src/test/resources/google/registry/reporting/icann/whois_counts_test_cloud_sql.sql index d79df769f..0c5a8a584 100644 --- a/core/src/test/resources/google/registry/reporting/icann/whois_counts_test_cloud_sql.sql +++ b/core/src/test/resources/google/registry/reporting/icann/whois_counts_test_cloud_sql.sql @@ -23,6 +23,7 @@ SELECT CASE WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries' WHEN SUBSTR(requestPath, 0, 7) = '/whois/' THEN 'web-whois-queries' + WHEN SUBSTR(requestPath, 0, 6) = '/rdap/' THEN 'rdap-queries' END AS metricName, COUNT(requestPath) AS count FROM