Upgrade activity reporting queries to StandardSQL

This also brings the SQL template parameters in-line with the anticipated Bigquery dataset format, and switches from DateTime to the more appropriate LocalDate (since we only need monthly granularity).

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=162747692

javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java

@@ -19,7 +19,7 @@ import static com.google.common.truth.Truth.assertThat;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
-import org.joda.time.DateTime;
+import org.joda.time.LocalDate;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -34,7 +34,7 @@ public class ActivityReportingQueryBuilderTest {
         ImmutableList.of(
             ActivityReportingQueryBuilder.REGISTRAR_OPERATING_STATUS,
             ActivityReportingQueryBuilder.DNS_COUNTS,
-            ActivityReportingQueryBuilder.MONTHLY_LOGS,
+            ActivityReportingQueryBuilder.MONTHLY_LOGS_TABLE,
             ActivityReportingQueryBuilder.EPP_METRICS,
             ActivityReportingQueryBuilder.WHOIS_COUNTS,
             "activity_report_aggregation");
@@ -46,7 +46,8 @@ public class ActivityReportingQueryBuilderTest {
     }
     ImmutableMap<String, String> testQueries = testQueryBuilder.build();
     ImmutableMap<String, String> queries =
-        ActivityReportingQueryBuilder.getQueryMap(new DateTime(2017, 05, 15, 0, 0));
+        ActivityReportingQueryBuilder.getQueryMap(
+            new LocalDate(2017, 05, 15), "domain-registry-alpha");
     for (String query : queryNames) {
       assertThat(queries.get(query)).isEqualTo(testQueries.get(query));
     }

javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql

@@ -11,8 +11,10 @@
   -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   -- See the License for the specific language governing permissions and
   -- limitations under the License.
+
   -- This query pulls from all intermediary tables to create the activity
   -- report csv, via a table transpose and sum over all activity report fields.
+
 SELECT
   Tld.tld AS tld,
   SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars,
@@ -79,10 +81,10 @@ LEFT OUTER JOIN (
     metricName,
     count FROM
     -- BEGIN INTERMEDIARY DATA SOURCES --
-    [activity_reporting.registrar_operating_status],
-    [activity_reporting.dns_counts],
-    [activity_reporting.epp_metrics],
-    [activity_reporting.whois_counts],
+    [icann_reporting.registrar_operating_status],
+    [icann_reporting.dns_counts],
+    [icann_reporting.epp_metrics],
+    [icann_reporting.whois_counts],
     -- END INTERMEDIARY DATA SOURCES --
     ) AS TldMetrics
 ON

javatests/google/registry/reporting/testdata/dns_counts_test.sql

@@ -14,20 +14,16 @@
 
   -- Query for DNS metrics.
 
-  -- Our DNS provider exports logs for all queries received. However, these
-  -- tables only have a TTL of 7 days. We make daily exports of the data
-  -- relevant to us, which allows us to get the full month's UDP and TCP
-  -- queries when generating activity reports.
+  -- This is a no-op until after we transition to Google Cloud DNS, which
+  -- will likely export metrics via Stackdriver.
 
 SELECT
   -- DNS metrics apply to all tlds, which requires the 'null' magic value.
   STRING(NULL) AS tld,
   metricName,
-  -- TODO(b/63388735): Change this to actually query the DNS tables when ready.
-  -1 AS count,
-FROM (
-  SELECT
-    'dns-udp-queries' AS metricName),
-  (
-  SELECT
-    'dns-tcp-queries' AS metricName)
+  -- TODO(b/63388735): Change this to actually query Google Cloud DNS when ready.
+  -1 AS count
+FROM ((
+  SELECT 'dns-udp-queries' AS metricName)
+  UNION ALL
+  (SELECT 'dns-tcp-queries' AS metricName))

javatests/google/registry/reporting/testdata/epp_metrics_test.sql

@@ -14,51 +14,44 @@
 
   -- Query FlowReporter JSON log messages and calculate SRS metrics.
 
-  -- We use regex's over the monthly appengine logs to determine how many
-  -- EPP requests we received for each command.
+  -- We use ugly regex's over the monthly appengine logs to determine how many
+  -- EPP requests we received for each command. For example:
+  -- {"commandType":"check"...,"targetIds":["ais.a.how"],
+  -- "tld":"","tlds":["a.how"],"icannActivityReportField":"srs-dom-check"}
 
 SELECT
-  tld,
+  -- Remove quotation marks from tld fields.
+  REGEXP_EXTRACT(tld, '^"(.*)"$') AS tld,
   activityReportField AS metricName,
-  -- Manual INTEGER cast to work around a BigQuery bug (b/14560012).
-  INTEGER(COUNT(*)) AS count,
-FROM
-  -- Flatten the "tld" column (repeated) so that domain checks for names
-  -- across multiple TLDs are counted towards each checked TLD as though
-  -- there were one copy of this row per TLD (the effect of flattening).
-  FLATTEN((
+  COUNT(*) AS count
+FROM (
+  SELECT
+    -- TODO(b/32486667): Replace with JSON.parse() UDF when available for views
+    SPLIT(
+    REGEXP_EXTRACT(JSON_EXTRACT(json, '$.tlds'), r'^\[(.*)\]$')) AS tlds,
+    JSON_EXTRACT_SCALAR(json,
+      '$.resourceType') AS resourceType,
+    JSON_EXTRACT_SCALAR(json,
+      '$.icannActivityReportField') AS activityReportField
+  FROM (
     SELECT
-      -- Use some ugly regex hackery to convert JSON list of strings into
-      -- repeated string values, since there's no built-in for this.
-      -- TODO(b/20829992): replace with "JSON.parse()" inside a JS UDF
-      --   once we can use GoogleSQL; example in b/37629674#comment2.
-      -- e.g. JSON:"{"commandType":"check"...,"targetIds":["ais.a.how"],
-      -- "tld":"","tlds":["a.how"],"icannActivityReportField":"srs-dom-check"}
-      REGEXP_EXTRACT(
-        SPLIT(
-          REGEXP_EXTRACT(
-            JSON_EXTRACT(json, '$.tlds'),
-            r'^\[(.*)\]$')),
-        '^"(.*)"$') AS tld,
-      -- TODO(b/XXX): remove rawTlds after June 2017 (see below).
-      JSON_EXTRACT_SCALAR(json, '$.resourceType') AS resourceType,
-      JSON_EXTRACT_SCALAR(json, '$.icannActivityReportField')
-        AS activityReportField,
-    FROM (
-      SELECT
-        -- Extract JSON payload following log signature.
-        REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
-          AS json,
-      FROM
-        [monthly_logs.monthly_logs_table]
-      WHERE logMessage CONTAINS 'FLOW-LOG-SIGNATURE-METADATA'
-    )
-  ),
-  -- Second argument to flatten (see above).
-  tld)
--- Exclude cases that can't be tabulated correctly - activity report field
--- is null/empty, or the TLD is null/empty even though it's a domain flow.
+      -- Extract the logged JSON payload.
+      REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
+      AS json
+    FROM `domain-registry-alpha.icann_reporting.monthly_logs` AS logs
+    JOIN
+      UNNEST(logs.logMessage) AS logMessage
+    WHERE
+      logMessage LIKE "%FLOW-LOG-SIGNATURE-METADATA%")) AS regexes
+JOIN
+  -- Unnest the JSON-parsed tlds.
+  UNNEST(regexes.tlds) AS tld
+-- Exclude cases that can't be tabulated correctly, where activityReportField
+-- is null/empty, or TLD is null/empty despite being a domain flow.
 WHERE
-  activityReportField != '' AND (tld != '' OR resourceType != 'domain')
-GROUP BY tld, metricName
-ORDER BY tld, metricName
+  activityReportField != ''
+  AND (tld != '' OR resourceType != 'domain')
+GROUP BY
+  tld, metricName
+ORDER BY
+  tld, metricName

javatests/google/registry/reporting/testdata/monthly_logs_test.sql

@@ -14,15 +14,16 @@
 
   -- Query to fetch AppEngine request logs for the report month.
 
-  -- START_OF_MONTH and END_OF_MONTH should be in YYYY-MM-01 format.
+  -- START_OF_MONTH and END_OF_MONTH should be in YYYYMM01 format.
 
 SELECT
   protoPayload.resource AS requestPath,
-  protoPayload.line.logMessage AS logMessage,
+  ARRAY(
+  SELECT
+    logMessage
+  FROM
+    UNNEST(protoPayload.line)) AS logMessage
 FROM
-  TABLE_DATE_RANGE_STRICT(
-    [appengine_logs.appengine_googleapis_com_request_log_],
-    TIMESTAMP('2017-05-01'),
-    -- End timestamp is inclusive, so subtract 1 day from the
-    -- timestamp representing the start of the next month.
-    DATE_ADD(TIMESTAMP('2017-06-01'), -1, 'DAY'))
+  `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
+WHERE
+  _TABLE_SUFFIX BETWEEN '20170501' AND '20170601'

javatests/google/registry/reporting/testdata/registrar_operating_status_test.sql

@@ -26,7 +26,7 @@ SELECT
     THEN 'pre-ramp-up-registrars'
   -- The import process is imprecise; filter out invalid rows.
   ELSE 'not-applicable' END AS metricName,
-  INTEGER(COUNT(registrar_id)) AS count
+  COUNT(registrar_id) AS count
 FROM
-  [registrar_data.registrar_status]
+  `domain-registry-alpha.registrar_data.registrar_status`
 GROUP BY metricName

javatests/google/registry/reporting/testdata/whois_counts_test.sql

@@ -18,16 +18,15 @@
   -- counts the number of hits via both endpoints (port 43 and the web).
 
 SELECT
-  -- Whois applies to all TLDs, hence the 'null' magic value.
   STRING(NULL) AS tld,
-  -- Whois queries over port 43 get forwarded by the proxy to /_dr/whois,
-  -- while web queries come in via /whois/<params>.
   CASE
     WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries'
-    WHEN LEFT(requestPath, 7) = '/whois/' THEN 'web-whois-queries'
+    WHEN SUBSTR(requestPath, 0, 7) = '/whois/' THEN 'web-whois-queries'
   END AS metricName,
-  INTEGER(COUNT(requestPath)) AS count,
+  COUNT(requestPath) AS count
 FROM
-  [monthly_logs.monthly_logs_table]
-GROUP BY metricName
-HAVING metricName IS NOT NULL
+  `domain-registry-alpha.monthly_logs.monthly_logs`
+GROUP BY
+  metricName
+HAVING
+  metricName IS NOT NULL