Optimize activity report SQL

We only log FLOW-LOG-SIGNATURE-METADATA from one place- FlowRunner. As a
result, we can swap the generalized regex for a prefix-only regex, saving a
<strong>lot</strong> of processing for our epp query (which is the most
expensive of the bunch).

I've also changed the test dates from 2017-05 to 2017-06, allowing us to copy-paste
the test data into Bigquery to verify their function. The reason for 2017-06 in particular is because June was the first month that populated all the metadata necessary to generate these reports.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=165391715

java/google/registry/reporting/ActivityReportingQueryBuilder.java

@@ -103,6 +103,10 @@ public final class ActivityReportingQueryBuilder {
             .put("PROJECT_ID", projectId)
             .put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
             .put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS))
+            // All metadata logs for reporting come from google.registry.flows.FlowReporter.
+            .put(
+                "METADATA_LOG_PREFIX",
+                "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
             .build();
     queriesBuilder.put(getTableName(EPP_METRICS), eppQuery);
 

java/google/registry/reporting/sql/epp_metrics.sql

@@ -43,7 +43,7 @@ FROM (
     JOIN
       UNNEST(logs.logMessage) AS logMessage
     WHERE
-      logMessage LIKE "%FLOW-LOG-SIGNATURE-METADATA%")) AS regexes
+      STARTS_WITH(logMessage, "%METADATA_LOG_PREFIX%"))) AS regexes
 JOIN
   -- Unnest the JSON-parsed tlds.
   UNNEST(regexes.tlds) AS tld

javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java

@@ -29,7 +29,7 @@ public class ActivityReportingQueryBuilderTest {
 
   private ActivityReportingQueryBuilder getQueryBuilder() {
     ActivityReportingQueryBuilder queryBuilder = new ActivityReportingQueryBuilder();
-    queryBuilder.yearMonth = "2017-05";
+    queryBuilder.yearMonth = "2017-06";
     queryBuilder.projectId = "domain-registry-alpha";
     return queryBuilder;
   }
@@ -40,7 +40,7 @@ public class ActivityReportingQueryBuilderTest {
     assertThat(queryBuilder.getActivityReportQuery())
         .isEqualTo(
             "#standardSQL\nSELECT * FROM "
-                + "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201705`");
+                + "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201706`");
   }
 
   @Test
@@ -59,7 +59,7 @@ public class ActivityReportingQueryBuilderTest {
     for (String queryName : queryNames) {
       String testFilename = String.format("%s_test.sql", queryName);
       testQueryBuilder.put(
-          String.format("%s_201705", queryName), ReportingTestData.getString(testFilename));
+          String.format("%s_201706", queryName), ReportingTestData.getString(testFilename));
     }
     ImmutableMap<String, String> expectedQueries = testQueryBuilder.build();
     ImmutableMap<String, String> actualQueries = queryBuilder.getViewQueryMap();

javatests/google/registry/reporting/IcannHttpReporterTest.java

@@ -82,9 +82,9 @@ public class IcannHttpReporterTest {
   @Test
   public void testSuccess() throws Exception {
     IcannHttpReporter reporter = createReporter();
-    reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS);
+    reporter.send(FAKE_PAYLOAD, "test", "2017-06", ReportType.TRANSACTIONS);
 
-    assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2016-06");
+    assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2017-06");
     Map<String, List<String>> headers = mockRequest.getHeaders();
     String userPass = "test_ry:fakePass";
     String expectedAuth =
@@ -98,7 +98,7 @@ public class IcannHttpReporterTest {
     IcannHttpReporter reporter = createReporter();
     reporter.httpTransport = createMockTransport(IIRDEA_BAD_XML);
     try {
-      reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS);
+      reporter.send(FAKE_PAYLOAD, "test", "2017-06", ReportType.TRANSACTIONS);
       assertWithMessage("Expected InternalServerErrorException to be thrown").fail();
     } catch (InternalServerErrorException expected) {
       assertThat(expected).hasMessageThat().isEqualTo("The structure of the report is invalid.");

javatests/google/registry/reporting/IcannReportingStagingActionTest.java

@@ -64,9 +64,9 @@ public class IcannReportingStagingActionTest {
     IcannReportingStagingAction action = new IcannReportingStagingAction();
     queryBuilder = new ActivityReportingQueryBuilder();
     queryBuilder.projectId = "test-project";
-    queryBuilder.yearMonth = "2017-05";
+    queryBuilder.yearMonth = "2017-06";
     action.reportingBucket = "test-bucket";
-    action.yearMonth = "2017-05";
+    action.yearMonth = "2017-06";
     action.subdir = Optional.absent();
     action.queryBuilder = queryBuilder;
     action.bigquery = bigquery;
@@ -103,12 +103,12 @@ public class IcannReportingStagingActionTest {
     byte[] generatedFile1 =
         readGcsFile(
             gcsService,
-            new GcsFilename("test-bucket/icann/monthly/2017-05", "fooTld-activity-201705.csv"));
+            new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv"));
     assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
     byte[] generatedFile2 =
         readGcsFile(
             gcsService,
-            new GcsFilename("test-bucket/icann/monthly/2017-05", "barTld-activity-201705.csv"));
+            new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv"));
     assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
   }
 

javatests/google/registry/reporting/IcannReportingUploadActionTest.java

@@ -55,14 +55,14 @@ public class IcannReportingUploadActionTest {
   private final FakeResponse response = new FakeResponse();
   private final GcsService gcsService = GcsServiceFactory.createGcsService();
   private final GcsFilename reportFile =
-      new GcsFilename("basin/icann/monthly/2017-05", "test-transactions-201705.csv");
+      new GcsFilename("basin/icann/monthly/2017-06", "test-transactions-201706.csv");
 
   private IcannReportingUploadAction createAction() {
     IcannReportingUploadAction action = new IcannReportingUploadAction();
     action.icannReporter = mockReporter;
     action.gcsUtils = new GcsUtils(gcsService, 1024);
     action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
-    action.yearMonth = "2017-05";
+    action.yearMonth = "2017-06";
     action.reportType = TRANSACTIONS;
     action.subdir = Optional.absent();
     action.tld = "test";
@@ -81,7 +81,7 @@ public class IcannReportingUploadActionTest {
   public void testSuccess() throws Exception {
     IcannReportingUploadAction action = createAction();
     action.run();
-    verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS);
+    verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
     verifyNoMoreInteractions(mockReporter);
     assertThat(((FakeResponse) action.response).getPayload())
         .isEqualTo("OK, sending: test,csv\n13,37");
@@ -93,9 +93,9 @@ public class IcannReportingUploadActionTest {
     doThrow(new IOException("Expected exception."))
         .doNothing()
         .when(mockReporter)
-        .send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS);
+        .send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
     action.run();
-    verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS);
+    verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
     verifyNoMoreInteractions(mockReporter);
     assertThat(((FakeResponse) action.response).getPayload())
         .isEqualTo("OK, sending: test,csv\n13,37");
@@ -162,8 +162,8 @@ public class IcannReportingUploadActionTest {
 
   @Test
   public void testSuccess_CreateFilename() throws Exception{
-    assertThat(IcannReportingUploadAction.createFilename("test", "2017-05", ACTIVITY))
+    assertThat(IcannReportingUploadAction.createFilename("test", "2017-06", ACTIVITY))
-        .isEqualTo("test-activity-201705.csv");
+        .isEqualTo("test-activity-201706.csv");
     assertThat(IcannReportingUploadAction.createFilename("foo", "1234-56", TRANSACTIONS))
         .isEqualTo("foo-transactions-123456.csv");
   }
@@ -172,11 +172,11 @@ public class IcannReportingUploadActionTest {
   public void testSuccess_CreateBucketname() throws Exception{
     assertThat(
         IcannReportingUploadAction
-            .createReportingBucketName("gs://my-reporting", Optional.<String>absent(), "2017-05"))
+            .createReportingBucketName("gs://my-reporting", Optional.<String>absent(), "2017-06"))
-        .isEqualTo("gs://my-reporting/icann/monthly/2017-05");
+        .isEqualTo("gs://my-reporting/icann/monthly/2017-06");
     assertThat(
         IcannReportingUploadAction
-            .createReportingBucketName("gs://my-reporting", Optional.of("manual"), "2017-05"))
+            .createReportingBucketName("gs://my-reporting", Optional.of("manual"), "2017-06"))
         .isEqualTo("gs://my-reporting/manual");
   }
 }

javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql

@@ -82,16 +82,16 @@ CROSS JOIN(
     SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 as count
     UNION ALL
     SELECT * FROM
-    `domain-registry-alpha.icann_reporting.registrar_operating_status_201705`
+    `domain-registry-alpha.icann_reporting.registrar_operating_status_201706`
     UNION ALL
     SELECT * FROM
-    `domain-registry-alpha.icann_reporting.dns_counts_201705`
+    `domain-registry-alpha.icann_reporting.dns_counts_201706`
     UNION ALL
     SELECT * FROM
-    `domain-registry-alpha.icann_reporting.epp_metrics_201705`
+    `domain-registry-alpha.icann_reporting.epp_metrics_201706`
     UNION ALL
     SELECT * FROM
-    `domain-registry-alpha.icann_reporting.whois_counts_201705`
+    `domain-registry-alpha.icann_reporting.whois_counts_201706`
     -- END INTERMEDIARY DATA SOURCES --
     )) AS TldMetrics
 WHERE RealTlds.tld = TldMetrics.tld OR TldMetrics.tld IS NULL

javatests/google/registry/reporting/testdata/epp_metrics_test.sql

@@ -39,11 +39,11 @@ FROM (
       -- Extract the logged JSON payload.
       REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
       AS json
-    FROM `domain-registry-alpha.icann_reporting.monthly_logs_201705` AS logs
+    FROM `domain-registry-alpha.icann_reporting.monthly_logs_201706` AS logs
     JOIN
       UNNEST(logs.logMessage) AS logMessage
     WHERE
-      logMessage LIKE "%FLOW-LOG-SIGNATURE-METADATA%")) AS regexes
+      STARTS_WITH(logMessage, "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA"))) AS regexes
 JOIN
   -- Unnest the JSON-parsed tlds.
   UNNEST(regexes.tlds) AS tld

javatests/google/registry/reporting/testdata/monthly_logs_test.sql

@@ -27,4 +27,4 @@ SELECT
 FROM
   `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
 WHERE
-  _TABLE_SUFFIX BETWEEN '20170501' AND '20170601'
+  _TABLE_SUFFIX BETWEEN '20170601' AND '20170701'

javatests/google/registry/reporting/testdata/whois_counts_test.sql

@@ -26,7 +26,7 @@ SELECT
   END AS metricName,
   COUNT(requestPath) AS count
 FROM
-  `domain-registry-alpha.icann_reporting.monthly_logs_201705`
+  `domain-registry-alpha.icann_reporting.monthly_logs_201706`
 GROUP BY
   metricName
 HAVING