diff --git a/java/google/registry/reporting/ActivityReportingQueryBuilder.java b/java/google/registry/reporting/ActivityReportingQueryBuilder.java index 90516216b..85a1ebb97 100644 --- a/java/google/registry/reporting/ActivityReportingQueryBuilder.java +++ b/java/google/registry/reporting/ActivityReportingQueryBuilder.java @@ -103,6 +103,10 @@ public final class ActivityReportingQueryBuilder { .put("PROJECT_ID", projectId) .put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET) .put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS)) + // All metadata logs for reporting come from google.registry.flows.FlowReporter. + .put( + "METADATA_LOG_PREFIX", + "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA") .build(); queriesBuilder.put(getTableName(EPP_METRICS), eppQuery); diff --git a/java/google/registry/reporting/sql/epp_metrics.sql b/java/google/registry/reporting/sql/epp_metrics.sql index b14918e17..91151fbf1 100644 --- a/java/google/registry/reporting/sql/epp_metrics.sql +++ b/java/google/registry/reporting/sql/epp_metrics.sql @@ -43,7 +43,7 @@ FROM ( JOIN UNNEST(logs.logMessage) AS logMessage WHERE - logMessage LIKE "%FLOW-LOG-SIGNATURE-METADATA%")) AS regexes + STARTS_WITH(logMessage, "%METADATA_LOG_PREFIX%"))) AS regexes JOIN -- Unnest the JSON-parsed tlds. UNNEST(regexes.tlds) AS tld diff --git a/javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java b/javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java index cec4c3eec..b8734f5fe 100644 --- a/javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java +++ b/javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java @@ -29,7 +29,7 @@ public class ActivityReportingQueryBuilderTest { private ActivityReportingQueryBuilder getQueryBuilder() { ActivityReportingQueryBuilder queryBuilder = new ActivityReportingQueryBuilder(); - queryBuilder.yearMonth = "2017-05"; + queryBuilder.yearMonth = "2017-06"; queryBuilder.projectId = "domain-registry-alpha"; return queryBuilder; } @@ -40,7 +40,7 @@ public class ActivityReportingQueryBuilderTest { assertThat(queryBuilder.getActivityReportQuery()) .isEqualTo( "#standardSQL\nSELECT * FROM " - + "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201705`"); + + "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201706`"); } @Test @@ -59,7 +59,7 @@ public class ActivityReportingQueryBuilderTest { for (String queryName : queryNames) { String testFilename = String.format("%s_test.sql", queryName); testQueryBuilder.put( - String.format("%s_201705", queryName), ReportingTestData.getString(testFilename)); + String.format("%s_201706", queryName), ReportingTestData.getString(testFilename)); } ImmutableMap expectedQueries = testQueryBuilder.build(); ImmutableMap actualQueries = queryBuilder.getViewQueryMap(); diff --git a/javatests/google/registry/reporting/IcannHttpReporterTest.java b/javatests/google/registry/reporting/IcannHttpReporterTest.java index 9ef23202f..b3398f1a2 100644 --- a/javatests/google/registry/reporting/IcannHttpReporterTest.java +++ b/javatests/google/registry/reporting/IcannHttpReporterTest.java @@ -82,9 +82,9 @@ public class IcannHttpReporterTest { @Test public void testSuccess() throws Exception { IcannHttpReporter reporter = createReporter(); - reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS); + reporter.send(FAKE_PAYLOAD, "test", "2017-06", ReportType.TRANSACTIONS); - assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2016-06"); + assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2017-06"); Map> headers = mockRequest.getHeaders(); String userPass = "test_ry:fakePass"; String expectedAuth = @@ -98,7 +98,7 @@ public class IcannHttpReporterTest { IcannHttpReporter reporter = createReporter(); reporter.httpTransport = createMockTransport(IIRDEA_BAD_XML); try { - reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS); + reporter.send(FAKE_PAYLOAD, "test", "2017-06", ReportType.TRANSACTIONS); assertWithMessage("Expected InternalServerErrorException to be thrown").fail(); } catch (InternalServerErrorException expected) { assertThat(expected).hasMessageThat().isEqualTo("The structure of the report is invalid."); diff --git a/javatests/google/registry/reporting/IcannReportingStagingActionTest.java b/javatests/google/registry/reporting/IcannReportingStagingActionTest.java index 00fdbef65..9c3b79c5e 100644 --- a/javatests/google/registry/reporting/IcannReportingStagingActionTest.java +++ b/javatests/google/registry/reporting/IcannReportingStagingActionTest.java @@ -64,9 +64,9 @@ public class IcannReportingStagingActionTest { IcannReportingStagingAction action = new IcannReportingStagingAction(); queryBuilder = new ActivityReportingQueryBuilder(); queryBuilder.projectId = "test-project"; - queryBuilder.yearMonth = "2017-05"; + queryBuilder.yearMonth = "2017-06"; action.reportingBucket = "test-bucket"; - action.yearMonth = "2017-05"; + action.yearMonth = "2017-06"; action.subdir = Optional.absent(); action.queryBuilder = queryBuilder; action.bigquery = bigquery; @@ -103,12 +103,12 @@ public class IcannReportingStagingActionTest { byte[] generatedFile1 = readGcsFile( gcsService, - new GcsFilename("test-bucket/icann/monthly/2017-05", "fooTld-activity-201705.csv")); + new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv")); assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1); byte[] generatedFile2 = readGcsFile( gcsService, - new GcsFilename("test-bucket/icann/monthly/2017-05", "barTld-activity-201705.csv")); + new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv")); assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2); } diff --git a/javatests/google/registry/reporting/IcannReportingUploadActionTest.java b/javatests/google/registry/reporting/IcannReportingUploadActionTest.java index b2a4ce9a8..b497312e6 100644 --- a/javatests/google/registry/reporting/IcannReportingUploadActionTest.java +++ b/javatests/google/registry/reporting/IcannReportingUploadActionTest.java @@ -55,14 +55,14 @@ public class IcannReportingUploadActionTest { private final FakeResponse response = new FakeResponse(); private final GcsService gcsService = GcsServiceFactory.createGcsService(); private final GcsFilename reportFile = - new GcsFilename("basin/icann/monthly/2017-05", "test-transactions-201705.csv"); + new GcsFilename("basin/icann/monthly/2017-06", "test-transactions-201706.csv"); private IcannReportingUploadAction createAction() { IcannReportingUploadAction action = new IcannReportingUploadAction(); action.icannReporter = mockReporter; action.gcsUtils = new GcsUtils(gcsService, 1024); action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); - action.yearMonth = "2017-05"; + action.yearMonth = "2017-06"; action.reportType = TRANSACTIONS; action.subdir = Optional.absent(); action.tld = "test"; @@ -81,7 +81,7 @@ public class IcannReportingUploadActionTest { public void testSuccess() throws Exception { IcannReportingUploadAction action = createAction(); action.run(); - verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS); + verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS); verifyNoMoreInteractions(mockReporter); assertThat(((FakeResponse) action.response).getPayload()) .isEqualTo("OK, sending: test,csv\n13,37"); @@ -93,9 +93,9 @@ public class IcannReportingUploadActionTest { doThrow(new IOException("Expected exception.")) .doNothing() .when(mockReporter) - .send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS); + .send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS); action.run(); - verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS); + verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS); verifyNoMoreInteractions(mockReporter); assertThat(((FakeResponse) action.response).getPayload()) .isEqualTo("OK, sending: test,csv\n13,37"); @@ -162,8 +162,8 @@ public class IcannReportingUploadActionTest { @Test public void testSuccess_CreateFilename() throws Exception{ - assertThat(IcannReportingUploadAction.createFilename("test", "2017-05", ACTIVITY)) - .isEqualTo("test-activity-201705.csv"); + assertThat(IcannReportingUploadAction.createFilename("test", "2017-06", ACTIVITY)) + .isEqualTo("test-activity-201706.csv"); assertThat(IcannReportingUploadAction.createFilename("foo", "1234-56", TRANSACTIONS)) .isEqualTo("foo-transactions-123456.csv"); } @@ -172,11 +172,11 @@ public class IcannReportingUploadActionTest { public void testSuccess_CreateBucketname() throws Exception{ assertThat( IcannReportingUploadAction - .createReportingBucketName("gs://my-reporting", Optional.absent(), "2017-05")) - .isEqualTo("gs://my-reporting/icann/monthly/2017-05"); + .createReportingBucketName("gs://my-reporting", Optional.absent(), "2017-06")) + .isEqualTo("gs://my-reporting/icann/monthly/2017-06"); assertThat( IcannReportingUploadAction - .createReportingBucketName("gs://my-reporting", Optional.of("manual"), "2017-05")) + .createReportingBucketName("gs://my-reporting", Optional.of("manual"), "2017-06")) .isEqualTo("gs://my-reporting/manual"); } } diff --git a/javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql b/javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql index 06f4ca910..3aa648fc1 100644 --- a/javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql +++ b/javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql @@ -82,16 +82,16 @@ CROSS JOIN( SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 as count UNION ALL SELECT * FROM - `domain-registry-alpha.icann_reporting.registrar_operating_status_201705` + `domain-registry-alpha.icann_reporting.registrar_operating_status_201706` UNION ALL SELECT * FROM - `domain-registry-alpha.icann_reporting.dns_counts_201705` + `domain-registry-alpha.icann_reporting.dns_counts_201706` UNION ALL SELECT * FROM - `domain-registry-alpha.icann_reporting.epp_metrics_201705` + `domain-registry-alpha.icann_reporting.epp_metrics_201706` UNION ALL SELECT * FROM - `domain-registry-alpha.icann_reporting.whois_counts_201705` + `domain-registry-alpha.icann_reporting.whois_counts_201706` -- END INTERMEDIARY DATA SOURCES -- )) AS TldMetrics WHERE RealTlds.tld = TldMetrics.tld OR TldMetrics.tld IS NULL diff --git a/javatests/google/registry/reporting/testdata/epp_metrics_test.sql b/javatests/google/registry/reporting/testdata/epp_metrics_test.sql index 50f940ae6..c3bc7f9ad 100644 --- a/javatests/google/registry/reporting/testdata/epp_metrics_test.sql +++ b/javatests/google/registry/reporting/testdata/epp_metrics_test.sql @@ -39,11 +39,11 @@ FROM ( -- Extract the logged JSON payload. REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') AS json - FROM `domain-registry-alpha.icann_reporting.monthly_logs_201705` AS logs + FROM `domain-registry-alpha.icann_reporting.monthly_logs_201706` AS logs JOIN UNNEST(logs.logMessage) AS logMessage WHERE - logMessage LIKE "%FLOW-LOG-SIGNATURE-METADATA%")) AS regexes + STARTS_WITH(logMessage, "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA"))) AS regexes JOIN -- Unnest the JSON-parsed tlds. UNNEST(regexes.tlds) AS tld diff --git a/javatests/google/registry/reporting/testdata/monthly_logs_test.sql b/javatests/google/registry/reporting/testdata/monthly_logs_test.sql index ad5e65024..f75ff0a5a 100644 --- a/javatests/google/registry/reporting/testdata/monthly_logs_test.sql +++ b/javatests/google/registry/reporting/testdata/monthly_logs_test.sql @@ -27,4 +27,4 @@ SELECT FROM `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*` WHERE - _TABLE_SUFFIX BETWEEN '20170501' AND '20170601' + _TABLE_SUFFIX BETWEEN '20170601' AND '20170701' diff --git a/javatests/google/registry/reporting/testdata/whois_counts_test.sql b/javatests/google/registry/reporting/testdata/whois_counts_test.sql index d2051932e..55ee56f04 100644 --- a/javatests/google/registry/reporting/testdata/whois_counts_test.sql +++ b/javatests/google/registry/reporting/testdata/whois_counts_test.sql @@ -26,7 +26,7 @@ SELECT END AS metricName, COUNT(requestPath) AS count FROM - `domain-registry-alpha.icann_reporting.monthly_logs_201705` + `domain-registry-alpha.icann_reporting.monthly_logs_201706` GROUP BY metricName HAVING