zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-13 07:57:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

Optimize activity report SQL

Browse source 
We only log FLOW-LOG-SIGNATURE-METADATA from one place- FlowRunner. As a
result, we can swap the generalized regex for a prefix-only regex, saving a
<strong>lot</strong> of processing for our epp query (which is the most
expensive of the bunch).

I've also changed the test dates from 2017-05 to 2017-06, allowing us to copy-paste
the test data into Bigquery to verify their function. The reason for 2017-06 in particular is because June was the first month that populated all the metadata necessary to generate these reports.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=165391715
This commit is contained in:
larryruili 2017-08-15 19:11:05 -07:00 • committed by Ben McIlwain
parent 9e7c996081
commit 38abe9fa48
10 changed files with 33 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

4
java/google/registry/reporting/ActivityReportingQueryBuilder.java
View file

@ -103,6 +103,10 @@ public final class ActivityReportingQueryBuilder {
.put("PROJECT_ID", projectId)
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
.put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS))
// All metadata logs for reporting come from google.registry.flows.FlowReporter.
.put(
"METADATA_LOG_PREFIX",
"google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
.build();
queriesBuilder.put(getTableName(EPP_METRICS), eppQuery);

2
java/google/registry/reporting/sql/epp_metrics.sql
View file

@ -43,7 +43,7 @@ FROM (
JOIN
UNNEST(logs.logMessage) AS logMessage
WHERE
logMessage LIKE "%FLOW-LOG-SIGNATURE-METADATA%")) AS regexes
STARTS_WITH(logMessage, "%METADATA_LOG_PREFIX%"))) AS regexes
JOIN
-- Unnest the JSON-parsed tlds.
UNNEST(regexes.tlds) AS tld

6
javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java
View file

@ -29,7 +29,7 @@ public class ActivityReportingQueryBuilderTest {
private ActivityReportingQueryBuilder getQueryBuilder() {
ActivityReportingQueryBuilder queryBuilder = new ActivityReportingQueryBuilder();
queryBuilder.yearMonth = "2017-05";
queryBuilder.yearMonth = "2017-06";
queryBuilder.projectId = "domain-registry-alpha";
return queryBuilder;
}
@ -40,7 +40,7 @@ public class ActivityReportingQueryBuilderTest {
assertThat(queryBuilder.getActivityReportQuery())
.isEqualTo(
"#standardSQL\nSELECT * FROM "
+ "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201705`");
+ "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201706`");
}
@Test
@ -59,7 +59,7 @@ public class ActivityReportingQueryBuilderTest {
for (String queryName : queryNames) {
String testFilename = String.format("%s_test.sql", queryName);
testQueryBuilder.put(
String.format("%s_201705", queryName), ReportingTestData.getString(testFilename));
String.format("%s_201706", queryName), ReportingTestData.getString(testFilename));
}
ImmutableMap<String, String> expectedQueries = testQueryBuilder.build();
ImmutableMap<String, String> actualQueries = queryBuilder.getViewQueryMap();

6
javatests/google/registry/reporting/IcannHttpReporterTest.java
View file

@ -82,9 +82,9 @@ public class IcannHttpReporterTest {
@Test
public void testSuccess() throws Exception {
IcannHttpReporter reporter = createReporter();
reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS);
reporter.send(FAKE_PAYLOAD, "test", "2017-06", ReportType.TRANSACTIONS);
assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2016-06");
assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2017-06");
Map<String, List<String>> headers = mockRequest.getHeaders();
String userPass = "test_ry:fakePass";
String expectedAuth =
@ -98,7 +98,7 @@ public class IcannHttpReporterTest {
IcannHttpReporter reporter = createReporter();
reporter.httpTransport = createMockTransport(IIRDEA_BAD_XML);
try {
reporter.send(FAKE_PAYLOAD, "test", "2016-06", ReportType.TRANSACTIONS);
reporter.send(FAKE_PAYLOAD, "test", "2017-06", ReportType.TRANSACTIONS);
assertWithMessage("Expected InternalServerErrorException to be thrown").fail();
} catch (InternalServerErrorException expected) {
assertThat(expected).hasMessageThat().isEqualTo("The structure of the report is invalid.");

8
javatests/google/registry/reporting/IcannReportingStagingActionTest.java
View file

@ -64,9 +64,9 @@ public class IcannReportingStagingActionTest {
IcannReportingStagingAction action = new IcannReportingStagingAction();
queryBuilder = new ActivityReportingQueryBuilder();
queryBuilder.projectId = "test-project";
queryBuilder.yearMonth = "2017-05";
queryBuilder.yearMonth = "2017-06";
action.reportingBucket = "test-bucket";
action.yearMonth = "2017-05";
action.yearMonth = "2017-06";
action.subdir = Optional.absent();
action.queryBuilder = queryBuilder;
action.bigquery = bigquery;
@ -103,12 +103,12 @@ public class IcannReportingStagingActionTest {
byte[] generatedFile1 =
readGcsFile(
gcsService,
new GcsFilename("test-bucket/icann/monthly/2017-05", "fooTld-activity-201705.csv"));
new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv"));
assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
byte[] generatedFile2 =
readGcsFile(
gcsService,
new GcsFilename("test-bucket/icann/monthly/2017-05", "barTld-activity-201705.csv"));
new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv"));
assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
}

20
javatests/google/registry/reporting/IcannReportingUploadActionTest.java
View file

@ -55,14 +55,14 @@ public class IcannReportingUploadActionTest {
private final FakeResponse response = new FakeResponse();
private final GcsService gcsService = GcsServiceFactory.createGcsService();
private final GcsFilename reportFile =
new GcsFilename("basin/icann/monthly/2017-05", "test-transactions-201705.csv");
new GcsFilename("basin/icann/monthly/2017-06", "test-transactions-201706.csv");
private IcannReportingUploadAction createAction() {
IcannReportingUploadAction action = new IcannReportingUploadAction();
action.icannReporter = mockReporter;
action.gcsUtils = new GcsUtils(gcsService, 1024);
action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
action.yearMonth = "2017-05";
action.yearMonth = "2017-06";
action.reportType = TRANSACTIONS;
action.subdir = Optional.absent();
action.tld = "test";
@ -81,7 +81,7 @@ public class IcannReportingUploadActionTest {
public void testSuccess() throws Exception {
IcannReportingUploadAction action = createAction();
action.run();
verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS);
verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
verifyNoMoreInteractions(mockReporter);
assertThat(((FakeResponse) action.response).getPayload())
.isEqualTo("OK, sending: test,csv\n13,37");
@ -93,9 +93,9 @@ public class IcannReportingUploadActionTest {
doThrow(new IOException("Expected exception."))
.doNothing()
.when(mockReporter)
.send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS);
.send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
action.run();
verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS);
verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
verifyNoMoreInteractions(mockReporter);
assertThat(((FakeResponse) action.response).getPayload())
.isEqualTo("OK, sending: test,csv\n13,37");
@ -162,8 +162,8 @@ public class IcannReportingUploadActionTest {
@Test
public void testSuccess_CreateFilename() throws Exception{
assertThat(IcannReportingUploadAction.createFilename("test", "2017-05", ACTIVITY))
.isEqualTo("test-activity-201705.csv");
assertThat(IcannReportingUploadAction.createFilename("test", "2017-06", ACTIVITY))
.isEqualTo("test-activity-201706.csv");
assertThat(IcannReportingUploadAction.createFilename("foo", "1234-56", TRANSACTIONS))
.isEqualTo("foo-transactions-123456.csv");
}
@ -172,11 +172,11 @@ public class IcannReportingUploadActionTest {
public void testSuccess_CreateBucketname() throws Exception{
assertThat(
IcannReportingUploadAction
.createReportingBucketName("gs://my-reporting", Optional.<String>absent(), "2017-05"))
.isEqualTo("gs://my-reporting/icann/monthly/2017-05");
.createReportingBucketName("gs://my-reporting", Optional.<String>absent(), "2017-06"))
.isEqualTo("gs://my-reporting/icann/monthly/2017-06");
assertThat(
IcannReportingUploadAction
.createReportingBucketName("gs://my-reporting", Optional.of("manual"), "2017-05"))
.createReportingBucketName("gs://my-reporting", Optional.of("manual"), "2017-06"))
.isEqualTo("gs://my-reporting/manual");
}
}

8
javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql vendored
View file

@ -82,16 +82,16 @@ CROSS JOIN(
SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 as count
UNION ALL
SELECT * FROM
`domain-registry-alpha.icann_reporting.registrar_operating_status_201705`
`domain-registry-alpha.icann_reporting.registrar_operating_status_201706`
UNION ALL
SELECT * FROM
`domain-registry-alpha.icann_reporting.dns_counts_201705`
`domain-registry-alpha.icann_reporting.dns_counts_201706`
UNION ALL
SELECT * FROM
`domain-registry-alpha.icann_reporting.epp_metrics_201705`
`domain-registry-alpha.icann_reporting.epp_metrics_201706`
UNION ALL
SELECT * FROM
`domain-registry-alpha.icann_reporting.whois_counts_201705`
`domain-registry-alpha.icann_reporting.whois_counts_201706`
-- END INTERMEDIARY DATA SOURCES --
)) AS TldMetrics
WHERE RealTlds.tld = TldMetrics.tld OR TldMetrics.tld IS NULL

4
javatests/google/registry/reporting/testdata/epp_metrics_test.sql vendored
View file

@ -39,11 +39,11 @@ FROM (
-- Extract the logged JSON payload.
REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
AS json
FROM `domain-registry-alpha.icann_reporting.monthly_logs_201705` AS logs
FROM `domain-registry-alpha.icann_reporting.monthly_logs_201706` AS logs
JOIN
UNNEST(logs.logMessage) AS logMessage
WHERE
logMessage LIKE "%FLOW-LOG-SIGNATURE-METADATA%")) AS regexes
STARTS_WITH(logMessage, "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA"))) AS regexes
JOIN
-- Unnest the JSON-parsed tlds.
UNNEST(regexes.tlds) AS tld

2
javatests/google/registry/reporting/testdata/monthly_logs_test.sql vendored
View file

@ -27,4 +27,4 @@ SELECT
FROM
`domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
WHERE
_TABLE_SUFFIX BETWEEN '20170501' AND '20170601'
_TABLE_SUFFIX BETWEEN '20170601' AND '20170701'

2
javatests/google/registry/reporting/testdata/whois_counts_test.sql vendored
View file

@ -26,7 +26,7 @@ SELECT
END AS metricName,
COUNT(requestPath) AS count
FROM
`domain-registry-alpha.icann_reporting.monthly_logs_201705`
`domain-registry-alpha.icann_reporting.monthly_logs_201706`
GROUP BY
metricName
HAVING