zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-08-02 16:02:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add activity report generation code

Browse source 
This adds Bigquery API client code to generate the activity reports from our
now standardSQL queries. The naming mirrors that of RDE (Staging generates the
reports and uploads them to GCS).

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=164656344
This commit is contained in:
larryruili 2017-08-08 15:46:59 -07:00 • committed by Ben McIlwain
parent 2547313ef9
commit 477617eec9
25 changed files with 612 additions and 117 deletions
Download patch file Download diff file Expand all files Collapse all files

1
javatests/google/registry/module/backend/testdata/backend_routing.txt vendored
View file

@ -14,6 +14,7 @@ PATH CLASS METHOD
/_dr/task/exportDomainLists ExportDomainListsAction POST n INTERNAL APP IGNORED
/_dr/task/exportReservedTerms ExportReservedTermsAction POST n INTERNAL APP IGNORED
/_dr/task/exportSnapshot ExportSnapshotAction POST y INTERNAL APP IGNORED
/_dr/task/icannReportingStaging IcannReportingStagingAction POST n INTERNAL APP IGNORED
/_dr/task/icannReportingUpload IcannReportingUploadAction POST n INTERNAL,API APP ADMIN
/_dr/task/importRdeContacts RdeContactImportAction GET n INTERNAL APP IGNORED
/_dr/task/importRdeDomains RdeDomainImportAction GET n INTERNAL APP IGNORED

38
javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java
View file

@ -19,7 +19,6 @@ import static com.google.common.truth.Truth.assertThat;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import org.joda.time.LocalDate;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@ -28,28 +27,43 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class ActivityReportingQueryBuilderTest {
private ActivityReportingQueryBuilder getQueryBuilder() {
ActivityReportingQueryBuilder queryBuilder = new ActivityReportingQueryBuilder();
queryBuilder.yearMonth = "2017-05";
queryBuilder.projectId = "domain-registry-alpha";
return queryBuilder;
}
@Test
public void testQueryMatch() throws IOException {
public void testAggregateQueryMatch() throws IOException {
ActivityReportingQueryBuilder queryBuilder = getQueryBuilder();
assertThat(queryBuilder.getActivityReportQuery())
.isEqualTo(
"#standardSQL\nSELECT * FROM "
+ "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201705`");
}
@Test
public void testIntermediaryQueryMatch() throws IOException {
ActivityReportingQueryBuilder queryBuilder = getQueryBuilder();
ImmutableList<String> queryNames =
ImmutableList.of(
ActivityReportingQueryBuilder.REGISTRAR_OPERATING_STATUS,
ActivityReportingQueryBuilder.DNS_COUNTS,
ActivityReportingQueryBuilder.MONTHLY_LOGS_TABLE,
ActivityReportingQueryBuilder.MONTHLY_LOGS,
ActivityReportingQueryBuilder.EPP_METRICS,
ActivityReportingQueryBuilder.WHOIS_COUNTS,
"activity_report_aggregation");
ActivityReportingQueryBuilder.ACTIVITY_REPORT_AGGREGATION);
ImmutableMap.Builder<String, String> testQueryBuilder = ImmutableMap.builder();
for (String queryName : queryNames) {
String testFilename = String.format("%s_test.sql", queryName);
testQueryBuilder.put(queryName, ReportingTestData.getString(testFilename));
}
ImmutableMap<String, String> testQueries = testQueryBuilder.build();
ImmutableMap<String, String> queries =
ActivityReportingQueryBuilder.getQueryMap(
new LocalDate(2017, 05, 15), "domain-registry-alpha");
for (String query : queryNames) {
assertThat(queries.get(query)).isEqualTo(testQueries.get(query));
testQueryBuilder.put(
String.format("%s_201705", queryName), ReportingTestData.getString(testFilename));
}
ImmutableMap<String, String> expectedQueries = testQueryBuilder.build();
ImmutableMap<String, String> actualQueries = queryBuilder.getViewQueryMap();
assertThat(actualQueries).isEqualTo(expectedQueries);
}
}

2
javatests/google/registry/reporting/BUILD
View file

@ -12,11 +12,13 @@ java_library(
srcs = glob(["*.java"]),
resources = glob(["testdata/*"]),
deps = [
"//java/google/registry/bigquery",
"//java/google/registry/gcs",
"//java/google/registry/reporting",
"//java/google/registry/request",
"//java/google/registry/util",
"//javatests/google/registry/testing",
"@com_google_apis_google_api_services_bigquery",
"@com_google_appengine_tools_appengine_gcs_client",
"@com_google_code_findbugs_jsr305",
"@com_google_dagger",

149
javatests/google/registry/reporting/IcannReportingStagingActionTest.java Normal file
View file

@ -0,0 +1,149 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.reporting;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.testing.GcsTestingUtils.readGcsFile;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableTable;
import com.google.common.util.concurrent.ListenableFuture;
import google.registry.bigquery.BigqueryConnection;
import google.registry.bigquery.BigqueryConnection.DestinationTable;
import google.registry.bigquery.BigqueryUtils.TableType;
import google.registry.gcs.GcsUtils;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeResponse;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/**
* Unit tests for {@link google.registry.reporting.IcannReportingStagingAction}.
*/
@RunWith(JUnit4.class)
public class IcannReportingStagingActionTest {
BigqueryConnection bigquery = mock(BigqueryConnection.class);
FakeResponse response = new FakeResponse();
ActivityReportingQueryBuilder queryBuilder;
GcsService gcsService = GcsServiceFactory.createGcsService();
@Rule
public final AppEngineRule appEngine = AppEngineRule.builder()
.withDatastore()
.withLocalModules()
.build();
private IcannReportingStagingAction createAction() {
IcannReportingStagingAction action = new IcannReportingStagingAction();
queryBuilder = new ActivityReportingQueryBuilder();
queryBuilder.projectId = "test-project";
queryBuilder.yearMonth = "2017-05";
action.reportingBucket = "test-bucket";
action.yearMonth = "2017-05";
action.subdir = Optional.absent();
action.queryBuilder = queryBuilder;
action.bigquery = bigquery;
action.gcsUtils = new GcsUtils(gcsService, 1024);
action.response = response;
return action;
}
@Test
public void testRunSuccess() throws Exception {
when(bigquery.query(any(), any())).thenReturn(fakeFuture());
DestinationTable.Builder tableBuilder = new DestinationTable.Builder()
.datasetId("testdataset")
.type(TableType.TABLE)
.name("tablename")
.overwrite(true);
when(bigquery.buildDestinationTable(any())).thenReturn(tableBuilder);
ImmutableTable<Integer, TableFieldSchema, Object> reportTable =
new ImmutableTable.Builder<Integer, TableFieldSchema, Object>()
.put(1, new TableFieldSchema().setName("tld"), "fooTld")
.put(1, new TableFieldSchema().setName("fooField"), "12")
.put(1, new TableFieldSchema().setName("barField"), "34")
.put(2, new TableFieldSchema().setName("tld"), "barTld")
.put(2, new TableFieldSchema().setName("fooField"), "56")
.put(2, new TableFieldSchema().setName("barField"), "78")
.build();
when(bigquery.queryToLocalTableSync(any())).thenReturn(reportTable);
IcannReportingStagingAction action = createAction();
action.run();
String expectedReport1 = "fooField,barField\r\n12,34";
String expectedReport2 = "fooField,barField\r\n56,78";
byte[] generatedFile1 =
readGcsFile(
gcsService,
new GcsFilename("test-bucket/icann/monthly/2017-05", "fooTld-activity-201705.csv"));
assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
byte[] generatedFile2 =
readGcsFile(
gcsService,
new GcsFilename("test-bucket/icann/monthly/2017-05", "barTld-activity-201705.csv"));
assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
}
private ListenableFuture<DestinationTable> fakeFuture() {
return new ListenableFuture<DestinationTable>() {
@Override
public void addListener(Runnable runnable, Executor executor) {
// No-op
}
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
return false;
}
@Override
public boolean isCancelled() {
return false;
}
@Override
public boolean isDone() {
return false;
}
@Override
public DestinationTable get() throws InterruptedException, ExecutionException {
return null;
}
@Override
public DestinationTable get(long timeout, TimeUnit unit)
throws InterruptedException, ExecutionException, TimeoutException {
return null;
}
};
}
}

35
javatests/google/registry/reporting/IcannReportingUploadActionTest.java
View file

@ -16,6 +16,7 @@ package google.registry.reporting;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static google.registry.reporting.IcannReportingModule.ReportType.ACTIVITY;
import static google.registry.reporting.IcannReportingModule.ReportType.TRANSACTIONS;
import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.testing.GcsTestingUtils.writeGcsFile;
@ -54,18 +55,18 @@ public class IcannReportingUploadActionTest {
private final FakeResponse response = new FakeResponse();
private final GcsService gcsService = GcsServiceFactory.createGcsService();
private final GcsFilename reportFile =
new GcsFilename("basin/icann/monthly", "test-transactions-201706.csv");
new GcsFilename("basin/icann/monthly/2017-05", "test-transactions-201705.csv");
private IcannReportingUploadAction createAction() {
IcannReportingUploadAction action = new IcannReportingUploadAction();
action.icannReporter = mockReporter;
action.gcsUtils = new GcsUtils(gcsService, 1024);
action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
action.yearMonth = "2017-06";
action.yearMonth = "2017-05";
action.reportType = TRANSACTIONS;
action.subdir = Optional.absent();
action.tld = "test";
action.reportingBucket = "basin";
action.icannReportingBucket = "basin";
action.response = response;
return action;
}
@ -80,7 +81,7 @@ public class IcannReportingUploadActionTest {
public void testSuccess() throws Exception {
IcannReportingUploadAction action = createAction();
action.run();
verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS);
verifyNoMoreInteractions(mockReporter);
assertThat(((FakeResponse) action.response).getPayload())
.isEqualTo("OK, sending: test,csv\n13,37");
@ -92,9 +93,9 @@ public class IcannReportingUploadActionTest {
doThrow(new IOException("Expected exception."))
.doNothing()
.when(mockReporter)
.send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
.send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS);
action.run();
verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-05", TRANSACTIONS);
verifyNoMoreInteractions(mockReporter);
assertThat(((FakeResponse) action.response).getPayload())
.isEqualTo("OK, sending: test,csv\n13,37");
@ -155,7 +156,27 @@ public class IcannReportingUploadActionTest {
.hasMessageThat()
.isEqualTo(
"ICANN report object test-transactions-123456.csv "
+ "in bucket basin/icann/monthly not found");
+ "in bucket basin/icann/monthly/1234-56 not found");
}
}
@Test
public void testSuccess_CreateFilename() throws Exception{
assertThat(IcannReportingUploadAction.createFilename("test", "2017-05", ACTIVITY))
.isEqualTo("test-activity-201705.csv");
assertThat(IcannReportingUploadAction.createFilename("foo", "1234-56", TRANSACTIONS))
.isEqualTo("foo-transactions-123456.csv");
}
@Test
public void testSuccess_CreateBucketname() throws Exception{
assertThat(
IcannReportingUploadAction
.createReportingBucketName("gs://my-reporting", Optional.absent(), "2017-05"))
.isEqualTo("gs://my-reporting/icann/monthly/2017-05");
assertThat(
IcannReportingUploadAction
.createReportingBucketName("gs://my-reporting", Optional.of("manual"), "2017-05"))
.isEqualTo("gs://my-reporting/manual");
}
}

59
javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql vendored
View file

@ -1,3 +1,4 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
@ -16,7 +17,7 @@
-- report csv, via a table transpose and sum over all activity report fields.
SELECT
Tld.tld AS tld,
RealTlds.tld AS tld,
SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars,
SUM(IF(metricName = 'ramp-up-registrars', count, 0)) AS ramp_up_registrars,
SUM(IF(metricName = 'pre-ramp-up-registrars', count, 0)) AS pre_ramp_up_registrars,
@ -59,37 +60,41 @@ SELECT
SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query,
SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject,
SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request,
SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update,
SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update
-- Cross join a list of all TLDs against TLD-specific metrics and then
-- filter so that only metrics with that TLD or a NULL TLD are counted
-- towards a given TLD.
FROM (
SELECT
tldStr AS tld
FROM
[latest_snapshot.Registry]
-- Include all real TLDs that are not in pre-delegation testing.
WHERE
tldType = 'REAL'
OMIT
RECORD IF SOME(tldStateTransitions.tldState = 'PDT') ) AS Tld
-- TODO(larryruili): Use LEFT JOIN on Tld.tld = TldMetrics.tld instead.
-- Also obsoletes dummy data.
LEFT OUTER JOIN (
SELECT
SELECT tldStr as tld
FROM `domain-registry-alpha.latest_datastore_views.Registry`
WHERE tldType = 'REAL'
) as RealTlds
CROSS JOIN(
SELECT
tld,
metricName,
count FROM
count
FROM
(
-- BEGIN INTERMEDIARY DATA SOURCES --
[icann_reporting.registrar_operating_status],
[icann_reporting.dns_counts],
[icann_reporting.epp_metrics],
[icann_reporting.whois_counts],
-- Dummy data source to ensure all TLDs appear in report, even if
-- they have no recorded metrics for the month.
SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 as count
UNION ALL
SELECT * FROM
`domain-registry-alpha.icann_reporting.registrar_operating_status_201705`
UNION ALL
SELECT * FROM
`domain-registry-alpha.icann_reporting.dns_counts_201705`
UNION ALL
SELECT * FROM
`domain-registry-alpha.icann_reporting.epp_metrics_201705`
UNION ALL
SELECT * FROM
`domain-registry-alpha.icann_reporting.whois_counts_201705`
-- END INTERMEDIARY DATA SOURCES --
) AS TldMetrics
ON
Tld.tld = TldMetrics.tld
GROUP BY
tld
ORDER BY
tld
)) AS TldMetrics
WHERE RealTlds.tld = TldMetrics.tld OR TldMetrics.tld IS NULL
GROUP BY tld
ORDER BY tld

1
javatests/google/registry/reporting/testdata/dns_counts_test.sql vendored
View file

@ -1,3 +1,4 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");

3
javatests/google/registry/reporting/testdata/epp_metrics_test.sql vendored
View file

@ -1,3 +1,4 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
@ -38,7 +39,7 @@ FROM (
-- Extract the logged JSON payload.
REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
AS json
FROM `domain-registry-alpha.icann_reporting.monthly_logs` AS logs
FROM `domain-registry-alpha.icann_reporting.monthly_logs_201705` AS logs
JOIN
UNNEST(logs.logMessage) AS logMessage
WHERE

1
javatests/google/registry/reporting/testdata/monthly_logs_test.sql vendored
View file

@ -1,3 +1,4 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");

1
javatests/google/registry/reporting/testdata/registrar_operating_status_test.sql vendored
View file

@ -1,3 +1,4 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");

3
javatests/google/registry/reporting/testdata/whois_counts_test.sql vendored
View file

@ -1,3 +1,4 @@
#standardSQL
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
@ -25,7 +26,7 @@ SELECT
END AS metricName,
COUNT(requestPath) AS count
FROM
`domain-registry-alpha.monthly_logs.monthly_logs`
`domain-registry-alpha.icann_reporting.monthly_logs_201705`
GROUP BY
metricName
HAVING