diff --git a/java/google/registry/reporting/ActivityReportingQueryBuilder.java b/java/google/registry/reporting/ActivityReportingQueryBuilder.java
index 159a25cb3..61a6e5231 100644
--- a/java/google/registry/reporting/ActivityReportingQueryBuilder.java
+++ b/java/google/registry/reporting/ActivityReportingQueryBuilder.java
@@ -36,12 +36,12 @@ import org.joda.time.format.DateTimeFormatter;
public final class ActivityReportingQueryBuilder implements QueryBuilder {
// Names for intermediary tables for overall activity reporting query.
- static final String ACTIVITY_REPORT_AGGREGATION = "activity_report_aggregation";
- static final String MONTHLY_LOGS = "monthly_logs";
static final String REGISTRAR_OPERATING_STATUS = "registrar_operating_status";
static final String DNS_COUNTS = "dns_counts";
+ static final String MONTHLY_LOGS = "monthly_logs";
static final String EPP_METRICS = "epp_metrics";
static final String WHOIS_COUNTS = "whois_counts";
+ static final String ACTIVITY_REPORT_AGGREGATION = "activity_report_aggregation";
@Inject @Config("projectId") String projectId;
@Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
@@ -81,7 +81,6 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
.build();
queriesBuilder.put(getTableName(REGISTRAR_OPERATING_STATUS), operationalRegistrarsQuery);
- // TODO(b/62626209): Make this use the CloudDNS counts instead.
String dnsCountsQuery =
SqlTemplate.create(getQueryFromFile("dns_counts.sql")).build();
queriesBuilder.put(getTableName(DNS_COUNTS), dnsCountsQuery);
@@ -135,6 +134,7 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
return queriesBuilder.build();
}
+
/** Returns the table name of the query, suffixed with the yearMonth in _YYYYMM format. */
private String getTableName(String queryName) {
return String.format("%s_%s", queryName, yearMonth.replace("-", ""));
@@ -149,4 +149,3 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
return Resources.getResource(ActivityReportingQueryBuilder.class, "sql/" + filename);
}
}
-
diff --git a/java/google/registry/reporting/IcannHttpReporter.java b/java/google/registry/reporting/IcannHttpReporter.java
index ea7842d6d..fca16bea9 100644
--- a/java/google/registry/reporting/IcannHttpReporter.java
+++ b/java/google/registry/reporting/IcannHttpReporter.java
@@ -14,7 +14,9 @@
package google.registry.reporting;
+import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.net.MediaType.CSV_UTF_8;
+import static google.registry.model.registry.Registries.assertTldExists;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.api.client.http.ByteArrayContent;
@@ -23,6 +25,8 @@ import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
+import com.google.common.base.Ascii;
+import com.google.common.base.Splitter;
import com.google.common.io.ByteStreams;
import google.registry.config.RegistryConfig.Config;
import google.registry.keyring.api.KeyModule.Key;
@@ -35,16 +39,22 @@ import google.registry.xjc.iirdea.XjcIirdeaResult;
import google.registry.xml.XmlException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.util.List;
import javax.inject.Inject;
+import org.joda.time.YearMonth;
+import org.joda.time.format.DateTimeFormat;
/**
* Class that uploads a CSV file to ICANN's endpoint via an HTTP PUT call.
*
- * It uses basic authorization credentials as specified in the "Registry Interfaces" draft.
+ *
It uses basic authorization credentials as specified in the "Registry Interfaces" draft.
+ *
+ *
Note that there's a lot of hard-coded logic extracting parameters from the report filenames.
+ * These are safe, as long as they follow the tld-reportType-yearMonth.csv filename format.
*
* @see IcannReportingUploadAction
- * @see
- * ICANN Reporting Specification
+ * @see ICANN
+ * Reporting Specification
*/
public class IcannHttpReporter {
@@ -57,29 +67,24 @@ public class IcannHttpReporter {
@Inject IcannHttpReporter() {}
/** Uploads {@code reportBytes} to ICANN. */
- public void send(
- byte[] reportBytes,
- String tld,
- String yearMonth,
- ReportType reportType) throws XmlException, IOException {
- GenericUrl uploadUrl = new GenericUrl(makeUrl(tld, yearMonth, reportType));
+ public void send(byte[] reportBytes, String reportFilename) throws XmlException, IOException {
+ validateReportFilename(reportFilename);
+ GenericUrl uploadUrl = new GenericUrl(makeUrl(reportFilename));
HttpRequest request =
httpTransport
.createRequestFactory()
.buildPutRequest(uploadUrl, new ByteArrayContent(CSV_UTF_8.toString(), reportBytes));
HttpHeaders headers = request.getHeaders();
- headers.setBasicAuthentication(tld + "_ry", password);
+ headers.setBasicAuthentication(getTld(reportFilename) + "_ry", password);
headers.setContentType(CSV_UTF_8.toString());
request.setHeaders(headers);
request.setFollowRedirects(false);
HttpResponse response = null;
logger.infofmt(
- "Sending %s report to %s with content length %s",
- reportType,
- uploadUrl.toString(),
- request.getContent().getLength());
+ "Sending report to %s with content length %s",
+ uploadUrl.toString(), request.getContent().getLength());
try {
response = request.execute();
byte[] content;
@@ -117,9 +122,31 @@ public class IcannHttpReporter {
return result;
}
- private String makeUrl(String tld, String yearMonth, ReportType reportType) {
- String urlPrefix = getUrlPrefix(reportType);
- return String.format("%s/%s/%s", urlPrefix, tld, yearMonth);
+ /** Verifies a given report filename matches the pattern tld-reportType-yyyyMM.csv. */
+ private void validateReportFilename(String filename) {
+ checkArgument(
+ filename.matches("[a-z0-9.\\-]+-((activity)|(transactions))-[0-9]{6}\\.csv"),
+ "Expected file format: tld-reportType-yyyyMM.csv, got %s instead",
+ filename);
+ assertTldExists(getTld(filename));
+ }
+
+ private String getTld(String filename) {
+ // Extract the TLD, up to second-to-last hyphen in the filename (works with international TLDs)
+ return filename.substring(0, filename.lastIndexOf('-', filename.lastIndexOf('-') - 1));
+ }
+
+ private String makeUrl(String filename) {
+ // Filename is in the format tld-reportType-yearMonth.csv
+ String tld = getTld(filename);
+ // Remove the tld- prefix and csv suffix
+ String remainder = filename.substring(tld.length() + 1, filename.length() - 4);
+ List elements = Splitter.on('-').splitToList(remainder);
+ ReportType reportType = ReportType.valueOf(Ascii.toUpperCase(elements.get(0)));
+ // Re-add hyphen between year and month, because ICANN is inconsistent between filename and URL
+ String yearMonth =
+ YearMonth.parse(elements.get(1), DateTimeFormat.forPattern("yyyyMM")).toString("yyyy-MM");
+ return String.format("%s/%s/%s", getUrlPrefix(reportType), tld, yearMonth);
}
private String getUrlPrefix(ReportType reportType) {
@@ -131,7 +158,7 @@ public class IcannHttpReporter {
default:
throw new IllegalStateException(
String.format(
- "Received invalid reportType! Expected ACTIVITY or TRANSACTIONS, got %s.",
+ "Received invalid reportTypes! Expected ACTIVITY or TRANSACTIONS, got %s.",
reportType));
}
}
diff --git a/java/google/registry/reporting/IcannReportingModule.java b/java/google/registry/reporting/IcannReportingModule.java
index 8ee0c6ca0..77aa5370e 100644
--- a/java/google/registry/reporting/IcannReportingModule.java
+++ b/java/google/registry/reporting/IcannReportingModule.java
@@ -14,22 +14,24 @@
package google.registry.reporting;
-import static google.registry.request.RequestParameters.extractEnumParameter;
+import static google.registry.request.RequestParameters.extractOptionalEnumParameter;
import static google.registry.request.RequestParameters.extractOptionalParameter;
-import static google.registry.request.RequestParameters.extractRequiredParameter;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.MoreExecutors;
import dagger.Module;
import dagger.Provides;
import google.registry.bigquery.BigqueryConnection;
+import google.registry.request.HttpException.BadRequestException;
import google.registry.request.Parameter;
+import google.registry.util.Clock;
import java.util.Optional;
-import java.util.concurrent.Executors;
import javax.servlet.http.HttpServletRequest;
import org.joda.time.Duration;
+import org.joda.time.format.DateTimeFormat;
/** Module for dependencies required by ICANN monthly transactions/activity reporting. */
@Module
@@ -41,43 +43,78 @@ public final class IcannReportingModule {
ACTIVITY
}
+ static final String PARAM_OPTIONAL_YEAR_MONTH = "yearMonthOptional";
static final String PARAM_YEAR_MONTH = "yearMonth";
- static final String PARAM_REPORT_TYPE = "reportType";
+ static final String PARAM_OPTIONAL_SUBDIR = "subdirOptional";
static final String PARAM_SUBDIR = "subdir";
+ static final String PARAM_REPORT_TYPE = "reportType";
static final String ICANN_REPORTING_DATA_SET = "icann_reporting";
static final String DATASTORE_EXPORT_DATA_SET = "latest_datastore_export";
- private static final String BIGQUERY_SCOPE = "https://www.googleapis.com/auth/bigquery";
+ static final String MANIFEST_FILE_NAME = "MANIFEST.txt";
+ private static final String DEFAULT_SUBDIR = "icann/monthly";
+ private static final String BIGQUERY_SCOPE = "https://www.googleapis.com/auth/cloud-platform";
+ /** Extracts an optional yearMonth in yyyy-MM format from the request. */
+ @Provides
+ @Parameter(PARAM_OPTIONAL_YEAR_MONTH)
+ static Optional provideYearMonthOptional(HttpServletRequest req) {
+ return extractOptionalParameter(req, PARAM_YEAR_MONTH);
+ }
+
+ /** Provides the yearMonth in yyyy-MM format, defaults to one month prior to run time. */
@Provides
@Parameter(PARAM_YEAR_MONTH)
- static String provideYearMonth(HttpServletRequest req) {
- return extractRequiredParameter(req, PARAM_YEAR_MONTH);
+ static String provideYearMonth(
+ @Parameter(PARAM_OPTIONAL_YEAR_MONTH) Optional yearMonthOptional, Clock clock) {
+ String yearMonth =
+ yearMonthOptional.orElse(
+ DateTimeFormat.forPattern("yyyy-MM").print(clock.nowUtc().minusMonths(1)));
+ if (!yearMonth.matches("[0-9]{4}-[0-9]{2}")) {
+ throw new BadRequestException(
+ String.format("yearMonth must be in yyyy-MM format, got %s instead", yearMonth));
+ }
+ return yearMonth;
}
+ /** Provides an optional subdirectory to store/upload reports to, extracted from the request. */
@Provides
- @Parameter(PARAM_REPORT_TYPE)
- static ReportType provideReportType(HttpServletRequest req) {
- return extractEnumParameter(req, ReportType.class, PARAM_REPORT_TYPE);
- }
-
- @Provides
- @Parameter(PARAM_SUBDIR)
- static Optional provideSubdir(HttpServletRequest req) {
+ @Parameter(PARAM_OPTIONAL_SUBDIR)
+ static Optional provideSubdirOptional(HttpServletRequest req) {
return extractOptionalParameter(req, PARAM_SUBDIR);
}
+ /** Provides the subdirectory to store/upload reports to, defaults to icann/monthly/yearMonth. */
@Provides
- static QueryBuilder provideQueryBuilder(
- @Parameter(PARAM_REPORT_TYPE) ReportType reportType,
- ActivityReportingQueryBuilder activityBuilder,
- TransactionsReportingQueryBuilder transactionsBuilder) {
- return reportType == ReportType.ACTIVITY ? activityBuilder : transactionsBuilder;
+ @Parameter(PARAM_SUBDIR)
+ static String provideSubdir(
+ @Parameter(PARAM_OPTIONAL_SUBDIR) Optional subdirOptional,
+ @Parameter(PARAM_YEAR_MONTH) String yearMonth) {
+ String subdir = subdirOptional.orElse(String.format("%s/%s", DEFAULT_SUBDIR, yearMonth));
+ if (subdir.startsWith("/") || subdir.endsWith("/")) {
+ throw new BadRequestException(
+ String.format("subdir must not start or end with a \"/\", got %s instead.", subdir));
+ }
+ return subdir;
+ }
+
+ /** Provides an optional reportType to store/upload reports to, extracted from the request. */
+ @Provides
+ static Optional provideReportTypeOptional(HttpServletRequest req) {
+ return extractOptionalEnumParameter(req, ReportType.class, PARAM_REPORT_TYPE);
+ }
+
+ /** Provides a list of reportTypes specified. If absent, we default to both report types. */
+ @Provides
+ @Parameter(PARAM_REPORT_TYPE)
+ static ImmutableList provideReportTypes(Optional reportTypeOptional) {
+ return reportTypeOptional.map(ImmutableList::of)
+ .orElseGet(() -> ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS));
}
/**
* Constructs a BigqueryConnection with default settings.
*
- * We use Bigquery to generate activity reports via large aggregate SQL queries.
+ *
We use Bigquery to generate ICANN monthly reports via large aggregate SQL queries.
*
* @see ActivityReportingQueryBuilder
* @see google.registry.tools.BigqueryParameters for justifications of defaults.
@@ -87,13 +124,14 @@ public final class IcannReportingModule {
try {
GoogleCredential credential = GoogleCredential
.getApplicationDefault(transport, new JacksonFactory());
- BigqueryConnection connection = new BigqueryConnection.Builder()
- .setExecutorService(Executors.newFixedThreadPool(20))
- .setCredential(credential.createScoped(ImmutableList.of(BIGQUERY_SCOPE)))
- .setDatasetId(ICANN_REPORTING_DATA_SET)
- .setOverwrite(true)
- .setPollInterval(Duration.standardSeconds(1))
- .build();
+ BigqueryConnection connection =
+ new BigqueryConnection.Builder()
+ .setExecutorService(MoreExecutors.newDirectExecutorService())
+ .setCredential(credential.createScoped(ImmutableList.of(BIGQUERY_SCOPE)))
+ .setDatasetId(ICANN_REPORTING_DATA_SET)
+ .setOverwrite(true)
+ .setPollInterval(Duration.standardSeconds(1))
+ .build();
connection.initialize();
return connection;
} catch (Throwable e) {
@@ -101,3 +139,4 @@ public final class IcannReportingModule {
}
}
}
+
diff --git a/java/google/registry/reporting/IcannReportingStager.java b/java/google/registry/reporting/IcannReportingStager.java
new file mode 100644
index 000000000..04ab07ec0
--- /dev/null
+++ b/java/google/registry/reporting/IcannReportingStager.java
@@ -0,0 +1,264 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static google.registry.reporting.IcannReportingModule.MANIFEST_FILE_NAME;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.api.services.bigquery.model.TableFieldSchema;
+import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableTable;
+import com.google.common.collect.ListMultimap;
+import google.registry.bigquery.BigqueryConnection;
+import google.registry.bigquery.BigqueryUtils.TableType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
+import google.registry.reporting.IcannReportingModule.ReportType;
+import google.registry.request.Parameter;
+import google.registry.util.FormattingLogger;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+import javax.inject.Inject;
+
+/**
+ * Class containing methods for staging ICANN monthly reports on GCS.
+ *
+ *
The main entrypoint is stageReports, which generates a given type of reports.
+ */
+public class IcannReportingStager {
+
+ private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+
+ @Inject @Config("icannReportingBucket") String reportingBucket;
+ @Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
+
+ @Inject
+ @Parameter(IcannReportingModule.PARAM_SUBDIR)
+ String subdir;
+
+ @Inject ActivityReportingQueryBuilder activityQueryBuilder;
+ @Inject TransactionsReportingQueryBuilder transactionsQueryBuilder;
+ @Inject GcsUtils gcsUtils;
+ @Inject BigqueryConnection bigquery;
+
+ @Inject
+ IcannReportingStager() {}
+
+ /**
+ * Creates and stores reports of a given type on GCS.
+ *
+ *
This is factored out to facilitate choosing which reports to upload,
+ */
+ ImmutableList stageReports(ReportType reportType) throws Exception {
+ QueryBuilder queryBuilder =
+ (reportType == ReportType.ACTIVITY) ? activityQueryBuilder : transactionsQueryBuilder;
+
+
+ ImmutableMap viewQueryMap = queryBuilder.getViewQueryMap();
+ // Generate intermediary views
+ for (Entry entry : viewQueryMap.entrySet()) {
+ createIntermediaryTableView(entry.getKey(), entry.getValue(), reportType);
+ }
+
+ // Get an in-memory table of the aggregate query's result
+ ImmutableTable reportTable =
+ bigquery.queryToLocalTableSync(queryBuilder.getReportQuery());
+
+ // Get report headers from the table schema and convert into CSV format
+ String headerRow = constructRow(getHeaders(reportTable.columnKeySet()));
+ logger.infofmt("Headers: %s", headerRow);
+
+ return (reportType == ReportType.ACTIVITY)
+ ? stageActivityReports(headerRow, reportTable.rowMap().values())
+ : stageTransactionsReports(headerRow, reportTable.rowMap().values());
+ }
+
+ private void createIntermediaryTableView(String queryName, String query, ReportType reportType)
+ throws ExecutionException, InterruptedException {
+ // Later views depend on the results of earlier ones, so query everything synchronously
+ logger.infofmt("Generating intermediary view %s", queryName);
+ bigquery.query(
+ query,
+ bigquery.buildDestinationTable(queryName)
+ .description(String.format(
+ "An intermediary view to generate %s reports for this month.", reportType))
+ .type(TableType.VIEW)
+ .build()
+ ).get();
+ }
+
+ private Iterable getHeaders(ImmutableSet fields) {
+ return fields
+ .stream()
+ .map((schema) -> schema.getName().replace('_', '-'))
+ .collect(ImmutableList.toImmutableList());
+ }
+
+ /** Creates and stores activity reports on GCS, returns a list of files stored. */
+ private ImmutableList stageActivityReports(
+ String headerRow, ImmutableCollection> rows)
+ throws IOException {
+ ImmutableList.Builder manifestBuilder = new ImmutableList.Builder<>();
+ // Create a report csv for each tld from query table, and upload to GCS
+ for (Map row : rows) {
+ // Get the tld (first cell in each row)
+ String tld = row.values().iterator().next().toString();
+ if (isNullOrEmpty(tld)) {
+ throw new RuntimeException("Found an empty row in the activity report table!");
+ }
+ ImmutableList rowStrings = ImmutableList.of(constructRow(row.values()));
+ // Create and upload the activity report with a single row
+ manifestBuilder.add(
+ saveReportToGcs(tld, createReport(headerRow, rowStrings), ReportType.ACTIVITY));
+ }
+ return manifestBuilder.build();
+ }
+
+ /** Creates and stores transactions reports on GCS, returns a list of files stored. */
+ private ImmutableList stageTransactionsReports(
+ String headerRow, ImmutableCollection> rows)
+ throws IOException {
+ // Map from tld to rows
+ ListMultimap tldToRows = ArrayListMultimap.create();
+ // Map from tld to totals
+ HashMap> tldToTotals = new HashMap<>();
+ for (Map row : rows) {
+ // Get the tld (first cell in each row)
+ String tld = row.values().iterator().next().toString();
+ if (isNullOrEmpty(tld)) {
+ throw new RuntimeException("Found an empty row in the transactions report table!");
+ }
+ tldToRows.put(tld, constructRow(row.values()));
+ // Construct totals for each tld, skipping non-summable columns (TLD, registrar name, iana-id)
+ if (!tldToTotals.containsKey(tld)) {
+ tldToTotals.put(tld, new ArrayList<>(Collections.nCopies(row.values().size() - 3, 0)));
+ }
+ addToTotal(tldToTotals.get(tld), row);
+ }
+ ImmutableList.Builder manifestBuilder = new ImmutableList.Builder<>();
+ // Create and upload a transactions report for each tld via its rows
+ for (String tld : tldToRows.keySet()) {
+ // Append the totals row
+ tldToRows.put(tld, constructTotalRow(tldToTotals.get(tld)));
+ manifestBuilder.add(
+ saveReportToGcs(
+ tld, createReport(headerRow, tldToRows.get(tld)), ReportType.TRANSACTIONS));
+ }
+ return manifestBuilder.build();
+ }
+
+ /** Adds a row's values to an existing list of integers (totals). */
+ private void addToTotal(List totals, Map row) {
+ List rowVals =
+ row.values()
+ .stream()
+ // Ignore TLD, Registrar name and IANA id
+ .skip(3)
+ .map((Object o) -> Integer.parseInt(o.toString()))
+ .collect(Collectors.toList());
+ checkState(
+ rowVals.size() == totals.size(),
+ "Number of elements in totals not equal to number of elements in row!");
+ for (int i = 0; i < rowVals.size(); i++) {
+ totals.set(i, totals.get(i) + rowVals.get(i));
+ }
+ }
+
+ /** Returns a list of integers (totals) as a comma separated string. */
+ private String constructTotalRow(List totals) {
+ StringBuilder rowString = new StringBuilder("Totals,,");
+ rowString.append(
+ totals.stream().map((Integer i) -> i.toString()).collect(Collectors.joining(",")));
+ return rowString.toString();
+ }
+
+ /**
+ * Makes a row of the report by appending the string representation of all objects in an iterable
+ * with commas separating individual fields.
+ *
+ * This discards the first object, which is assumed to be the TLD field.
+ * */
+ private String constructRow(Iterable iterable) {
+ Iterator rowIter = iterable.iterator();
+ StringBuilder rowString = new StringBuilder();
+ // Skip the TLD column
+ rowIter.next();
+ while (rowIter.hasNext()) {
+ rowString.append(String.format("%s,", rowIter.next().toString()));
+ }
+ // Remove trailing comma
+ rowString.deleteCharAt(rowString.length() - 1);
+ return rowString.toString();
+ }
+
+ /**
+ * Constructs a report given its headers and rows as a string.
+ *
+ *
Note that activity reports will only have one row, while transactions reports may have
+ * multiple rows.
+ */
+ private String createReport(String headers, List rows) {
+ StringBuilder reportCsv = new StringBuilder(headers);
+ for (String row : rows) {
+ // Add CRLF between rows per ICANN specification
+ reportCsv.append("\r\n");
+ reportCsv.append(row);
+ }
+ logger.infofmt("Created report:\n%s", reportCsv.toString());
+ return reportCsv.toString();
+ }
+
+ /** Stores a report on GCS, returning the name of the file stored. */
+ private String saveReportToGcs(String tld, String reportCsv, ReportType reportType)
+ throws IOException {
+ // Upload resulting CSV file to GCS
+ byte[] reportBytes = reportCsv.getBytes(UTF_8);
+ String reportFilename = ReportingUtils.createFilename(tld, yearMonth, reportType);
+ String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
+ final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
+ gcsUtils.createFromBytes(gcsFilename, reportBytes);
+ logger.infofmt(
+ "Wrote %d bytes to file location %s",
+ reportBytes.length,
+ gcsFilename.toString());
+ return reportFilename;
+ }
+
+ /** Creates and stores a manifest file on GCS, indicating which reports were generated. */
+ void createAndUploadManifest(ImmutableList filenames) throws IOException {
+ String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
+ final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
+ StringBuilder manifestString = new StringBuilder();
+ filenames.forEach((filename) -> manifestString.append(filename).append("\n"));
+ gcsUtils.createFromBytes(gcsFilename, manifestString.toString().getBytes(UTF_8));
+ logger.infofmt(
+ "Wrote %d filenames to manifest at %s", filenames.size(), gcsFilename.toString());
+ }
+}
diff --git a/java/google/registry/reporting/IcannReportingStagingAction.java b/java/google/registry/reporting/IcannReportingStagingAction.java
index d5de89a19..c1e47bc44 100644
--- a/java/google/registry/reporting/IcannReportingStagingAction.java
+++ b/java/google/registry/reporting/IcannReportingStagingAction.java
@@ -14,99 +14,72 @@
package google.registry.reporting;
-import static com.google.common.base.Strings.isNullOrEmpty;
import static google.registry.request.Action.Method.POST;
-import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_OK;
-import com.google.api.services.bigquery.model.TableFieldSchema;
-import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.base.Throwables;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableTable;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.ListMultimap;
import com.google.common.net.MediaType;
-import google.registry.bigquery.BigqueryConnection;
-import google.registry.bigquery.BigqueryUtils.TableType;
-import google.registry.config.RegistryConfig.Config;
-import google.registry.gcs.GcsUtils;
import google.registry.reporting.IcannReportingModule.ReportType;
import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.FormattingLogger;
-import java.io.IOException;
-import java.io.OutputStream;
import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Optional;
-import java.util.concurrent.ExecutionException;
import javax.inject.Inject;
/**
* Action that generates monthly ICANN activity and transactions reports.
*
- * The reports are then uploaded to GCS under
- * gs://domain-registry-reporting/icann/monthly/YYYY-MM
+ *
The reports are stored in GCS under gs://[project-id]-reporting/[subdir]. We also store a
+ * MANIFEST.txt file that contains a list of filenames generated, to facilitate subsequent uploads.
+ *
+ *
Parameters:
+ *
+ *
yearMonth: the reporting month in yyyy-MM format. Defaults to the previous month at runtime
+ * (i.e. a run on 2017-09-01 defaults to 2017-08's reports).
+ *
+ *
subdir: the subdirectory of gs://[project-id]-reporting/ to upload to. For example:
+ * "manual/dir" means reports will be stored under gs://[project-id]-reporting/manual/dir. Defaults
+ * to "icann/monthly/[yearMonth]".
+ *
+ *
reportTypes: the type of reports to generate. You can specify either 'activity' or
+ * 'transactions'. Defaults to generating both.
*/
-@Action(
- path = IcannReportingStagingAction.PATH,
- method = POST,
- auth = Auth.AUTH_INTERNAL_ONLY
-)
+@Action(path = IcannReportingStagingAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_ONLY)
public final class IcannReportingStagingAction implements Runnable {
static final String PATH = "/_dr/task/icannReportingStaging";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
- @Inject @Config("icannReportingBucket") String reportingBucket;
- @Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
- @Inject @Parameter(IcannReportingModule.PARAM_SUBDIR) Optional subdir;
- @Inject @Parameter(IcannReportingModule.PARAM_REPORT_TYPE) ReportType reportType;
- @Inject QueryBuilder queryBuilder;
- @Inject BigqueryConnection bigquery;
- @Inject GcsUtils gcsUtils;
+ @Inject
+ @Parameter(IcannReportingModule.PARAM_REPORT_TYPE)
+ ImmutableList reportTypes;
+
+ @Inject IcannReportingStager stager;
@Inject Response response;
@Inject IcannReportingStagingAction() {}
@Override
public void run() {
try {
- ImmutableMap viewQueryMap = queryBuilder.getViewQueryMap();
- // Generate intermediary views
- for (Entry entry : viewQueryMap.entrySet()) {
- createIntermediaryTableView(entry.getKey(), entry.getValue());
+ ImmutableList.Builder manifestedFilesBuilder = new ImmutableList.Builder<>();
+ for (ReportType reportType : reportTypes) {
+ manifestedFilesBuilder.addAll(stager.stageReports(reportType));
}
+ ImmutableList manifestedFiles = manifestedFilesBuilder.build();
+ stager.createAndUploadManifest(manifestedFiles);
- // Get an in-memory table of the aggregate query's result
- ImmutableTable reportTable =
- bigquery.queryToLocalTableSync(queryBuilder.getReportQuery());
-
- // Get report headers from the table schema and convert into CSV format
- String headerRow = constructRow(getHeaders(reportTable.columnKeySet()));
- logger.infofmt("Headers: %s", headerRow);
-
- if (reportType == ReportType.ACTIVITY) {
- stageActivityReports(headerRow, reportTable.rowMap().values());
- } else {
- stageTransactionsReports(headerRow, reportTable.rowMap().values());
- }
+ logger.infofmt("Completed staging %d report files.", manifestedFiles.size());
response.setStatus(SC_OK);
response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
response.setPayload("Completed staging action.");
} catch (Exception e) {
- logger.warning(Throwables.getStackTraceAsString(e));
+ logger.severe("Reporting staging action failed!");
+ logger.severe(Throwables.getStackTraceAsString(e));
response.setStatus(SC_INTERNAL_SERVER_ERROR);
response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
response.setPayload(
@@ -114,109 +87,4 @@ public final class IcannReportingStagingAction implements Runnable {
Arrays.toString(e.getStackTrace())));
}
}
-
- private void createIntermediaryTableView(String queryName, String query)
- throws ExecutionException, InterruptedException {
- // Later views depend on the results of earlier ones, so query everything synchronously
- bigquery.query(
- query,
- bigquery.buildDestinationTable(queryName)
- .description(String.format(
- "An intermediary view to generate %s reports for this month.", reportType))
- .type(TableType.VIEW)
- .build()
- ).get();
- }
-
- private Iterable getHeaders(ImmutableSet fields) {
- return Iterables.transform(fields, schema -> schema.getName().replace('_', '-'));
- }
-
- private void stageActivityReports (
- String headerRow, ImmutableCollection> rows)
- throws IOException {
- // Create a report csv for each tld from query table, and upload to GCS
- for (Map row : rows) {
- // Get the tld (first cell in each row)
- String tld = row.values().iterator().next().toString();
- if (isNullOrEmpty(tld)) {
- throw new RuntimeException("Found an empty row in the activity report table!");
- }
- ImmutableList rowStrings = ImmutableList.of(constructRow(row.values()));
- // Create and upload the activity report with a single row
- uploadReport(tld, createReport(headerRow, rowStrings));
- }
- }
-
- private void stageTransactionsReports(
- String headerRow, ImmutableCollection> rows)
- throws IOException {
- // Map from tld to rows
- ListMultimap tldToRows = ArrayListMultimap.create();
- for (Map row : rows) {
- // Get the tld (first cell in each row)
- String tld = row.values().iterator().next().toString();
- if (isNullOrEmpty(tld)) {
- throw new RuntimeException("Found an empty row in the activity report table!");
- }
- tldToRows.put(tld, constructRow(row.values()));
- }
- // Create and upload a transactions report for each tld via its rows
- for (String tld : tldToRows.keySet()) {
- uploadReport(tld, createReport(headerRow, tldToRows.get(tld)));
- }
- }
-
- /**
- * Makes a row of the report by appending the string representation of all objects in an iterable
- * with commas separating individual fields.
- *
- * This discards the first object, which is assumed to be the TLD field.
- * */
- private String constructRow(Iterable iterable) {
- Iterator rowIter = iterable.iterator();
- StringBuilder rowString = new StringBuilder();
- // Skip the TLD column
- rowIter.next();
- while (rowIter.hasNext()) {
- rowString.append(String.format("%s,", rowIter.next().toString()));
- }
- // Remove trailing comma
- rowString.deleteCharAt(rowString.length() - 1);
- return rowString.toString();
- }
-
- /**
- * Constructs a report given its headers and rows as a string.
- *
- *
Note that activity reports will only have one row, while transactions reports may have
- * multiple rows.
- */
- private String createReport(String headers, List rows) {
- StringBuilder reportCsv = new StringBuilder(headers);
- for (String row : rows) {
- // Add CRLF between rows per ICANN specification
- reportCsv.append("\r\n");
- reportCsv.append(row);
- }
- logger.infofmt("Created %s report:\n%s", reportType, reportCsv.toString());
- return reportCsv.toString();
- }
-
- private void uploadReport(String tld, String reportCsv) throws IOException {
- // Upload resulting CSV file to GCS
- byte[] reportBytes = reportCsv.getBytes(UTF_8);
- String reportFilename =
- IcannReportingUploadAction.createFilename(tld, yearMonth, reportType);
- String reportBucketname =
- IcannReportingUploadAction.createReportingBucketName(reportingBucket, subdir, yearMonth);
- final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
- try (OutputStream gcsOutput = gcsUtils.openOutputStream(gcsFilename)) {
- gcsOutput.write(reportBytes);
- }
- logger.infofmt(
- "Wrote %d bytes to file location %s",
- reportBytes.length,
- gcsFilename.toString());
- }
}
diff --git a/java/google/registry/reporting/IcannReportingUploadAction.java b/java/google/registry/reporting/IcannReportingUploadAction.java
index 417d89b43..6b6a2e646 100644
--- a/java/google/registry/reporting/IcannReportingUploadAction.java
+++ b/java/google/registry/reporting/IcannReportingUploadAction.java
@@ -14,115 +14,108 @@
package google.registry.reporting;
-import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
-import static google.registry.model.registry.Registries.assertTldExists;
+import static google.registry.reporting.IcannReportingModule.MANIFEST_FILE_NAME;
import static google.registry.request.Action.Method.POST;
+import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
import com.google.common.io.ByteStreams;
import google.registry.config.RegistryConfig.Config;
import google.registry.gcs.GcsUtils;
-import google.registry.reporting.IcannReportingModule.ReportType;
import google.registry.request.Action;
import google.registry.request.Parameter;
-import google.registry.request.RequestParameters;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.FormattingLogger;
import google.registry.util.Retrier;
import java.io.IOException;
import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.Optional;
import javax.inject.Inject;
/**
- * Action that uploads the monthly transaction and activity reports from Cloud Storage to ICANN via
- * an HTTP PUT.
+ * Action that uploads the monthly activity/transactions reports from GCS to ICANN via an HTTP PUT.
*
+ * This should be run after {@link IcannReportingStagingAction}, which writes out the month's
+ * reports and a MANIFEST.txt file. This action reads the filenames from the MANIFEST.txt, and
+ * attempts to upload every file in the manifest to ICANN's endpoint.
+ *
+ *
Parameters:
+ *
+ *
subdir: the subdirectory of gs://[project-id]-reporting/ to retrieve reports from. For
+ * example: "manual/dir" means reports will be stored under gs://[project-id]-reporting/manual/dir.
+ * Defaults to "icann/monthly/[last month in yyyy-MM format]".
*/
-@Action(
- path = IcannReportingUploadAction.PATH,
- method = POST,
- auth = Auth.AUTH_INTERNAL_OR_ADMIN
-)
+@Action(path = IcannReportingUploadAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_OR_ADMIN)
public final class IcannReportingUploadAction implements Runnable {
static final String PATH = "/_dr/task/icannReportingUpload";
- static final String DEFAULT_SUBDIR = "icann/monthly";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
- @Inject @Config("icannReportingBucket") String icannReportingBucket;
- @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
- @Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
- @Inject @Parameter(IcannReportingModule.PARAM_REPORT_TYPE) ReportType reportType;
- @Inject @Parameter(IcannReportingModule.PARAM_SUBDIR) Optional subdir;
+ @Inject
+ @Config("icannReportingBucket")
+ String reportingBucket;
+
+ @Inject
+ @Parameter(IcannReportingModule.PARAM_SUBDIR)
+ String subdir;
+
@Inject GcsUtils gcsUtils;
@Inject IcannHttpReporter icannReporter;
- @Inject Response response;
@Inject Retrier retrier;
+ @Inject Response response;
@Inject
IcannReportingUploadAction() {}
@Override
public void run() {
- validateParams();
- String reportFilename = createFilename(tld, yearMonth, reportType);
- String reportBucketname = createReportingBucketName(icannReportingBucket, subdir, yearMonth);
- logger.infofmt("Reading ICANN report %s from bucket %s", reportFilename, reportBucketname);
- final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
- checkState(
- gcsUtils.existsAndNotEmpty(gcsFilename),
- "ICANN report object %s in bucket %s not found",
- gcsFilename.getObjectName(),
- gcsFilename.getBucketName());
+ String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
+ ImmutableList manifestedFiles = getManifestedFiles(reportBucketname);
+ // Report on all manifested files
+ for (String reportFilename : manifestedFiles) {
+ logger.infofmt("Reading ICANN report %s from bucket %s", reportFilename, reportBucketname);
+ final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
+ verifyFileExists(gcsFilename);
+ retrier.callWithRetry(
+ () -> {
+ final byte[] payload = readBytesFromGcs(gcsFilename);
+ icannReporter.send(payload, reportFilename);
+ response.setContentType(PLAIN_TEXT_UTF_8);
+ response.setPayload(String.format("OK, sending: %s", new String(payload, UTF_8)));
+ return null;
+ },
+ IOException.class);
+ }
+ }
- retrier.callWithRetry(
- () -> {
- final byte[] payload = readReportFromGcs(gcsFilename);
- icannReporter.send(payload, tld, yearMonth, reportType);
- response.setContentType(PLAIN_TEXT_UTF_8);
- response.setPayload(
- String.format("OK, sending: %s", new String(payload, StandardCharsets.UTF_8)));
- return null;
- },
+ private ImmutableList getManifestedFiles(String reportBucketname) {
+ GcsFilename manifestFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
+ verifyFileExists(manifestFilename);
+ return retrier.callWithRetry(
+ () ->
+ ImmutableList.copyOf(
+ Splitter.on('\n')
+ .omitEmptyStrings()
+ .split(new String(readBytesFromGcs(manifestFilename), UTF_8))),
IOException.class);
}
- private byte[] readReportFromGcs(GcsFilename reportFilename) throws IOException {
+ private byte[] readBytesFromGcs(GcsFilename reportFilename) throws IOException {
try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) {
return ByteStreams.toByteArray(gcsInput);
}
}
- static String createFilename(String tld, String yearMonth, ReportType reportType) {
- // Report files use YYYYMM naming instead of standard YYYY-MM, per ICANN requirements.
- String fileYearMonth = yearMonth.substring(0, 4) + yearMonth.substring(5, 7);
- return String.format("%s-%s-%s.csv", tld, reportType.toString().toLowerCase(), fileYearMonth);
- }
-
- static String createReportingBucketName(
- String reportingBucket, Optional subdir, String yearMonth) {
- return subdir.isPresent()
- ? String.format("%s/%s", reportingBucket, subdir.get())
- : String.format("%s/%s/%s", reportingBucket, DEFAULT_SUBDIR, yearMonth);
-
- }
-
- private void validateParams() {
- assertTldExists(tld);
- checkState(
- yearMonth.matches("[0-9]{4}-[0-9]{2}"),
- "yearMonth must be in YYYY-MM format, got %s instead.",
- yearMonth);
- if (subdir.isPresent()) {
- checkState(
- !subdir.get().startsWith("/") && !subdir.get().endsWith("/"),
- "subdir must not start or end with a \"/\", got %s instead.",
- subdir.get());
- }
+ private void verifyFileExists(GcsFilename gcsFilename) {
+ checkArgument(
+ gcsUtils.existsAndNotEmpty(gcsFilename),
+ "Object %s in bucket %s not found",
+ gcsFilename.getObjectName(),
+ gcsFilename.getBucketName());
}
}
diff --git a/java/google/registry/reporting/ReportingUtils.java b/java/google/registry/reporting/ReportingUtils.java
new file mode 100644
index 000000000..7303b4f94
--- /dev/null
+++ b/java/google/registry/reporting/ReportingUtils.java
@@ -0,0 +1,34 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import com.google.common.base.Ascii;
+import google.registry.reporting.IcannReportingModule.ReportType;
+
+/** Static utils for reporting. */
+public final class ReportingUtils {
+
+ /** Generates a report filename in accord with ICANN's specifications. */
+ static String createFilename(String tld, String yearMonth, ReportType reportType) {
+ // Report files use YYYYMM naming instead of standard YYYY-MM, per ICANN requirements.
+ return String.format(
+ "%s-%s-%s.csv", tld, Ascii.toLowerCase(reportType.toString()), yearMonth.replace("-", ""));
+ }
+
+ /** Constructs the bucket name to store/upload reports to. */
+ static String createReportingBucketName(String reportingBucket, String subdir) {
+ return String.format("%s/%s", reportingBucket, subdir);
+ }
+}
diff --git a/java/google/registry/reporting/TransactionsReportingQueryBuilder.java b/java/google/registry/reporting/TransactionsReportingQueryBuilder.java
index 8b5c2cd7b..f4bdf8590 100644
--- a/java/google/registry/reporting/TransactionsReportingQueryBuilder.java
+++ b/java/google/registry/reporting/TransactionsReportingQueryBuilder.java
@@ -158,6 +158,8 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
String aggregateQuery =
SqlTemplate.create(getQueryFromFile("transactions_report_aggregation.sql"))
.put("PROJECT_ID", projectId)
+ .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+ .put("REGISTRY_TABLE", "Registry")
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
.put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID))
.put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS))
diff --git a/java/google/registry/reporting/sql/activity_report_aggregation.sql b/java/google/registry/reporting/sql/activity_report_aggregation.sql
index 873781d65..67d9ab01f 100644
--- a/java/google/registry/reporting/sql/activity_report_aggregation.sql
+++ b/java/google/registry/reporting/sql/activity_report_aggregation.sql
@@ -19,8 +19,6 @@
SELECT
RealTlds.tld AS tld,
SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars,
- SUM(IF(metricName = 'ramp-up-registrars', count, 0)) AS ramp_up_registrars,
- SUM(IF(metricName = 'pre-ramp-up-registrars', count, 0)) AS pre_ramp_up_registrars,
-- We use the Centralized Zone Data Service.
"CZDS" AS zfa_passwords,
SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries,
@@ -65,7 +63,7 @@ SELECT
-- filter so that only metrics with that TLD or a NULL TLD are counted
-- towards a given TLD.
FROM (
-SELECT tldStr as tld
+SELECT tldStr AS tld
FROM `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
WHERE tldType = 'REAL'
) as RealTlds
diff --git a/java/google/registry/reporting/sql/dns_counts.sql b/java/google/registry/reporting/sql/dns_counts.sql
index 62d822a6b..85293dbb3 100644
--- a/java/google/registry/reporting/sql/dns_counts.sql
+++ b/java/google/registry/reporting/sql/dns_counts.sql
@@ -15,14 +15,13 @@
-- Query for DNS metrics.
- -- This is a no-op until after we transition to Google Cloud DNS, which
- -- will likely export metrics via Stackdriver.
+ -- You must configure this yourself to enable activity reporting, according
+ -- to whatever metrics your DNS provider makes available. We hope to make
+ -- this available in the open-source build in the near future.
SELECT
- -- DNS metrics apply to all tlds, which requires the 'null' magic value.
STRING(NULL) AS tld,
metricName,
- -- TODO(b/63388735): Change this to actually query Google Cloud DNS when ready.
-1 AS count
FROM ((
SELECT 'dns-udp-queries' AS metricName)
diff --git a/java/google/registry/reporting/sql/registrar_iana_id.sql b/java/google/registry/reporting/sql/registrar_iana_id.sql
index 0693e82c3..c6726eb1b 100644
--- a/java/google/registry/reporting/sql/registrar_iana_id.sql
+++ b/java/google/registry/reporting/sql/registrar_iana_id.sql
@@ -26,5 +26,5 @@ FROM
UNNEST(allowedTlds) as allowed_tlds
WHERE (type = 'REAL' OR type = 'INTERNAL')
-- Filter out prober data
-AND NOT ENDS_WITH(allowed_tlds, "test")
+AND NOT ENDS_WITH(allowed_tlds, ".test")
ORDER BY tld, registrarName
diff --git a/java/google/registry/reporting/sql/registrar_operating_status.sql b/java/google/registry/reporting/sql/registrar_operating_status.sql
index 63b9e40f1..c29da8102 100644
--- a/java/google/registry/reporting/sql/registrar_operating_status.sql
+++ b/java/google/registry/reporting/sql/registrar_operating_status.sql
@@ -23,5 +23,5 @@ SELECT
FROM
`%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
WHERE
- type = 'REAL'
+ (type = 'REAL' OR type = 'INTERNAL')
GROUP BY metricName
diff --git a/java/google/registry/reporting/sql/total_domains.sql b/java/google/registry/reporting/sql/total_domains.sql
index f3c1e6361..46a6523b5 100644
--- a/java/google/registry/reporting/sql/total_domains.sql
+++ b/java/google/registry/reporting/sql/total_domains.sql
@@ -32,7 +32,7 @@ JOIN
ON
currentSponsorClientId = registrar_table.__key__.name
WHERE
- domain_table._d = "DomainResource"
- AND (registrar_table.type = "REAL" OR registrar_table.type = "INTERNAL")
+ domain_table._d = 'DomainResource'
+ AND (registrar_table.type = 'REAL' OR registrar_table.type = 'INTERNAL')
GROUP BY tld, registrarName
ORDER BY tld, registrarName
diff --git a/java/google/registry/reporting/sql/transaction_counts.sql b/java/google/registry/reporting/sql/transaction_counts.sql
index 481017818..e9d730ab4 100644
--- a/java/google/registry/reporting/sql/transaction_counts.sql
+++ b/java/google/registry/reporting/sql/transaction_counts.sql
@@ -65,8 +65,6 @@ FROM (
WHERE reportingTime
BETWEEN TIMESTAMP('%EARLIEST_REPORT_TIME%')
AND TIMESTAMP('%LATEST_REPORT_TIME%')
- -- Ignore prober data
- AND NOT ENDS_WITH(tld, "test")
GROUP BY
tld,
clientId,
diff --git a/java/google/registry/reporting/sql/transactions_report_aggregation.sql b/java/google/registry/reporting/sql/transactions_report_aggregation.sql
index e68274995..9b4567e5d 100644
--- a/java/google/registry/reporting/sql/transactions_report_aggregation.sql
+++ b/java/google/registry/reporting/sql/transactions_report_aggregation.sql
@@ -20,7 +20,8 @@
SELECT
registrars.tld as tld,
- registrars.registrar_name as registrar_name,
+ -- Surround registrar names with quotes to handle names containing a comma.
+ FORMAT("\"%s\"", registrars.registrar_name) as registrar_name,
registrars.iana_id as iana_id,
SUM(IF(metrics.metricName = 'TOTAL_DOMAINS', metrics.metricValue, 0)) AS total_domains,
SUM(IF(metrics.metricName = 'TOTAL_NAMESERVERS', metrics.metricValue, 0)) AS total_nameservers,
@@ -62,9 +63,16 @@ SELECT
0 AS agp_exemptions_granted,
0 AS agp_exempted_domains,
SUM(IF(metrics.metricName = 'ATTEMPTED_ADDS', metrics.metricValue, 0)) AS attempted_adds
-FROM (
- SELECT *
- FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%REGISTRAR_IANA_ID_TABLE%`) AS registrars
+FROM
+-- Only produce reports for real TLDs
+(SELECT tldStr AS tld
+ FROM `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
+ WHERE tldType = 'REAL') AS registries
+JOIN
+(SELECT *
+ FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%REGISTRAR_IANA_ID_TABLE%`)
+ AS registrars
+ON registries.tld = registrars.tld
-- We LEFT JOIN to produce reports even if the registrar made no transactions
LEFT OUTER JOIN (
-- Gather all intermediary data views
diff --git a/java/google/registry/request/RequestParameters.java b/java/google/registry/request/RequestParameters.java
index 8f6e4d28c..502176613 100644
--- a/java/google/registry/request/RequestParameters.java
+++ b/java/google/registry/request/RequestParameters.java
@@ -98,6 +98,20 @@ public final class RequestParameters {
return parameters == null ? ImmutableSet.of() : ImmutableSet.copyOf(parameters);
}
+ /**
+ * Returns the first GET or POST parameter associated with {@code name}, absent otherwise.
+ *
+ * @throws BadRequestException if request parameter named {@code name} is not equal to any of the
+ * values in {@code enumClass}
+ */
+ public static > Optional extractOptionalEnumParameter(
+ HttpServletRequest req, Class enumClass, String name) {
+ String stringParam = req.getParameter(name);
+ return isNullOrEmpty(stringParam)
+ ? Optional.empty()
+ : Optional.of(extractEnumParameter(req, enumClass, name));
+ }
+
/**
* Returns the first GET or POST parameter associated with {@code name}.
*
diff --git a/javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java b/javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java
index 508806272..64c0d6014 100644
--- a/javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java
+++ b/javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java
@@ -15,7 +15,6 @@
package google.registry.reporting;
import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -30,7 +29,7 @@ public class ActivityReportingQueryBuilderTest {
private ActivityReportingQueryBuilder getQueryBuilder() {
ActivityReportingQueryBuilder queryBuilder = new ActivityReportingQueryBuilder();
- queryBuilder.yearMonth = "2017-06";
+ queryBuilder.yearMonth = "2017-09";
queryBuilder.projectId = "domain-registry-alpha";
return queryBuilder;
}
@@ -41,28 +40,28 @@ public class ActivityReportingQueryBuilderTest {
assertThat(queryBuilder.getReportQuery())
.isEqualTo(
"#standardSQL\nSELECT * FROM "
- + "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201706`");
+ + "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201709`");
}
@Test
public void testIntermediaryQueryMatch() throws IOException {
- ActivityReportingQueryBuilder queryBuilder = getQueryBuilder();
- ImmutableList queryNames =
+ ImmutableList expectedQueryNames =
ImmutableList.of(
ActivityReportingQueryBuilder.REGISTRAR_OPERATING_STATUS,
- ActivityReportingQueryBuilder.DNS_COUNTS,
ActivityReportingQueryBuilder.MONTHLY_LOGS,
+ ActivityReportingQueryBuilder.DNS_COUNTS,
ActivityReportingQueryBuilder.EPP_METRICS,
ActivityReportingQueryBuilder.WHOIS_COUNTS,
ActivityReportingQueryBuilder.ACTIVITY_REPORT_AGGREGATION);
+ ActivityReportingQueryBuilder queryBuilder = getQueryBuilder();
ImmutableMap actualQueries = queryBuilder.getViewQueryMap();
- for (String queryName : queryNames) {
- String actualTableName = String.format("%s_201706", queryName);
+ for (String queryName : expectedQueryNames) {
+ String actualTableName = String.format("%s_201709", queryName);
String testFilename = String.format("%s_test.sql", queryName);
assertThat(actualQueries.get(actualTableName))
.isEqualTo(ReportingTestData.getString(testFilename));
}
}
-}
+}
diff --git a/javatests/google/registry/reporting/IcannHttpReporterTest.java b/javatests/google/registry/reporting/IcannHttpReporterTest.java
index 9d9dc2f7f..7952a6e45 100644
--- a/javatests/google/registry/reporting/IcannHttpReporterTest.java
+++ b/javatests/google/registry/reporting/IcannHttpReporterTest.java
@@ -18,7 +18,7 @@ import static com.google.common.net.MediaType.CSV_UTF_8;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
-import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.testing.DatastoreHelper.createTld;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.api.client.http.LowLevelHttpRequest;
@@ -29,11 +29,14 @@ import com.google.api.client.testing.http.MockLowLevelHttpResponse;
import com.google.api.client.util.Base64;
import com.google.api.client.util.StringUtils;
import com.google.common.io.ByteSource;
-import google.registry.reporting.IcannReportingModule.ReportType;
import google.registry.request.HttpException.InternalServerErrorException;
+import google.registry.testing.AppEngineRule;
+import google.registry.testing.ExceptionRule;
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -46,9 +49,14 @@ public class IcannHttpReporterTest {
private static final ByteSource IIRDEA_GOOD_XML = ReportingTestData.get("iirdea_good.xml");
private static final ByteSource IIRDEA_BAD_XML = ReportingTestData.get("iirdea_bad.xml");
+ private static final byte[] FAKE_PAYLOAD = "test,csv\n1,2".getBytes(UTF_8);
private MockLowLevelHttpRequest mockRequest;
+ @Rule public final ExceptionRule thrown = new ExceptionRule();
+
+ @Rule public AppEngineRule appEngineRule = new AppEngineRule.Builder().withDatastore().build();
+
private MockHttpTransport createMockTransport (final ByteSource iirdeaResponse) {
return new MockHttpTransport() {
@Override
@@ -69,7 +77,11 @@ public class IcannHttpReporterTest {
};
}
- private static final byte[] FAKE_PAYLOAD = "test,csv\n1,2".getBytes(UTF_8);
+ @Before
+ public void setUp() {
+ createTld("test");
+ createTld("xn--abc123");
+ }
private IcannHttpReporter createReporter() {
IcannHttpReporter reporter = new IcannHttpReporter();
@@ -83,7 +95,7 @@ public class IcannHttpReporterTest {
@Test
public void testSuccess() throws Exception {
IcannHttpReporter reporter = createReporter();
- reporter.send(FAKE_PAYLOAD, "test", "2017-06", ReportType.TRANSACTIONS);
+ reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv");
assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2017-06");
Map> headers = mockRequest.getHeaders();
@@ -94,15 +106,65 @@ public class IcannHttpReporterTest {
assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString());
}
+ @Test
+ public void testSuccess_internationalTld() throws Exception {
+ IcannHttpReporter reporter = createReporter();
+ reporter.send(FAKE_PAYLOAD, "xn--abc123-transactions-201706.csv");
+
+ assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/xn--abc123/2017-06");
+ Map> headers = mockRequest.getHeaders();
+ String userPass = "xn--abc123_ry:fakePass";
+ String expectedAuth =
+ String.format("Basic %s", Base64.encodeBase64String(StringUtils.getBytesUtf8(userPass)));
+ assertThat(headers.get("authorization")).containsExactly(expectedAuth);
+ assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString());
+ }
+
@Test
public void testFail_BadIirdeaResponse() throws Exception {
IcannHttpReporter reporter = createReporter();
reporter.httpTransport = createMockTransport(IIRDEA_BAD_XML);
try {
- reporter.send(FAKE_PAYLOAD, "test", "2017-06", ReportType.TRANSACTIONS);
+ reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv");
assertWithMessage("Expected InternalServerErrorException to be thrown").fail();
} catch (InternalServerErrorException expected) {
assertThat(expected).hasMessageThat().isEqualTo("The structure of the report is invalid.");
}
}
+
+ @Test
+ public void testFail_invalidFilename_nonSixDigitYearMonth() throws Exception {
+ thrown.expect(
+ IllegalArgumentException.class,
+ "Expected file format: tld-reportType-yyyyMM.csv, got test-transactions-20176.csv instead");
+ IcannHttpReporter reporter = createReporter();
+ reporter.send(FAKE_PAYLOAD, "test-transactions-20176.csv");
+ }
+
+ @Test
+ public void testFail_invalidFilename_notActivityOrTransactions() throws Exception {
+ thrown.expect(
+ IllegalArgumentException.class,
+ "Expected file format: tld-reportType-yyyyMM.csv, got test-invalid-201706.csv instead");
+ IcannHttpReporter reporter = createReporter();
+ reporter.send(FAKE_PAYLOAD, "test-invalid-201706.csv");
+ }
+
+ @Test
+ public void testFail_invalidFilename_invalidTldName() throws Exception {
+ thrown.expect(
+ IllegalArgumentException.class,
+ "Expected file format: tld-reportType-yyyyMM.csv, got n!-n-activity-201706.csv instead");
+ IcannHttpReporter reporter = createReporter();
+ reporter.send(FAKE_PAYLOAD, "n!-n-activity-201706.csv");
+ }
+
+ @Test
+ public void testFail_invalidFilename_tldDoesntExist() throws Exception {
+ thrown.expect(
+ IllegalArgumentException.class,
+ "TLD hello does not exist");
+ IcannHttpReporter reporter = createReporter();
+ reporter.send(FAKE_PAYLOAD, "hello-activity-201706.csv");
+ }
}
diff --git a/javatests/google/registry/reporting/IcannReportingModuleTest.java b/javatests/google/registry/reporting/IcannReportingModuleTest.java
new file mode 100644
index 000000000..228b46463
--- /dev/null
+++ b/javatests/google/registry/reporting/IcannReportingModuleTest.java
@@ -0,0 +1,97 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.Mockito.mock;
+
+import google.registry.reporting.IcannReportingModule.ReportType;
+import google.registry.request.HttpException.BadRequestException;
+import google.registry.testing.ExceptionRule;
+import google.registry.testing.FakeClock;
+import google.registry.util.Clock;
+import java.util.Optional;
+import javax.servlet.http.HttpServletRequest;
+import org.joda.time.DateTime;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link google.registry.reporting.IcannReportingModule}. */
+@RunWith(JUnit4.class)
+public class IcannReportingModuleTest {
+
+ HttpServletRequest req = mock(HttpServletRequest.class);
+ Clock clock;
+
+ @Rule public final ExceptionRule thrown = new ExceptionRule();
+
+ @Before
+ public void setUp() {
+ clock = new FakeClock(DateTime.parse("2017-07-01TZ"));
+ }
+
+ @Test
+ public void testEmptyYearMonth_returnsCurrentDate() {
+ assertThat(IcannReportingModule.provideYearMonth(Optional.empty(), clock)).isEqualTo("2017-06");
+ }
+
+ @Test
+ public void testGivenYearMonth_returnsThatMonth() {
+ assertThat(IcannReportingModule.provideYearMonth(Optional.of("2017-05"), clock))
+ .isEqualTo("2017-05");
+ }
+
+ @Test
+ public void testInvalidYearMonth_throwsException() {
+ thrown.expect(
+ BadRequestException.class, "yearMonth must be in yyyy-MM format, got 201705 instead");
+ IcannReportingModule.provideYearMonth(Optional.of("201705"), clock);
+ }
+
+ @Test
+ public void testEmptySubDir_returnsDefaultSubdir() {
+ assertThat(IcannReportingModule.provideSubdir(Optional.empty(), "2017-06"))
+ .isEqualTo("icann/monthly/2017-06");
+ }
+
+ @Test
+ public void testGivenSubdir_returnsManualSubdir() {
+ assertThat(IcannReportingModule.provideSubdir(Optional.of("manual/dir"), "2017-06"))
+ .isEqualTo("manual/dir");
+ }
+
+ @Test
+ public void testInvalidSubdir_throwsException() {
+ thrown.expect(
+ BadRequestException.class,
+ "subdir must not start or end with a \"/\", got /whoops instead.");
+ IcannReportingModule.provideSubdir(Optional.of("/whoops"), "2017-06");
+ }
+
+ @Test
+ public void testGivenReportType_returnsReportType() {
+ assertThat(IcannReportingModule.provideReportTypes(Optional.of(ReportType.ACTIVITY)))
+ .containsExactly(ReportType.ACTIVITY);
+ }
+
+ @Test
+ public void testNoReportType_returnsBothReportTypes() {
+ assertThat(IcannReportingModule.provideReportTypes(Optional.empty()))
+ .containsExactly(ReportType.ACTIVITY, ReportType.TRANSACTIONS);
+ }
+}
diff --git a/javatests/google/registry/reporting/IcannReportingStagerTest.java b/javatests/google/registry/reporting/IcannReportingStagerTest.java
new file mode 100644
index 000000000..12486922d
--- /dev/null
+++ b/javatests/google/registry/reporting/IcannReportingStagerTest.java
@@ -0,0 +1,212 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.GcsTestingUtils.readGcsFile;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.TableFieldSchema;
+import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableTable;
+import com.google.common.util.concurrent.ListenableFuture;
+import google.registry.bigquery.BigqueryConnection;
+import google.registry.bigquery.BigqueryConnection.DestinationTable;
+import google.registry.bigquery.BigqueryUtils.TableType;
+import google.registry.gcs.GcsUtils;
+import google.registry.reporting.IcannReportingModule.ReportType;
+import google.registry.testing.AppEngineRule;
+import google.registry.testing.FakeResponse;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link google.registry.reporting.IcannReportingStager}. */
+@RunWith(JUnit4.class)
+public class IcannReportingStagerTest {
+
+ BigqueryConnection bigquery = mock(BigqueryConnection.class);
+ FakeResponse response = new FakeResponse();
+ GcsService gcsService = GcsServiceFactory.createGcsService();
+
+ @Rule
+ public final AppEngineRule appEngine = AppEngineRule.builder()
+ .withDatastore()
+ .withLocalModules()
+ .build();
+
+ private IcannReportingStager createStager() {
+ IcannReportingStager action = new IcannReportingStager();
+ ActivityReportingQueryBuilder activityBuilder = new ActivityReportingQueryBuilder();
+ activityBuilder.projectId = "test-project";
+ activityBuilder.yearMonth = "2017-06";
+ action.activityQueryBuilder = activityBuilder;
+ TransactionsReportingQueryBuilder transactionsBuilder = new TransactionsReportingQueryBuilder();
+ transactionsBuilder.projectId = "test-project";
+ transactionsBuilder.yearMonth = "2017-06";
+ action.transactionsQueryBuilder = transactionsBuilder;
+ action.reportingBucket = "test-bucket";
+ action.yearMonth = "2017-06";
+ action.subdir = "icann/monthly/2017-06";
+ action.bigquery = bigquery;
+ action.gcsUtils = new GcsUtils(gcsService, 1024);
+ return action;
+ }
+
+ private void setUpBigquery() {
+ when(bigquery.query(any(String.class), any(DestinationTable.class))).thenReturn(fakeFuture());
+ DestinationTable.Builder tableBuilder = new DestinationTable.Builder()
+ .datasetId("testdataset")
+ .type(TableType.TABLE)
+ .name("tablename")
+ .overwrite(true);
+ when(bigquery.buildDestinationTable(any(String.class))).thenReturn(tableBuilder);
+ }
+
+ @Test
+ public void testRunSuccess_activityReport() throws Exception {
+ setUpBigquery();
+ ImmutableTable activityReportTable =
+ new ImmutableTable.Builder()
+ .put(1, new TableFieldSchema().setName("tld"), "fooTld")
+ .put(1, new TableFieldSchema().setName("fooField"), "12")
+ .put(1, new TableFieldSchema().setName("barField"), "34")
+ .put(2, new TableFieldSchema().setName("tld"), "barTld")
+ .put(2, new TableFieldSchema().setName("fooField"), "56")
+ .put(2, new TableFieldSchema().setName("barField"), "78")
+ .build();
+ when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(activityReportTable);
+ IcannReportingStager stager = createStager();
+ stager.stageReports(ReportType.ACTIVITY);
+
+ String expectedReport1 = "fooField,barField\r\n12,34";
+ String expectedReport2 = "fooField,barField\r\n56,78";
+ byte[] generatedFile1 =
+ readGcsFile(
+ gcsService,
+ new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv"));
+ assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
+ byte[] generatedFile2 =
+ readGcsFile(
+ gcsService,
+ new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv"));
+ assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
+ }
+
+ @Test
+ public void testRunSuccess_transactionsReport() throws Exception {
+ setUpBigquery();
+ /*
+ The fake table result looks like:
+ tld registrar iana field
+ 1 fooTld reg1 123 10
+ 2 fooTld reg2 456 20
+ 3 barTld reg1 123 30
+ */
+ ImmutableTable transactionReportTable =
+ new ImmutableTable.Builder()
+ .put(1, new TableFieldSchema().setName("tld"), "fooTld")
+ .put(1, new TableFieldSchema().setName("registrar"), "\"reg1\"")
+ .put(1, new TableFieldSchema().setName("iana"), "123")
+ .put(1, new TableFieldSchema().setName("field"), "10")
+ .put(2, new TableFieldSchema().setName("tld"), "fooTld")
+ .put(2, new TableFieldSchema().setName("registrar"), "\"reg2\"")
+ .put(2, new TableFieldSchema().setName("iana"), "456")
+ .put(2, new TableFieldSchema().setName("field"), "20")
+ .put(3, new TableFieldSchema().setName("tld"), "barTld")
+ .put(3, new TableFieldSchema().setName("registrar"), "\"reg1\"")
+ .put(3, new TableFieldSchema().setName("iana"), "123")
+ .put(3, new TableFieldSchema().setName("field"), "30")
+ .build();
+ when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(transactionReportTable);
+ IcannReportingStager stager = createStager();
+ stager.stageReports(ReportType.TRANSACTIONS);
+
+ String expectedReport1 =
+ "registrar,iana,field\r\n\"reg1\",123,10\r\n\"reg2\",456,20\r\nTotals,,30";
+ String expectedReport2 = "registrar,iana,field\r\n\"reg1\",123,30\r\nTotals,,30";
+ byte[] generatedFile1 =
+ readGcsFile(
+ gcsService,
+ new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv"));
+ assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
+ byte[] generatedFile2 =
+ readGcsFile(
+ gcsService,
+ new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv"));
+ assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
+ }
+
+ @Test
+ public void testRunSuccess_createAndUploadManifest() throws Exception {
+ IcannReportingStager stager = createStager();
+ ImmutableList filenames =
+ ImmutableList.of("fooTld-transactions-201706.csv", "barTld-activity-201706.csv");
+ stager.createAndUploadManifest(filenames);
+
+ String expectedManifest = "fooTld-transactions-201706.csv\nbarTld-activity-201706.csv\n";
+ byte[] generatedManifest =
+ readGcsFile(
+ gcsService, new GcsFilename("test-bucket/icann/monthly/2017-06", "MANIFEST.txt"));
+ assertThat(new String(generatedManifest, UTF_8)).isEqualTo(expectedManifest);
+ }
+
+ private ListenableFuture fakeFuture() {
+ return new ListenableFuture() {
+ @Override
+ public void addListener(Runnable runnable, Executor executor) {
+ // No-op
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ return false;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ @Override
+ public boolean isDone() {
+ return false;
+ }
+
+ @Override
+ public DestinationTable get() throws InterruptedException, ExecutionException {
+ return null;
+ }
+
+ @Override
+ public DestinationTable get(long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ return null;
+ }
+ };
+ }
+}
+
diff --git a/javatests/google/registry/reporting/IcannReportingStagingActionTest.java b/javatests/google/registry/reporting/IcannReportingStagingActionTest.java
index 07d697f11..482a6cde4 100644
--- a/javatests/google/registry/reporting/IcannReportingStagingActionTest.java
+++ b/javatests/google/registry/reporting/IcannReportingStagingActionTest.java
@@ -14,32 +14,15 @@
package google.registry.reporting;
-import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
-import static google.registry.testing.GcsTestingUtils.readGcsFile;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import com.google.api.services.bigquery.model.TableFieldSchema;
-import com.google.appengine.tools.cloudstorage.GcsFilename;
-import com.google.appengine.tools.cloudstorage.GcsService;
-import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
-import com.google.common.collect.ImmutableTable;
-import com.google.common.util.concurrent.ListenableFuture;
-import google.registry.bigquery.BigqueryConnection;
-import google.registry.bigquery.BigqueryConnection.DestinationTable;
-import google.registry.bigquery.BigqueryUtils.TableType;
-import google.registry.gcs.GcsUtils;
+import com.google.common.collect.ImmutableList;
import google.registry.reporting.IcannReportingModule.ReportType;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeResponse;
-import java.util.Optional;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -51,9 +34,8 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class IcannReportingStagingActionTest {
- BigqueryConnection bigquery = mock(BigqueryConnection.class);
FakeResponse response = new FakeResponse();
- GcsService gcsService = GcsServiceFactory.createGcsService();
+ IcannReportingStager stager = mock(IcannReportingStager.class);
@Rule
public final AppEngineRule appEngine = AppEngineRule.builder()
@@ -61,143 +43,36 @@ public class IcannReportingStagingActionTest {
.withLocalModules()
.build();
- private IcannReportingStagingAction createAction(ReportType reportType) {
+ @Before
+ public void setUp() throws Exception {
+ when(stager.stageReports(ReportType.ACTIVITY)).thenReturn(ImmutableList.of("a", "b"));
+ when(stager.stageReports(ReportType.TRANSACTIONS)).thenReturn(ImmutableList.of("c", "d"));
+ }
+
+ private IcannReportingStagingAction createAction(ImmutableList reportingMode) {
IcannReportingStagingAction action = new IcannReportingStagingAction();
- if (reportType == ReportType.ACTIVITY) {
- ActivityReportingQueryBuilder activityBuilder = new ActivityReportingQueryBuilder();
- activityBuilder.projectId = "test-project";
- activityBuilder.yearMonth = "2017-06";
- action.queryBuilder = activityBuilder;
- } else {
- TransactionsReportingQueryBuilder transactionsBuilder =
- new TransactionsReportingQueryBuilder();
- transactionsBuilder.projectId = "test-project";
- transactionsBuilder.yearMonth = "2017-06";
- action.queryBuilder = transactionsBuilder;
- }
- action.reportType = reportType;
- action.reportingBucket = "test-bucket";
- action.yearMonth = "2017-06";
- action.subdir = Optional.empty();
- action.bigquery = bigquery;
- action.gcsUtils = new GcsUtils(gcsService, 1024);
+ action.reportTypes = reportingMode;
action.response = response;
+ action.stager = stager;
return action;
}
- private void setUpBigquery() {
- when(bigquery.query(any(String.class), any(DestinationTable.class))).thenReturn(fakeFuture());
- DestinationTable.Builder tableBuilder = new DestinationTable.Builder()
- .datasetId("testdataset")
- .type(TableType.TABLE)
- .name("tablename")
- .overwrite(true);
- when(bigquery.buildDestinationTable(any(String.class))).thenReturn(tableBuilder);
+ @Test
+ public void testActivityReportingMode_onlyStagesActivityReports() throws Exception {
+ IcannReportingStagingAction action = createAction(ImmutableList.of(ReportType.ACTIVITY));
+ action.run();
+ verify(stager).stageReports(ReportType.ACTIVITY);
+ verify(stager).createAndUploadManifest(ImmutableList.of("a", "b"));
}
@Test
- public void testRunSuccess_activityReport() throws Exception {
- setUpBigquery();
- ImmutableTable activityReportTable =
- new ImmutableTable.Builder()
- .put(1, new TableFieldSchema().setName("tld"), "fooTld")
- .put(1, new TableFieldSchema().setName("fooField"), "12")
- .put(1, new TableFieldSchema().setName("barField"), "34")
- .put(2, new TableFieldSchema().setName("tld"), "barTld")
- .put(2, new TableFieldSchema().setName("fooField"), "56")
- .put(2, new TableFieldSchema().setName("barField"), "78")
- .build();
- when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(activityReportTable);
- IcannReportingStagingAction action = createAction(ReportType.ACTIVITY);
+ public void testAbsentReportingMode_stagesBothReports() throws Exception {
+ IcannReportingStagingAction action =
+ createAction(ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS));
action.run();
-
- String expectedReport1 = "fooField,barField\r\n12,34";
- String expectedReport2 = "fooField,barField\r\n56,78";
- byte[] generatedFile1 =
- readGcsFile(
- gcsService,
- new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv"));
- assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
- byte[] generatedFile2 =
- readGcsFile(
- gcsService,
- new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv"));
- assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
- }
-
- @Test
- public void testRunSuccess_transactionsReport() throws Exception {
- setUpBigquery();
- /*
- The fake table result looks like:
- tld registrar field
- 1 fooTld reg1 10
- 2 fooTld reg2 20
- 3 barTld reg1 30
- */
- ImmutableTable transactionReportTable =
- new ImmutableTable.Builder()
- .put(1, new TableFieldSchema().setName("tld"), "fooTld")
- .put(1, new TableFieldSchema().setName("registrar"), "reg1")
- .put(1, new TableFieldSchema().setName("field"), "10")
- .put(2, new TableFieldSchema().setName("tld"), "fooTld")
- .put(2, new TableFieldSchema().setName("registrar"), "reg2")
- .put(2, new TableFieldSchema().setName("field"), "20")
- .put(3, new TableFieldSchema().setName("tld"), "barTld")
- .put(3, new TableFieldSchema().setName("registrar"), "reg1")
- .put(3, new TableFieldSchema().setName("field"), "30")
- .build();
- when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(transactionReportTable);
- IcannReportingStagingAction action = createAction(ReportType.TRANSACTIONS);
- action.reportType = ReportType.TRANSACTIONS;
- action.run();
-
- String expectedReport1 = "registrar,field\r\nreg1,10\r\nreg2,20";
- String expectedReport2 = "registrar,field\r\nreg1,30";
- byte[] generatedFile1 =
- readGcsFile(
- gcsService,
- new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv"));
- assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
- byte[] generatedFile2 =
- readGcsFile(
- gcsService,
- new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv"));
- assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
- }
-
- private ListenableFuture fakeFuture() {
- return new ListenableFuture() {
- @Override
- public void addListener(Runnable runnable, Executor executor) {
- // No-op
- }
-
- @Override
- public boolean cancel(boolean mayInterruptIfRunning) {
- return false;
- }
-
- @Override
- public boolean isCancelled() {
- return false;
- }
-
- @Override
- public boolean isDone() {
- return false;
- }
-
- @Override
- public DestinationTable get() throws InterruptedException, ExecutionException {
- return null;
- }
-
- @Override
- public DestinationTable get(long timeout, TimeUnit unit)
- throws InterruptedException, ExecutionException, TimeoutException {
- return null;
- }
- };
+ verify(stager).stageReports(ReportType.ACTIVITY);
+ verify(stager).stageReports(ReportType.TRANSACTIONS);
+ verify(stager).createAndUploadManifest(ImmutableList.of("a", "b", "c", "d"));
}
}
+
diff --git a/javatests/google/registry/reporting/IcannReportingUploadActionTest.java b/javatests/google/registry/reporting/IcannReportingUploadActionTest.java
index 2fac153b9..8e9f5d8f0 100644
--- a/javatests/google/registry/reporting/IcannReportingUploadActionTest.java
+++ b/javatests/google/registry/reporting/IcannReportingUploadActionTest.java
@@ -16,10 +16,6 @@ package google.registry.reporting;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
-import static com.google.common.truth.Truth8.assertThat;
-import static google.registry.reporting.IcannReportingModule.ReportType.ACTIVITY;
-import static google.registry.reporting.IcannReportingModule.ReportType.TRANSACTIONS;
-import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.testing.GcsTestingUtils.writeGcsFile;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.mockito.Mockito.doThrow;
@@ -38,7 +34,6 @@ import google.registry.testing.FakeResponse;
import google.registry.testing.FakeSleeper;
import google.registry.util.Retrier;
import java.io.IOException;
-import java.util.Optional;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -52,37 +47,37 @@ public class IcannReportingUploadActionTest {
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build();
private static final byte[] FAKE_PAYLOAD = "test,csv\n13,37".getBytes(UTF_8);
+ private static final byte[] MANIFEST_PAYLOAD = "test-transactions-201706.csv\n".getBytes(UTF_8);
private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class);
private final FakeResponse response = new FakeResponse();
private final GcsService gcsService = GcsServiceFactory.createGcsService();
private final GcsFilename reportFile =
new GcsFilename("basin/icann/monthly/2017-06", "test-transactions-201706.csv");
+ private final GcsFilename manifestFile =
+ new GcsFilename("basin/icann/monthly/2017-06", "MANIFEST.txt");
private IcannReportingUploadAction createAction() {
IcannReportingUploadAction action = new IcannReportingUploadAction();
action.icannReporter = mockReporter;
action.gcsUtils = new GcsUtils(gcsService, 1024);
action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
- action.yearMonth = "2017-06";
- action.reportType = TRANSACTIONS;
- action.subdir = Optional.empty();
- action.tld = "test";
- action.icannReportingBucket = "basin";
+ action.subdir = "icann/monthly/2017-06";
+ action.reportingBucket = "basin";
action.response = response;
return action;
}
@Before
public void before() throws Exception {
- createTld("test");
writeGcsFile(gcsService, reportFile, FAKE_PAYLOAD);
+ writeGcsFile(gcsService, manifestFile, MANIFEST_PAYLOAD);
}
@Test
public void testSuccess() throws Exception {
IcannReportingUploadAction action = createAction();
action.run();
- verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+ verify(mockReporter).send(FAKE_PAYLOAD, "test-transactions-201706.csv");
verifyNoMoreInteractions(mockReporter);
assertThat(((FakeResponse) action.response).getPayload())
.isEqualTo("OK, sending: test,csv\n13,37");
@@ -94,90 +89,26 @@ public class IcannReportingUploadActionTest {
doThrow(new IOException("Expected exception."))
.doNothing()
.when(mockReporter)
- .send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+ .send(FAKE_PAYLOAD, "test-transactions-201706.csv");
action.run();
- verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+ verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test-transactions-201706.csv");
verifyNoMoreInteractions(mockReporter);
assertThat(((FakeResponse) action.response).getPayload())
.isEqualTo("OK, sending: test,csv\n13,37");
}
@Test
- public void testFail_NonexisistentTld() throws Exception {
+ public void testFail_FileNotFound() throws Exception {
IcannReportingUploadAction action = createAction();
- action.tld = "invalidTld";
+ action.subdir = "somewhere/else";
try {
action.run();
- assertWithMessage("Expected IllegalArgumentException to be thrown").fail();
+ assertWithMessage("Expected IllegalStateException to be thrown").fail();
} catch (IllegalArgumentException expected) {
assertThat(expected)
.hasMessageThat()
- .isEqualTo("TLD invalidTld does not exist");
+ .isEqualTo("Object MANIFEST.txt in bucket basin/somewhere/else not found");
}
}
-
- @Test
- public void testFail_InvalidYearMonth() throws Exception {
- IcannReportingUploadAction action = createAction();
- action.yearMonth = "2017-3";
- try {
- action.run();
- assertWithMessage("Expected IllegalStateException to be thrown").fail();
- } catch (IllegalStateException expected) {
- assertThat(expected)
- .hasMessageThat()
- .isEqualTo("yearMonth must be in YYYY-MM format, got 2017-3 instead.");
- }
- }
-
- @Test
- public void testFail_InvalidSubdir() throws Exception {
- IcannReportingUploadAction action = createAction();
- action.subdir = Optional.of("/subdir/with/slash");
- try {
- action.run();
- assertWithMessage("Expected IllegalStateException to be thrown").fail();
- } catch (IllegalStateException expected) {
- assertThat(expected)
- .hasMessageThat()
- .isEqualTo(
- "subdir must not start or end with a \"/\", got /subdir/with/slash instead.");
- }
- }
-
- @Test
- public void testFail_FileNotFound() throws Exception {
- IcannReportingUploadAction action = createAction();
- action.yearMonth = "1234-56";
- try {
- action.run();
- assertWithMessage("Expected IllegalStateException to be thrown").fail();
- } catch (IllegalStateException expected) {
- assertThat(expected)
- .hasMessageThat()
- .isEqualTo(
- "ICANN report object test-transactions-123456.csv "
- + "in bucket basin/icann/monthly/1234-56 not found");
- }
- }
-
- @Test
- public void testSuccess_CreateFilename() throws Exception{
- assertThat(IcannReportingUploadAction.createFilename("test", "2017-06", ACTIVITY))
- .isEqualTo("test-activity-201706.csv");
- assertThat(IcannReportingUploadAction.createFilename("foo", "1234-56", TRANSACTIONS))
- .isEqualTo("foo-transactions-123456.csv");
- }
-
- @Test
- public void testSuccess_CreateBucketname() throws Exception{
- assertThat(
- IcannReportingUploadAction
- .createReportingBucketName("gs://my-reporting", Optional.empty(), "2017-06"))
- .isEqualTo("gs://my-reporting/icann/monthly/2017-06");
- assertThat(
- IcannReportingUploadAction
- .createReportingBucketName("gs://my-reporting", Optional.of("manual"), "2017-06"))
- .isEqualTo("gs://my-reporting/manual");
- }
}
+
diff --git a/javatests/google/registry/reporting/ReportingUtilsTest.java b/javatests/google/registry/reporting/ReportingUtilsTest.java
new file mode 100644
index 000000000..b30906fcf
--- /dev/null
+++ b/javatests/google/registry/reporting/ReportingUtilsTest.java
@@ -0,0 +1,40 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import google.registry.reporting.IcannReportingModule.ReportType;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link google.registry.reporting.ReportingUtils}. */
+@RunWith(JUnit4.class)
+public class ReportingUtilsTest {
+ @Test
+ public void testCreateFilename_success() {
+ assertThat(ReportingUtils.createFilename("test", "2017-06", ReportType.ACTIVITY))
+ .isEqualTo("test-activity-201706.csv");
+ assertThat(ReportingUtils.createFilename("foo", "2017-06", ReportType.TRANSACTIONS))
+ .isEqualTo("foo-transactions-201706.csv");
+ }
+
+ @Test
+ public void testCreateBucketName_success() {
+ assertThat(ReportingUtils.createReportingBucketName("gs://domain-registry-basin", "my/subdir"))
+ .isEqualTo("gs://domain-registry-basin/my/subdir");
+ }
+}
diff --git a/javatests/google/registry/reporting/TransactionsReportingQueryBuilderTest.java b/javatests/google/registry/reporting/TransactionsReportingQueryBuilderTest.java
index 65b2b69c4..64368187e 100644
--- a/javatests/google/registry/reporting/TransactionsReportingQueryBuilderTest.java
+++ b/javatests/google/registry/reporting/TransactionsReportingQueryBuilderTest.java
@@ -15,7 +15,6 @@
package google.registry.reporting;
import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -30,7 +29,7 @@ public class TransactionsReportingQueryBuilderTest {
private TransactionsReportingQueryBuilder getQueryBuilder() {
TransactionsReportingQueryBuilder queryBuilder = new TransactionsReportingQueryBuilder();
- queryBuilder.yearMonth = "2017-06";
+ queryBuilder.yearMonth = "2017-09";
queryBuilder.projectId = "domain-registry-alpha";
return queryBuilder;
}
@@ -41,13 +40,12 @@ public class TransactionsReportingQueryBuilderTest {
assertThat(queryBuilder.getReportQuery())
.isEqualTo(
"#standardSQL\nSELECT * FROM "
- + "`domain-registry-alpha.icann_reporting.transactions_report_aggregation_201706`");
+ + "`domain-registry-alpha.icann_reporting.transactions_report_aggregation_201709`");
}
@Test
public void testIntermediaryQueryMatch() throws IOException {
- TransactionsReportingQueryBuilder queryBuilder = getQueryBuilder();
- ImmutableList queryNames =
+ ImmutableList expectedQueryNames =
ImmutableList.of(
TransactionsReportingQueryBuilder.TRANSACTIONS_REPORT_AGGREGATION,
TransactionsReportingQueryBuilder.REGISTRAR_IANA_ID,
@@ -57,9 +55,10 @@ public class TransactionsReportingQueryBuilderTest {
TransactionsReportingQueryBuilder.TRANSACTION_TRANSFER_LOSING,
TransactionsReportingQueryBuilder.ATTEMPTED_ADDS);
+ TransactionsReportingQueryBuilder queryBuilder = getQueryBuilder();
ImmutableMap actualQueries = queryBuilder.getViewQueryMap();
- for (String queryName : queryNames) {
- String actualTableName = String.format("%s_201706", queryName);
+ for (String queryName : expectedQueryNames) {
+ String actualTableName = String.format("%s_201709", queryName);
String testFilename = String.format("%s_test.sql", queryName);
assertThat(actualQueries.get(actualTableName))
.isEqualTo(ReportingTestData.getString(testFilename));
diff --git a/javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql b/javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql
index a0dbcda86..9f8097c2a 100644
--- a/javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql
+++ b/javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql
@@ -19,8 +19,6 @@
SELECT
RealTlds.tld AS tld,
SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars,
- SUM(IF(metricName = 'ramp-up-registrars', count, 0)) AS ramp_up_registrars,
- SUM(IF(metricName = 'pre-ramp-up-registrars', count, 0)) AS pre_ramp_up_registrars,
-- We use the Centralized Zone Data Service.
"CZDS" AS zfa_passwords,
SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries,
@@ -65,7 +63,7 @@ SELECT
-- filter so that only metrics with that TLD or a NULL TLD are counted
-- towards a given TLD.
FROM (
-SELECT tldStr as tld
+SELECT tldStr AS tld
FROM `domain-registry-alpha.latest_datastore_export.Registry`
WHERE tldType = 'REAL'
) as RealTlds
@@ -82,16 +80,16 @@ CROSS JOIN(
SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 as count
UNION ALL
SELECT * FROM
- `domain-registry-alpha.icann_reporting.registrar_operating_status_201706`
+ `domain-registry-alpha.icann_reporting.registrar_operating_status_201709`
UNION ALL
SELECT * FROM
- `domain-registry-alpha.icann_reporting.dns_counts_201706`
+ `domain-registry-alpha.icann_reporting.dns_counts_201709`
UNION ALL
SELECT * FROM
- `domain-registry-alpha.icann_reporting.epp_metrics_201706`
+ `domain-registry-alpha.icann_reporting.epp_metrics_201709`
UNION ALL
SELECT * FROM
- `domain-registry-alpha.icann_reporting.whois_counts_201706`
+ `domain-registry-alpha.icann_reporting.whois_counts_201709`
-- END INTERMEDIARY DATA SOURCES --
)) AS TldMetrics
WHERE RealTlds.tld = TldMetrics.tld OR TldMetrics.tld IS NULL
diff --git a/javatests/google/registry/reporting/testdata/attempted_adds_test.sql b/javatests/google/registry/reporting/testdata/attempted_adds_test.sql
index b9423f2f2..031b701c7 100644
--- a/javatests/google/registry/reporting/testdata/attempted_adds_test.sql
+++ b/javatests/google/registry/reporting/testdata/attempted_adds_test.sql
@@ -52,8 +52,8 @@ FROM (
FROM
`domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
WHERE _TABLE_SUFFIX
- BETWEEN '20170601'
- AND '20170630')
+ BETWEEN '20170901'
+ AND '20170930')
JOIN UNNEST(logMessage) AS logMessages
-- Look for metadata logs from epp and registrar console requests
WHERE requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr')
diff --git a/javatests/google/registry/reporting/testdata/dns_counts_internal_test.sql b/javatests/google/registry/reporting/testdata/dns_counts_internal_test.sql
new file mode 100644
index 000000000..d75d53ab6
--- /dev/null
+++ b/javatests/google/registry/reporting/testdata/dns_counts_internal_test.sql
@@ -0,0 +1,24 @@
+#standardSQL
+ -- Copyright 2017 The Nomulus Authors. All Rights Reserved.
+ --
+ -- Licensed under the Apache License, Version 2.0 (the "License");
+ -- you may not use this file except in compliance with the License.
+ -- You may obtain a copy of the License at
+ --
+ -- http://www.apache.org/licenses/LICENSE-2.0
+ --
+ -- Unless required by applicable law or agreed to in writing, software
+ -- distributed under the License is distributed on an "AS IS" BASIS,
+ -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ -- See the License for the specific language governing permissions and
+ -- limitations under the License.
+
+ -- Retrieve per-TLD DNS query counts.
+
+ -- This is a hack to enable using DNS counts from the internal-only #plx
+ -- workflow. See other references to b/67301320 in the codebase to see the
+ -- full extent of the hackery.
+ -- TODO(b/67301320): Delete this when we can make open-source DNS metrics.
+
+SELECT *
+FROM `domain-registry-alpha.icann_reporting.dns_counts_from_plx`
diff --git a/javatests/google/registry/reporting/testdata/dns_counts_test.sql b/javatests/google/registry/reporting/testdata/dns_counts_test.sql
index 62d822a6b..85293dbb3 100644
--- a/javatests/google/registry/reporting/testdata/dns_counts_test.sql
+++ b/javatests/google/registry/reporting/testdata/dns_counts_test.sql
@@ -15,14 +15,13 @@
-- Query for DNS metrics.
- -- This is a no-op until after we transition to Google Cloud DNS, which
- -- will likely export metrics via Stackdriver.
+ -- You must configure this yourself to enable activity reporting, according
+ -- to whatever metrics your DNS provider makes available. We hope to make
+ -- this available in the open-source build in the near future.
SELECT
- -- DNS metrics apply to all tlds, which requires the 'null' magic value.
STRING(NULL) AS tld,
metricName,
- -- TODO(b/63388735): Change this to actually query Google Cloud DNS when ready.
-1 AS count
FROM ((
SELECT 'dns-udp-queries' AS metricName)
diff --git a/javatests/google/registry/reporting/testdata/epp_metrics_test.sql b/javatests/google/registry/reporting/testdata/epp_metrics_test.sql
index c3bc7f9ad..62b74e172 100644
--- a/javatests/google/registry/reporting/testdata/epp_metrics_test.sql
+++ b/javatests/google/registry/reporting/testdata/epp_metrics_test.sql
@@ -39,7 +39,7 @@ FROM (
-- Extract the logged JSON payload.
REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
AS json
- FROM `domain-registry-alpha.icann_reporting.monthly_logs_201706` AS logs
+ FROM `domain-registry-alpha.icann_reporting.monthly_logs_201709` AS logs
JOIN
UNNEST(logs.logMessage) AS logMessage
WHERE
diff --git a/javatests/google/registry/reporting/testdata/monthly_logs_test.sql b/javatests/google/registry/reporting/testdata/monthly_logs_test.sql
index 70a5495b7..86660f304 100644
--- a/javatests/google/registry/reporting/testdata/monthly_logs_test.sql
+++ b/javatests/google/registry/reporting/testdata/monthly_logs_test.sql
@@ -27,4 +27,4 @@ SELECT
FROM
`domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
WHERE
- _TABLE_SUFFIX BETWEEN '20170601' AND '20170630'
+ _TABLE_SUFFIX BETWEEN '20170901' AND '20170930'
diff --git a/javatests/google/registry/reporting/testdata/registrar_iana_id_test.sql b/javatests/google/registry/reporting/testdata/registrar_iana_id_test.sql
index b1fed99f1..7b6db0947 100644
--- a/javatests/google/registry/reporting/testdata/registrar_iana_id_test.sql
+++ b/javatests/google/registry/reporting/testdata/registrar_iana_id_test.sql
@@ -26,5 +26,5 @@ FROM
UNNEST(allowedTlds) as allowed_tlds
WHERE (type = 'REAL' OR type = 'INTERNAL')
-- Filter out prober data
-AND NOT ENDS_WITH(allowed_tlds, "test")
+AND NOT ENDS_WITH(allowed_tlds, ".test")
ORDER BY tld, registrarName
diff --git a/javatests/google/registry/reporting/testdata/registrar_operating_status_test.sql b/javatests/google/registry/reporting/testdata/registrar_operating_status_test.sql
index bf1a3ed94..01e69ff2c 100644
--- a/javatests/google/registry/reporting/testdata/registrar_operating_status_test.sql
+++ b/javatests/google/registry/reporting/testdata/registrar_operating_status_test.sql
@@ -23,5 +23,5 @@ SELECT
FROM
`domain-registry-alpha.latest_datastore_export.Registrar`
WHERE
- type = 'REAL'
+ (type = 'REAL' OR type = 'INTERNAL')
GROUP BY metricName
diff --git a/javatests/google/registry/reporting/testdata/total_domains_test.sql b/javatests/google/registry/reporting/testdata/total_domains_test.sql
index 0d6db99fb..af516f0bf 100644
--- a/javatests/google/registry/reporting/testdata/total_domains_test.sql
+++ b/javatests/google/registry/reporting/testdata/total_domains_test.sql
@@ -32,7 +32,7 @@ JOIN
ON
currentSponsorClientId = registrar_table.__key__.name
WHERE
- domain_table._d = "DomainResource"
- AND (registrar_table.type = "REAL" OR registrar_table.type = "INTERNAL")
+ domain_table._d = 'DomainResource'
+ AND (registrar_table.type = 'REAL' OR registrar_table.type = 'INTERNAL')
GROUP BY tld, registrarName
ORDER BY tld, registrarName
diff --git a/javatests/google/registry/reporting/testdata/total_nameservers_test.sql b/javatests/google/registry/reporting/testdata/total_nameservers_test.sql
index 4c9f9e4bf..0f9839b87 100644
--- a/javatests/google/registry/reporting/testdata/total_nameservers_test.sql
+++ b/javatests/google/registry/reporting/testdata/total_nameservers_test.sql
@@ -45,12 +45,12 @@ JOIN (
`domain-registry-alpha.latest_datastore_export.DomainBase`,
UNNEST(nsHosts) AS hosts
WHERE _d = 'DomainResource'
- AND creationTime <= TIMESTAMP("2017-06-30 23:59:59")
- AND deletionTime > TIMESTAMP("2017-06-30 23:59:59") ) AS domain_table
+ AND creationTime <= TIMESTAMP("2017-09-30 23:59:59")
+ AND deletionTime > TIMESTAMP("2017-09-30 23:59:59") ) AS domain_table
ON
host_table.__key__.name = domain_table.referencedHostName
-WHERE creationTime <= TIMESTAMP("2017-06-30 23:59:59")
-AND deletionTime > TIMESTAMP("2017-06-30 23:59:59")
+WHERE creationTime <= TIMESTAMP("2017-09-30 23:59:59")
+AND deletionTime > TIMESTAMP("2017-09-30 23:59:59")
GROUP BY tld, registrarName
ORDER BY tld, registrarName
diff --git a/javatests/google/registry/reporting/testdata/transaction_counts_test.sql b/javatests/google/registry/reporting/testdata/transaction_counts_test.sql
index 78e235ad6..250910e5a 100644
--- a/javatests/google/registry/reporting/testdata/transaction_counts_test.sql
+++ b/javatests/google/registry/reporting/testdata/transaction_counts_test.sql
@@ -63,10 +63,8 @@ FROM (
WHERE entries.domainTransactionRecords IS NOT NULL )
-- Only look at this month's data
WHERE reportingTime
- BETWEEN TIMESTAMP('2017-06-01 00:00:00')
- AND TIMESTAMP('2017-06-30 23:59:59')
- -- Ignore prober data
- AND NOT ENDS_WITH(tld, "test")
+ BETWEEN TIMESTAMP('2017-09-01 00:00:00')
+ AND TIMESTAMP('2017-09-30 23:59:59')
GROUP BY
tld,
clientId,
diff --git a/javatests/google/registry/reporting/testdata/transaction_transfer_losing_test.sql b/javatests/google/registry/reporting/testdata/transaction_transfer_losing_test.sql
index 82ff14824..2264de1fe 100644
--- a/javatests/google/registry/reporting/testdata/transaction_transfer_losing_test.sql
+++ b/javatests/google/registry/reporting/testdata/transaction_transfer_losing_test.sql
@@ -63,10 +63,8 @@ FROM (
WHERE entries.domainTransactionRecords IS NOT NULL )
-- Only look at this month's data
WHERE reportingTime
- BETWEEN TIMESTAMP('2017-06-01 00:00:00')
- AND TIMESTAMP('2017-06-30 23:59:59')
- -- Ignore prober data
- AND NOT ENDS_WITH(tld, "test")
+ BETWEEN TIMESTAMP('2017-09-01 00:00:00')
+ AND TIMESTAMP('2017-09-30 23:59:59')
GROUP BY
tld,
clientId,
diff --git a/javatests/google/registry/reporting/testdata/transactions_report_aggregation_test.sql b/javatests/google/registry/reporting/testdata/transactions_report_aggregation_test.sql
index 795285f0f..523b3f702 100644
--- a/javatests/google/registry/reporting/testdata/transactions_report_aggregation_test.sql
+++ b/javatests/google/registry/reporting/testdata/transactions_report_aggregation_test.sql
@@ -20,7 +20,8 @@
SELECT
registrars.tld as tld,
- registrars.registrar_name as registrar_name,
+ -- Surround registrar names with quotes to handle names containing a comma.
+ FORMAT("\"%s\"", registrars.registrar_name) as registrar_name,
registrars.iana_id as iana_id,
SUM(IF(metrics.metricName = 'TOTAL_DOMAINS', metrics.metricValue, 0)) AS total_domains,
SUM(IF(metrics.metricName = 'TOTAL_NAMESERVERS', metrics.metricValue, 0)) AS total_nameservers,
@@ -62,26 +63,33 @@ SELECT
0 AS agp_exemptions_granted,
0 AS agp_exempted_domains,
SUM(IF(metrics.metricName = 'ATTEMPTED_ADDS', metrics.metricValue, 0)) AS attempted_adds
-FROM (
- SELECT *
- FROM `domain-registry-alpha.icann_reporting.registrar_iana_id_201706`) AS registrars
+FROM
+-- Only produce reports for real TLDs
+(SELECT tldStr AS tld
+ FROM `domain-registry-alpha.latest_datastore_export.Registry`
+ WHERE tldType = 'REAL') AS registries
+JOIN
+(SELECT *
+ FROM `domain-registry-alpha.icann_reporting.registrar_iana_id_201709`)
+ AS registrars
+ON registries.tld = registrars.tld
-- We LEFT JOIN to produce reports even if the registrar made no transactions
LEFT OUTER JOIN (
-- Gather all intermediary data views
SELECT *
- FROM `domain-registry-alpha.icann_reporting.total_domains_201706`
+ FROM `domain-registry-alpha.icann_reporting.total_domains_201709`
UNION ALL
SELECT *
- FROM `domain-registry-alpha.icann_reporting.total_nameservers_201706`
+ FROM `domain-registry-alpha.icann_reporting.total_nameservers_201709`
UNION ALL
SELECT *
- FROM `domain-registry-alpha.icann_reporting.transaction_counts_201706`
+ FROM `domain-registry-alpha.icann_reporting.transaction_counts_201709`
UNION ALL
SELECT *
- FROM `domain-registry-alpha.icann_reporting.transaction_transfer_losing_201706`
+ FROM `domain-registry-alpha.icann_reporting.transaction_transfer_losing_201709`
UNION ALL
SELECT *
- FROM `domain-registry-alpha.icann_reporting.attempted_adds_201706` ) AS metrics
+ FROM `domain-registry-alpha.icann_reporting.attempted_adds_201709` ) AS metrics
-- Join on tld and registrar name
ON registrars.tld = metrics.tld
AND registrars.registrar_name = metrics.registrar_name
diff --git a/javatests/google/registry/reporting/testdata/whois_counts_test.sql b/javatests/google/registry/reporting/testdata/whois_counts_test.sql
index 55ee56f04..8aa1caea0 100644
--- a/javatests/google/registry/reporting/testdata/whois_counts_test.sql
+++ b/javatests/google/registry/reporting/testdata/whois_counts_test.sql
@@ -26,7 +26,7 @@ SELECT
END AS metricName,
COUNT(requestPath) AS count
FROM
- `domain-registry-alpha.icann_reporting.monthly_logs_201706`
+ `domain-registry-alpha.icann_reporting.monthly_logs_201709`
GROUP BY
metricName
HAVING
diff --git a/javatests/google/registry/request/RequestParametersTest.java b/javatests/google/registry/request/RequestParametersTest.java
index 743d8dc85..3921a2444 100644
--- a/javatests/google/registry/request/RequestParametersTest.java
+++ b/javatests/google/registry/request/RequestParametersTest.java
@@ -19,6 +19,7 @@ import static com.google.common.truth.Truth8.assertThat;
import static google.registry.request.RequestParameters.extractBooleanParameter;
import static google.registry.request.RequestParameters.extractEnumParameter;
import static google.registry.request.RequestParameters.extractOptionalDatetimeParameter;
+import static google.registry.request.RequestParameters.extractOptionalEnumParameter;
import static google.registry.request.RequestParameters.extractOptionalParameter;
import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
import static google.registry.request.RequestParameters.extractRequiredParameter;
@@ -147,6 +148,25 @@ public class RequestParametersTest {
extractEnumParameter(req, Club.class, "spin");
}
+ @Test
+ public void testOptionalExtractEnumValue_givenValue_returnsValue() throws Exception {
+ when(req.getParameter("spin")).thenReturn("DANCE");
+ assertThat(extractOptionalEnumParameter(req, Club.class, "spin")).hasValue(Club.DANCE);
+ }
+
+ @Test
+ public void testOptionalExtractEnumValue_noValue_returnsAbsent() throws Exception {
+ when(req.getParameter("spin")).thenReturn("");
+ assertThat(extractOptionalEnumParameter(req, Club.class, "spin")).isEmpty();
+ }
+
+ @Test
+ public void testOptionalExtractEnumValue_nonExistentValue_throwsBadRequest() throws Exception {
+ when(req.getParameter("spin")).thenReturn("sing");
+ thrown.expect(BadRequestException.class, "spin");
+ extractOptionalEnumParameter(req, Club.class, "spin");
+ }
+
@Test
public void testExtractRequiredDatetimeParameter_correctValue_works() throws Exception {
when(req.getParameter("timeParam")).thenReturn("2015-08-27T13:25:34.123Z");