Prepare ICANN reporting for production

This originally started as a small change, but quickly grew into a major refactor as I realized the original parameter structure wasn't conducive to a cron task and manual re-runs.

The changes are as follows:
1. Adds DNS metrics to activity reports, thanks to Nick's work with the Zoneman Dremel -> #plx workflow.
2. Surrounds registrar names in transactions reports with quotes, to escape possible commas.
3. Factors out the report generation logic into IcannReportingStager.
4. Assigns default values to the three main parameters
  - yearMonth defaults to the previous month
  - subdir defaults to "icann/monthly/yearMonth", i.e. "gs://domain-registry-reporting/icann/monthly/yyyy-MM"
  - reportType defaults to both reports
5. Adds "Total" row generation logic to transactions reports
  - This was a previously overlooked requirement.
6. Adds "MANIFEST.txt" generation and upload logic.
  - The MANIFEST lists out which files need to be uploaded in the subdirectory.
7. Increases urlfetch timeout from 5s to 10s in backend tasks.
  - Backend tasks should be more latency tolerant anyway, and this reduces the number of incorrect timeouts we see for services like Bigquery which might take some time to respond.

TESTED=Extensive testing in alpha, and ran FOSS test.
TODO: send out an e-mail for report generation and upload, and add reporting to cron.xml

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=172738344

java/google/registry/reporting/ActivityReportingQueryBuilder.java

@@ -36,12 +36,12 @@ import org.joda.time.format.DateTimeFormatter;
 public final class ActivityReportingQueryBuilder implements QueryBuilder {
 
   // Names for intermediary tables for overall activity reporting query.
-  static final String ACTIVITY_REPORT_AGGREGATION = "activity_report_aggregation";
-  static final String MONTHLY_LOGS = "monthly_logs";
   static final String REGISTRAR_OPERATING_STATUS = "registrar_operating_status";
   static final String DNS_COUNTS = "dns_counts";
+  static final String MONTHLY_LOGS = "monthly_logs";
   static final String EPP_METRICS = "epp_metrics";
   static final String WHOIS_COUNTS = "whois_counts";
+  static final String ACTIVITY_REPORT_AGGREGATION = "activity_report_aggregation";
 
   @Inject @Config("projectId") String projectId;
   @Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
@@ -81,7 +81,6 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
             .build();
     queriesBuilder.put(getTableName(REGISTRAR_OPERATING_STATUS), operationalRegistrarsQuery);
 
-    // TODO(b/62626209): Make this use the CloudDNS counts instead.
     String dnsCountsQuery =
         SqlTemplate.create(getQueryFromFile("dns_counts.sql")).build();
     queriesBuilder.put(getTableName(DNS_COUNTS), dnsCountsQuery);
@@ -135,6 +134,7 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
     return queriesBuilder.build();
   }
 
+
   /** Returns the table name of the query, suffixed with the yearMonth in _YYYYMM format. */
   private String getTableName(String queryName) {
     return String.format("%s_%s", queryName, yearMonth.replace("-", ""));
@@ -149,4 +149,3 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
     return Resources.getResource(ActivityReportingQueryBuilder.class, "sql/" + filename);
   }
 }
-

java/google/registry/reporting/IcannHttpReporter.java

@@ -14,7 +14,9 @@
 
 package google.registry.reporting;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.net.MediaType.CSV_UTF_8;
+import static google.registry.model.registry.Registries.assertTldExists;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.client.http.ByteArrayContent;
@@ -23,6 +25,8 @@ import com.google.api.client.http.HttpHeaders;
 import com.google.api.client.http.HttpRequest;
 import com.google.api.client.http.HttpResponse;
 import com.google.api.client.http.HttpTransport;
+import com.google.common.base.Ascii;
+import com.google.common.base.Splitter;
 import com.google.common.io.ByteStreams;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
@@ -35,16 +39,22 @@ import google.registry.xjc.iirdea.XjcIirdeaResult;
 import google.registry.xml.XmlException;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.util.List;
 import javax.inject.Inject;
+import org.joda.time.YearMonth;
+import org.joda.time.format.DateTimeFormat;
 
 /**
  * Class that uploads a CSV file to ICANN's endpoint via an HTTP PUT call.
  *
- * <p> It uses basic authorization credentials as specified in the "Registry Interfaces" draft.
+ * <p>It uses basic authorization credentials as specified in the "Registry Interfaces" draft.
+ *
+ * <p>Note that there's a lot of hard-coded logic extracting parameters from the report filenames.
+ * These are safe, as long as they follow the tld-reportType-yearMonth.csv filename format.
  *
  * @see IcannReportingUploadAction
- * @see <a href=https://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-07#page-9>
+ * @see <a href=https://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-07#page-9>ICANN
- *   ICANN Reporting Specification</a>
+ *     Reporting Specification</a>
  */
 public class IcannHttpReporter {
 
@@ -57,29 +67,24 @@ public class IcannHttpReporter {
   @Inject IcannHttpReporter() {}
 
   /** Uploads {@code reportBytes} to ICANN. */
-  public void send(
+  public void send(byte[] reportBytes, String reportFilename) throws XmlException, IOException {
-      byte[] reportBytes,
+    validateReportFilename(reportFilename);
-      String tld,
+    GenericUrl uploadUrl = new GenericUrl(makeUrl(reportFilename));
-      String yearMonth,
-      ReportType reportType) throws XmlException, IOException {
-    GenericUrl uploadUrl = new GenericUrl(makeUrl(tld, yearMonth, reportType));
     HttpRequest request =
         httpTransport
             .createRequestFactory()
             .buildPutRequest(uploadUrl, new ByteArrayContent(CSV_UTF_8.toString(), reportBytes));
 
     HttpHeaders headers = request.getHeaders();
-    headers.setBasicAuthentication(tld + "_ry", password);
+    headers.setBasicAuthentication(getTld(reportFilename) + "_ry", password);
     headers.setContentType(CSV_UTF_8.toString());
     request.setHeaders(headers);
     request.setFollowRedirects(false);
 
     HttpResponse response = null;
     logger.infofmt(
-        "Sending %s report to %s with content length %s",
+        "Sending report to %s with content length %s",
-        reportType,
+        uploadUrl.toString(), request.getContent().getLength());
-        uploadUrl.toString(),
-        request.getContent().getLength());
     try {
       response = request.execute();
       byte[] content;
@@ -117,9 +122,31 @@ public class IcannHttpReporter {
     return result;
   }
 
-  private String makeUrl(String tld, String yearMonth, ReportType reportType) {
+  /** Verifies a given report filename matches the pattern tld-reportType-yyyyMM.csv. */
-    String urlPrefix = getUrlPrefix(reportType);
+  private void validateReportFilename(String filename) {
-    return String.format("%s/%s/%s", urlPrefix, tld, yearMonth);
+    checkArgument(
+        filename.matches("[a-z0-9.\\-]+-((activity)|(transactions))-[0-9]{6}\\.csv"),
+        "Expected file format: tld-reportType-yyyyMM.csv, got %s instead",
+        filename);
+    assertTldExists(getTld(filename));
+  }
+
+  private String getTld(String filename) {
+    // Extract the TLD, up to second-to-last hyphen in the filename (works with international TLDs)
+    return filename.substring(0, filename.lastIndexOf('-', filename.lastIndexOf('-') - 1));
+  }
+
+  private String makeUrl(String filename) {
+    // Filename is in the format tld-reportType-yearMonth.csv
+    String tld = getTld(filename);
+    // Remove the tld- prefix and csv suffix
+    String remainder = filename.substring(tld.length() + 1, filename.length() - 4);
+    List<String> elements = Splitter.on('-').splitToList(remainder);
+    ReportType reportType = ReportType.valueOf(Ascii.toUpperCase(elements.get(0)));
+    // Re-add hyphen between year and month, because ICANN is inconsistent between filename and URL
+    String yearMonth =
+        YearMonth.parse(elements.get(1), DateTimeFormat.forPattern("yyyyMM")).toString("yyyy-MM");
+    return String.format("%s/%s/%s", getUrlPrefix(reportType), tld, yearMonth);
   }
 
   private String getUrlPrefix(ReportType reportType) {
@@ -131,7 +158,7 @@ public class IcannHttpReporter {
       default:
         throw new IllegalStateException(
             String.format(
-                "Received invalid reportType! Expected ACTIVITY or TRANSACTIONS, got %s.",
+                "Received invalid reportTypes! Expected ACTIVITY or TRANSACTIONS, got %s.",
                 reportType));
     }
   }

java/google/registry/reporting/IcannReportingModule.java

@@ -14,22 +14,24 @@
 
 package google.registry.reporting;
 
-import static google.registry.request.RequestParameters.extractEnumParameter;
+import static google.registry.request.RequestParameters.extractOptionalEnumParameter;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
-import static google.registry.request.RequestParameters.extractRequiredParameter;
 
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.MoreExecutors;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.bigquery.BigqueryConnection;
+import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.Parameter;
+import google.registry.util.Clock;
 import java.util.Optional;
-import java.util.concurrent.Executors;
 import javax.servlet.http.HttpServletRequest;
 import org.joda.time.Duration;
+import org.joda.time.format.DateTimeFormat;
 
 /** Module for dependencies required by ICANN monthly transactions/activity reporting. */
 @Module
@@ -41,43 +43,78 @@ public final class IcannReportingModule {
     ACTIVITY
   }
 
+  static final String PARAM_OPTIONAL_YEAR_MONTH = "yearMonthOptional";
   static final String PARAM_YEAR_MONTH = "yearMonth";
-  static final String PARAM_REPORT_TYPE = "reportType";
+  static final String PARAM_OPTIONAL_SUBDIR = "subdirOptional";
   static final String PARAM_SUBDIR = "subdir";
+  static final String PARAM_REPORT_TYPE = "reportType";
   static final String ICANN_REPORTING_DATA_SET = "icann_reporting";
   static final String DATASTORE_EXPORT_DATA_SET = "latest_datastore_export";
-  private static final String BIGQUERY_SCOPE =  "https://www.googleapis.com/auth/bigquery";
+  static final String MANIFEST_FILE_NAME = "MANIFEST.txt";
+  private static final String DEFAULT_SUBDIR = "icann/monthly";
+  private static final String BIGQUERY_SCOPE = "https://www.googleapis.com/auth/cloud-platform";
 
+  /** Extracts an optional yearMonth in yyyy-MM format from the request. */
+  @Provides
+  @Parameter(PARAM_OPTIONAL_YEAR_MONTH)
+  static Optional<String> provideYearMonthOptional(HttpServletRequest req) {
+    return extractOptionalParameter(req, PARAM_YEAR_MONTH);
+  }
+
+  /** Provides the yearMonth in yyyy-MM format, defaults to one month prior to run time. */
   @Provides
   @Parameter(PARAM_YEAR_MONTH)
-  static String provideYearMonth(HttpServletRequest req) {
+  static String provideYearMonth(
-    return extractRequiredParameter(req, PARAM_YEAR_MONTH);
+      @Parameter(PARAM_OPTIONAL_YEAR_MONTH) Optional<String> yearMonthOptional, Clock clock) {
+    String yearMonth =
+        yearMonthOptional.orElse(
+            DateTimeFormat.forPattern("yyyy-MM").print(clock.nowUtc().minusMonths(1)));
+    if (!yearMonth.matches("[0-9]{4}-[0-9]{2}")) {
+      throw new BadRequestException(
+          String.format("yearMonth must be in yyyy-MM format, got %s instead", yearMonth));
+    }
+    return yearMonth;
   }
 
+  /** Provides an optional subdirectory to store/upload reports to, extracted from the request. */
   @Provides
-  @Parameter(PARAM_REPORT_TYPE)
+  @Parameter(PARAM_OPTIONAL_SUBDIR)
-  static ReportType provideReportType(HttpServletRequest req) {
+  static Optional<String> provideSubdirOptional(HttpServletRequest req) {
-    return extractEnumParameter(req, ReportType.class, PARAM_REPORT_TYPE);
-  }
-
-  @Provides
-  @Parameter(PARAM_SUBDIR)
-  static Optional<String> provideSubdir(HttpServletRequest req) {
     return extractOptionalParameter(req, PARAM_SUBDIR);
   }
 
+  /** Provides the subdirectory to store/upload reports to, defaults to icann/monthly/yearMonth. */
   @Provides
-  static QueryBuilder provideQueryBuilder(
+  @Parameter(PARAM_SUBDIR)
-      @Parameter(PARAM_REPORT_TYPE) ReportType reportType,
+  static String provideSubdir(
-      ActivityReportingQueryBuilder activityBuilder,
+      @Parameter(PARAM_OPTIONAL_SUBDIR) Optional<String> subdirOptional,
-      TransactionsReportingQueryBuilder transactionsBuilder) {
+      @Parameter(PARAM_YEAR_MONTH) String yearMonth) {
-    return reportType == ReportType.ACTIVITY ? activityBuilder : transactionsBuilder;
+    String subdir = subdirOptional.orElse(String.format("%s/%s", DEFAULT_SUBDIR, yearMonth));
+    if (subdir.startsWith("/") || subdir.endsWith("/")) {
+      throw new BadRequestException(
+          String.format("subdir must not start or end with a \"/\", got %s instead.", subdir));
+    }
+    return subdir;
+  }
+
+  /** Provides an optional reportType to store/upload reports to, extracted from the request. */
+  @Provides
+  static Optional<ReportType> provideReportTypeOptional(HttpServletRequest req) {
+    return extractOptionalEnumParameter(req, ReportType.class, PARAM_REPORT_TYPE);
+  }
+
+  /** Provides a list of reportTypes specified. If absent, we default to both report types. */
+  @Provides
+  @Parameter(PARAM_REPORT_TYPE)
+  static ImmutableList<ReportType> provideReportTypes(Optional<ReportType> reportTypeOptional) {
+    return reportTypeOptional.map(ImmutableList::of)
+        .orElseGet(() -> ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS));
   }
 
   /**
    * Constructs a BigqueryConnection with default settings.
    *
-   * <p> We use Bigquery to generate activity reports via large aggregate SQL queries.
+   * <p>We use Bigquery to generate ICANN monthly reports via large aggregate SQL queries.
    *
    * @see ActivityReportingQueryBuilder
    * @see google.registry.tools.BigqueryParameters for justifications of defaults.
@@ -87,8 +124,9 @@ public final class IcannReportingModule {
     try {
       GoogleCredential credential = GoogleCredential
           .getApplicationDefault(transport, new JacksonFactory());
-      BigqueryConnection connection = new BigqueryConnection.Builder()
+      BigqueryConnection connection =
-          .setExecutorService(Executors.newFixedThreadPool(20))
+          new BigqueryConnection.Builder()
+              .setExecutorService(MoreExecutors.newDirectExecutorService())
               .setCredential(credential.createScoped(ImmutableList.of(BIGQUERY_SCOPE)))
               .setDatasetId(ICANN_REPORTING_DATA_SET)
               .setOverwrite(true)
@@ -101,3 +139,4 @@ public final class IcannReportingModule {
     }
   }
 }
+

java/google/registry/reporting/IcannReportingStager.java

@@ -0,0 +1,264 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static google.registry.reporting.IcannReportingModule.MANIFEST_FILE_NAME;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.api.services.bigquery.model.TableFieldSchema;
+import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableTable;
+import com.google.common.collect.ListMultimap;
+import google.registry.bigquery.BigqueryConnection;
+import google.registry.bigquery.BigqueryUtils.TableType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
+import google.registry.reporting.IcannReportingModule.ReportType;
+import google.registry.request.Parameter;
+import google.registry.util.FormattingLogger;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+import javax.inject.Inject;
+
+/**
+ * Class containing methods for staging ICANN monthly reports on GCS.
+ *
+ * <p>The main entrypoint is stageReports, which generates a given type of reports.
+ */
+public class IcannReportingStager {
+
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+
+  @Inject @Config("icannReportingBucket") String reportingBucket;
+  @Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
+
+  @Inject
+  @Parameter(IcannReportingModule.PARAM_SUBDIR)
+  String subdir;
+
+  @Inject ActivityReportingQueryBuilder activityQueryBuilder;
+  @Inject TransactionsReportingQueryBuilder transactionsQueryBuilder;
+  @Inject GcsUtils gcsUtils;
+  @Inject BigqueryConnection bigquery;
+
+  @Inject
+  IcannReportingStager() {}
+
+  /**
+   * Creates and stores reports of a given type on GCS.
+   *
+   * <p>This is factored out to facilitate choosing which reports to upload,
+   */
+  ImmutableList<String> stageReports(ReportType reportType) throws Exception {
+    QueryBuilder queryBuilder =
+        (reportType == ReportType.ACTIVITY) ? activityQueryBuilder : transactionsQueryBuilder;
+
+
+    ImmutableMap<String, String> viewQueryMap = queryBuilder.getViewQueryMap();
+    // Generate intermediary views
+    for (Entry<String, String> entry : viewQueryMap.entrySet()) {
+      createIntermediaryTableView(entry.getKey(), entry.getValue(), reportType);
+    }
+
+    // Get an in-memory table of the aggregate query's result
+    ImmutableTable<Integer, TableFieldSchema, Object> reportTable =
+        bigquery.queryToLocalTableSync(queryBuilder.getReportQuery());
+
+    // Get report headers from the table schema and convert into CSV format
+    String headerRow = constructRow(getHeaders(reportTable.columnKeySet()));
+    logger.infofmt("Headers: %s", headerRow);
+
+    return (reportType == ReportType.ACTIVITY)
+        ? stageActivityReports(headerRow, reportTable.rowMap().values())
+        : stageTransactionsReports(headerRow, reportTable.rowMap().values());
+  }
+
+  private void createIntermediaryTableView(String queryName, String query, ReportType reportType)
+      throws ExecutionException, InterruptedException {
+    // Later views depend on the results of earlier ones, so query everything synchronously
+    logger.infofmt("Generating intermediary view %s", queryName);
+    bigquery.query(
+        query,
+        bigquery.buildDestinationTable(queryName)
+            .description(String.format(
+                "An intermediary view to generate %s reports for this month.", reportType))
+            .type(TableType.VIEW)
+            .build()
+    ).get();
+  }
+
+  private Iterable<String> getHeaders(ImmutableSet<TableFieldSchema> fields) {
+    return fields
+        .stream()
+        .map((schema) -> schema.getName().replace('_', '-'))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  /** Creates and stores activity reports on GCS, returns a list of files stored. */
+  private ImmutableList<String> stageActivityReports(
+      String headerRow, ImmutableCollection<Map<TableFieldSchema, Object>> rows)
+      throws IOException {
+    ImmutableList.Builder<String> manifestBuilder = new ImmutableList.Builder<>();
+    // Create a report csv for each tld from query table, and upload to GCS
+    for (Map<TableFieldSchema, Object> row : rows) {
+      // Get the tld (first cell in each row)
+      String tld = row.values().iterator().next().toString();
+      if (isNullOrEmpty(tld)) {
+        throw new RuntimeException("Found an empty row in the activity report table!");
+      }
+      ImmutableList<String> rowStrings = ImmutableList.of(constructRow(row.values()));
+      // Create and upload the activity report with a single row
+      manifestBuilder.add(
+          saveReportToGcs(tld, createReport(headerRow, rowStrings), ReportType.ACTIVITY));
+    }
+    return manifestBuilder.build();
+  }
+
+  /** Creates and stores transactions reports on GCS, returns a list of files stored. */
+  private ImmutableList<String> stageTransactionsReports(
+      String headerRow, ImmutableCollection<Map<TableFieldSchema, Object>> rows)
+      throws IOException {
+    // Map from tld to rows
+    ListMultimap<String, String> tldToRows = ArrayListMultimap.create();
+    // Map from tld to totals
+    HashMap<String, List<Integer>> tldToTotals = new HashMap<>();
+    for (Map<TableFieldSchema, Object> row : rows) {
+      // Get the tld (first cell in each row)
+      String tld = row.values().iterator().next().toString();
+      if (isNullOrEmpty(tld)) {
+        throw new RuntimeException("Found an empty row in the transactions report table!");
+      }
+      tldToRows.put(tld, constructRow(row.values()));
+      // Construct totals for each tld, skipping non-summable columns (TLD, registrar name, iana-id)
+      if (!tldToTotals.containsKey(tld)) {
+        tldToTotals.put(tld, new ArrayList<>(Collections.nCopies(row.values().size() - 3, 0)));
+      }
+      addToTotal(tldToTotals.get(tld), row);
+    }
+    ImmutableList.Builder<String> manifestBuilder = new ImmutableList.Builder<>();
+    // Create and upload a transactions report for each tld via its rows
+    for (String tld : tldToRows.keySet()) {
+      // Append the totals row
+      tldToRows.put(tld, constructTotalRow(tldToTotals.get(tld)));
+      manifestBuilder.add(
+          saveReportToGcs(
+              tld, createReport(headerRow, tldToRows.get(tld)), ReportType.TRANSACTIONS));
+    }
+    return manifestBuilder.build();
+  }
+
+  /** Adds a row's values to an existing list of integers (totals). */
+  private void addToTotal(List<Integer> totals, Map<TableFieldSchema, Object> row) {
+    List<Integer> rowVals =
+        row.values()
+            .stream()
+            // Ignore TLD, Registrar name and IANA id
+            .skip(3)
+            .map((Object o) -> Integer.parseInt(o.toString()))
+            .collect(Collectors.toList());
+    checkState(
+        rowVals.size() == totals.size(),
+        "Number of elements in totals not equal to number of elements in row!");
+    for (int i = 0; i < rowVals.size(); i++) {
+      totals.set(i, totals.get(i) + rowVals.get(i));
+    }
+  }
+
+  /** Returns a list of integers (totals) as a comma separated string. */
+  private String constructTotalRow(List<Integer> totals) {
+    StringBuilder rowString = new StringBuilder("Totals,,");
+    rowString.append(
+        totals.stream().map((Integer i) -> i.toString()).collect(Collectors.joining(",")));
+    return rowString.toString();
+  }
+
+  /**
+   * Makes a row of the report by appending the string representation of all objects in an iterable
+   * with commas separating individual fields.
+   *
+   * <p>This discards the first object, which is assumed to be the TLD field.
+   * */
+  private String constructRow(Iterable<? extends Object> iterable) {
+    Iterator<? extends Object> rowIter = iterable.iterator();
+    StringBuilder rowString = new StringBuilder();
+    // Skip the TLD column
+    rowIter.next();
+    while (rowIter.hasNext()) {
+      rowString.append(String.format("%s,", rowIter.next().toString()));
+    }
+    // Remove trailing comma
+    rowString.deleteCharAt(rowString.length() - 1);
+    return rowString.toString();
+  }
+
+  /**
+   * Constructs a report given its headers and rows as a string.
+   *
+   * <p>Note that activity reports will only have one row, while transactions reports may have
+   * multiple rows.
+   */
+  private String createReport(String headers, List<String> rows) {
+    StringBuilder reportCsv = new StringBuilder(headers);
+    for (String row : rows) {
+      // Add CRLF between rows per ICANN specification
+      reportCsv.append("\r\n");
+      reportCsv.append(row);
+    }
+    logger.infofmt("Created report:\n%s", reportCsv.toString());
+    return reportCsv.toString();
+  }
+
+  /** Stores a report on GCS, returning the name of the file stored. */
+  private String saveReportToGcs(String tld, String reportCsv, ReportType reportType)
+      throws IOException {
+    // Upload resulting CSV file to GCS
+    byte[] reportBytes = reportCsv.getBytes(UTF_8);
+    String reportFilename = ReportingUtils.createFilename(tld, yearMonth, reportType);
+    String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
+    final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
+    gcsUtils.createFromBytes(gcsFilename, reportBytes);
+    logger.infofmt(
+        "Wrote %d bytes to file location %s",
+        reportBytes.length,
+        gcsFilename.toString());
+    return reportFilename;
+  }
+
+  /** Creates and stores a manifest file on GCS, indicating which reports were generated. */
+  void createAndUploadManifest(ImmutableList<String> filenames) throws IOException {
+    String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
+    final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
+    StringBuilder manifestString = new StringBuilder();
+    filenames.forEach((filename) -> manifestString.append(filename).append("\n"));
+    gcsUtils.createFromBytes(gcsFilename, manifestString.toString().getBytes(UTF_8));
+    logger.infofmt(
+        "Wrote %d filenames to manifest at %s", filenames.size(), gcsFilename.toString());
+  }
+}

java/google/registry/reporting/IcannReportingStagingAction.java

@@ -14,99 +14,72 @@
 
 package google.registry.reporting;
 
-import static com.google.common.base.Strings.isNullOrEmpty;
 import static google.registry.request.Action.Method.POST;
-import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.api.services.bigquery.model.TableFieldSchema;
-import com.google.appengine.tools.cloudstorage.GcsFilename;
 import com.google.common.base.Throwables;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableTable;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.ListMultimap;
 import com.google.common.net.MediaType;
-import google.registry.bigquery.BigqueryConnection;
-import google.registry.bigquery.BigqueryUtils.TableType;
-import google.registry.config.RegistryConfig.Config;
-import google.registry.gcs.GcsUtils;
 import google.registry.reporting.IcannReportingModule.ReportType;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.FormattingLogger;
-import java.io.IOException;
-import java.io.OutputStream;
 import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Optional;
-import java.util.concurrent.ExecutionException;
 import javax.inject.Inject;
 
 /**
  * Action that generates monthly ICANN activity and transactions reports.
  *
- * <p> The reports are then uploaded to GCS under
+ * <p>The reports are stored in GCS under gs://[project-id]-reporting/[subdir]. We also store a
- * gs://domain-registry-reporting/icann/monthly/YYYY-MM
+ * MANIFEST.txt file that contains a list of filenames generated, to facilitate subsequent uploads.
+ *
+ * <p>Parameters:
+ *
+ * <p>yearMonth: the reporting month in yyyy-MM format. Defaults to the previous month at runtime
+ * (i.e. a run on 2017-09-01 defaults to 2017-08's reports).
+ *
+ * <p>subdir: the subdirectory of gs://[project-id]-reporting/ to upload to. For example:
+ * "manual/dir" means reports will be stored under gs://[project-id]-reporting/manual/dir. Defaults
+ * to "icann/monthly/[yearMonth]".
+ *
+ * <p>reportTypes: the type of reports to generate. You can specify either 'activity' or
+ * 'transactions'. Defaults to generating both.
  */
-@Action(
+@Action(path = IcannReportingStagingAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_ONLY)
-    path = IcannReportingStagingAction.PATH,
-    method = POST,
-    auth = Auth.AUTH_INTERNAL_ONLY
-)
 public final class IcannReportingStagingAction implements Runnable {
 
   static final String PATH = "/_dr/task/icannReportingStaging";
 
   private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
 
-  @Inject @Config("icannReportingBucket") String reportingBucket;
+  @Inject
-  @Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
+  @Parameter(IcannReportingModule.PARAM_REPORT_TYPE)
-  @Inject @Parameter(IcannReportingModule.PARAM_SUBDIR) Optional<String> subdir;
+  ImmutableList<ReportType> reportTypes;
-  @Inject @Parameter(IcannReportingModule.PARAM_REPORT_TYPE) ReportType reportType;
+
-  @Inject QueryBuilder queryBuilder;
+  @Inject IcannReportingStager stager;
-  @Inject BigqueryConnection bigquery;
-  @Inject GcsUtils gcsUtils;
   @Inject Response response;
   @Inject IcannReportingStagingAction() {}
 
   @Override
   public void run() {
     try {
-      ImmutableMap<String, String> viewQueryMap = queryBuilder.getViewQueryMap();
+      ImmutableList.Builder<String> manifestedFilesBuilder = new ImmutableList.Builder<>();
-      // Generate intermediary views
+      for (ReportType reportType : reportTypes) {
-      for (Entry<String, String> entry : viewQueryMap.entrySet()) {
+        manifestedFilesBuilder.addAll(stager.stageReports(reportType));
-        createIntermediaryTableView(entry.getKey(), entry.getValue());
       }
+      ImmutableList<String> manifestedFiles = manifestedFilesBuilder.build();
+      stager.createAndUploadManifest(manifestedFiles);
 
-      // Get an in-memory table of the aggregate query's result
+      logger.infofmt("Completed staging %d report files.", manifestedFiles.size());
-      ImmutableTable<Integer, TableFieldSchema, Object> reportTable =
-          bigquery.queryToLocalTableSync(queryBuilder.getReportQuery());
-
-      // Get report headers from the table schema and convert into CSV format
-      String headerRow = constructRow(getHeaders(reportTable.columnKeySet()));
-      logger.infofmt("Headers: %s", headerRow);
-
-      if (reportType == ReportType.ACTIVITY) {
-        stageActivityReports(headerRow, reportTable.rowMap().values());
-      } else {
-        stageTransactionsReports(headerRow, reportTable.rowMap().values());
-      }
       response.setStatus(SC_OK);
       response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
       response.setPayload("Completed staging action.");
     } catch (Exception e) {
-      logger.warning(Throwables.getStackTraceAsString(e));
+      logger.severe("Reporting staging action failed!");
+      logger.severe(Throwables.getStackTraceAsString(e));
       response.setStatus(SC_INTERNAL_SERVER_ERROR);
       response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
       response.setPayload(
@@ -114,109 +87,4 @@ public final class IcannReportingStagingAction implements Runnable {
               Arrays.toString(e.getStackTrace())));
     }
   }
-
-  private void createIntermediaryTableView(String queryName, String query)
-      throws ExecutionException, InterruptedException {
-    // Later views depend on the results of earlier ones, so query everything synchronously
-    bigquery.query(
-        query,
-        bigquery.buildDestinationTable(queryName)
-            .description(String.format(
-                "An intermediary view to generate %s reports for this month.", reportType))
-            .type(TableType.VIEW)
-            .build()
-    ).get();
-  }
-
-  private Iterable<String> getHeaders(ImmutableSet<TableFieldSchema> fields) {
-    return Iterables.transform(fields, schema -> schema.getName().replace('_', '-'));
-  }
-
-  private void stageActivityReports (
-      String headerRow, ImmutableCollection<Map<TableFieldSchema, Object>> rows)
-      throws IOException {
-    // Create a report csv for each tld from query table, and upload to GCS
-    for (Map<TableFieldSchema, Object> row : rows) {
-      // Get the tld (first cell in each row)
-      String tld = row.values().iterator().next().toString();
-      if (isNullOrEmpty(tld)) {
-        throw new RuntimeException("Found an empty row in the activity report table!");
-      }
-      ImmutableList<String> rowStrings = ImmutableList.of(constructRow(row.values()));
-      // Create and upload the activity report with a single row
-      uploadReport(tld, createReport(headerRow, rowStrings));
-    }
-  }
-
-  private void stageTransactionsReports(
-      String headerRow, ImmutableCollection<Map<TableFieldSchema, Object>> rows)
-      throws IOException {
-    // Map from tld to rows
-    ListMultimap<String, String> tldToRows = ArrayListMultimap.create();
-    for (Map<TableFieldSchema, Object> row : rows) {
-      // Get the tld (first cell in each row)
-      String tld = row.values().iterator().next().toString();
-      if (isNullOrEmpty(tld)) {
-        throw new RuntimeException("Found an empty row in the activity report table!");
-      }
-      tldToRows.put(tld, constructRow(row.values()));
-    }
-    // Create and upload a transactions report for each tld via its rows
-    for (String tld : tldToRows.keySet()) {
-      uploadReport(tld, createReport(headerRow, tldToRows.get(tld)));
-    }
-  }
-
-  /**
-   * Makes a row of the report by appending the string representation of all objects in an iterable
-   * with commas separating individual fields.
-   *
-   * <p>This discards the first object, which is assumed to be the TLD field.
-   * */
-  private String constructRow(Iterable<? extends Object> iterable) {
-    Iterator<? extends Object> rowIter = iterable.iterator();
-    StringBuilder rowString = new StringBuilder();
-    // Skip the TLD column
-    rowIter.next();
-    while (rowIter.hasNext()) {
-      rowString.append(String.format("%s,", rowIter.next().toString()));
-    }
-    // Remove trailing comma
-    rowString.deleteCharAt(rowString.length() - 1);
-    return rowString.toString();
-  }
-
-  /**
-   * Constructs a report given its headers and rows as a string.
-   *
-   * <p>Note that activity reports will only have one row, while transactions reports may have
-   * multiple rows.
-   */
-  private String createReport(String headers, List<String> rows) {
-    StringBuilder reportCsv = new StringBuilder(headers);
-    for (String row : rows) {
-      // Add CRLF between rows per ICANN specification
-      reportCsv.append("\r\n");
-      reportCsv.append(row);
-    }
-    logger.infofmt("Created %s report:\n%s", reportType, reportCsv.toString());
-    return reportCsv.toString();
-  }
-
-  private void uploadReport(String tld, String reportCsv) throws IOException {
-    // Upload resulting CSV file to GCS
-    byte[] reportBytes = reportCsv.getBytes(UTF_8);
-    String reportFilename =
-        IcannReportingUploadAction.createFilename(tld, yearMonth, reportType);
-    String reportBucketname =
-        IcannReportingUploadAction.createReportingBucketName(reportingBucket, subdir, yearMonth);
-    final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
-    try (OutputStream gcsOutput = gcsUtils.openOutputStream(gcsFilename)) {
-      gcsOutput.write(reportBytes);
-    }
-    logger.infofmt(
-        "Wrote %d bytes to file location %s",
-        reportBytes.length,
-        gcsFilename.toString());
-  }
 }

java/google/registry/reporting/IcannReportingUploadAction.java

@@ -14,115 +14,108 @@
 
 package google.registry.reporting;
 
-import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
-import static google.registry.model.registry.Registries.assertTldExists;
+import static google.registry.reporting.IcannReportingModule.MANIFEST_FILE_NAME;
 import static google.registry.request.Action.Method.POST;
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
 import com.google.common.io.ByteStreams;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
-import google.registry.reporting.IcannReportingModule.ReportType;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
-import google.registry.request.RequestParameters;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.FormattingLogger;
 import google.registry.util.Retrier;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.Optional;
 import javax.inject.Inject;
 
 /**
- * Action that uploads the monthly transaction and activity reports from Cloud Storage to ICANN via
+ * Action that uploads the monthly activity/transactions reports from GCS to ICANN via an HTTP PUT.
- * an HTTP PUT.
  *
+ * <p>This should be run after {@link IcannReportingStagingAction}, which writes out the month's
+ * reports and a MANIFEST.txt file. This action reads the filenames from the MANIFEST.txt, and
+ * attempts to upload every file in the manifest to ICANN's endpoint.
+ *
+ * <p>Parameters:
+ *
+ * <p>subdir: the subdirectory of gs://[project-id]-reporting/ to retrieve reports from. For
+ * example: "manual/dir" means reports will be stored under gs://[project-id]-reporting/manual/dir.
+ * Defaults to "icann/monthly/[last month in yyyy-MM format]".
  */
-@Action(
+@Action(path = IcannReportingUploadAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_OR_ADMIN)
-  path = IcannReportingUploadAction.PATH,
-  method = POST,
-  auth = Auth.AUTH_INTERNAL_OR_ADMIN
-)
 public final class IcannReportingUploadAction implements Runnable {
 
   static final String PATH = "/_dr/task/icannReportingUpload";
-  static final String DEFAULT_SUBDIR = "icann/monthly";
 
   private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
 
-  @Inject @Config("icannReportingBucket") String icannReportingBucket;
+  @Inject
-  @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
+  @Config("icannReportingBucket")
-  @Inject @Parameter(IcannReportingModule.PARAM_YEAR_MONTH) String yearMonth;
+  String reportingBucket;
-  @Inject @Parameter(IcannReportingModule.PARAM_REPORT_TYPE) ReportType reportType;
+
-  @Inject @Parameter(IcannReportingModule.PARAM_SUBDIR) Optional<String> subdir;
+  @Inject
+  @Parameter(IcannReportingModule.PARAM_SUBDIR)
+  String subdir;
+
   @Inject GcsUtils gcsUtils;
   @Inject IcannHttpReporter icannReporter;
-  @Inject Response response;
   @Inject Retrier retrier;
+  @Inject Response response;
 
   @Inject
   IcannReportingUploadAction() {}
 
   @Override
   public void run() {
-    validateParams();
+    String reportBucketname = ReportingUtils.createReportingBucketName(reportingBucket, subdir);
-    String reportFilename = createFilename(tld, yearMonth, reportType);
+    ImmutableList<String> manifestedFiles = getManifestedFiles(reportBucketname);
-    String reportBucketname = createReportingBucketName(icannReportingBucket, subdir, yearMonth);
+    // Report on all manifested files
+    for (String reportFilename : manifestedFiles) {
       logger.infofmt("Reading ICANN report %s from bucket %s", reportFilename, reportBucketname);
       final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
-    checkState(
+      verifyFileExists(gcsFilename);
-        gcsUtils.existsAndNotEmpty(gcsFilename),
-        "ICANN report object %s in bucket %s not found",
-        gcsFilename.getObjectName(),
-        gcsFilename.getBucketName());
-
       retrier.callWithRetry(
           () -> {
-          final byte[] payload = readReportFromGcs(gcsFilename);
+            final byte[] payload = readBytesFromGcs(gcsFilename);
-          icannReporter.send(payload, tld, yearMonth, reportType);
+            icannReporter.send(payload, reportFilename);
             response.setContentType(PLAIN_TEXT_UTF_8);
-          response.setPayload(
+            response.setPayload(String.format("OK, sending: %s", new String(payload, UTF_8)));
-              String.format("OK, sending: %s", new String(payload, StandardCharsets.UTF_8)));
             return null;
           },
           IOException.class);
     }
+  }
 
-  private byte[] readReportFromGcs(GcsFilename reportFilename) throws IOException {
+  private ImmutableList<String> getManifestedFiles(String reportBucketname) {
+    GcsFilename manifestFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
+    verifyFileExists(manifestFilename);
+    return retrier.callWithRetry(
+        () ->
+            ImmutableList.copyOf(
+                Splitter.on('\n')
+                    .omitEmptyStrings()
+                    .split(new String(readBytesFromGcs(manifestFilename), UTF_8))),
+        IOException.class);
+  }
+
+  private byte[] readBytesFromGcs(GcsFilename reportFilename) throws IOException {
     try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) {
       return ByteStreams.toByteArray(gcsInput);
     }
   }
 
-  static String createFilename(String tld, String yearMonth, ReportType reportType) {
+  private void verifyFileExists(GcsFilename gcsFilename) {
-    // Report files use YYYYMM naming instead of standard YYYY-MM, per ICANN requirements.
+    checkArgument(
-    String fileYearMonth = yearMonth.substring(0, 4) + yearMonth.substring(5, 7);
+        gcsUtils.existsAndNotEmpty(gcsFilename),
-    return String.format("%s-%s-%s.csv", tld, reportType.toString().toLowerCase(), fileYearMonth);
+        "Object %s in bucket %s not found",
-  }
+        gcsFilename.getObjectName(),
-
+        gcsFilename.getBucketName());
-  static String createReportingBucketName(
-      String reportingBucket, Optional<String> subdir, String yearMonth) {
-    return subdir.isPresent()
-        ? String.format("%s/%s", reportingBucket, subdir.get())
-        : String.format("%s/%s/%s", reportingBucket, DEFAULT_SUBDIR, yearMonth);
-
-  }
-
-  private void validateParams() {
-    assertTldExists(tld);
-    checkState(
-        yearMonth.matches("[0-9]{4}-[0-9]{2}"),
-        "yearMonth must be in YYYY-MM format, got %s instead.",
-        yearMonth);
-    if (subdir.isPresent()) {
-      checkState(
-          !subdir.get().startsWith("/") && !subdir.get().endsWith("/"),
-          "subdir must not start or end with a \"/\", got %s instead.",
-          subdir.get());
-    }
   }
 }

java/google/registry/reporting/ReportingUtils.java

@@ -0,0 +1,34 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import com.google.common.base.Ascii;
+import google.registry.reporting.IcannReportingModule.ReportType;
+
+/** Static utils for reporting. */
+public final class ReportingUtils {
+
+  /** Generates a report filename in accord with ICANN's specifications. */
+  static String createFilename(String tld, String yearMonth, ReportType reportType) {
+    // Report files use YYYYMM naming instead of standard YYYY-MM, per ICANN requirements.
+    return String.format(
+        "%s-%s-%s.csv", tld, Ascii.toLowerCase(reportType.toString()), yearMonth.replace("-", ""));
+  }
+
+  /** Constructs the bucket name to store/upload reports to. */
+  static String createReportingBucketName(String reportingBucket, String subdir) {
+    return String.format("%s/%s", reportingBucket, subdir);
+  }
+}

java/google/registry/reporting/TransactionsReportingQueryBuilder.java

@@ -158,6 +158,8 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
     String aggregateQuery =
         SqlTemplate.create(getQueryFromFile("transactions_report_aggregation.sql"))
             .put("PROJECT_ID", projectId)
+            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+            .put("REGISTRY_TABLE", "Registry")
             .put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
             .put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID))
             .put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS))

java/google/registry/reporting/sql/activity_report_aggregation.sql

@@ -19,8 +19,6 @@
 SELECT
   RealTlds.tld AS tld,
   SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars,
-  SUM(IF(metricName = 'ramp-up-registrars', count, 0)) AS ramp_up_registrars,
-  SUM(IF(metricName = 'pre-ramp-up-registrars', count, 0)) AS pre_ramp_up_registrars,
   -- We use the Centralized Zone Data Service.
   "CZDS" AS zfa_passwords,
   SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries,
@@ -65,7 +63,7 @@ SELECT
   -- filter so that only metrics with that TLD or a NULL TLD are counted
   -- towards a given TLD.
 FROM (
-SELECT tldStr as tld
+SELECT tldStr AS tld
 FROM `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
 WHERE tldType = 'REAL'
 ) as RealTlds

java/google/registry/reporting/sql/dns_counts.sql

@@ -15,14 +15,13 @@
 
   -- Query for DNS metrics.
 
-  -- This is a no-op until after we transition to Google Cloud DNS, which
+  -- You must configure this yourself to enable activity reporting, according
-  -- will likely export metrics via Stackdriver.
+  -- to whatever metrics your DNS provider makes available. We hope to make
+  -- this available in the open-source build in the near future.
 
 SELECT
-  -- DNS metrics apply to all tlds, which requires the 'null' magic value.
   STRING(NULL) AS tld,
   metricName,
-  -- TODO(b/63388735): Change this to actually query Google Cloud DNS when ready.
   -1 AS count
 FROM ((
   SELECT 'dns-udp-queries' AS metricName)

java/google/registry/reporting/sql/registrar_iana_id.sql

@@ -26,5 +26,5 @@ FROM
   UNNEST(allowedTlds) as allowed_tlds
 WHERE (type = 'REAL' OR type = 'INTERNAL')
 -- Filter out prober data
-AND NOT ENDS_WITH(allowed_tlds, "test")
+AND NOT ENDS_WITH(allowed_tlds, ".test")
 ORDER BY tld, registrarName

java/google/registry/reporting/sql/registrar_operating_status.sql

@@ -23,5 +23,5 @@ SELECT
 FROM
   `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%`
 WHERE
-  type = 'REAL'
+  (type = 'REAL' OR type = 'INTERNAL')
 GROUP BY metricName

java/google/registry/reporting/sql/total_domains.sql

@@ -32,7 +32,7 @@ JOIN
 ON
   currentSponsorClientId = registrar_table.__key__.name
 WHERE
-  domain_table._d = "DomainResource"
+  domain_table._d = 'DomainResource'
-  AND (registrar_table.type = "REAL" OR registrar_table.type = "INTERNAL")
+  AND (registrar_table.type = 'REAL' OR registrar_table.type = 'INTERNAL')
 GROUP BY tld, registrarName
 ORDER BY tld, registrarName

java/google/registry/reporting/sql/transaction_counts.sql

@@ -65,8 +65,6 @@ FROM (
   WHERE reportingTime
   BETWEEN TIMESTAMP('%EARLIEST_REPORT_TIME%')
   AND TIMESTAMP('%LATEST_REPORT_TIME%')
-    -- Ignore prober data
-  AND NOT ENDS_WITH(tld, "test")
   GROUP BY
     tld,
     clientId,

java/google/registry/reporting/sql/transactions_report_aggregation.sql

@@ -20,7 +20,8 @@
 
 SELECT
   registrars.tld as tld,
-  registrars.registrar_name as registrar_name,
+  -- Surround registrar names with quotes to handle names containing a comma.
+  FORMAT("\"%s\"", registrars.registrar_name) as registrar_name,
   registrars.iana_id as iana_id,
   SUM(IF(metrics.metricName = 'TOTAL_DOMAINS', metrics.metricValue, 0)) AS total_domains,
   SUM(IF(metrics.metricName = 'TOTAL_NAMESERVERS', metrics.metricValue, 0)) AS total_nameservers,
@@ -62,9 +63,16 @@ SELECT
   0 AS agp_exemptions_granted,
   0 AS agp_exempted_domains,
   SUM(IF(metrics.metricName = 'ATTEMPTED_ADDS', metrics.metricValue, 0)) AS attempted_adds
-FROM (
+FROM
-  SELECT *
+-- Only produce reports for real TLDs
-  FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%REGISTRAR_IANA_ID_TABLE%`) AS registrars
+(SELECT tldStr AS tld
+  FROM `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
+  WHERE tldType = 'REAL') AS registries
+JOIN
+(SELECT *
+  FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%REGISTRAR_IANA_ID_TABLE%`)
+  AS registrars
+ON registries.tld = registrars.tld
 -- We LEFT JOIN to produce reports even if the registrar made no transactions
 LEFT OUTER JOIN (
   -- Gather all intermediary data views

java/google/registry/request/RequestParameters.java

@@ -98,6 +98,20 @@ public final class RequestParameters {
     return parameters == null ? ImmutableSet.<String>of() : ImmutableSet.copyOf(parameters);
   }
 
+  /**
+   * Returns the first GET or POST parameter associated with {@code name}, absent otherwise.
+   *
+   * @throws BadRequestException if request parameter named {@code name} is not equal to any of the
+   *     values in {@code enumClass}
+   */
+  public static <C extends Enum<C>> Optional<C> extractOptionalEnumParameter(
+      HttpServletRequest req, Class<C> enumClass, String name) {
+    String stringParam = req.getParameter(name);
+    return isNullOrEmpty(stringParam)
+        ? Optional.empty()
+        : Optional.of(extractEnumParameter(req, enumClass, name));
+  }
+
   /**
    * Returns the first GET or POST parameter associated with {@code name}.
    *

javatests/google/registry/reporting/ActivityReportingQueryBuilderTest.java

@@ -15,7 +15,6 @@
 package google.registry.reporting;
 
 import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -30,7 +29,7 @@ public class ActivityReportingQueryBuilderTest {
 
   private ActivityReportingQueryBuilder getQueryBuilder() {
     ActivityReportingQueryBuilder queryBuilder = new ActivityReportingQueryBuilder();
-    queryBuilder.yearMonth = "2017-06";
+    queryBuilder.yearMonth = "2017-09";
     queryBuilder.projectId = "domain-registry-alpha";
     return queryBuilder;
   }
@@ -41,28 +40,28 @@ public class ActivityReportingQueryBuilderTest {
     assertThat(queryBuilder.getReportQuery())
         .isEqualTo(
             "#standardSQL\nSELECT * FROM "
-                + "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201706`");
+                + "`domain-registry-alpha.icann_reporting.activity_report_aggregation_201709`");
   }
 
   @Test
   public void testIntermediaryQueryMatch() throws IOException {
-    ActivityReportingQueryBuilder queryBuilder = getQueryBuilder();
+    ImmutableList<String> expectedQueryNames =
-    ImmutableList<String> queryNames =
         ImmutableList.of(
             ActivityReportingQueryBuilder.REGISTRAR_OPERATING_STATUS,
-            ActivityReportingQueryBuilder.DNS_COUNTS,
             ActivityReportingQueryBuilder.MONTHLY_LOGS,
+            ActivityReportingQueryBuilder.DNS_COUNTS,
             ActivityReportingQueryBuilder.EPP_METRICS,
             ActivityReportingQueryBuilder.WHOIS_COUNTS,
             ActivityReportingQueryBuilder.ACTIVITY_REPORT_AGGREGATION);
 
+    ActivityReportingQueryBuilder queryBuilder = getQueryBuilder();
     ImmutableMap<String, String> actualQueries = queryBuilder.getViewQueryMap();
-    for (String queryName : queryNames) {
+    for (String queryName : expectedQueryNames) {
-      String actualTableName = String.format("%s_201706", queryName);
+      String actualTableName = String.format("%s_201709", queryName);
       String testFilename = String.format("%s_test.sql", queryName);
       assertThat(actualQueries.get(actualTableName))
           .isEqualTo(ReportingTestData.getString(testFilename));
     }
   }
-}
 
+}

javatests/google/registry/reporting/IcannHttpReporterTest.java

@@ -18,7 +18,7 @@ import static com.google.common.net.MediaType.CSV_UTF_8;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
-import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.testing.DatastoreHelper.createTld;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.client.http.LowLevelHttpRequest;
@@ -29,11 +29,14 @@ import com.google.api.client.testing.http.MockLowLevelHttpResponse;
 import com.google.api.client.util.Base64;
 import com.google.api.client.util.StringUtils;
 import com.google.common.io.ByteSource;
-import google.registry.reporting.IcannReportingModule.ReportType;
 import google.registry.request.HttpException.InternalServerErrorException;
+import google.registry.testing.AppEngineRule;
+import google.registry.testing.ExceptionRule;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -46,9 +49,14 @@ public class IcannHttpReporterTest {
 
   private static final ByteSource IIRDEA_GOOD_XML = ReportingTestData.get("iirdea_good.xml");
   private static final ByteSource IIRDEA_BAD_XML = ReportingTestData.get("iirdea_bad.xml");
+  private static final byte[] FAKE_PAYLOAD = "test,csv\n1,2".getBytes(UTF_8);
 
   private MockLowLevelHttpRequest mockRequest;
 
+  @Rule public final ExceptionRule thrown = new ExceptionRule();
+
+  @Rule public AppEngineRule appEngineRule = new AppEngineRule.Builder().withDatastore().build();
+
   private MockHttpTransport createMockTransport (final ByteSource iirdeaResponse) {
     return new MockHttpTransport() {
       @Override
@@ -69,7 +77,11 @@ public class IcannHttpReporterTest {
     };
   }
 
-  private static final byte[] FAKE_PAYLOAD = "test,csv\n1,2".getBytes(UTF_8);
+  @Before
+  public void setUp() {
+    createTld("test");
+    createTld("xn--abc123");
+  }
 
   private IcannHttpReporter createReporter() {
     IcannHttpReporter reporter = new IcannHttpReporter();
@@ -83,7 +95,7 @@ public class IcannHttpReporterTest {
   @Test
   public void testSuccess() throws Exception {
     IcannHttpReporter reporter = createReporter();
-    reporter.send(FAKE_PAYLOAD, "test", "2017-06", ReportType.TRANSACTIONS);
+    reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv");
 
     assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2017-06");
     Map<String, List<String>> headers = mockRequest.getHeaders();
@@ -94,15 +106,65 @@ public class IcannHttpReporterTest {
     assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString());
   }
 
+  @Test
+  public void testSuccess_internationalTld() throws Exception {
+    IcannHttpReporter reporter = createReporter();
+    reporter.send(FAKE_PAYLOAD, "xn--abc123-transactions-201706.csv");
+
+    assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/xn--abc123/2017-06");
+    Map<String, List<String>> headers = mockRequest.getHeaders();
+    String userPass = "xn--abc123_ry:fakePass";
+    String expectedAuth =
+        String.format("Basic %s", Base64.encodeBase64String(StringUtils.getBytesUtf8(userPass)));
+    assertThat(headers.get("authorization")).containsExactly(expectedAuth);
+    assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString());
+  }
+
   @Test
   public void testFail_BadIirdeaResponse() throws Exception {
     IcannHttpReporter reporter = createReporter();
     reporter.httpTransport = createMockTransport(IIRDEA_BAD_XML);
     try {
-      reporter.send(FAKE_PAYLOAD, "test", "2017-06", ReportType.TRANSACTIONS);
+      reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv");
       assertWithMessage("Expected InternalServerErrorException to be thrown").fail();
     } catch (InternalServerErrorException expected) {
       assertThat(expected).hasMessageThat().isEqualTo("The structure of the report is invalid.");
     }
   }
+
+  @Test
+  public void testFail_invalidFilename_nonSixDigitYearMonth() throws Exception {
+    thrown.expect(
+        IllegalArgumentException.class,
+        "Expected file format: tld-reportType-yyyyMM.csv, got test-transactions-20176.csv instead");
+    IcannHttpReporter reporter = createReporter();
+    reporter.send(FAKE_PAYLOAD, "test-transactions-20176.csv");
+  }
+
+  @Test
+  public void testFail_invalidFilename_notActivityOrTransactions() throws Exception {
+    thrown.expect(
+        IllegalArgumentException.class,
+        "Expected file format: tld-reportType-yyyyMM.csv, got test-invalid-201706.csv instead");
+    IcannHttpReporter reporter = createReporter();
+    reporter.send(FAKE_PAYLOAD, "test-invalid-201706.csv");
+  }
+
+  @Test
+  public void testFail_invalidFilename_invalidTldName() throws Exception {
+    thrown.expect(
+        IllegalArgumentException.class,
+        "Expected file format: tld-reportType-yyyyMM.csv, got n!-n-activity-201706.csv instead");
+    IcannHttpReporter reporter = createReporter();
+    reporter.send(FAKE_PAYLOAD, "n!-n-activity-201706.csv");
+  }
+
+  @Test
+  public void testFail_invalidFilename_tldDoesntExist() throws Exception {
+    thrown.expect(
+        IllegalArgumentException.class,
+        "TLD hello does not exist");
+    IcannHttpReporter reporter = createReporter();
+    reporter.send(FAKE_PAYLOAD, "hello-activity-201706.csv");
+  }
 }

javatests/google/registry/reporting/IcannReportingModuleTest.java

@@ -0,0 +1,97 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.Mockito.mock;
+
+import google.registry.reporting.IcannReportingModule.ReportType;
+import google.registry.request.HttpException.BadRequestException;
+import google.registry.testing.ExceptionRule;
+import google.registry.testing.FakeClock;
+import google.registry.util.Clock;
+import java.util.Optional;
+import javax.servlet.http.HttpServletRequest;
+import org.joda.time.DateTime;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link google.registry.reporting.IcannReportingModule}. */
+@RunWith(JUnit4.class)
+public class IcannReportingModuleTest {
+
+  HttpServletRequest req = mock(HttpServletRequest.class);
+  Clock clock;
+
+  @Rule public final ExceptionRule thrown = new ExceptionRule();
+
+  @Before
+  public void setUp() {
+    clock = new FakeClock(DateTime.parse("2017-07-01TZ"));
+  }
+
+  @Test
+  public void testEmptyYearMonth_returnsCurrentDate() {
+    assertThat(IcannReportingModule.provideYearMonth(Optional.empty(), clock)).isEqualTo("2017-06");
+  }
+
+  @Test
+  public void testGivenYearMonth_returnsThatMonth() {
+    assertThat(IcannReportingModule.provideYearMonth(Optional.of("2017-05"), clock))
+        .isEqualTo("2017-05");
+  }
+
+  @Test
+  public void testInvalidYearMonth_throwsException() {
+    thrown.expect(
+        BadRequestException.class, "yearMonth must be in yyyy-MM format, got 201705 instead");
+    IcannReportingModule.provideYearMonth(Optional.of("201705"), clock);
+  }
+
+  @Test
+  public void testEmptySubDir_returnsDefaultSubdir() {
+    assertThat(IcannReportingModule.provideSubdir(Optional.empty(), "2017-06"))
+        .isEqualTo("icann/monthly/2017-06");
+  }
+
+  @Test
+  public void testGivenSubdir_returnsManualSubdir() {
+    assertThat(IcannReportingModule.provideSubdir(Optional.of("manual/dir"), "2017-06"))
+        .isEqualTo("manual/dir");
+  }
+
+  @Test
+  public void testInvalidSubdir_throwsException() {
+    thrown.expect(
+        BadRequestException.class,
+        "subdir must not start or end with a \"/\", got /whoops instead.");
+    IcannReportingModule.provideSubdir(Optional.of("/whoops"), "2017-06");
+  }
+
+  @Test
+  public void testGivenReportType_returnsReportType() {
+    assertThat(IcannReportingModule.provideReportTypes(Optional.of(ReportType.ACTIVITY)))
+        .containsExactly(ReportType.ACTIVITY);
+  }
+
+  @Test
+  public void testNoReportType_returnsBothReportTypes() {
+    assertThat(IcannReportingModule.provideReportTypes(Optional.empty()))
+        .containsExactly(ReportType.ACTIVITY, ReportType.TRANSACTIONS);
+  }
+}

javatests/google/registry/reporting/IcannReportingStagerTest.java

@@ -0,0 +1,212 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.GcsTestingUtils.readGcsFile;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.TableFieldSchema;
+import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableTable;
+import com.google.common.util.concurrent.ListenableFuture;
+import google.registry.bigquery.BigqueryConnection;
+import google.registry.bigquery.BigqueryConnection.DestinationTable;
+import google.registry.bigquery.BigqueryUtils.TableType;
+import google.registry.gcs.GcsUtils;
+import google.registry.reporting.IcannReportingModule.ReportType;
+import google.registry.testing.AppEngineRule;
+import google.registry.testing.FakeResponse;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link google.registry.reporting.IcannReportingStager}. */
+@RunWith(JUnit4.class)
+public class IcannReportingStagerTest {
+
+  BigqueryConnection bigquery = mock(BigqueryConnection.class);
+  FakeResponse response = new FakeResponse();
+  GcsService gcsService = GcsServiceFactory.createGcsService();
+
+  @Rule
+  public final AppEngineRule appEngine = AppEngineRule.builder()
+      .withDatastore()
+      .withLocalModules()
+      .build();
+
+  private IcannReportingStager createStager() {
+    IcannReportingStager action = new IcannReportingStager();
+    ActivityReportingQueryBuilder activityBuilder = new ActivityReportingQueryBuilder();
+    activityBuilder.projectId = "test-project";
+    activityBuilder.yearMonth = "2017-06";
+    action.activityQueryBuilder = activityBuilder;
+    TransactionsReportingQueryBuilder transactionsBuilder = new TransactionsReportingQueryBuilder();
+    transactionsBuilder.projectId = "test-project";
+    transactionsBuilder.yearMonth = "2017-06";
+    action.transactionsQueryBuilder = transactionsBuilder;
+    action.reportingBucket = "test-bucket";
+    action.yearMonth = "2017-06";
+    action.subdir = "icann/monthly/2017-06";
+    action.bigquery = bigquery;
+    action.gcsUtils = new GcsUtils(gcsService, 1024);
+    return action;
+  }
+
+  private void setUpBigquery() {
+    when(bigquery.query(any(String.class), any(DestinationTable.class))).thenReturn(fakeFuture());
+    DestinationTable.Builder tableBuilder = new DestinationTable.Builder()
+        .datasetId("testdataset")
+        .type(TableType.TABLE)
+        .name("tablename")
+        .overwrite(true);
+    when(bigquery.buildDestinationTable(any(String.class))).thenReturn(tableBuilder);
+  }
+
+  @Test
+  public void testRunSuccess_activityReport() throws Exception {
+    setUpBigquery();
+    ImmutableTable<Integer, TableFieldSchema, Object> activityReportTable =
+        new ImmutableTable.Builder<Integer, TableFieldSchema, Object>()
+            .put(1, new TableFieldSchema().setName("tld"), "fooTld")
+            .put(1, new TableFieldSchema().setName("fooField"), "12")
+            .put(1, new TableFieldSchema().setName("barField"), "34")
+            .put(2, new TableFieldSchema().setName("tld"), "barTld")
+            .put(2, new TableFieldSchema().setName("fooField"), "56")
+            .put(2, new TableFieldSchema().setName("barField"), "78")
+            .build();
+    when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(activityReportTable);
+    IcannReportingStager stager = createStager();
+    stager.stageReports(ReportType.ACTIVITY);
+
+    String expectedReport1 = "fooField,barField\r\n12,34";
+    String expectedReport2 = "fooField,barField\r\n56,78";
+    byte[] generatedFile1 =
+        readGcsFile(
+            gcsService,
+            new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv"));
+    assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
+    byte[] generatedFile2 =
+        readGcsFile(
+            gcsService,
+            new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv"));
+    assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
+  }
+
+  @Test
+  public void testRunSuccess_transactionsReport() throws Exception {
+    setUpBigquery();
+    /*
+      The fake table result looks like:
+         tld     registrar iana   field
+       1 fooTld  reg1      123    10
+       2 fooTld  reg2      456    20
+       3 barTld  reg1      123    30
+    */
+    ImmutableTable<Integer, TableFieldSchema, Object> transactionReportTable =
+        new ImmutableTable.Builder<Integer, TableFieldSchema, Object>()
+            .put(1, new TableFieldSchema().setName("tld"), "fooTld")
+            .put(1, new TableFieldSchema().setName("registrar"), "\"reg1\"")
+            .put(1, new TableFieldSchema().setName("iana"), "123")
+            .put(1, new TableFieldSchema().setName("field"), "10")
+            .put(2, new TableFieldSchema().setName("tld"), "fooTld")
+            .put(2, new TableFieldSchema().setName("registrar"), "\"reg2\"")
+            .put(2, new TableFieldSchema().setName("iana"), "456")
+            .put(2, new TableFieldSchema().setName("field"), "20")
+            .put(3, new TableFieldSchema().setName("tld"), "barTld")
+            .put(3, new TableFieldSchema().setName("registrar"), "\"reg1\"")
+            .put(3, new TableFieldSchema().setName("iana"), "123")
+            .put(3, new TableFieldSchema().setName("field"), "30")
+            .build();
+    when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(transactionReportTable);
+    IcannReportingStager stager = createStager();
+    stager.stageReports(ReportType.TRANSACTIONS);
+
+    String expectedReport1 =
+        "registrar,iana,field\r\n\"reg1\",123,10\r\n\"reg2\",456,20\r\nTotals,,30";
+    String expectedReport2 = "registrar,iana,field\r\n\"reg1\",123,30\r\nTotals,,30";
+    byte[] generatedFile1 =
+        readGcsFile(
+            gcsService,
+            new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv"));
+    assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
+    byte[] generatedFile2 =
+        readGcsFile(
+            gcsService,
+            new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv"));
+    assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
+  }
+
+  @Test
+  public void testRunSuccess_createAndUploadManifest() throws Exception {
+    IcannReportingStager stager = createStager();
+    ImmutableList<String> filenames =
+        ImmutableList.of("fooTld-transactions-201706.csv", "barTld-activity-201706.csv");
+    stager.createAndUploadManifest(filenames);
+
+    String expectedManifest = "fooTld-transactions-201706.csv\nbarTld-activity-201706.csv\n";
+    byte[] generatedManifest =
+        readGcsFile(
+            gcsService, new GcsFilename("test-bucket/icann/monthly/2017-06", "MANIFEST.txt"));
+    assertThat(new String(generatedManifest, UTF_8)).isEqualTo(expectedManifest);
+  }
+
+  private ListenableFuture<DestinationTable> fakeFuture() {
+    return new ListenableFuture<DestinationTable>() {
+      @Override
+      public void addListener(Runnable runnable, Executor executor) {
+        // No-op
+      }
+
+      @Override
+      public boolean cancel(boolean mayInterruptIfRunning) {
+        return false;
+      }
+
+      @Override
+      public boolean isCancelled() {
+        return false;
+      }
+
+      @Override
+      public boolean isDone() {
+        return false;
+      }
+
+      @Override
+      public DestinationTable get() throws InterruptedException, ExecutionException {
+        return null;
+      }
+
+      @Override
+      public DestinationTable get(long timeout, TimeUnit unit)
+          throws InterruptedException, ExecutionException, TimeoutException {
+        return null;
+      }
+    };
+  }
+}
+

javatests/google/registry/reporting/IcannReportingStagingActionTest.java

@@ -14,32 +14,15 @@
 
 package google.registry.reporting;
 
-import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
-import static google.registry.testing.GcsTestingUtils.readGcsFile;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import com.google.api.services.bigquery.model.TableFieldSchema;
+import com.google.common.collect.ImmutableList;
-import com.google.appengine.tools.cloudstorage.GcsFilename;
-import com.google.appengine.tools.cloudstorage.GcsService;
-import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
-import com.google.common.collect.ImmutableTable;
-import com.google.common.util.concurrent.ListenableFuture;
-import google.registry.bigquery.BigqueryConnection;
-import google.registry.bigquery.BigqueryConnection.DestinationTable;
-import google.registry.bigquery.BigqueryUtils.TableType;
-import google.registry.gcs.GcsUtils;
 import google.registry.reporting.IcannReportingModule.ReportType;
 import google.registry.testing.AppEngineRule;
 import google.registry.testing.FakeResponse;
-import java.util.Optional;
+import org.junit.Before;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -51,9 +34,8 @@ import org.junit.runners.JUnit4;
 @RunWith(JUnit4.class)
 public class IcannReportingStagingActionTest {
 
-  BigqueryConnection bigquery = mock(BigqueryConnection.class);
   FakeResponse response = new FakeResponse();
-  GcsService gcsService = GcsServiceFactory.createGcsService();
+  IcannReportingStager stager = mock(IcannReportingStager.class);
 
   @Rule
   public final AppEngineRule appEngine = AppEngineRule.builder()
@@ -61,143 +43,36 @@ public class IcannReportingStagingActionTest {
       .withLocalModules()
       .build();
 
-  private IcannReportingStagingAction createAction(ReportType reportType) {
+  @Before
-    IcannReportingStagingAction action = new IcannReportingStagingAction();
+  public void setUp() throws Exception {
-    if (reportType == ReportType.ACTIVITY) {
+    when(stager.stageReports(ReportType.ACTIVITY)).thenReturn(ImmutableList.of("a", "b"));
-      ActivityReportingQueryBuilder activityBuilder = new ActivityReportingQueryBuilder();
+    when(stager.stageReports(ReportType.TRANSACTIONS)).thenReturn(ImmutableList.of("c", "d"));
-      activityBuilder.projectId = "test-project";
-      activityBuilder.yearMonth = "2017-06";
-      action.queryBuilder = activityBuilder;
-    } else {
-      TransactionsReportingQueryBuilder transactionsBuilder =
-          new TransactionsReportingQueryBuilder();
-      transactionsBuilder.projectId = "test-project";
-      transactionsBuilder.yearMonth = "2017-06";
-      action.queryBuilder = transactionsBuilder;
   }
-    action.reportType = reportType;
+
-    action.reportingBucket = "test-bucket";
+  private IcannReportingStagingAction createAction(ImmutableList<ReportType> reportingMode) {
-    action.yearMonth = "2017-06";
+    IcannReportingStagingAction action = new IcannReportingStagingAction();
-    action.subdir = Optional.empty();
+    action.reportTypes = reportingMode;
-    action.bigquery = bigquery;
-    action.gcsUtils = new GcsUtils(gcsService, 1024);
     action.response = response;
+    action.stager = stager;
     return action;
   }
 
-  private void setUpBigquery() {
+  @Test
-    when(bigquery.query(any(String.class), any(DestinationTable.class))).thenReturn(fakeFuture());
+  public void testActivityReportingMode_onlyStagesActivityReports() throws Exception {
-    DestinationTable.Builder tableBuilder = new DestinationTable.Builder()
+    IcannReportingStagingAction action = createAction(ImmutableList.of(ReportType.ACTIVITY));
-        .datasetId("testdataset")
+    action.run();
-        .type(TableType.TABLE)
+    verify(stager).stageReports(ReportType.ACTIVITY);
-        .name("tablename")
+    verify(stager).createAndUploadManifest(ImmutableList.of("a", "b"));
-        .overwrite(true);
-    when(bigquery.buildDestinationTable(any(String.class))).thenReturn(tableBuilder);
   }
 
   @Test
-  public void testRunSuccess_activityReport() throws Exception {
+  public void testAbsentReportingMode_stagesBothReports() throws Exception {
-    setUpBigquery();
+    IcannReportingStagingAction action =
-    ImmutableTable<Integer, TableFieldSchema, Object> activityReportTable =
+        createAction(ImmutableList.of(ReportType.ACTIVITY, ReportType.TRANSACTIONS));
-        new ImmutableTable.Builder<Integer, TableFieldSchema, Object>()
-            .put(1, new TableFieldSchema().setName("tld"), "fooTld")
-            .put(1, new TableFieldSchema().setName("fooField"), "12")
-            .put(1, new TableFieldSchema().setName("barField"), "34")
-            .put(2, new TableFieldSchema().setName("tld"), "barTld")
-            .put(2, new TableFieldSchema().setName("fooField"), "56")
-            .put(2, new TableFieldSchema().setName("barField"), "78")
-            .build();
-    when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(activityReportTable);
-    IcannReportingStagingAction action = createAction(ReportType.ACTIVITY);
     action.run();
-
+    verify(stager).stageReports(ReportType.ACTIVITY);
-    String expectedReport1 = "fooField,barField\r\n12,34";
+    verify(stager).stageReports(ReportType.TRANSACTIONS);
-    String expectedReport2 = "fooField,barField\r\n56,78";
+    verify(stager).createAndUploadManifest(ImmutableList.of("a", "b", "c", "d"));
-    byte[] generatedFile1 =
-        readGcsFile(
-            gcsService,
-            new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv"));
-    assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
-    byte[] generatedFile2 =
-        readGcsFile(
-            gcsService,
-            new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv"));
-    assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
-  }
-
-  @Test
-  public void testRunSuccess_transactionsReport() throws Exception {
-    setUpBigquery();
-    /*
-       The fake table result looks like:
-          tld     registrar  field
-        1 fooTld  reg1       10
-        2 fooTld  reg2       20
-        3 barTld  reg1       30
-     */
-    ImmutableTable<Integer, TableFieldSchema, Object> transactionReportTable =
-        new ImmutableTable.Builder<Integer, TableFieldSchema, Object>()
-            .put(1, new TableFieldSchema().setName("tld"), "fooTld")
-            .put(1, new TableFieldSchema().setName("registrar"), "reg1")
-            .put(1, new TableFieldSchema().setName("field"), "10")
-            .put(2, new TableFieldSchema().setName("tld"), "fooTld")
-            .put(2, new TableFieldSchema().setName("registrar"), "reg2")
-            .put(2, new TableFieldSchema().setName("field"), "20")
-            .put(3, new TableFieldSchema().setName("tld"), "barTld")
-            .put(3, new TableFieldSchema().setName("registrar"), "reg1")
-            .put(3, new TableFieldSchema().setName("field"), "30")
-            .build();
-    when(bigquery.queryToLocalTableSync(any(String.class))).thenReturn(transactionReportTable);
-    IcannReportingStagingAction action = createAction(ReportType.TRANSACTIONS);
-    action.reportType = ReportType.TRANSACTIONS;
-    action.run();
-
-    String expectedReport1 = "registrar,field\r\nreg1,10\r\nreg2,20";
-    String expectedReport2 = "registrar,field\r\nreg1,30";
-    byte[] generatedFile1 =
-        readGcsFile(
-            gcsService,
-            new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv"));
-    assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
-    byte[] generatedFile2 =
-        readGcsFile(
-            gcsService,
-            new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv"));
-    assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
-  }
-
-  private ListenableFuture<DestinationTable> fakeFuture() {
-    return new ListenableFuture<DestinationTable>() {
-      @Override
-      public void addListener(Runnable runnable, Executor executor) {
-        // No-op
-      }
-
-      @Override
-      public boolean cancel(boolean mayInterruptIfRunning) {
-        return false;
-      }
-
-      @Override
-      public boolean isCancelled() {
-        return false;
-      }
-
-      @Override
-      public boolean isDone() {
-        return false;
-      }
-
-      @Override
-      public DestinationTable get() throws InterruptedException, ExecutionException {
-        return null;
-      }
-
-      @Override
-      public DestinationTable get(long timeout, TimeUnit unit)
-          throws InterruptedException, ExecutionException, TimeoutException {
-        return null;
-      }
-    };
   }
 }
+

javatests/google/registry/reporting/IcannReportingUploadActionTest.java

@@ -16,10 +16,6 @@ package google.registry.reporting;
 
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
-import static com.google.common.truth.Truth8.assertThat;
-import static google.registry.reporting.IcannReportingModule.ReportType.ACTIVITY;
-import static google.registry.reporting.IcannReportingModule.ReportType.TRANSACTIONS;
-import static google.registry.testing.DatastoreHelper.createTld;
 import static google.registry.testing.GcsTestingUtils.writeGcsFile;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.mockito.Mockito.doThrow;
@@ -38,7 +34,6 @@ import google.registry.testing.FakeResponse;
 import google.registry.testing.FakeSleeper;
 import google.registry.util.Retrier;
 import java.io.IOException;
-import java.util.Optional;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -52,37 +47,37 @@ public class IcannReportingUploadActionTest {
   @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build();
 
   private static final byte[] FAKE_PAYLOAD = "test,csv\n13,37".getBytes(UTF_8);
+  private static final byte[] MANIFEST_PAYLOAD = "test-transactions-201706.csv\n".getBytes(UTF_8);
   private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class);
   private final FakeResponse response = new FakeResponse();
   private final GcsService gcsService = GcsServiceFactory.createGcsService();
   private final GcsFilename reportFile =
       new GcsFilename("basin/icann/monthly/2017-06", "test-transactions-201706.csv");
+  private final GcsFilename manifestFile =
+      new GcsFilename("basin/icann/monthly/2017-06", "MANIFEST.txt");
 
   private IcannReportingUploadAction createAction() {
     IcannReportingUploadAction action = new IcannReportingUploadAction();
     action.icannReporter = mockReporter;
     action.gcsUtils = new GcsUtils(gcsService, 1024);
     action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
-    action.yearMonth = "2017-06";
+    action.subdir = "icann/monthly/2017-06";
-    action.reportType = TRANSACTIONS;
+    action.reportingBucket = "basin";
-    action.subdir = Optional.empty();
-    action.tld = "test";
-    action.icannReportingBucket = "basin";
     action.response = response;
     return action;
   }
 
   @Before
   public void before() throws Exception {
-    createTld("test");
     writeGcsFile(gcsService, reportFile, FAKE_PAYLOAD);
+    writeGcsFile(gcsService, manifestFile, MANIFEST_PAYLOAD);
   }
 
   @Test
   public void testSuccess() throws Exception {
     IcannReportingUploadAction action = createAction();
     action.run();
-    verify(mockReporter).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+    verify(mockReporter).send(FAKE_PAYLOAD, "test-transactions-201706.csv");
     verifyNoMoreInteractions(mockReporter);
     assertThat(((FakeResponse) action.response).getPayload())
         .isEqualTo("OK, sending: test,csv\n13,37");
@@ -94,90 +89,26 @@ public class IcannReportingUploadActionTest {
     doThrow(new IOException("Expected exception."))
         .doNothing()
         .when(mockReporter)
-        .send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+        .send(FAKE_PAYLOAD, "test-transactions-201706.csv");
     action.run();
-    verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test", "2017-06", TRANSACTIONS);
+    verify(mockReporter, times(2)).send(FAKE_PAYLOAD, "test-transactions-201706.csv");
     verifyNoMoreInteractions(mockReporter);
     assertThat(((FakeResponse) action.response).getPayload())
         .isEqualTo("OK, sending: test,csv\n13,37");
   }
 
   @Test
-  public void testFail_NonexisistentTld() throws Exception {
+  public void testFail_FileNotFound() throws Exception {
     IcannReportingUploadAction action = createAction();
-    action.tld = "invalidTld";
+    action.subdir = "somewhere/else";
     try {
       action.run();
-      assertWithMessage("Expected IllegalArgumentException to be thrown").fail();
+      assertWithMessage("Expected IllegalStateException to be thrown").fail();
     } catch (IllegalArgumentException expected) {
       assertThat(expected)
           .hasMessageThat()
-          .isEqualTo("TLD invalidTld does not exist");
+          .isEqualTo("Object MANIFEST.txt in bucket basin/somewhere/else not found");
     }
   }
-
-  @Test
-  public void testFail_InvalidYearMonth() throws Exception {
-    IcannReportingUploadAction action = createAction();
-    action.yearMonth = "2017-3";
-    try {
-      action.run();
-      assertWithMessage("Expected IllegalStateException to be thrown").fail();
-    } catch (IllegalStateException expected) {
-      assertThat(expected)
-          .hasMessageThat()
-          .isEqualTo("yearMonth must be in YYYY-MM format, got 2017-3 instead.");
-    }
-  }
-
-  @Test
-  public void testFail_InvalidSubdir() throws Exception {
-    IcannReportingUploadAction action = createAction();
-    action.subdir = Optional.of("/subdir/with/slash");
-    try {
-      action.run();
-      assertWithMessage("Expected IllegalStateException to be thrown").fail();
-    } catch (IllegalStateException expected) {
-      assertThat(expected)
-          .hasMessageThat()
-          .isEqualTo(
-              "subdir must not start or end with a \"/\", got /subdir/with/slash instead.");
-    }
-  }
-
-  @Test
-  public void testFail_FileNotFound() throws Exception {
-    IcannReportingUploadAction action = createAction();
-    action.yearMonth = "1234-56";
-    try {
-      action.run();
-      assertWithMessage("Expected IllegalStateException to be thrown").fail();
-    } catch (IllegalStateException expected) {
-      assertThat(expected)
-          .hasMessageThat()
-          .isEqualTo(
-              "ICANN report object test-transactions-123456.csv "
-                  + "in bucket basin/icann/monthly/1234-56 not found");
-    }
-  }
-
-  @Test
-  public void testSuccess_CreateFilename() throws Exception{
-    assertThat(IcannReportingUploadAction.createFilename("test", "2017-06", ACTIVITY))
-        .isEqualTo("test-activity-201706.csv");
-    assertThat(IcannReportingUploadAction.createFilename("foo", "1234-56", TRANSACTIONS))
-        .isEqualTo("foo-transactions-123456.csv");
-  }
-
-  @Test
-  public void testSuccess_CreateBucketname() throws Exception{
-    assertThat(
-        IcannReportingUploadAction
-            .createReportingBucketName("gs://my-reporting", Optional.<String>empty(), "2017-06"))
-        .isEqualTo("gs://my-reporting/icann/monthly/2017-06");
-    assertThat(
-        IcannReportingUploadAction
-            .createReportingBucketName("gs://my-reporting", Optional.of("manual"), "2017-06"))
-        .isEqualTo("gs://my-reporting/manual");
-  }
 }
+

javatests/google/registry/reporting/ReportingUtilsTest.java

@@ -0,0 +1,40 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.reporting;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import google.registry.reporting.IcannReportingModule.ReportType;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link google.registry.reporting.ReportingUtils}. */
+@RunWith(JUnit4.class)
+public class ReportingUtilsTest {
+  @Test
+  public void testCreateFilename_success() {
+    assertThat(ReportingUtils.createFilename("test", "2017-06", ReportType.ACTIVITY))
+        .isEqualTo("test-activity-201706.csv");
+    assertThat(ReportingUtils.createFilename("foo", "2017-06", ReportType.TRANSACTIONS))
+        .isEqualTo("foo-transactions-201706.csv");
+  }
+
+  @Test
+  public void testCreateBucketName_success() {
+    assertThat(ReportingUtils.createReportingBucketName("gs://domain-registry-basin", "my/subdir"))
+        .isEqualTo("gs://domain-registry-basin/my/subdir");
+  }
+}

javatests/google/registry/reporting/TransactionsReportingQueryBuilderTest.java

@@ -15,7 +15,6 @@
 package google.registry.reporting;
 
 import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -30,7 +29,7 @@ public class TransactionsReportingQueryBuilderTest {
 
   private TransactionsReportingQueryBuilder getQueryBuilder() {
     TransactionsReportingQueryBuilder queryBuilder = new TransactionsReportingQueryBuilder();
-    queryBuilder.yearMonth = "2017-06";
+    queryBuilder.yearMonth = "2017-09";
     queryBuilder.projectId = "domain-registry-alpha";
     return queryBuilder;
   }
@@ -41,13 +40,12 @@ public class TransactionsReportingQueryBuilderTest {
     assertThat(queryBuilder.getReportQuery())
         .isEqualTo(
             "#standardSQL\nSELECT * FROM "
-                + "`domain-registry-alpha.icann_reporting.transactions_report_aggregation_201706`");
+                + "`domain-registry-alpha.icann_reporting.transactions_report_aggregation_201709`");
   }
 
   @Test
   public void testIntermediaryQueryMatch() throws IOException {
-    TransactionsReportingQueryBuilder queryBuilder = getQueryBuilder();
+    ImmutableList<String> expectedQueryNames =
-    ImmutableList<String> queryNames =
         ImmutableList.of(
             TransactionsReportingQueryBuilder.TRANSACTIONS_REPORT_AGGREGATION,
             TransactionsReportingQueryBuilder.REGISTRAR_IANA_ID,
@@ -57,9 +55,10 @@ public class TransactionsReportingQueryBuilderTest {
             TransactionsReportingQueryBuilder.TRANSACTION_TRANSFER_LOSING,
             TransactionsReportingQueryBuilder.ATTEMPTED_ADDS);
 
+    TransactionsReportingQueryBuilder queryBuilder = getQueryBuilder();
     ImmutableMap<String, String> actualQueries = queryBuilder.getViewQueryMap();
-    for (String queryName : queryNames) {
+    for (String queryName : expectedQueryNames) {
-      String actualTableName = String.format("%s_201706", queryName);
+      String actualTableName = String.format("%s_201709", queryName);
       String testFilename = String.format("%s_test.sql", queryName);
       assertThat(actualQueries.get(actualTableName))
           .isEqualTo(ReportingTestData.getString(testFilename));

javatests/google/registry/reporting/testdata/activity_report_aggregation_test.sql

@@ -19,8 +19,6 @@
 SELECT
   RealTlds.tld AS tld,
   SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars,
-  SUM(IF(metricName = 'ramp-up-registrars', count, 0)) AS ramp_up_registrars,
-  SUM(IF(metricName = 'pre-ramp-up-registrars', count, 0)) AS pre_ramp_up_registrars,
   -- We use the Centralized Zone Data Service.
   "CZDS" AS zfa_passwords,
   SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries,
@@ -65,7 +63,7 @@ SELECT
   -- filter so that only metrics with that TLD or a NULL TLD are counted
   -- towards a given TLD.
 FROM (
-SELECT tldStr as tld
+SELECT tldStr AS tld
 FROM `domain-registry-alpha.latest_datastore_export.Registry`
 WHERE tldType = 'REAL'
 ) as RealTlds
@@ -82,16 +80,16 @@ CROSS JOIN(
     SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 as count
     UNION ALL
     SELECT * FROM
-    `domain-registry-alpha.icann_reporting.registrar_operating_status_201706`
+    `domain-registry-alpha.icann_reporting.registrar_operating_status_201709`
     UNION ALL
     SELECT * FROM
-    `domain-registry-alpha.icann_reporting.dns_counts_201706`
+    `domain-registry-alpha.icann_reporting.dns_counts_201709`
     UNION ALL
     SELECT * FROM
-    `domain-registry-alpha.icann_reporting.epp_metrics_201706`
+    `domain-registry-alpha.icann_reporting.epp_metrics_201709`
     UNION ALL
     SELECT * FROM
-    `domain-registry-alpha.icann_reporting.whois_counts_201706`
+    `domain-registry-alpha.icann_reporting.whois_counts_201709`
     -- END INTERMEDIARY DATA SOURCES --
     )) AS TldMetrics
 WHERE RealTlds.tld = TldMetrics.tld OR TldMetrics.tld IS NULL

javatests/google/registry/reporting/testdata/attempted_adds_test.sql

@@ -52,8 +52,8 @@ FROM (
       FROM
         `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
       WHERE _TABLE_SUFFIX
-      BETWEEN '20170601'
+      BETWEEN '20170901'
-      AND '20170630')
+      AND '20170930')
     JOIN UNNEST(logMessage) AS logMessages
     -- Look for metadata logs from epp and registrar console requests
     WHERE requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr')

javatests/google/registry/reporting/testdata/dns_counts_internal_test.sql

@@ -0,0 +1,24 @@
+#standardSQL
+  -- Copyright 2017 The Nomulus Authors. All Rights Reserved.
+  --
+  -- Licensed under the Apache License, Version 2.0 (the "License");
+  -- you may not use this file except in compliance with the License.
+  -- You may obtain a copy of the License at
+  --
+  --     http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+
+  -- Retrieve per-TLD DNS query counts.
+
+  -- This is a hack to enable using DNS counts from the internal-only #plx
+  -- workflow. See other references to b/67301320 in the codebase to see the
+  -- full extent of the hackery.
+  -- TODO(b/67301320): Delete this when we can make open-source DNS metrics.
+
+SELECT *
+FROM `domain-registry-alpha.icann_reporting.dns_counts_from_plx`

javatests/google/registry/reporting/testdata/dns_counts_test.sql

@@ -15,14 +15,13 @@
 
   -- Query for DNS metrics.
 
-  -- This is a no-op until after we transition to Google Cloud DNS, which
+  -- You must configure this yourself to enable activity reporting, according
-  -- will likely export metrics via Stackdriver.
+  -- to whatever metrics your DNS provider makes available. We hope to make
+  -- this available in the open-source build in the near future.
 
 SELECT
-  -- DNS metrics apply to all tlds, which requires the 'null' magic value.
   STRING(NULL) AS tld,
   metricName,
-  -- TODO(b/63388735): Change this to actually query Google Cloud DNS when ready.
   -1 AS count
 FROM ((
   SELECT 'dns-udp-queries' AS metricName)

javatests/google/registry/reporting/testdata/epp_metrics_test.sql

@@ -39,7 +39,7 @@ FROM (
       -- Extract the logged JSON payload.
       REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
       AS json
-    FROM `domain-registry-alpha.icann_reporting.monthly_logs_201706` AS logs
+    FROM `domain-registry-alpha.icann_reporting.monthly_logs_201709` AS logs
     JOIN
       UNNEST(logs.logMessage) AS logMessage
     WHERE

javatests/google/registry/reporting/testdata/monthly_logs_test.sql

@@ -27,4 +27,4 @@ SELECT
 FROM
   `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
 WHERE
-  _TABLE_SUFFIX BETWEEN '20170601' AND '20170630'
+  _TABLE_SUFFIX BETWEEN '20170901' AND '20170930'

javatests/google/registry/reporting/testdata/registrar_iana_id_test.sql

@@ -26,5 +26,5 @@ FROM
   UNNEST(allowedTlds) as allowed_tlds
 WHERE (type = 'REAL' OR type = 'INTERNAL')
 -- Filter out prober data
-AND NOT ENDS_WITH(allowed_tlds, "test")
+AND NOT ENDS_WITH(allowed_tlds, ".test")
 ORDER BY tld, registrarName

javatests/google/registry/reporting/testdata/registrar_operating_status_test.sql

@@ -23,5 +23,5 @@ SELECT
 FROM
   `domain-registry-alpha.latest_datastore_export.Registrar`
 WHERE
-  type = 'REAL'
+  (type = 'REAL' OR type = 'INTERNAL')
 GROUP BY metricName

javatests/google/registry/reporting/testdata/total_domains_test.sql

@@ -32,7 +32,7 @@ JOIN
 ON
   currentSponsorClientId = registrar_table.__key__.name
 WHERE
-  domain_table._d = "DomainResource"
+  domain_table._d = 'DomainResource'
-  AND (registrar_table.type = "REAL" OR registrar_table.type = "INTERNAL")
+  AND (registrar_table.type = 'REAL' OR registrar_table.type = 'INTERNAL')
 GROUP BY tld, registrarName
 ORDER BY tld, registrarName

javatests/google/registry/reporting/testdata/total_nameservers_test.sql

@@ -45,12 +45,12 @@ JOIN (
     `domain-registry-alpha.latest_datastore_export.DomainBase`,
     UNNEST(nsHosts) AS hosts
   WHERE _d = 'DomainResource'
-  AND creationTime <= TIMESTAMP("2017-06-30 23:59:59")
+  AND creationTime <= TIMESTAMP("2017-09-30 23:59:59")
-  AND deletionTime > TIMESTAMP("2017-06-30 23:59:59") ) AS domain_table
+  AND deletionTime > TIMESTAMP("2017-09-30 23:59:59") ) AS domain_table
 ON
   host_table.__key__.name = domain_table.referencedHostName
-WHERE creationTime <= TIMESTAMP("2017-06-30 23:59:59")
+WHERE creationTime <= TIMESTAMP("2017-09-30 23:59:59")
-AND deletionTime > TIMESTAMP("2017-06-30 23:59:59")
+AND deletionTime > TIMESTAMP("2017-09-30 23:59:59")
 GROUP BY tld, registrarName
 ORDER BY tld, registrarName
 

javatests/google/registry/reporting/testdata/transaction_counts_test.sql

@@ -63,10 +63,8 @@ FROM (
     WHERE entries.domainTransactionRecords IS NOT NULL )
   -- Only look at this month's data
   WHERE reportingTime
-  BETWEEN TIMESTAMP('2017-06-01 00:00:00')
+  BETWEEN TIMESTAMP('2017-09-01 00:00:00')
-  AND TIMESTAMP('2017-06-30 23:59:59')
+  AND TIMESTAMP('2017-09-30 23:59:59')
-    -- Ignore prober data
-  AND NOT ENDS_WITH(tld, "test")
   GROUP BY
     tld,
     clientId,

javatests/google/registry/reporting/testdata/transaction_transfer_losing_test.sql

@@ -63,10 +63,8 @@ FROM (
     WHERE entries.domainTransactionRecords IS NOT NULL )
   -- Only look at this month's data
   WHERE reportingTime
-  BETWEEN TIMESTAMP('2017-06-01 00:00:00')
+  BETWEEN TIMESTAMP('2017-09-01 00:00:00')
-  AND TIMESTAMP('2017-06-30 23:59:59')
+  AND TIMESTAMP('2017-09-30 23:59:59')
-    -- Ignore prober data
-  AND NOT ENDS_WITH(tld, "test")
   GROUP BY
     tld,
     clientId,

javatests/google/registry/reporting/testdata/transactions_report_aggregation_test.sql

@@ -20,7 +20,8 @@
 
 SELECT
   registrars.tld as tld,
-  registrars.registrar_name as registrar_name,
+  -- Surround registrar names with quotes to handle names containing a comma.
+  FORMAT("\"%s\"", registrars.registrar_name) as registrar_name,
   registrars.iana_id as iana_id,
   SUM(IF(metrics.metricName = 'TOTAL_DOMAINS', metrics.metricValue, 0)) AS total_domains,
   SUM(IF(metrics.metricName = 'TOTAL_NAMESERVERS', metrics.metricValue, 0)) AS total_nameservers,
@@ -62,26 +63,33 @@ SELECT
   0 AS agp_exemptions_granted,
   0 AS agp_exempted_domains,
   SUM(IF(metrics.metricName = 'ATTEMPTED_ADDS', metrics.metricValue, 0)) AS attempted_adds
-FROM (
+FROM
-  SELECT *
+-- Only produce reports for real TLDs
-  FROM `domain-registry-alpha.icann_reporting.registrar_iana_id_201706`) AS registrars
+(SELECT tldStr AS tld
+  FROM `domain-registry-alpha.latest_datastore_export.Registry`
+  WHERE tldType = 'REAL') AS registries
+JOIN
+(SELECT *
+  FROM `domain-registry-alpha.icann_reporting.registrar_iana_id_201709`)
+  AS registrars
+ON registries.tld = registrars.tld
 -- We LEFT JOIN to produce reports even if the registrar made no transactions
 LEFT OUTER JOIN (
   -- Gather all intermediary data views
   SELECT *
-  FROM `domain-registry-alpha.icann_reporting.total_domains_201706`
+  FROM `domain-registry-alpha.icann_reporting.total_domains_201709`
   UNION ALL
   SELECT *
-  FROM `domain-registry-alpha.icann_reporting.total_nameservers_201706`
+  FROM `domain-registry-alpha.icann_reporting.total_nameservers_201709`
   UNION ALL
   SELECT *
-  FROM `domain-registry-alpha.icann_reporting.transaction_counts_201706`
+  FROM `domain-registry-alpha.icann_reporting.transaction_counts_201709`
   UNION ALL
   SELECT *
-  FROM `domain-registry-alpha.icann_reporting.transaction_transfer_losing_201706`
+  FROM `domain-registry-alpha.icann_reporting.transaction_transfer_losing_201709`
   UNION ALL
   SELECT *
-  FROM `domain-registry-alpha.icann_reporting.attempted_adds_201706` ) AS metrics
+  FROM `domain-registry-alpha.icann_reporting.attempted_adds_201709` ) AS metrics
 -- Join on tld and registrar name
 ON registrars.tld = metrics.tld
 AND registrars.registrar_name = metrics.registrar_name

javatests/google/registry/reporting/testdata/whois_counts_test.sql

@@ -26,7 +26,7 @@ SELECT
   END AS metricName,
   COUNT(requestPath) AS count
 FROM
-  `domain-registry-alpha.icann_reporting.monthly_logs_201706`
+  `domain-registry-alpha.icann_reporting.monthly_logs_201709`
 GROUP BY
   metricName
 HAVING

javatests/google/registry/request/RequestParametersTest.java

@@ -19,6 +19,7 @@ import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.request.RequestParameters.extractBooleanParameter;
 import static google.registry.request.RequestParameters.extractEnumParameter;
 import static google.registry.request.RequestParameters.extractOptionalDatetimeParameter;
+import static google.registry.request.RequestParameters.extractOptionalEnumParameter;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
@@ -147,6 +148,25 @@ public class RequestParametersTest {
     extractEnumParameter(req, Club.class, "spin");
   }
 
+  @Test
+  public void testOptionalExtractEnumValue_givenValue_returnsValue() throws Exception {
+    when(req.getParameter("spin")).thenReturn("DANCE");
+    assertThat(extractOptionalEnumParameter(req, Club.class, "spin")).hasValue(Club.DANCE);
+  }
+
+  @Test
+  public void testOptionalExtractEnumValue_noValue_returnsAbsent() throws Exception {
+    when(req.getParameter("spin")).thenReturn("");
+    assertThat(extractOptionalEnumParameter(req, Club.class, "spin")).isEmpty();
+  }
+
+  @Test
+  public void testOptionalExtractEnumValue_nonExistentValue_throwsBadRequest() throws Exception {
+    when(req.getParameter("spin")).thenReturn("sing");
+    thrown.expect(BadRequestException.class, "spin");
+    extractOptionalEnumParameter(req, Club.class, "spin");
+  }
+
   @Test
   public void testExtractRequiredDatetimeParameter_correctValue_works() throws Exception {
     when(req.getParameter("timeParam")).thenReturn("2015-08-27T13:25:34.123Z");