Migrate to new Sheets v4 api

This moves us from the oudated google/data XML api to the OnePlatform REST/JSON api, finally silencing the deprecation warnings we've been seeing.

The synchronization algorithm diffs the spreadsheet's current values with its internally sourced values, adding the row to a batch update request if there's a discrepancy. Additional internal data are added as an append operation to the end of the sheet, and any extraneous spreadsheet data is cleared from the spreadsheet.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=169273590

java/google/registry/export/sheet/BUILD

@@ -8,7 +8,6 @@ java_library(
     name = "sheet",
     srcs = glob(["*.java"]),
     deps = [
-        "//java/com/google/gdata:spreadsheet",
         "//java/google/registry/config",
         "//java/google/registry/model",
         "//java/google/registry/request",
@@ -17,11 +16,12 @@ java_library(
         "//java/google/registry/util",
         "//third_party/java/objectify:objectify-v4_1",
         "@com_google_api_client",
+        "@com_google_apis_google_api_services_sheets",
         "@com_google_appengine_api_1_0_sdk",
         "@com_google_code_findbugs_jsr305",
         "@com_google_dagger",
-        "@com_google_gdata_core",
         "@com_google_guava",
+        "@com_google_http_client",
         "@javax_servlet_api",
         "@joda_time",
     ],

java/google/registry/export/sheet/SheetSynchronizer.java

@@ -14,31 +14,32 @@
 
 package google.registry.export.sheet;
 
+import static com.google.common.base.Strings.nullToEmpty;
+
+import com.google.api.services.sheets.v4.Sheets;
+import com.google.api.services.sheets.v4.model.AppendValuesResponse;
+import com.google.api.services.sheets.v4.model.BatchUpdateValuesRequest;
+import com.google.api.services.sheets.v4.model.BatchUpdateValuesResponse;
+import com.google.api.services.sheets.v4.model.ClearValuesRequest;
+import com.google.api.services.sheets.v4.model.ClearValuesResponse;
+import com.google.api.services.sheets.v4.model.ValueRange;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.gdata.client.spreadsheet.SpreadsheetService;
-import com.google.gdata.data.spreadsheet.CustomElementCollection;
-import com.google.gdata.data.spreadsheet.ListEntry;
-import com.google.gdata.data.spreadsheet.ListFeed;
-import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
-import com.google.gdata.data.spreadsheet.WorksheetEntry;
-import com.google.gdata.util.ServiceException;
-import google.registry.util.Retrier;
+import google.registry.util.FormattingLogger;
 import java.io.IOException;
-import java.net.URL;
+import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.Callable;
 import javax.inject.Inject;
 
 /** Generic data synchronization utility for Google Spreadsheets. */
 class SheetSynchronizer {
 
-  private static final String SPREADSHEET_URL_PREFIX =
-      "https://spreadsheets.google.com/feeds/spreadsheets/";
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
 
-  @Inject SpreadsheetService spreadsheetService;
+  private static final String SHEET_NAME = "Registrars";
+
+  @Inject Sheets sheetsService;
   @Inject SheetSynchronizer() {}
-  @Inject Retrier retrier;
 
   /**
    * Replace the contents of a Google Spreadsheet with {@code data}.
@@ -50,10 +51,10 @@ class SheetSynchronizer {
    * credential email address.
    *
    * <p>The algorithm works by first assuming that the spreadsheet is sorted in the same way that
-   * {@code data} is sorted. It then iterates through the existing rows and comparing them to the
-   * items in {@code data}. Iteration continues until we either run out of rows, or items in
-   * {@code data}. If there's any rows remaining, they'll be deleted. If instead, items remain in
-   * data, they'll be inserted.
+   * {@code data} is sorted (i.e. registrar name order). It then iterates through the existing rows
+   * and comparing them to the items in {@code data}. Iteration continues until we either run out of
+   * rows, or items in {@code data}. If there's any rows remaining, they'll be deleted. If instead,
+   * items remain in data, they'll be appended to the end of the sheet.
    *
    * @param spreadsheetId The ID of your spreadsheet. This can be obtained by opening the Google
    *     spreadsheet in your browser and copying the ID from the URL.
@@ -61,62 +62,127 @@ class SheetSynchronizer {
    *     spreadsheet. Each row is a map, where the key must be exactly the same as the column header
    *     cell in the spreadsheet, and value is an arbitrary object which will be converted to a
    *     string before storing it in the spreadsheet.
-   * @throws IOException error communicating with the GData service.
-   * @throws ServiceException if a system error occurred when retrieving the entry.
-   * @throws com.google.gdata.util.ParseException error parsing the returned entry.
-   * @throws com.google.gdata.util.ResourceNotFoundException if an entry URL is not valid.
-   * @throws com.google.gdata.util.ServiceForbiddenException if the GData service cannot get the
-   *     entry resource due to access constraints.
-   * @see <a href="https://developers.google.com/google-apps/spreadsheets/">Google Sheets API</a>
+   * @throws IOException if encountering an error communicating with the Sheets service.
+   * @see <a href="https://developers.google.com/sheets/">Google Sheets API v4</a>
    */
   void synchronize(String spreadsheetId, ImmutableList<ImmutableMap<String, String>> data)
-      throws IOException, ServiceException {
-    URL url = new URL(SPREADSHEET_URL_PREFIX + spreadsheetId);
-    SpreadsheetEntry spreadsheet = spreadsheetService.getEntry(url, SpreadsheetEntry.class);
-    WorksheetEntry worksheet = spreadsheet.getWorksheets().get(0);
-    worksheet.setRowCount(data.size() + 1); // account for header row
-    worksheet = worksheet.update();
-    final ListFeed listFeed =
-        spreadsheetService.getFeed(worksheet.getListFeedUrl(), ListFeed.class);
-    List<ListEntry> entries = listFeed.getEntries();
-    int commonSize = Math.min(entries.size(), data.size());
-    for (int i = 0; i < commonSize; i++) {
-      final ListEntry entry = entries.get(i);
-      CustomElementCollection elements = entry.getCustomElements();
+      throws IOException {
+
+    // Get the existing sheet's values
+    ValueRange sheetValues =
+        sheetsService.spreadsheets().values().get(spreadsheetId, SHEET_NAME).execute();
+    List<List<Object>> originalVals = sheetValues.getValues();
+
+    // Assemble headers from the sheet
+    ImmutableList.Builder<String> headersBuilder = new ImmutableList.Builder<>();
+    for (Object headerCell : originalVals.get(0)) {
+      headersBuilder.add(headerCell.toString());
+    }
+    ImmutableList<String> headers = headersBuilder.build();
+    // Pop off the headers row
+    originalVals.remove(0);
+
+    List<ValueRange> updates = new ArrayList<>();
+    int minSize = Math.min(originalVals.size(), data.size());
+    for (int i = 0; i < minSize; i++) {
       boolean mutated = false;
-      for (ImmutableMap.Entry<String, String> cell : data.get(i).entrySet()) {
-        if (!cell.getValue().equals(elements.getValue(cell.getKey()))) {
+      List<Object> cellRow = originalVals.get(i);
+      // If the row isn't full, pad it with empty strings until it is
+      while (cellRow.size() < headers.size()) {
+        cellRow.add("");
+      }
+      for (int j = 0; j < headers.size(); j++) {
+        // Look for the value corresponding to the row and header indices in data
+        String dataField = data.get(i).get(headers.get(j));
+        // If the cell's header matches a data header, and the values aren't equal, mutate it
+        if (dataField != null && !cellRow.get(j).toString().equals(dataField)) {
           mutated = true;
-          elements.setValueLocal(cell.getKey(), cell.getValue());
+          originalVals.get(i).set(j, dataField);
         }
       }
       if (mutated) {
-        // Wrap in a retrier to deal with transient HTTP failures, which are IOExceptions wrapped
-        // in RuntimeExceptions.
-        retrier.callWithRetry(new Callable<Void>() {
-          @Override
-          public Void call() throws Exception {
-            entry.update();
-            return null;
-          }}, RuntimeException.class);
+        ValueRange rowUpdate =
+            new ValueRange()
+                .setValues(originalVals.subList(i, i + 1))
+                .setRange(getCellRange(i));
+        updates.add(rowUpdate);
       }
     }
-    if (data.size() > entries.size()) {
-      for (int i = entries.size(); i < data.size(); i++) {
-        final ListEntry entry = listFeed.createEntry();
-        CustomElementCollection elements = entry.getCustomElements();
-        for (ImmutableMap.Entry<String, String> cell : data.get(i).entrySet()) {
-          elements.setValueLocal(cell.getKey(), cell.getValue());
+    // Update the mutated cells if necessary
+    if (!updates.isEmpty()) {
+      BatchUpdateValuesRequest updateRequest = new BatchUpdateValuesRequest()
+          .setValueInputOption("RAW")
+          .setData(updates);
+
+      BatchUpdateValuesResponse response =
+          sheetsService.spreadsheets().values().batchUpdate(spreadsheetId, updateRequest).execute();
+      Integer cellsUpdated = response.getTotalUpdatedCells();
+      logger.infofmt("Updated %d originalVals", cellsUpdated != null ? cellsUpdated : 0);
+    }
+
+    // Append extra rows if necessary
+    if (data.size() > originalVals.size()) {
+      ImmutableList.Builder<List<Object>> valsBuilder = new ImmutableList.Builder<>();
+      for (int i = originalVals.size(); i < data.size(); i++) {
+        ImmutableList.Builder<Object> rowBuilder = new ImmutableList.Builder<>();
+        for (String header : headers) {
+          rowBuilder.add(nullToEmpty(data.get(i).get(header)));
         }
-        // Wrap in a retrier to deal with transient HTTP failures, which are IOExceptions wrapped
-        // in RuntimeExceptions.
-        retrier.callWithRetry(new Callable<Void>() {
-          @Override
-          public Void call() throws Exception {
-            listFeed.insert(entry);
-            return null;
-          }}, RuntimeException.class);
+        valsBuilder.add(rowBuilder.build());
       }
+      // Start the append at index originalVals.size (where the previous operation left off)
+      ValueRange appendUpdate = new ValueRange().setValues(valsBuilder.build());
+      AppendValuesResponse appendResponse = sheetsService
+          .spreadsheets()
+          .values()
+          .append(spreadsheetId, getCellRange(originalVals.size()), appendUpdate)
+          .setValueInputOption("RAW")
+          .setInsertDataOption("INSERT_ROWS")
+          .execute();
+      logger.infofmt(
+          "Appended %d rows to range %s",
+          data.size() - originalVals.size(), appendResponse.getTableRange());
+    // Clear the extra rows if necessary
+    } else if (data.size() < originalVals.size()) {
+      // Clear other rows if there's more originalVals on the sheet than live data.
+      ClearValuesResponse clearResponse =
+          sheetsService
+              .spreadsheets()
+              .values()
+              .clear(
+                  spreadsheetId,
+                  getRowRange(data.size(), originalVals.size()),
+                  new ClearValuesRequest())
+              .execute();
+      logger.infofmt(
+          "Cleared %d rows from range %s",
+          originalVals.size() - data.size(), clearResponse.getClearedRange());
     }
   }
+
+  /**
+   * Returns an A1 representation of the cell indicating the top-left corner of the update.
+   *
+   * <p>Updates and appends can specify either a complete range (i.e. A1:B4), or just the top-left
+   * corner of the request. For the latter case, the data fills in based on the primary dimension
+   * (either row-major or column-major order, default is row-major.) For simplicity, we just specify
+   * a single cell for these requests.
+   *
+   * @see <a href="https://developers.google.com/sheets/api/guides/values#writing_to_a_single_range">
+   *   Writing to a single range</a>
+   */
+  private String getCellRange(int rowNum) {
+    // We add 1 to rowNum to compensate for Sheet's 1-indexing, and 1 to offset for the header
+    return String.format("%s!A%d", SHEET_NAME, rowNum + 2);
+  }
+
+  /**
+   * Returns an A1 representation of the cell indicating an entire set of rows.
+   *
+   * <p>Clear requests require a specific range, and we always want to clear an entire row at a time
+   * (to avoid leaving any data behind).
+   */
+  private String getRowRange(int firstRow, int lastRow) {
+    return String.format("%s!%d:%d", SHEET_NAME, firstRow + 2, lastRow + 2);
+  }
 }

java/google/registry/export/sheet/SheetsServiceModule.java

@@ -15,24 +15,28 @@
 package google.registry.export.sheet;
 
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.services.sheets.v4.Sheets;
 import com.google.common.collect.ImmutableList;
-import com.google.gdata.client.spreadsheet.SpreadsheetService;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.config.RegistryConfig.Config;
 
-/** Dagger module for {@link SpreadsheetService}. */
+/** Dagger module for {@link Sheets}. */
 @Module
-public final class SpreadsheetServiceModule {
+public final class SheetsServiceModule {
 
-  private static final String APPLICATION_NAME = "google-registry-v1";
   private static final ImmutableList<String> SCOPES = ImmutableList.of(
-      "https://spreadsheets.google.com/feeds",
-      "https://docs.google.com/feeds");
-
+      "https://www.googleapis.com/auth/spreadsheets");
   @Provides
-  static SpreadsheetService provideSpreadsheetService(GoogleCredential credential) {
-    SpreadsheetService service = new SpreadsheetService(APPLICATION_NAME);
-    service.setOAuth2Credentials(credential.createScoped(SCOPES));
-    return service;
+  static Sheets provideSheets(
+      HttpTransport transport,
+      JsonFactory jsonFactory,
+      @Config("projectId") String projectId,
+      GoogleCredential credential) {
+    return new Sheets.Builder(transport, jsonFactory, credential.createScoped(SCOPES))
+        .setApplicationName(projectId)
+        .build();
   }
 }

java/google/registry/export/sheet/SyncRegistrarsSheet.java

@@ -34,7 +34,6 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Ordering;
-import com.google.gdata.util.ServiceException;
 import com.googlecode.objectify.VoidWork;
 import google.registry.model.common.Cursor;
 import google.registry.model.registrar.Registrar;
@@ -74,7 +73,7 @@ class SyncRegistrarsSheet {
   }
 
   /** Performs the synchronization operation. */
-  void run(String spreadsheetId) throws IOException, ServiceException {
+  void run(String spreadsheetId) throws IOException {
     final DateTime executionTime = clock.nowUtc();
     sheetSynchronizer.synchronize(
         spreadsheetId,

java/google/registry/export/sheet/SyncRegistrarsSheetAction.java

@@ -28,7 +28,6 @@ import com.google.appengine.api.modules.ModulesServiceFactory;
 import com.google.appengine.api.taskqueue.TaskHandle;
 import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.common.base.Optional;
-import com.google.gdata.util.ServiceException;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
@@ -133,6 +132,7 @@ public class SyncRegistrarsSheetAction implements Runnable {
         return;
       }
     }
+
     String sheetLockName = String.format("%s: %s", LOCK_NAME, sheetId.get());
     Callable<Void> runner = new Callable<Void>() {
       @Nullable
@@ -141,7 +141,7 @@ public class SyncRegistrarsSheetAction implements Runnable {
         try {
           syncRegistrarsSheet.run(sheetId.get());
           Result.OK.send(response, null);
-        } catch (IOException | ServiceException e) {
+        } catch (IOException e) {
           Result.FAILED.send(response, e);
         }
         return null;

java/google/registry/module/backend/BackendComponent.java

@@ -19,7 +19,7 @@ import google.registry.bigquery.BigqueryModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.dns.writer.VoidDnsWriterModule;
 import google.registry.export.DriveModule;
-import google.registry.export.sheet.SpreadsheetServiceModule;
+import google.registry.export.sheet.SheetsServiceModule;
 import google.registry.gcs.GcsServiceModule;
 import google.registry.groups.DirectoryModule;
 import google.registry.groups.GroupsModule;
@@ -67,7 +67,7 @@ import javax.inject.Singleton;
         KeyringModule.class,
         KmsModule.class,
         ModulesServiceModule.class,
-        SpreadsheetServiceModule.class,
+        SheetsServiceModule.class,
         StackdriverModule.class,
         SystemClockModule.class,
         SystemSleeperModule.class,

java/google/registry/repositories.bzl

@@ -37,6 +37,7 @@ def domain_registry_repositories(
     omit_com_google_apis_google_api_services_drive=False,
     omit_com_google_apis_google_api_services_groupssettings=False,
     omit_com_google_apis_google_api_services_monitoring=False,
+    omit_com_google_apis_google_api_services_sheets=False,
     omit_com_google_apis_google_api_services_storage=False,
     omit_com_google_appengine_api_1_0_sdk=False,
     omit_com_google_appengine_api_labs=False,
@@ -141,6 +142,8 @@ def domain_registry_repositories(
     com_google_apis_google_api_services_groupssettings()
   if not omit_com_google_apis_google_api_services_monitoring:
     com_google_apis_google_api_services_monitoring()
+  if not omit_com_google_apis_google_api_services_sheets:
+    com_google_apis_google_api_services_sheets()
   if not omit_com_google_apis_google_api_services_storage:
     com_google_apis_google_api_services_storage()
   if not omit_com_google_appengine_api_1_0_sdk:
@@ -494,6 +497,19 @@ def com_google_apis_google_api_services_monitoring():
       deps = ["@com_google_api_client"],
   )
 
+def com_google_apis_google_api_services_sheets():
+  java_import_external(
+      name = "com_google_apis_google_api_services_sheets",
+      jar_sha256 = "67529b9efceb1a16b72c6aa0822d50f4f6e8c3c84972a5a37ca6e7bdc19064ba",
+      jar_urls = [
+          "http://domain-registry-maven.storage.googleapis.com/repo1.maven.org/maven2/com/google/apis/google-api-services-sheets/v4-rev483-1.22.0/google-api-services-sheets-v4-rev483-1.22.0.jar",
+          "http://repo1.maven.org/maven2/com/google/apis/google-api-services-sheets/v4-rev483-1.22.0/google-api-services-sheets-v4-rev483-1.22.0.jar",
+          "http://maven.ibiblio.org/maven2/com/google/apis/google-api-services-sheets/v4-rev483-1.22.0/google-api-services-sheets-v4-rev483-1.22.0.jar",
+      ],
+      licenses = ["notice"],  # The Apache Software License, Version 2.0
+      deps = ["@com_google_api_client"],
+  )
+
 def com_google_apis_google_api_services_storage():
   java_import_external(
       name = "com_google_apis_google_api_services_storage",