mirror of
https://github.com/google/nomulus.git
synced 2025-05-14 00:17:20 +02:00
Migrate to new Sheets v4 api
This moves us from the oudated google/data XML api to the OnePlatform REST/JSON api, finally silencing the deprecation warnings we've been seeing. The synchronization algorithm diffs the spreadsheet's current values with its internally sourced values, adding the row to a batch update request if there's a discrepancy. Additional internal data are added as an append operation to the end of the sheet, and any extraneous spreadsheet data is cleared from the spreadsheet. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=169273590
This commit is contained in:
larryruili
jianglai
parent
67116c5fa1
commit
578673141c
9 changed files with 305 additions and 213 deletions
|
|
@ -8,7 +8,6 @@ java_library(
|
|||
name = "sheet",
|
||||
srcs = glob(["*.java"]),
|
||||
deps = [
|
||||
"//java/com/google/gdata:spreadsheet",
|
||||
"//java/google/registry/config",
|
||||
"//java/google/registry/model",
|
||||
"//java/google/registry/request",
|
||||
|
|
@ -17,11 +16,12 @@ java_library(
|
|||
"//java/google/registry/util",
|
||||
"//third_party/java/objectify:objectify-v4_1",
|
||||
"@com_google_api_client",
|
||||
"@com_google_apis_google_api_services_sheets",
|
||||
"@com_google_appengine_api_1_0_sdk",
|
||||
"@com_google_code_findbugs_jsr305",
|
||||
"@com_google_dagger",
|
||||
"@com_google_gdata_core",
|
||||
"@com_google_guava",
|
||||
"@com_google_http_client",
|
||||
"@javax_servlet_api",
|
||||
"@joda_time",
|
||||
],
|
||||
|
|
|
|||
|
|
@ -14,31 +14,32 @@
|
|||
|
||||
package google.registry.export.sheet;
|
||||
|
||||
import static com.google.common.base.Strings.nullToEmpty;
|
||||
|
||||
import com.google.api.services.sheets.v4.Sheets;
|
||||
import com.google.api.services.sheets.v4.model.AppendValuesResponse;
|
||||
import com.google.api.services.sheets.v4.model.BatchUpdateValuesRequest;
|
||||
import com.google.api.services.sheets.v4.model.BatchUpdateValuesResponse;
|
||||
import com.google.api.services.sheets.v4.model.ClearValuesRequest;
|
||||
import com.google.api.services.sheets.v4.model.ClearValuesResponse;
|
||||
import com.google.api.services.sheets.v4.model.ValueRange;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.gdata.client.spreadsheet.SpreadsheetService;
|
||||
import com.google.gdata.data.spreadsheet.CustomElementCollection;
|
||||
import com.google.gdata.data.spreadsheet.ListEntry;
|
||||
import com.google.gdata.data.spreadsheet.ListFeed;
|
||||
import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
|
||||
import com.google.gdata.data.spreadsheet.WorksheetEntry;
|
||||
import com.google.gdata.util.ServiceException;
|
||||
import google.registry.util.Retrier;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/** Generic data synchronization utility for Google Spreadsheets. */
|
||||
class SheetSynchronizer {
|
||||
|
||||
private static final String SPREADSHEET_URL_PREFIX =
|
||||
"https://spreadsheets.google.com/feeds/spreadsheets/";
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@Inject SpreadsheetService spreadsheetService;
|
||||
private static final String SHEET_NAME = "Registrars";
|
||||
|
||||
@Inject Sheets sheetsService;
|
||||
@Inject SheetSynchronizer() {}
|
||||
@Inject Retrier retrier;
|
||||
|
||||
/**
|
||||
* Replace the contents of a Google Spreadsheet with {@code data}.
|
||||
|
|
@ -50,10 +51,10 @@ class SheetSynchronizer {
|
|||
* credential email address.
|
||||
*
|
||||
* <p>The algorithm works by first assuming that the spreadsheet is sorted in the same way that
|
||||
* {@code data} is sorted. It then iterates through the existing rows and comparing them to the
|
||||
* items in {@code data}. Iteration continues until we either run out of rows, or items in
|
||||
* {@code data}. If there's any rows remaining, they'll be deleted. If instead, items remain in
|
||||
* data, they'll be inserted.
|
||||
* {@code data} is sorted (i.e. registrar name order). It then iterates through the existing rows
|
||||
* and comparing them to the items in {@code data}. Iteration continues until we either run out of
|
||||
* rows, or items in {@code data}. If there's any rows remaining, they'll be deleted. If instead,
|
||||
* items remain in data, they'll be appended to the end of the sheet.
|
||||
*
|
||||
* @param spreadsheetId The ID of your spreadsheet. This can be obtained by opening the Google
|
||||
* spreadsheet in your browser and copying the ID from the URL.
|
||||
|
|
@ -61,62 +62,127 @@ class SheetSynchronizer {
|
|||
* spreadsheet. Each row is a map, where the key must be exactly the same as the column header
|
||||
* cell in the spreadsheet, and value is an arbitrary object which will be converted to a
|
||||
* string before storing it in the spreadsheet.
|
||||
* @throws IOException error communicating with the GData service.
|
||||
* @throws ServiceException if a system error occurred when retrieving the entry.
|
||||
* @throws com.google.gdata.util.ParseException error parsing the returned entry.
|
||||
* @throws com.google.gdata.util.ResourceNotFoundException if an entry URL is not valid.
|
||||
* @throws com.google.gdata.util.ServiceForbiddenException if the GData service cannot get the
|
||||
* entry resource due to access constraints.
|
||||
* @see <a href="https://developers.google.com/google-apps/spreadsheets/">Google Sheets API</a>
|
||||
* @throws IOException if encountering an error communicating with the Sheets service.
|
||||
* @see <a href="https://developers.google.com/sheets/">Google Sheets API v4</a>
|
||||
*/
|
||||
void synchronize(String spreadsheetId, ImmutableList<ImmutableMap<String, String>> data)
|
||||
throws IOException, ServiceException {
|
||||
URL url = new URL(SPREADSHEET_URL_PREFIX + spreadsheetId);
|
||||
SpreadsheetEntry spreadsheet = spreadsheetService.getEntry(url, SpreadsheetEntry.class);
|
||||
WorksheetEntry worksheet = spreadsheet.getWorksheets().get(0);
|
||||
worksheet.setRowCount(data.size() + 1); // account for header row
|
||||
worksheet = worksheet.update();
|
||||
final ListFeed listFeed =
|
||||
spreadsheetService.getFeed(worksheet.getListFeedUrl(), ListFeed.class);
|
||||
List<ListEntry> entries = listFeed.getEntries();
|
||||
int commonSize = Math.min(entries.size(), data.size());
|
||||
for (int i = 0; i < commonSize; i++) {
|
||||
final ListEntry entry = entries.get(i);
|
||||
CustomElementCollection elements = entry.getCustomElements();
|
||||
throws IOException {
|
||||
|
||||
// Get the existing sheet's values
|
||||
ValueRange sheetValues =
|
||||
sheetsService.spreadsheets().values().get(spreadsheetId, SHEET_NAME).execute();
|
||||
List<List<Object>> originalVals = sheetValues.getValues();
|
||||
|
||||
// Assemble headers from the sheet
|
||||
ImmutableList.Builder<String> headersBuilder = new ImmutableList.Builder<>();
|
||||
for (Object headerCell : originalVals.get(0)) {
|
||||
headersBuilder.add(headerCell.toString());
|
||||
}
|
||||
ImmutableList<String> headers = headersBuilder.build();
|
||||
// Pop off the headers row
|
||||
originalVals.remove(0);
|
||||
|
||||
List<ValueRange> updates = new ArrayList<>();
|
||||
int minSize = Math.min(originalVals.size(), data.size());
|
||||
for (int i = 0; i < minSize; i++) {
|
||||
boolean mutated = false;
|
||||
for (ImmutableMap.Entry<String, String> cell : data.get(i).entrySet()) {
|
||||
if (!cell.getValue().equals(elements.getValue(cell.getKey()))) {
|
||||
List<Object> cellRow = originalVals.get(i);
|
||||
// If the row isn't full, pad it with empty strings until it is
|
||||
while (cellRow.size() < headers.size()) {
|
||||
cellRow.add("");
|
||||
}
|
||||
for (int j = 0; j < headers.size(); j++) {
|
||||
// Look for the value corresponding to the row and header indices in data
|
||||
String dataField = data.get(i).get(headers.get(j));
|
||||
// If the cell's header matches a data header, and the values aren't equal, mutate it
|
||||
if (dataField != null && !cellRow.get(j).toString().equals(dataField)) {
|
||||
mutated = true;
|
||||
elements.setValueLocal(cell.getKey(), cell.getValue());
|
||||
originalVals.get(i).set(j, dataField);
|
||||
}
|
||||
}
|
||||
if (mutated) {
|
||||
// Wrap in a retrier to deal with transient HTTP failures, which are IOExceptions wrapped
|
||||
// in RuntimeExceptions.
|
||||
retrier.callWithRetry(new Callable<Void>() {
|
||||
@Override
|
||||
public Void call() throws Exception {
|
||||
entry.update();
|
||||
return null;
|
||||
}}, RuntimeException.class);
|
||||
ValueRange rowUpdate =
|
||||
new ValueRange()
|
||||
.setValues(originalVals.subList(i, i + 1))
|
||||
.setRange(getCellRange(i));
|
||||
updates.add(rowUpdate);
|
||||
}
|
||||
}
|
||||
if (data.size() > entries.size()) {
|
||||
for (int i = entries.size(); i < data.size(); i++) {
|
||||
final ListEntry entry = listFeed.createEntry();
|
||||
CustomElementCollection elements = entry.getCustomElements();
|
||||
for (ImmutableMap.Entry<String, String> cell : data.get(i).entrySet()) {
|
||||
elements.setValueLocal(cell.getKey(), cell.getValue());
|
||||
// Update the mutated cells if necessary
|
||||
if (!updates.isEmpty()) {
|
||||
BatchUpdateValuesRequest updateRequest = new BatchUpdateValuesRequest()
|
||||
.setValueInputOption("RAW")
|
||||
.setData(updates);
|
||||
|
||||
BatchUpdateValuesResponse response =
|
||||
sheetsService.spreadsheets().values().batchUpdate(spreadsheetId, updateRequest).execute();
|
||||
Integer cellsUpdated = response.getTotalUpdatedCells();
|
||||
logger.infofmt("Updated %d originalVals", cellsUpdated != null ? cellsUpdated : 0);
|
||||
}
|
||||
|
||||
// Append extra rows if necessary
|
||||
if (data.size() > originalVals.size()) {
|
||||
ImmutableList.Builder<List<Object>> valsBuilder = new ImmutableList.Builder<>();
|
||||
for (int i = originalVals.size(); i < data.size(); i++) {
|
||||
ImmutableList.Builder<Object> rowBuilder = new ImmutableList.Builder<>();
|
||||
for (String header : headers) {
|
||||
rowBuilder.add(nullToEmpty(data.get(i).get(header)));
|
||||
}
|
||||
// Wrap in a retrier to deal with transient HTTP failures, which are IOExceptions wrapped
|
||||
// in RuntimeExceptions.
|
||||
retrier.callWithRetry(new Callable<Void>() {
|
||||
@Override
|
||||
public Void call() throws Exception {
|
||||
listFeed.insert(entry);
|
||||
return null;
|
||||
}}, RuntimeException.class);
|
||||
valsBuilder.add(rowBuilder.build());
|
||||
}
|
||||
// Start the append at index originalVals.size (where the previous operation left off)
|
||||
ValueRange appendUpdate = new ValueRange().setValues(valsBuilder.build());
|
||||
AppendValuesResponse appendResponse = sheetsService
|
||||
.spreadsheets()
|
||||
.values()
|
||||
.append(spreadsheetId, getCellRange(originalVals.size()), appendUpdate)
|
||||
.setValueInputOption("RAW")
|
||||
.setInsertDataOption("INSERT_ROWS")
|
||||
.execute();
|
||||
logger.infofmt(
|
||||
"Appended %d rows to range %s",
|
||||
data.size() - originalVals.size(), appendResponse.getTableRange());
|
||||
// Clear the extra rows if necessary
|
||||
} else if (data.size() < originalVals.size()) {
|
||||
// Clear other rows if there's more originalVals on the sheet than live data.
|
||||
ClearValuesResponse clearResponse =
|
||||
sheetsService
|
||||
.spreadsheets()
|
||||
.values()
|
||||
.clear(
|
||||
spreadsheetId,
|
||||
getRowRange(data.size(), originalVals.size()),
|
||||
new ClearValuesRequest())
|
||||
.execute();
|
||||
logger.infofmt(
|
||||
"Cleared %d rows from range %s",
|
||||
originalVals.size() - data.size(), clearResponse.getClearedRange());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an A1 representation of the cell indicating the top-left corner of the update.
|
||||
*
|
||||
* <p>Updates and appends can specify either a complete range (i.e. A1:B4), or just the top-left
|
||||
* corner of the request. For the latter case, the data fills in based on the primary dimension
|
||||
* (either row-major or column-major order, default is row-major.) For simplicity, we just specify
|
||||
* a single cell for these requests.
|
||||
*
|
||||
* @see <a href="https://developers.google.com/sheets/api/guides/values#writing_to_a_single_range">
|
||||
* Writing to a single range</a>
|
||||
*/
|
||||
private String getCellRange(int rowNum) {
|
||||
// We add 1 to rowNum to compensate for Sheet's 1-indexing, and 1 to offset for the header
|
||||
return String.format("%s!A%d", SHEET_NAME, rowNum + 2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an A1 representation of the cell indicating an entire set of rows.
|
||||
*
|
||||
* <p>Clear requests require a specific range, and we always want to clear an entire row at a time
|
||||
* (to avoid leaving any data behind).
|
||||
*/
|
||||
private String getRowRange(int firstRow, int lastRow) {
|
||||
return String.format("%s!%d:%d", SHEET_NAME, firstRow + 2, lastRow + 2);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,24 +15,28 @@
|
|||
package google.registry.export.sheet;
|
||||
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
||||
import com.google.api.client.http.HttpTransport;
|
||||
import com.google.api.client.json.JsonFactory;
|
||||
import com.google.api.services.sheets.v4.Sheets;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.gdata.client.spreadsheet.SpreadsheetService;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
|
||||
/** Dagger module for {@link SpreadsheetService}. */
|
||||
/** Dagger module for {@link Sheets}. */
|
||||
@Module
|
||||
public final class SpreadsheetServiceModule {
|
||||
public final class SheetsServiceModule {
|
||||
|
||||
private static final String APPLICATION_NAME = "google-registry-v1";
|
||||
private static final ImmutableList<String> SCOPES = ImmutableList.of(
|
||||
"https://spreadsheets.google.com/feeds",
|
||||
"https://docs.google.com/feeds");
|
||||
|
||||
"https://www.googleapis.com/auth/spreadsheets");
|
||||
@Provides
|
||||
static SpreadsheetService provideSpreadsheetService(GoogleCredential credential) {
|
||||
SpreadsheetService service = new SpreadsheetService(APPLICATION_NAME);
|
||||
service.setOAuth2Credentials(credential.createScoped(SCOPES));
|
||||
return service;
|
||||
static Sheets provideSheets(
|
||||
HttpTransport transport,
|
||||
JsonFactory jsonFactory,
|
||||
@Config("projectId") String projectId,
|
||||
GoogleCredential credential) {
|
||||
return new Sheets.Builder(transport, jsonFactory, credential.createScoped(SCOPES))
|
||||
.setApplicationName(projectId)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -34,7 +34,6 @@ import com.google.common.collect.ImmutableList;
|
|||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSortedSet;
|
||||
import com.google.common.collect.Ordering;
|
||||
import com.google.gdata.util.ServiceException;
|
||||
import com.googlecode.objectify.VoidWork;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
|
|
@ -74,7 +73,7 @@ class SyncRegistrarsSheet {
|
|||
}
|
||||
|
||||
/** Performs the synchronization operation. */
|
||||
void run(String spreadsheetId) throws IOException, ServiceException {
|
||||
void run(String spreadsheetId) throws IOException {
|
||||
final DateTime executionTime = clock.nowUtc();
|
||||
sheetSynchronizer.synchronize(
|
||||
spreadsheetId,
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import com.google.appengine.api.modules.ModulesServiceFactory;
|
|||
import com.google.appengine.api.taskqueue.TaskHandle;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.common.base.Optional;
|
||||
import com.google.gdata.util.ServiceException;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
|
|
@ -133,6 +132,7 @@ public class SyncRegistrarsSheetAction implements Runnable {
|
|||
return;
|
||||
}
|
||||
}
|
||||
|
||||
String sheetLockName = String.format("%s: %s", LOCK_NAME, sheetId.get());
|
||||
Callable<Void> runner = new Callable<Void>() {
|
||||
@Nullable
|
||||
|
|
@ -141,7 +141,7 @@ public class SyncRegistrarsSheetAction implements Runnable {
|
|||
try {
|
||||
syncRegistrarsSheet.run(sheetId.get());
|
||||
Result.OK.send(response, null);
|
||||
} catch (IOException | ServiceException e) {
|
||||
} catch (IOException e) {
|
||||
Result.FAILED.send(response, e);
|
||||
}
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import google.registry.bigquery.BigqueryModule;
|
|||
import google.registry.config.RegistryConfig.ConfigModule;
|
||||
import google.registry.dns.writer.VoidDnsWriterModule;
|
||||
import google.registry.export.DriveModule;
|
||||
import google.registry.export.sheet.SpreadsheetServiceModule;
|
||||
import google.registry.export.sheet.SheetsServiceModule;
|
||||
import google.registry.gcs.GcsServiceModule;
|
||||
import google.registry.groups.DirectoryModule;
|
||||
import google.registry.groups.GroupsModule;
|
||||
|
|
@ -67,7 +67,7 @@ import javax.inject.Singleton;
|
|||
KeyringModule.class,
|
||||
KmsModule.class,
|
||||
ModulesServiceModule.class,
|
||||
SpreadsheetServiceModule.class,
|
||||
SheetsServiceModule.class,
|
||||
StackdriverModule.class,
|
||||
SystemClockModule.class,
|
||||
SystemSleeperModule.class,
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ def domain_registry_repositories(
|
|||
omit_com_google_apis_google_api_services_drive=False,
|
||||
omit_com_google_apis_google_api_services_groupssettings=False,
|
||||
omit_com_google_apis_google_api_services_monitoring=False,
|
||||
omit_com_google_apis_google_api_services_sheets=False,
|
||||
omit_com_google_apis_google_api_services_storage=False,
|
||||
omit_com_google_appengine_api_1_0_sdk=False,
|
||||
omit_com_google_appengine_api_labs=False,
|
||||
|
|
@ -141,6 +142,8 @@ def domain_registry_repositories(
|
|||
com_google_apis_google_api_services_groupssettings()
|
||||
if not omit_com_google_apis_google_api_services_monitoring:
|
||||
com_google_apis_google_api_services_monitoring()
|
||||
if not omit_com_google_apis_google_api_services_sheets:
|
||||
com_google_apis_google_api_services_sheets()
|
||||
if not omit_com_google_apis_google_api_services_storage:
|
||||
com_google_apis_google_api_services_storage()
|
||||
if not omit_com_google_appengine_api_1_0_sdk:
|
||||
|
|
@ -494,6 +497,19 @@ def com_google_apis_google_api_services_monitoring():
|
|||
deps = ["@com_google_api_client"],
|
||||
)
|
||||
|
||||
def com_google_apis_google_api_services_sheets():
|
||||
java_import_external(
|
||||
name = "com_google_apis_google_api_services_sheets",
|
||||
jar_sha256 = "67529b9efceb1a16b72c6aa0822d50f4f6e8c3c84972a5a37ca6e7bdc19064ba",
|
||||
jar_urls = [
|
||||
"http://domain-registry-maven.storage.googleapis.com/repo1.maven.org/maven2/com/google/apis/google-api-services-sheets/v4-rev483-1.22.0/google-api-services-sheets-v4-rev483-1.22.0.jar",
|
||||
"http://repo1.maven.org/maven2/com/google/apis/google-api-services-sheets/v4-rev483-1.22.0/google-api-services-sheets-v4-rev483-1.22.0.jar",
|
||||
"http://maven.ibiblio.org/maven2/com/google/apis/google-api-services-sheets/v4-rev483-1.22.0/google-api-services-sheets-v4-rev483-1.22.0.jar",
|
||||
],
|
||||
licenses = ["notice"], # The Apache Software License, Version 2.0
|
||||
deps = ["@com_google_api_client"],
|
||||
)
|
||||
|
||||
def com_google_apis_google_api_services_storage():
|
||||
java_import_external(
|
||||
name = "com_google_apis_google_api_services_storage",
|
||||
|
|
|
|||
|
|
@ -9,13 +9,12 @@ java_library(
|
|||
name = "sheet",
|
||||
srcs = glob(["*.java"]),
|
||||
deps = [
|
||||
"//java/com/google/gdata:spreadsheet",
|
||||
"//java/google/registry/config",
|
||||
"//java/google/registry/export/sheet",
|
||||
"//java/google/registry/model",
|
||||
"//java/google/registry/util",
|
||||
"//javatests/google/registry/testing",
|
||||
"//third_party/java/objectify:objectify-v4_1",
|
||||
"@com_google_apis_google_api_services_sheets",
|
||||
"@com_google_code_findbugs_jsr305",
|
||||
"@com_google_guava",
|
||||
"@com_google_truth",
|
||||
|
|
|
|||
|
|
@ -14,29 +14,25 @@
|
|||
|
||||
package google.registry.export.sheet;
|
||||
|
||||
import static com.google.common.collect.Lists.newArrayList;
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Matchers.eq;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.verifyZeroInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.services.sheets.v4.Sheets;
|
||||
import com.google.api.services.sheets.v4.model.AppendValuesResponse;
|
||||
import com.google.api.services.sheets.v4.model.BatchUpdateValuesRequest;
|
||||
import com.google.api.services.sheets.v4.model.BatchUpdateValuesResponse;
|
||||
import com.google.api.services.sheets.v4.model.ClearValuesRequest;
|
||||
import com.google.api.services.sheets.v4.model.ClearValuesResponse;
|
||||
import com.google.api.services.sheets.v4.model.ValueRange;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.gdata.client.spreadsheet.SpreadsheetService;
|
||||
import com.google.gdata.data.spreadsheet.CustomElementCollection;
|
||||
import com.google.gdata.data.spreadsheet.ListEntry;
|
||||
import com.google.gdata.data.spreadsheet.ListFeed;
|
||||
import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
|
||||
import com.google.gdata.data.spreadsheet.WorksheetEntry;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeSleeper;
|
||||
import google.registry.util.Retrier;
|
||||
import java.net.URL;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.After;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
|
@ -45,143 +41,155 @@ import org.junit.runners.JUnit4;
|
|||
/** Unit tests for {@link SheetSynchronizer}. */
|
||||
@RunWith(JUnit4.class)
|
||||
public class SheetSynchronizerTest {
|
||||
|
||||
private static final int MAX_RETRIES = 3;
|
||||
|
||||
private final SpreadsheetService spreadsheetService = mock(SpreadsheetService.class);
|
||||
private final SpreadsheetEntry spreadsheet = mock(SpreadsheetEntry.class);
|
||||
private final WorksheetEntry worksheet = mock(WorksheetEntry.class);
|
||||
private final ListFeed listFeed = mock(ListFeed.class);
|
||||
private final SheetSynchronizer sheetSynchronizer = new SheetSynchronizer();
|
||||
private final FakeSleeper sleeper =
|
||||
new FakeSleeper(new FakeClock(DateTime.parse("2000-01-01TZ")));
|
||||
private final Sheets sheetsService = mock(Sheets.class);
|
||||
private final Sheets.Spreadsheets spreadsheets = mock(Sheets.Spreadsheets.class);
|
||||
private final Sheets.Spreadsheets.Values values = mock(Sheets.Spreadsheets.Values.class);
|
||||
private final Sheets.Spreadsheets.Values.Get getReq = mock(Sheets.Spreadsheets.Values.Get.class);
|
||||
private final Sheets.Spreadsheets.Values.Append appendReq =
|
||||
mock(Sheets.Spreadsheets.Values.Append.class);
|
||||
private final Sheets.Spreadsheets.Values.BatchUpdate updateReq =
|
||||
mock(Sheets.Spreadsheets.Values.BatchUpdate.class);
|
||||
private final Sheets.Spreadsheets.Values.Clear clearReq =
|
||||
mock(Sheets.Spreadsheets.Values.Clear.class);
|
||||
|
||||
private List<List<Object>> existingSheet;
|
||||
private ImmutableList<ImmutableMap<String, String>> data;
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
sheetSynchronizer.spreadsheetService = spreadsheetService;
|
||||
sheetSynchronizer.retrier = new Retrier(sleeper, MAX_RETRIES);
|
||||
when(spreadsheetService.getEntry(any(URL.class), eq(SpreadsheetEntry.class)))
|
||||
.thenReturn(spreadsheet);
|
||||
when(spreadsheet.getWorksheets()).thenReturn(ImmutableList.of(worksheet));
|
||||
when(worksheet.getListFeedUrl()).thenReturn(new URL("http://example.com/spreadsheet"));
|
||||
when(spreadsheetService.getFeed(any(URL.class), eq(ListFeed.class))).thenReturn(listFeed);
|
||||
when(worksheet.update()).thenReturn(worksheet);
|
||||
sheetSynchronizer.sheetsService = sheetsService;
|
||||
when(sheetsService.spreadsheets()).thenReturn(spreadsheets);
|
||||
when(spreadsheets.values()).thenReturn(values);
|
||||
|
||||
when(values.get(any(String.class), any(String.class))).thenReturn(getReq);
|
||||
when(values.append(any(String.class), any(String.class), any(ValueRange.class)))
|
||||
.thenReturn(appendReq);
|
||||
when(values.clear(any(String.class), any(String.class), any(ClearValuesRequest.class)))
|
||||
.thenReturn(clearReq);
|
||||
when(values.batchUpdate(any(String.class), any(BatchUpdateValuesRequest.class)))
|
||||
.thenReturn(updateReq);
|
||||
|
||||
when(appendReq.execute()).thenReturn(new AppendValuesResponse());
|
||||
when(appendReq.setValueInputOption(any(String.class))).thenReturn(appendReq);
|
||||
when(appendReq.setInsertDataOption(any(String.class))).thenReturn(appendReq);
|
||||
when(clearReq.execute()).thenReturn(new ClearValuesResponse());
|
||||
when(updateReq.execute()).thenReturn(new BatchUpdateValuesResponse());
|
||||
|
||||
existingSheet = newArrayList();
|
||||
data = ImmutableList.of();
|
||||
ValueRange valueRange = new ValueRange().setValues(existingSheet);
|
||||
when(getReq.execute()).thenReturn(valueRange);
|
||||
}
|
||||
|
||||
@After
|
||||
public void after() throws Exception {
|
||||
verify(spreadsheetService)
|
||||
.getEntry(
|
||||
new URL("https://spreadsheets.google.com/feeds/spreadsheets/foobar"),
|
||||
SpreadsheetEntry.class);
|
||||
verify(spreadsheet).getWorksheets();
|
||||
verify(worksheet).getListFeedUrl();
|
||||
verify(spreadsheetService).getFeed(new URL("http://example.com/spreadsheet"), ListFeed.class);
|
||||
verify(listFeed).getEntries();
|
||||
verifyNoMoreInteractions(spreadsheetService, spreadsheet, worksheet, listFeed);
|
||||
// Explicitly constructs a List<Object> to avoid newArrayList typing to ArrayList<String>
|
||||
private List<Object> createRow(Object... elements) {
|
||||
List<Object> row = new ArrayList<>();
|
||||
row.addAll(Arrays.asList(elements));
|
||||
return row;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSynchronize_bothEmpty_doNothing() throws Exception {
|
||||
when(listFeed.getEntries()).thenReturn(ImmutableList.<ListEntry>of());
|
||||
sheetSynchronizer.synchronize("foobar", ImmutableList.<ImmutableMap<String, String>>of());
|
||||
verify(worksheet).setRowCount(1);
|
||||
verify(worksheet).update();
|
||||
public void testSynchronize_dataAndSheetEmpty_doNothing() throws Exception {
|
||||
existingSheet.add(createRow("a", "b"));
|
||||
sheetSynchronizer.synchronize("aSheetId", data);
|
||||
verifyZeroInteractions(appendReq);
|
||||
verifyZeroInteractions(clearReq);
|
||||
verifyZeroInteractions(updateReq);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSynchronize_bothContainSameRow_doNothing() throws Exception {
|
||||
ListEntry entry = makeListEntry(ImmutableMap.of("key", "value"));
|
||||
when(listFeed.getEntries()).thenReturn(ImmutableList.of(entry));
|
||||
sheetSynchronizer.synchronize("foobar", ImmutableList.of(ImmutableMap.of("key", "value")));
|
||||
verify(worksheet).setRowCount(2);
|
||||
verify(worksheet).update();
|
||||
verify(entry, atLeastOnce()).getCustomElements();
|
||||
verifyNoMoreInteractions(entry);
|
||||
public void testSynchronize_differentValues_updatesValues() throws Exception {
|
||||
existingSheet.add(createRow("a", "b"));
|
||||
existingSheet.add(createRow("diffVal1l", "diffVal2"));
|
||||
data = ImmutableList.of(ImmutableMap.of("a", "val1", "b", "val2"));
|
||||
sheetSynchronizer.synchronize("aSheetId", data);
|
||||
|
||||
verifyZeroInteractions(appendReq);
|
||||
verifyZeroInteractions(clearReq);
|
||||
|
||||
BatchUpdateValuesRequest expectedRequest = new BatchUpdateValuesRequest();
|
||||
List<List<Object>> expectedVals = newArrayList();
|
||||
expectedVals.add(createRow("val1", "val2"));
|
||||
expectedRequest.setData(
|
||||
newArrayList(new ValueRange().setRange("Registrars!A2").setValues(expectedVals)));
|
||||
expectedRequest.setValueInputOption("RAW");
|
||||
verify(values).batchUpdate("aSheetId", expectedRequest);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSynchronize_cellIsDifferent_updateRow() throws Exception {
|
||||
ListEntry entry = makeListEntry(ImmutableMap.of("key", "value"));
|
||||
when(listFeed.getEntries()).thenReturn(ImmutableList.of(entry));
|
||||
sheetSynchronizer.synchronize("foobar", ImmutableList.of(ImmutableMap.of("key", "new value")));
|
||||
verify(entry.getCustomElements()).setValueLocal("key", "new value");
|
||||
verify(entry).update();
|
||||
verify(worksheet).setRowCount(2);
|
||||
verify(worksheet).update();
|
||||
verify(entry, atLeastOnce()).getCustomElements();
|
||||
verifyNoMoreInteractions(entry);
|
||||
public void testSynchronize_unknownFields_doesntUpdate() throws Exception {
|
||||
existingSheet.add(createRow("a", "c", "b"));
|
||||
existingSheet.add(createRow("diffVal1", "sameVal", "diffVal2"));
|
||||
data = ImmutableList.of(ImmutableMap.of("a", "val1", "b", "val2", "d", "val3"));
|
||||
sheetSynchronizer.synchronize("aSheetId", data);
|
||||
|
||||
verifyZeroInteractions(appendReq);
|
||||
verifyZeroInteractions(clearReq);
|
||||
|
||||
BatchUpdateValuesRequest expectedRequest = new BatchUpdateValuesRequest();
|
||||
List<List<Object>> expectedVals = newArrayList();
|
||||
expectedVals.add(createRow("val1", "sameVal", "val2"));
|
||||
expectedRequest.setData(
|
||||
newArrayList(new ValueRange().setRange("Registrars!A2").setValues(expectedVals)));
|
||||
expectedRequest.setValueInputOption("RAW");
|
||||
verify(values).batchUpdate("aSheetId", expectedRequest);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSynchronize_cellIsDifferent_updateRow_retriesOnException() throws Exception {
|
||||
ListEntry entry = makeListEntry(ImmutableMap.of("key", "value"));
|
||||
when(listFeed.getEntries()).thenReturn(ImmutableList.of(entry));
|
||||
when(entry.update())
|
||||
.thenThrow(new RuntimeException())
|
||||
.thenThrow(new RuntimeException())
|
||||
.thenReturn(entry);
|
||||
sheetSynchronizer.synchronize("foobar", ImmutableList.of(ImmutableMap.of("key", "new value")));
|
||||
verify(entry.getCustomElements()).setValueLocal("key", "new value");
|
||||
verify(entry, times(3)).update();
|
||||
verify(worksheet).setRowCount(2);
|
||||
verify(worksheet).update();
|
||||
verify(entry, atLeastOnce()).getCustomElements();
|
||||
verifyNoMoreInteractions(entry);
|
||||
public void testSynchronize_notFullRow_getsPadded() throws Exception {
|
||||
existingSheet.add(createRow("a", "c", "b"));
|
||||
existingSheet.add(createRow("diffVal1", "diffVal2"));
|
||||
data = ImmutableList.of(ImmutableMap.of("a", "val1", "b", "paddedVal", "d", "val3"));
|
||||
sheetSynchronizer.synchronize("aSheetId", data);
|
||||
|
||||
verifyZeroInteractions(appendReq);
|
||||
verifyZeroInteractions(clearReq);
|
||||
|
||||
BatchUpdateValuesRequest expectedRequest = new BatchUpdateValuesRequest();
|
||||
List<List<Object>> expectedVals = newArrayList();
|
||||
expectedVals.add(createRow("val1", "diffVal2", "paddedVal"));
|
||||
expectedRequest.setData(
|
||||
newArrayList(new ValueRange().setRange("Registrars!A2").setValues(expectedVals)));
|
||||
expectedRequest.setValueInputOption("RAW");
|
||||
verify(values).batchUpdate("aSheetId", expectedRequest);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSynchronize_spreadsheetMissingRow_insertRow() throws Exception {
|
||||
ListEntry entry = makeListEntry(ImmutableMap.<String, String>of());
|
||||
when(listFeed.getEntries()).thenReturn(ImmutableList.<ListEntry>of());
|
||||
when(listFeed.createEntry()).thenReturn(entry);
|
||||
sheetSynchronizer.synchronize("foobar", ImmutableList.of(ImmutableMap.of("key", "value")));
|
||||
verify(entry.getCustomElements()).setValueLocal("key", "value");
|
||||
verify(listFeed).insert(entry);
|
||||
verify(worksheet).setRowCount(2);
|
||||
verify(worksheet).update();
|
||||
verify(listFeed).createEntry();
|
||||
verify(entry, atLeastOnce()).getCustomElements();
|
||||
verifyNoMoreInteractions(entry);
|
||||
public void testSynchronize_moreData_appendsValues() throws Exception {
|
||||
existingSheet.add(createRow("a", "b"));
|
||||
existingSheet.add(createRow("diffVal1", "diffVal2"));
|
||||
data = ImmutableList.of(
|
||||
ImmutableMap.of("a", "val1", "b", "val2"),
|
||||
ImmutableMap.of("a", "val3", "b", "val4"));
|
||||
sheetSynchronizer.synchronize("aSheetId", data);
|
||||
|
||||
verifyZeroInteractions(clearReq);
|
||||
|
||||
BatchUpdateValuesRequest expectedRequest = new BatchUpdateValuesRequest();
|
||||
List<List<Object>> updatedVals = newArrayList();
|
||||
updatedVals.add(createRow("val1", "val2"));
|
||||
expectedRequest.setData(
|
||||
newArrayList(
|
||||
new ValueRange().setRange("Registrars!A2").setValues(updatedVals)));
|
||||
expectedRequest.setValueInputOption("RAW");
|
||||
verify(values).batchUpdate("aSheetId", expectedRequest);
|
||||
|
||||
List<List<Object>> appendedVals = newArrayList();
|
||||
appendedVals.add(createRow("val3", "val4"));
|
||||
ValueRange appendRequest = new ValueRange().setValues(appendedVals);
|
||||
verify(values).append("aSheetId", "Registrars!A3", appendRequest);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSynchronize_spreadsheetMissingRow_insertRow_retriesOnException()
|
||||
throws Exception {
|
||||
ListEntry entry = makeListEntry(ImmutableMap.<String, String>of());
|
||||
when(listFeed.getEntries()).thenReturn(ImmutableList.<ListEntry>of());
|
||||
when(listFeed.createEntry()).thenReturn(entry);
|
||||
when(listFeed.insert(entry))
|
||||
.thenThrow(new RuntimeException())
|
||||
.thenThrow(new RuntimeException())
|
||||
.thenReturn(entry);
|
||||
sheetSynchronizer.synchronize("foobar", ImmutableList.of(ImmutableMap.of("key", "value")));
|
||||
verify(entry.getCustomElements()).setValueLocal("key", "value");
|
||||
verify(listFeed, times(3)).insert(entry);
|
||||
verify(worksheet).setRowCount(2);
|
||||
verify(worksheet).update();
|
||||
verify(listFeed).createEntry();
|
||||
verify(entry, atLeastOnce()).getCustomElements();
|
||||
verifyNoMoreInteractions(entry);
|
||||
}
|
||||
public void testSynchronize_lessData_clearsValues() throws Exception {
|
||||
existingSheet.add(createRow("a", "b"));
|
||||
existingSheet.add(createRow("val1", "val2"));
|
||||
existingSheet.add(createRow("diffVal3", "diffVal4"));
|
||||
data = ImmutableList.of(ImmutableMap.of("a", "val1", "b", "val2"));
|
||||
sheetSynchronizer.synchronize("aSheetId", data);
|
||||
|
||||
@Test
|
||||
public void testSynchronize_spreadsheetRowNoLongerInData_deleteRow() throws Exception {
|
||||
ListEntry entry = makeListEntry(ImmutableMap.of("key", "value"));
|
||||
when(listFeed.getEntries()).thenReturn(ImmutableList.of(entry));
|
||||
sheetSynchronizer.synchronize("foobar", ImmutableList.<ImmutableMap<String, String>>of());
|
||||
verify(worksheet).setRowCount(1);
|
||||
verify(worksheet).update();
|
||||
verifyNoMoreInteractions(entry);
|
||||
}
|
||||
|
||||
private static ListEntry makeListEntry(ImmutableMap<String, String> values) {
|
||||
CustomElementCollection collection = mock(CustomElementCollection.class);
|
||||
for (ImmutableMap.Entry<String, String> entry : values.entrySet()) {
|
||||
when(collection.getValue(eq(entry.getKey()))).thenReturn(entry.getValue());
|
||||
}
|
||||
ListEntry listEntry = mock(ListEntry.class);
|
||||
when(listEntry.getCustomElements()).thenReturn(collection);
|
||||
return listEntry;
|
||||
verify(values).clear("aSheetId", "Registrars!3:4", new ClearValuesRequest());
|
||||
verifyZeroInteractions(updateReq);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue