diff --git a/java/google/registry/export/sheet/BUILD b/java/google/registry/export/sheet/BUILD index 4ee364241..982d6be49 100644 --- a/java/google/registry/export/sheet/BUILD +++ b/java/google/registry/export/sheet/BUILD @@ -8,7 +8,6 @@ java_library( name = "sheet", srcs = glob(["*.java"]), deps = [ - "//java/com/google/gdata:spreadsheet", "//java/google/registry/config", "//java/google/registry/model", "//java/google/registry/request", @@ -17,11 +16,12 @@ java_library( "//java/google/registry/util", "//third_party/java/objectify:objectify-v4_1", "@com_google_api_client", + "@com_google_apis_google_api_services_sheets", "@com_google_appengine_api_1_0_sdk", "@com_google_code_findbugs_jsr305", "@com_google_dagger", - "@com_google_gdata_core", "@com_google_guava", + "@com_google_http_client", "@javax_servlet_api", "@joda_time", ], diff --git a/java/google/registry/export/sheet/SheetSynchronizer.java b/java/google/registry/export/sheet/SheetSynchronizer.java index 10342e5be..ae7b7a668 100644 --- a/java/google/registry/export/sheet/SheetSynchronizer.java +++ b/java/google/registry/export/sheet/SheetSynchronizer.java @@ -14,31 +14,32 @@ package google.registry.export.sheet; +import static com.google.common.base.Strings.nullToEmpty; + +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.model.AppendValuesResponse; +import com.google.api.services.sheets.v4.model.BatchUpdateValuesRequest; +import com.google.api.services.sheets.v4.model.BatchUpdateValuesResponse; +import com.google.api.services.sheets.v4.model.ClearValuesRequest; +import com.google.api.services.sheets.v4.model.ClearValuesResponse; +import com.google.api.services.sheets.v4.model.ValueRange; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.gdata.client.spreadsheet.SpreadsheetService; -import com.google.gdata.data.spreadsheet.CustomElementCollection; -import com.google.gdata.data.spreadsheet.ListEntry; -import com.google.gdata.data.spreadsheet.ListFeed; -import com.google.gdata.data.spreadsheet.SpreadsheetEntry; -import com.google.gdata.data.spreadsheet.WorksheetEntry; -import com.google.gdata.util.ServiceException; -import google.registry.util.Retrier; +import google.registry.util.FormattingLogger; import java.io.IOException; -import java.net.URL; +import java.util.ArrayList; import java.util.List; -import java.util.concurrent.Callable; import javax.inject.Inject; /** Generic data synchronization utility for Google Spreadsheets. */ class SheetSynchronizer { - private static final String SPREADSHEET_URL_PREFIX = - "https://spreadsheets.google.com/feeds/spreadsheets/"; + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); - @Inject SpreadsheetService spreadsheetService; + private static final String SHEET_NAME = "Registrars"; + + @Inject Sheets sheetsService; @Inject SheetSynchronizer() {} - @Inject Retrier retrier; /** * Replace the contents of a Google Spreadsheet with {@code data}. @@ -50,10 +51,10 @@ class SheetSynchronizer { * credential email address. * *

The algorithm works by first assuming that the spreadsheet is sorted in the same way that - * {@code data} is sorted. It then iterates through the existing rows and comparing them to the - * items in {@code data}. Iteration continues until we either run out of rows, or items in - * {@code data}. If there's any rows remaining, they'll be deleted. If instead, items remain in - * data, they'll be inserted. + * {@code data} is sorted (i.e. registrar name order). It then iterates through the existing rows + * and comparing them to the items in {@code data}. Iteration continues until we either run out of + * rows, or items in {@code data}. If there's any rows remaining, they'll be deleted. If instead, + * items remain in data, they'll be appended to the end of the sheet. * * @param spreadsheetId The ID of your spreadsheet. This can be obtained by opening the Google * spreadsheet in your browser and copying the ID from the URL. @@ -61,62 +62,127 @@ class SheetSynchronizer { * spreadsheet. Each row is a map, where the key must be exactly the same as the column header * cell in the spreadsheet, and value is an arbitrary object which will be converted to a * string before storing it in the spreadsheet. - * @throws IOException error communicating with the GData service. - * @throws ServiceException if a system error occurred when retrieving the entry. - * @throws com.google.gdata.util.ParseException error parsing the returned entry. - * @throws com.google.gdata.util.ResourceNotFoundException if an entry URL is not valid. - * @throws com.google.gdata.util.ServiceForbiddenException if the GData service cannot get the - * entry resource due to access constraints. - * @see Google Sheets API + * @throws IOException if encountering an error communicating with the Sheets service. + * @see Google Sheets API v4 */ void synchronize(String spreadsheetId, ImmutableList> data) - throws IOException, ServiceException { - URL url = new URL(SPREADSHEET_URL_PREFIX + spreadsheetId); - SpreadsheetEntry spreadsheet = spreadsheetService.getEntry(url, SpreadsheetEntry.class); - WorksheetEntry worksheet = spreadsheet.getWorksheets().get(0); - worksheet.setRowCount(data.size() + 1); // account for header row - worksheet = worksheet.update(); - final ListFeed listFeed = - spreadsheetService.getFeed(worksheet.getListFeedUrl(), ListFeed.class); - List entries = listFeed.getEntries(); - int commonSize = Math.min(entries.size(), data.size()); - for (int i = 0; i < commonSize; i++) { - final ListEntry entry = entries.get(i); - CustomElementCollection elements = entry.getCustomElements(); + throws IOException { + + // Get the existing sheet's values + ValueRange sheetValues = + sheetsService.spreadsheets().values().get(spreadsheetId, SHEET_NAME).execute(); + List> originalVals = sheetValues.getValues(); + + // Assemble headers from the sheet + ImmutableList.Builder headersBuilder = new ImmutableList.Builder<>(); + for (Object headerCell : originalVals.get(0)) { + headersBuilder.add(headerCell.toString()); + } + ImmutableList headers = headersBuilder.build(); + // Pop off the headers row + originalVals.remove(0); + + List updates = new ArrayList<>(); + int minSize = Math.min(originalVals.size(), data.size()); + for (int i = 0; i < minSize; i++) { boolean mutated = false; - for (ImmutableMap.Entry cell : data.get(i).entrySet()) { - if (!cell.getValue().equals(elements.getValue(cell.getKey()))) { + List cellRow = originalVals.get(i); + // If the row isn't full, pad it with empty strings until it is + while (cellRow.size() < headers.size()) { + cellRow.add(""); + } + for (int j = 0; j < headers.size(); j++) { + // Look for the value corresponding to the row and header indices in data + String dataField = data.get(i).get(headers.get(j)); + // If the cell's header matches a data header, and the values aren't equal, mutate it + if (dataField != null && !cellRow.get(j).toString().equals(dataField)) { mutated = true; - elements.setValueLocal(cell.getKey(), cell.getValue()); + originalVals.get(i).set(j, dataField); } } if (mutated) { - // Wrap in a retrier to deal with transient HTTP failures, which are IOExceptions wrapped - // in RuntimeExceptions. - retrier.callWithRetry(new Callable() { - @Override - public Void call() throws Exception { - entry.update(); - return null; - }}, RuntimeException.class); + ValueRange rowUpdate = + new ValueRange() + .setValues(originalVals.subList(i, i + 1)) + .setRange(getCellRange(i)); + updates.add(rowUpdate); } } - if (data.size() > entries.size()) { - for (int i = entries.size(); i < data.size(); i++) { - final ListEntry entry = listFeed.createEntry(); - CustomElementCollection elements = entry.getCustomElements(); - for (ImmutableMap.Entry cell : data.get(i).entrySet()) { - elements.setValueLocal(cell.getKey(), cell.getValue()); + // Update the mutated cells if necessary + if (!updates.isEmpty()) { + BatchUpdateValuesRequest updateRequest = new BatchUpdateValuesRequest() + .setValueInputOption("RAW") + .setData(updates); + + BatchUpdateValuesResponse response = + sheetsService.spreadsheets().values().batchUpdate(spreadsheetId, updateRequest).execute(); + Integer cellsUpdated = response.getTotalUpdatedCells(); + logger.infofmt("Updated %d originalVals", cellsUpdated != null ? cellsUpdated : 0); + } + + // Append extra rows if necessary + if (data.size() > originalVals.size()) { + ImmutableList.Builder> valsBuilder = new ImmutableList.Builder<>(); + for (int i = originalVals.size(); i < data.size(); i++) { + ImmutableList.Builder rowBuilder = new ImmutableList.Builder<>(); + for (String header : headers) { + rowBuilder.add(nullToEmpty(data.get(i).get(header))); } - // Wrap in a retrier to deal with transient HTTP failures, which are IOExceptions wrapped - // in RuntimeExceptions. - retrier.callWithRetry(new Callable() { - @Override - public Void call() throws Exception { - listFeed.insert(entry); - return null; - }}, RuntimeException.class); + valsBuilder.add(rowBuilder.build()); } + // Start the append at index originalVals.size (where the previous operation left off) + ValueRange appendUpdate = new ValueRange().setValues(valsBuilder.build()); + AppendValuesResponse appendResponse = sheetsService + .spreadsheets() + .values() + .append(spreadsheetId, getCellRange(originalVals.size()), appendUpdate) + .setValueInputOption("RAW") + .setInsertDataOption("INSERT_ROWS") + .execute(); + logger.infofmt( + "Appended %d rows to range %s", + data.size() - originalVals.size(), appendResponse.getTableRange()); + // Clear the extra rows if necessary + } else if (data.size() < originalVals.size()) { + // Clear other rows if there's more originalVals on the sheet than live data. + ClearValuesResponse clearResponse = + sheetsService + .spreadsheets() + .values() + .clear( + spreadsheetId, + getRowRange(data.size(), originalVals.size()), + new ClearValuesRequest()) + .execute(); + logger.infofmt( + "Cleared %d rows from range %s", + originalVals.size() - data.size(), clearResponse.getClearedRange()); } } + + /** + * Returns an A1 representation of the cell indicating the top-left corner of the update. + * + *

Updates and appends can specify either a complete range (i.e. A1:B4), or just the top-left + * corner of the request. For the latter case, the data fills in based on the primary dimension + * (either row-major or column-major order, default is row-major.) For simplicity, we just specify + * a single cell for these requests. + * + * @see + * Writing to a single range + */ + private String getCellRange(int rowNum) { + // We add 1 to rowNum to compensate for Sheet's 1-indexing, and 1 to offset for the header + return String.format("%s!A%d", SHEET_NAME, rowNum + 2); + } + + /** + * Returns an A1 representation of the cell indicating an entire set of rows. + * + *

Clear requests require a specific range, and we always want to clear an entire row at a time + * (to avoid leaving any data behind). + */ + private String getRowRange(int firstRow, int lastRow) { + return String.format("%s!%d:%d", SHEET_NAME, firstRow + 2, lastRow + 2); + } } diff --git a/java/google/registry/export/sheet/SpreadsheetServiceModule.java b/java/google/registry/export/sheet/SheetsServiceModule.java similarity index 58% rename from java/google/registry/export/sheet/SpreadsheetServiceModule.java rename to java/google/registry/export/sheet/SheetsServiceModule.java index 42cdbf0d2..e661752a4 100644 --- a/java/google/registry/export/sheet/SpreadsheetServiceModule.java +++ b/java/google/registry/export/sheet/SheetsServiceModule.java @@ -15,24 +15,28 @@ package google.registry.export.sheet; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.services.sheets.v4.Sheets; import com.google.common.collect.ImmutableList; -import com.google.gdata.client.spreadsheet.SpreadsheetService; import dagger.Module; import dagger.Provides; +import google.registry.config.RegistryConfig.Config; -/** Dagger module for {@link SpreadsheetService}. */ +/** Dagger module for {@link Sheets}. */ @Module -public final class SpreadsheetServiceModule { +public final class SheetsServiceModule { - private static final String APPLICATION_NAME = "google-registry-v1"; private static final ImmutableList SCOPES = ImmutableList.of( - "https://spreadsheets.google.com/feeds", - "https://docs.google.com/feeds"); - + "https://www.googleapis.com/auth/spreadsheets"); @Provides - static SpreadsheetService provideSpreadsheetService(GoogleCredential credential) { - SpreadsheetService service = new SpreadsheetService(APPLICATION_NAME); - service.setOAuth2Credentials(credential.createScoped(SCOPES)); - return service; + static Sheets provideSheets( + HttpTransport transport, + JsonFactory jsonFactory, + @Config("projectId") String projectId, + GoogleCredential credential) { + return new Sheets.Builder(transport, jsonFactory, credential.createScoped(SCOPES)) + .setApplicationName(projectId) + .build(); } } diff --git a/java/google/registry/export/sheet/SyncRegistrarsSheet.java b/java/google/registry/export/sheet/SyncRegistrarsSheet.java index 3f1fedd99..d4242095c 100644 --- a/java/google/registry/export/sheet/SyncRegistrarsSheet.java +++ b/java/google/registry/export/sheet/SyncRegistrarsSheet.java @@ -34,7 +34,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Ordering; -import com.google.gdata.util.ServiceException; import com.googlecode.objectify.VoidWork; import google.registry.model.common.Cursor; import google.registry.model.registrar.Registrar; @@ -74,7 +73,7 @@ class SyncRegistrarsSheet { } /** Performs the synchronization operation. */ - void run(String spreadsheetId) throws IOException, ServiceException { + void run(String spreadsheetId) throws IOException { final DateTime executionTime = clock.nowUtc(); sheetSynchronizer.synchronize( spreadsheetId, diff --git a/java/google/registry/export/sheet/SyncRegistrarsSheetAction.java b/java/google/registry/export/sheet/SyncRegistrarsSheetAction.java index 554b15d46..d391d4e3c 100644 --- a/java/google/registry/export/sheet/SyncRegistrarsSheetAction.java +++ b/java/google/registry/export/sheet/SyncRegistrarsSheetAction.java @@ -28,7 +28,6 @@ import com.google.appengine.api.modules.ModulesServiceFactory; import com.google.appengine.api.taskqueue.TaskHandle; import com.google.appengine.api.taskqueue.TaskOptions.Method; import com.google.common.base.Optional; -import com.google.gdata.util.ServiceException; import google.registry.config.RegistryConfig.Config; import google.registry.request.Action; import google.registry.request.Parameter; @@ -133,6 +132,7 @@ public class SyncRegistrarsSheetAction implements Runnable { return; } } + String sheetLockName = String.format("%s: %s", LOCK_NAME, sheetId.get()); Callable runner = new Callable() { @Nullable @@ -141,7 +141,7 @@ public class SyncRegistrarsSheetAction implements Runnable { try { syncRegistrarsSheet.run(sheetId.get()); Result.OK.send(response, null); - } catch (IOException | ServiceException e) { + } catch (IOException e) { Result.FAILED.send(response, e); } return null; diff --git a/java/google/registry/module/backend/BackendComponent.java b/java/google/registry/module/backend/BackendComponent.java index 0d76bfe25..179e3f9bf 100644 --- a/java/google/registry/module/backend/BackendComponent.java +++ b/java/google/registry/module/backend/BackendComponent.java @@ -19,7 +19,7 @@ import google.registry.bigquery.BigqueryModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.dns.writer.VoidDnsWriterModule; import google.registry.export.DriveModule; -import google.registry.export.sheet.SpreadsheetServiceModule; +import google.registry.export.sheet.SheetsServiceModule; import google.registry.gcs.GcsServiceModule; import google.registry.groups.DirectoryModule; import google.registry.groups.GroupsModule; @@ -67,7 +67,7 @@ import javax.inject.Singleton; KeyringModule.class, KmsModule.class, ModulesServiceModule.class, - SpreadsheetServiceModule.class, + SheetsServiceModule.class, StackdriverModule.class, SystemClockModule.class, SystemSleeperModule.class, diff --git a/java/google/registry/repositories.bzl b/java/google/registry/repositories.bzl index 3186290ee..7461956b8 100644 --- a/java/google/registry/repositories.bzl +++ b/java/google/registry/repositories.bzl @@ -37,6 +37,7 @@ def domain_registry_repositories( omit_com_google_apis_google_api_services_drive=False, omit_com_google_apis_google_api_services_groupssettings=False, omit_com_google_apis_google_api_services_monitoring=False, + omit_com_google_apis_google_api_services_sheets=False, omit_com_google_apis_google_api_services_storage=False, omit_com_google_appengine_api_1_0_sdk=False, omit_com_google_appengine_api_labs=False, @@ -141,6 +142,8 @@ def domain_registry_repositories( com_google_apis_google_api_services_groupssettings() if not omit_com_google_apis_google_api_services_monitoring: com_google_apis_google_api_services_monitoring() + if not omit_com_google_apis_google_api_services_sheets: + com_google_apis_google_api_services_sheets() if not omit_com_google_apis_google_api_services_storage: com_google_apis_google_api_services_storage() if not omit_com_google_appengine_api_1_0_sdk: @@ -494,6 +497,19 @@ def com_google_apis_google_api_services_monitoring(): deps = ["@com_google_api_client"], ) +def com_google_apis_google_api_services_sheets(): + java_import_external( + name = "com_google_apis_google_api_services_sheets", + jar_sha256 = "67529b9efceb1a16b72c6aa0822d50f4f6e8c3c84972a5a37ca6e7bdc19064ba", + jar_urls = [ + "http://domain-registry-maven.storage.googleapis.com/repo1.maven.org/maven2/com/google/apis/google-api-services-sheets/v4-rev483-1.22.0/google-api-services-sheets-v4-rev483-1.22.0.jar", + "http://repo1.maven.org/maven2/com/google/apis/google-api-services-sheets/v4-rev483-1.22.0/google-api-services-sheets-v4-rev483-1.22.0.jar", + "http://maven.ibiblio.org/maven2/com/google/apis/google-api-services-sheets/v4-rev483-1.22.0/google-api-services-sheets-v4-rev483-1.22.0.jar", + ], + licenses = ["notice"], # The Apache Software License, Version 2.0 + deps = ["@com_google_api_client"], + ) + def com_google_apis_google_api_services_storage(): java_import_external( name = "com_google_apis_google_api_services_storage", diff --git a/javatests/google/registry/export/sheet/BUILD b/javatests/google/registry/export/sheet/BUILD index 8dad8fda7..18d5d5e02 100644 --- a/javatests/google/registry/export/sheet/BUILD +++ b/javatests/google/registry/export/sheet/BUILD @@ -9,13 +9,12 @@ java_library( name = "sheet", srcs = glob(["*.java"]), deps = [ - "//java/com/google/gdata:spreadsheet", "//java/google/registry/config", "//java/google/registry/export/sheet", "//java/google/registry/model", - "//java/google/registry/util", "//javatests/google/registry/testing", "//third_party/java/objectify:objectify-v4_1", + "@com_google_apis_google_api_services_sheets", "@com_google_code_findbugs_jsr305", "@com_google_guava", "@com_google_truth", diff --git a/javatests/google/registry/export/sheet/SheetSynchronizerTest.java b/javatests/google/registry/export/sheet/SheetSynchronizerTest.java index 4981b1c19..7eb2dc4f8 100644 --- a/javatests/google/registry/export/sheet/SheetSynchronizerTest.java +++ b/javatests/google/registry/export/sheet/SheetSynchronizerTest.java @@ -14,29 +14,25 @@ package google.registry.export.sheet; +import static com.google.common.collect.Lists.newArrayList; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.model.AppendValuesResponse; +import com.google.api.services.sheets.v4.model.BatchUpdateValuesRequest; +import com.google.api.services.sheets.v4.model.BatchUpdateValuesResponse; +import com.google.api.services.sheets.v4.model.ClearValuesRequest; +import com.google.api.services.sheets.v4.model.ClearValuesResponse; +import com.google.api.services.sheets.v4.model.ValueRange; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.gdata.client.spreadsheet.SpreadsheetService; -import com.google.gdata.data.spreadsheet.CustomElementCollection; -import com.google.gdata.data.spreadsheet.ListEntry; -import com.google.gdata.data.spreadsheet.ListFeed; -import com.google.gdata.data.spreadsheet.SpreadsheetEntry; -import com.google.gdata.data.spreadsheet.WorksheetEntry; -import google.registry.testing.FakeClock; -import google.registry.testing.FakeSleeper; -import google.registry.util.Retrier; -import java.net.URL; -import org.joda.time.DateTime; -import org.junit.After; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -45,143 +41,155 @@ import org.junit.runners.JUnit4; /** Unit tests for {@link SheetSynchronizer}. */ @RunWith(JUnit4.class) public class SheetSynchronizerTest { - - private static final int MAX_RETRIES = 3; - - private final SpreadsheetService spreadsheetService = mock(SpreadsheetService.class); - private final SpreadsheetEntry spreadsheet = mock(SpreadsheetEntry.class); - private final WorksheetEntry worksheet = mock(WorksheetEntry.class); - private final ListFeed listFeed = mock(ListFeed.class); private final SheetSynchronizer sheetSynchronizer = new SheetSynchronizer(); - private final FakeSleeper sleeper = - new FakeSleeper(new FakeClock(DateTime.parse("2000-01-01TZ"))); + private final Sheets sheetsService = mock(Sheets.class); + private final Sheets.Spreadsheets spreadsheets = mock(Sheets.Spreadsheets.class); + private final Sheets.Spreadsheets.Values values = mock(Sheets.Spreadsheets.Values.class); + private final Sheets.Spreadsheets.Values.Get getReq = mock(Sheets.Spreadsheets.Values.Get.class); + private final Sheets.Spreadsheets.Values.Append appendReq = + mock(Sheets.Spreadsheets.Values.Append.class); + private final Sheets.Spreadsheets.Values.BatchUpdate updateReq = + mock(Sheets.Spreadsheets.Values.BatchUpdate.class); + private final Sheets.Spreadsheets.Values.Clear clearReq = + mock(Sheets.Spreadsheets.Values.Clear.class); + + private List> existingSheet; + private ImmutableList> data; @Before public void before() throws Exception { - sheetSynchronizer.spreadsheetService = spreadsheetService; - sheetSynchronizer.retrier = new Retrier(sleeper, MAX_RETRIES); - when(spreadsheetService.getEntry(any(URL.class), eq(SpreadsheetEntry.class))) - .thenReturn(spreadsheet); - when(spreadsheet.getWorksheets()).thenReturn(ImmutableList.of(worksheet)); - when(worksheet.getListFeedUrl()).thenReturn(new URL("http://example.com/spreadsheet")); - when(spreadsheetService.getFeed(any(URL.class), eq(ListFeed.class))).thenReturn(listFeed); - when(worksheet.update()).thenReturn(worksheet); + sheetSynchronizer.sheetsService = sheetsService; + when(sheetsService.spreadsheets()).thenReturn(spreadsheets); + when(spreadsheets.values()).thenReturn(values); + + when(values.get(any(String.class), any(String.class))).thenReturn(getReq); + when(values.append(any(String.class), any(String.class), any(ValueRange.class))) + .thenReturn(appendReq); + when(values.clear(any(String.class), any(String.class), any(ClearValuesRequest.class))) + .thenReturn(clearReq); + when(values.batchUpdate(any(String.class), any(BatchUpdateValuesRequest.class))) + .thenReturn(updateReq); + + when(appendReq.execute()).thenReturn(new AppendValuesResponse()); + when(appendReq.setValueInputOption(any(String.class))).thenReturn(appendReq); + when(appendReq.setInsertDataOption(any(String.class))).thenReturn(appendReq); + when(clearReq.execute()).thenReturn(new ClearValuesResponse()); + when(updateReq.execute()).thenReturn(new BatchUpdateValuesResponse()); + + existingSheet = newArrayList(); + data = ImmutableList.of(); + ValueRange valueRange = new ValueRange().setValues(existingSheet); + when(getReq.execute()).thenReturn(valueRange); } - @After - public void after() throws Exception { - verify(spreadsheetService) - .getEntry( - new URL("https://spreadsheets.google.com/feeds/spreadsheets/foobar"), - SpreadsheetEntry.class); - verify(spreadsheet).getWorksheets(); - verify(worksheet).getListFeedUrl(); - verify(spreadsheetService).getFeed(new URL("http://example.com/spreadsheet"), ListFeed.class); - verify(listFeed).getEntries(); - verifyNoMoreInteractions(spreadsheetService, spreadsheet, worksheet, listFeed); + // Explicitly constructs a List to avoid newArrayList typing to ArrayList + private List createRow(Object... elements) { + List row = new ArrayList<>(); + row.addAll(Arrays.asList(elements)); + return row; } @Test - public void testSynchronize_bothEmpty_doNothing() throws Exception { - when(listFeed.getEntries()).thenReturn(ImmutableList.of()); - sheetSynchronizer.synchronize("foobar", ImmutableList.>of()); - verify(worksheet).setRowCount(1); - verify(worksheet).update(); + public void testSynchronize_dataAndSheetEmpty_doNothing() throws Exception { + existingSheet.add(createRow("a", "b")); + sheetSynchronizer.synchronize("aSheetId", data); + verifyZeroInteractions(appendReq); + verifyZeroInteractions(clearReq); + verifyZeroInteractions(updateReq); } @Test - public void testSynchronize_bothContainSameRow_doNothing() throws Exception { - ListEntry entry = makeListEntry(ImmutableMap.of("key", "value")); - when(listFeed.getEntries()).thenReturn(ImmutableList.of(entry)); - sheetSynchronizer.synchronize("foobar", ImmutableList.of(ImmutableMap.of("key", "value"))); - verify(worksheet).setRowCount(2); - verify(worksheet).update(); - verify(entry, atLeastOnce()).getCustomElements(); - verifyNoMoreInteractions(entry); + public void testSynchronize_differentValues_updatesValues() throws Exception { + existingSheet.add(createRow("a", "b")); + existingSheet.add(createRow("diffVal1l", "diffVal2")); + data = ImmutableList.of(ImmutableMap.of("a", "val1", "b", "val2")); + sheetSynchronizer.synchronize("aSheetId", data); + + verifyZeroInteractions(appendReq); + verifyZeroInteractions(clearReq); + + BatchUpdateValuesRequest expectedRequest = new BatchUpdateValuesRequest(); + List> expectedVals = newArrayList(); + expectedVals.add(createRow("val1", "val2")); + expectedRequest.setData( + newArrayList(new ValueRange().setRange("Registrars!A2").setValues(expectedVals))); + expectedRequest.setValueInputOption("RAW"); + verify(values).batchUpdate("aSheetId", expectedRequest); } @Test - public void testSynchronize_cellIsDifferent_updateRow() throws Exception { - ListEntry entry = makeListEntry(ImmutableMap.of("key", "value")); - when(listFeed.getEntries()).thenReturn(ImmutableList.of(entry)); - sheetSynchronizer.synchronize("foobar", ImmutableList.of(ImmutableMap.of("key", "new value"))); - verify(entry.getCustomElements()).setValueLocal("key", "new value"); - verify(entry).update(); - verify(worksheet).setRowCount(2); - verify(worksheet).update(); - verify(entry, atLeastOnce()).getCustomElements(); - verifyNoMoreInteractions(entry); + public void testSynchronize_unknownFields_doesntUpdate() throws Exception { + existingSheet.add(createRow("a", "c", "b")); + existingSheet.add(createRow("diffVal1", "sameVal", "diffVal2")); + data = ImmutableList.of(ImmutableMap.of("a", "val1", "b", "val2", "d", "val3")); + sheetSynchronizer.synchronize("aSheetId", data); + + verifyZeroInteractions(appendReq); + verifyZeroInteractions(clearReq); + + BatchUpdateValuesRequest expectedRequest = new BatchUpdateValuesRequest(); + List> expectedVals = newArrayList(); + expectedVals.add(createRow("val1", "sameVal", "val2")); + expectedRequest.setData( + newArrayList(new ValueRange().setRange("Registrars!A2").setValues(expectedVals))); + expectedRequest.setValueInputOption("RAW"); + verify(values).batchUpdate("aSheetId", expectedRequest); } @Test - public void testSynchronize_cellIsDifferent_updateRow_retriesOnException() throws Exception { - ListEntry entry = makeListEntry(ImmutableMap.of("key", "value")); - when(listFeed.getEntries()).thenReturn(ImmutableList.of(entry)); - when(entry.update()) - .thenThrow(new RuntimeException()) - .thenThrow(new RuntimeException()) - .thenReturn(entry); - sheetSynchronizer.synchronize("foobar", ImmutableList.of(ImmutableMap.of("key", "new value"))); - verify(entry.getCustomElements()).setValueLocal("key", "new value"); - verify(entry, times(3)).update(); - verify(worksheet).setRowCount(2); - verify(worksheet).update(); - verify(entry, atLeastOnce()).getCustomElements(); - verifyNoMoreInteractions(entry); + public void testSynchronize_notFullRow_getsPadded() throws Exception { + existingSheet.add(createRow("a", "c", "b")); + existingSheet.add(createRow("diffVal1", "diffVal2")); + data = ImmutableList.of(ImmutableMap.of("a", "val1", "b", "paddedVal", "d", "val3")); + sheetSynchronizer.synchronize("aSheetId", data); + + verifyZeroInteractions(appendReq); + verifyZeroInteractions(clearReq); + + BatchUpdateValuesRequest expectedRequest = new BatchUpdateValuesRequest(); + List> expectedVals = newArrayList(); + expectedVals.add(createRow("val1", "diffVal2", "paddedVal")); + expectedRequest.setData( + newArrayList(new ValueRange().setRange("Registrars!A2").setValues(expectedVals))); + expectedRequest.setValueInputOption("RAW"); + verify(values).batchUpdate("aSheetId", expectedRequest); } @Test - public void testSynchronize_spreadsheetMissingRow_insertRow() throws Exception { - ListEntry entry = makeListEntry(ImmutableMap.of()); - when(listFeed.getEntries()).thenReturn(ImmutableList.of()); - when(listFeed.createEntry()).thenReturn(entry); - sheetSynchronizer.synchronize("foobar", ImmutableList.of(ImmutableMap.of("key", "value"))); - verify(entry.getCustomElements()).setValueLocal("key", "value"); - verify(listFeed).insert(entry); - verify(worksheet).setRowCount(2); - verify(worksheet).update(); - verify(listFeed).createEntry(); - verify(entry, atLeastOnce()).getCustomElements(); - verifyNoMoreInteractions(entry); + public void testSynchronize_moreData_appendsValues() throws Exception { + existingSheet.add(createRow("a", "b")); + existingSheet.add(createRow("diffVal1", "diffVal2")); + data = ImmutableList.of( + ImmutableMap.of("a", "val1", "b", "val2"), + ImmutableMap.of("a", "val3", "b", "val4")); + sheetSynchronizer.synchronize("aSheetId", data); + + verifyZeroInteractions(clearReq); + + BatchUpdateValuesRequest expectedRequest = new BatchUpdateValuesRequest(); + List> updatedVals = newArrayList(); + updatedVals.add(createRow("val1", "val2")); + expectedRequest.setData( + newArrayList( + new ValueRange().setRange("Registrars!A2").setValues(updatedVals))); + expectedRequest.setValueInputOption("RAW"); + verify(values).batchUpdate("aSheetId", expectedRequest); + + List> appendedVals = newArrayList(); + appendedVals.add(createRow("val3", "val4")); + ValueRange appendRequest = new ValueRange().setValues(appendedVals); + verify(values).append("aSheetId", "Registrars!A3", appendRequest); } @Test - public void testSynchronize_spreadsheetMissingRow_insertRow_retriesOnException() - throws Exception { - ListEntry entry = makeListEntry(ImmutableMap.of()); - when(listFeed.getEntries()).thenReturn(ImmutableList.of()); - when(listFeed.createEntry()).thenReturn(entry); - when(listFeed.insert(entry)) - .thenThrow(new RuntimeException()) - .thenThrow(new RuntimeException()) - .thenReturn(entry); - sheetSynchronizer.synchronize("foobar", ImmutableList.of(ImmutableMap.of("key", "value"))); - verify(entry.getCustomElements()).setValueLocal("key", "value"); - verify(listFeed, times(3)).insert(entry); - verify(worksheet).setRowCount(2); - verify(worksheet).update(); - verify(listFeed).createEntry(); - verify(entry, atLeastOnce()).getCustomElements(); - verifyNoMoreInteractions(entry); - } + public void testSynchronize_lessData_clearsValues() throws Exception { + existingSheet.add(createRow("a", "b")); + existingSheet.add(createRow("val1", "val2")); + existingSheet.add(createRow("diffVal3", "diffVal4")); + data = ImmutableList.of(ImmutableMap.of("a", "val1", "b", "val2")); + sheetSynchronizer.synchronize("aSheetId", data); - @Test - public void testSynchronize_spreadsheetRowNoLongerInData_deleteRow() throws Exception { - ListEntry entry = makeListEntry(ImmutableMap.of("key", "value")); - when(listFeed.getEntries()).thenReturn(ImmutableList.of(entry)); - sheetSynchronizer.synchronize("foobar", ImmutableList.>of()); - verify(worksheet).setRowCount(1); - verify(worksheet).update(); - verifyNoMoreInteractions(entry); - } - - private static ListEntry makeListEntry(ImmutableMap values) { - CustomElementCollection collection = mock(CustomElementCollection.class); - for (ImmutableMap.Entry entry : values.entrySet()) { - when(collection.getValue(eq(entry.getKey()))).thenReturn(entry.getValue()); - } - ListEntry listEntry = mock(ListEntry.class); - when(listEntry.getCustomElements()).thenReturn(collection); - return listEntry; + verify(values).clear("aSheetId", "Registrars!3:4", new ClearValuesRequest()); + verifyZeroInteractions(updateReq); } }