Delete deprecated legacy SQL 'latest_snapshot' dataset

We've migrated everything over to using the standard SQL view now. The legacy
version is just causing confusion and costing us resources.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=201966352
This commit is contained in:
mcilwain 2018-06-25 09:31:25 -07:00 committed by Ben McIlwain
parent aa11d5681f
commit 4cddbdacff
3 changed files with 17 additions and 26 deletions

View file

@ -35,7 +35,7 @@ import javax.inject.Inject;
* <li>The snapshot is exported to Google Cloud Storage (this action).
* <li>The {@link CheckSnapshotAction} polls until the export is completed.
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_datastore_export.
* </ol>
*/
@Action(

View file

@ -44,8 +44,7 @@ public class UpdateSnapshotViewAction implements Runnable {
static final String UPDATE_SNAPSHOT_TABLE_ID_PARAM = "table";
static final String UPDATE_SNAPSHOT_KIND_PARAM = "kind";
static final String LEGACY_LATEST_SNAPSHOT_DATASET = "latest_snapshot";
static final String STANDARD_LATEST_SNAPSHOT_DATASET = "latest_datastore_export";
private static final String TARGET_DATASET_NAME = "latest_datastore_export";
/** Servlet-specific details needed for enqueuing tasks against itself. */
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
@ -87,22 +86,15 @@ public class UpdateSnapshotViewAction implements Runnable {
@Override
public void run() {
try {
// TODO(b/32377148): Remove the legacySql view when migration complete.
SqlTemplate legacyTemplate =
SqlTemplate.create(
"#legacySQL\nSELECT * FROM [%PROJECT%:%SOURCE_DATASET%.%SOURCE_TABLE%]");
updateSnapshotView(
datasetId, tableId, kindName, LEGACY_LATEST_SNAPSHOT_DATASET, legacyTemplate, true);
SqlTemplate standardTemplate =
SqlTemplate sqlTemplate =
SqlTemplate.create(
"#standardSQL\nSELECT * FROM `%PROJECT%.%SOURCE_DATASET%.%SOURCE_TABLE%`");
updateSnapshotView(
datasetId, tableId, kindName, STANDARD_LATEST_SNAPSHOT_DATASET, standardTemplate, false);
updateSnapshotView(datasetId, tableId, kindName, TARGET_DATASET_NAME, sqlTemplate);
} catch (Throwable e) {
throw new InternalServerErrorException(
String.format("Could not update snapshot view for table %s", tableId), e);
String.format(
"Could not update snapshot view %s for table %s", TARGET_DATASET_NAME, tableId),
e);
}
}
@ -111,8 +103,7 @@ public class UpdateSnapshotViewAction implements Runnable {
String sourceTableId,
String kindName,
String viewDataset,
SqlTemplate viewQueryTemplate,
boolean useLegacySql)
SqlTemplate viewQueryTemplate)
throws IOException {
Bigquery bigquery = bigqueryFactory.create(projectId, viewDataset);
@ -126,7 +117,7 @@ public class UpdateSnapshotViewAction implements Runnable {
.setTableId(kindName))
.setView(
new ViewDefinition()
.setUseLegacySql(useLegacySql)
.setUseLegacySql(false)
.setQuery(
viewQueryTemplate
.put("PROJECT", projectId)

View file

@ -102,21 +102,18 @@ public class UpdateSnapshotViewActionTest {
InOrder factoryOrder = inOrder(bigqueryFactory);
// Check that the BigQuery factory was called in such a way that the dataset would be created
// if it didn't already exist.
factoryOrder.verify(bigqueryFactory).create("myproject", "latest_snapshot");
factoryOrder.verify(bigqueryFactory).create("myproject", "latest_datastore_export");
// Check that we updated both views
InOrder tableOrder = inOrder(bigqueryTables);
ArgumentCaptor<Table> tableArg = ArgumentCaptor.forClass(Table.class);
tableOrder.verify(bigqueryTables)
.update(eq("myproject"), eq("latest_snapshot"), eq("fookind"), tableArg.capture());
tableOrder.verify(bigqueryTables)
tableOrder
.verify(bigqueryTables)
.update(eq("myproject"), eq("latest_datastore_export"), eq("fookind"), tableArg.capture());
Iterable<String> actualQueries =
Iterables.transform(tableArg.getAllValues(), table -> table.getView().getQuery());
assertThat(actualQueries).containsExactly(
"#legacySQL\nSELECT * FROM [myproject:some_dataset.12345_fookind]",
"#standardSQL\nSELECT * FROM `myproject.some_dataset.12345_fookind`");
assertThat(actualQueries)
.containsExactly("#standardSQL\nSELECT * FROM `myproject.some_dataset.12345_fookind`");
}
@Test
@ -125,7 +122,10 @@ public class UpdateSnapshotViewActionTest {
.thenThrow(new IOException("I'm sorry Dave, I can't let you do that"));
InternalServerErrorException thrown =
assertThrows(InternalServerErrorException.class, action::run);
assertThat(thrown).hasMessageThat().contains("Could not update snapshot view for table");
assertThat(thrown)
.hasMessageThat()
.isEqualTo(
"Could not update snapshot view latest_datastore_export for table 12345_fookind");
assertThat(thrown).hasCauseThat().hasMessageThat().contains("I'm sorry Dave");
}
}