mirror of
https://github.com/google/nomulus.git
synced 2025-05-13 16:07:15 +02:00
Delete deprecated legacy SQL 'latest_snapshot' dataset
We've migrated everything over to using the standard SQL view now. The legacy version is just causing confusion and costing us resources. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=201966352
This commit is contained in:
parent
aa11d5681f
commit
4cddbdacff
3 changed files with 17 additions and 26 deletions
|
@ -35,7 +35,7 @@ import javax.inject.Inject;
|
|||
* <li>The snapshot is exported to Google Cloud Storage (this action).
|
||||
* <li>The {@link CheckSnapshotAction} polls until the export is completed.
|
||||
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
|
||||
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
|
||||
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_datastore_export.
|
||||
* </ol>
|
||||
*/
|
||||
@Action(
|
||||
|
|
|
@ -44,8 +44,7 @@ public class UpdateSnapshotViewAction implements Runnable {
|
|||
static final String UPDATE_SNAPSHOT_TABLE_ID_PARAM = "table";
|
||||
static final String UPDATE_SNAPSHOT_KIND_PARAM = "kind";
|
||||
|
||||
static final String LEGACY_LATEST_SNAPSHOT_DATASET = "latest_snapshot";
|
||||
static final String STANDARD_LATEST_SNAPSHOT_DATASET = "latest_datastore_export";
|
||||
private static final String TARGET_DATASET_NAME = "latest_datastore_export";
|
||||
|
||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
||||
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
|
||||
|
@ -87,22 +86,15 @@ public class UpdateSnapshotViewAction implements Runnable {
|
|||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
// TODO(b/32377148): Remove the legacySql view when migration complete.
|
||||
SqlTemplate legacyTemplate =
|
||||
SqlTemplate.create(
|
||||
"#legacySQL\nSELECT * FROM [%PROJECT%:%SOURCE_DATASET%.%SOURCE_TABLE%]");
|
||||
updateSnapshotView(
|
||||
datasetId, tableId, kindName, LEGACY_LATEST_SNAPSHOT_DATASET, legacyTemplate, true);
|
||||
|
||||
SqlTemplate standardTemplate =
|
||||
SqlTemplate sqlTemplate =
|
||||
SqlTemplate.create(
|
||||
"#standardSQL\nSELECT * FROM `%PROJECT%.%SOURCE_DATASET%.%SOURCE_TABLE%`");
|
||||
updateSnapshotView(
|
||||
datasetId, tableId, kindName, STANDARD_LATEST_SNAPSHOT_DATASET, standardTemplate, false);
|
||||
|
||||
updateSnapshotView(datasetId, tableId, kindName, TARGET_DATASET_NAME, sqlTemplate);
|
||||
} catch (Throwable e) {
|
||||
throw new InternalServerErrorException(
|
||||
String.format("Could not update snapshot view for table %s", tableId), e);
|
||||
String.format(
|
||||
"Could not update snapshot view %s for table %s", TARGET_DATASET_NAME, tableId),
|
||||
e);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -111,8 +103,7 @@ public class UpdateSnapshotViewAction implements Runnable {
|
|||
String sourceTableId,
|
||||
String kindName,
|
||||
String viewDataset,
|
||||
SqlTemplate viewQueryTemplate,
|
||||
boolean useLegacySql)
|
||||
SqlTemplate viewQueryTemplate)
|
||||
throws IOException {
|
||||
|
||||
Bigquery bigquery = bigqueryFactory.create(projectId, viewDataset);
|
||||
|
@ -126,7 +117,7 @@ public class UpdateSnapshotViewAction implements Runnable {
|
|||
.setTableId(kindName))
|
||||
.setView(
|
||||
new ViewDefinition()
|
||||
.setUseLegacySql(useLegacySql)
|
||||
.setUseLegacySql(false)
|
||||
.setQuery(
|
||||
viewQueryTemplate
|
||||
.put("PROJECT", projectId)
|
||||
|
|
|
@ -102,21 +102,18 @@ public class UpdateSnapshotViewActionTest {
|
|||
InOrder factoryOrder = inOrder(bigqueryFactory);
|
||||
// Check that the BigQuery factory was called in such a way that the dataset would be created
|
||||
// if it didn't already exist.
|
||||
factoryOrder.verify(bigqueryFactory).create("myproject", "latest_snapshot");
|
||||
factoryOrder.verify(bigqueryFactory).create("myproject", "latest_datastore_export");
|
||||
|
||||
// Check that we updated both views
|
||||
InOrder tableOrder = inOrder(bigqueryTables);
|
||||
ArgumentCaptor<Table> tableArg = ArgumentCaptor.forClass(Table.class);
|
||||
tableOrder.verify(bigqueryTables)
|
||||
.update(eq("myproject"), eq("latest_snapshot"), eq("fookind"), tableArg.capture());
|
||||
tableOrder.verify(bigqueryTables)
|
||||
tableOrder
|
||||
.verify(bigqueryTables)
|
||||
.update(eq("myproject"), eq("latest_datastore_export"), eq("fookind"), tableArg.capture());
|
||||
Iterable<String> actualQueries =
|
||||
Iterables.transform(tableArg.getAllValues(), table -> table.getView().getQuery());
|
||||
assertThat(actualQueries).containsExactly(
|
||||
"#legacySQL\nSELECT * FROM [myproject:some_dataset.12345_fookind]",
|
||||
"#standardSQL\nSELECT * FROM `myproject.some_dataset.12345_fookind`");
|
||||
assertThat(actualQueries)
|
||||
.containsExactly("#standardSQL\nSELECT * FROM `myproject.some_dataset.12345_fookind`");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
@ -125,7 +122,10 @@ public class UpdateSnapshotViewActionTest {
|
|||
.thenThrow(new IOException("I'm sorry Dave, I can't let you do that"));
|
||||
InternalServerErrorException thrown =
|
||||
assertThrows(InternalServerErrorException.class, action::run);
|
||||
assertThat(thrown).hasMessageThat().contains("Could not update snapshot view for table");
|
||||
assertThat(thrown)
|
||||
.hasMessageThat()
|
||||
.isEqualTo(
|
||||
"Could not update snapshot view latest_datastore_export for table 12345_fookind");
|
||||
assertThat(thrown).hasCauseThat().hasMessageThat().contains("I'm sorry Dave");
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue