mirror of
https://github.com/google/nomulus.git
synced 2025-05-13 16:07:15 +02:00
Delete deprecated legacy SQL 'latest_snapshot' dataset
We've migrated everything over to using the standard SQL view now. The legacy version is just causing confusion and costing us resources. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=201966352
This commit is contained in:
parent
aa11d5681f
commit
4cddbdacff
3 changed files with 17 additions and 26 deletions
|
@ -35,7 +35,7 @@ import javax.inject.Inject;
|
||||||
* <li>The snapshot is exported to Google Cloud Storage (this action).
|
* <li>The snapshot is exported to Google Cloud Storage (this action).
|
||||||
* <li>The {@link CheckSnapshotAction} polls until the export is completed.
|
* <li>The {@link CheckSnapshotAction} polls until the export is completed.
|
||||||
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
|
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
|
||||||
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
|
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_datastore_export.
|
||||||
* </ol>
|
* </ol>
|
||||||
*/
|
*/
|
||||||
@Action(
|
@Action(
|
||||||
|
|
|
@ -44,8 +44,7 @@ public class UpdateSnapshotViewAction implements Runnable {
|
||||||
static final String UPDATE_SNAPSHOT_TABLE_ID_PARAM = "table";
|
static final String UPDATE_SNAPSHOT_TABLE_ID_PARAM = "table";
|
||||||
static final String UPDATE_SNAPSHOT_KIND_PARAM = "kind";
|
static final String UPDATE_SNAPSHOT_KIND_PARAM = "kind";
|
||||||
|
|
||||||
static final String LEGACY_LATEST_SNAPSHOT_DATASET = "latest_snapshot";
|
private static final String TARGET_DATASET_NAME = "latest_datastore_export";
|
||||||
static final String STANDARD_LATEST_SNAPSHOT_DATASET = "latest_datastore_export";
|
|
||||||
|
|
||||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
||||||
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
|
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
|
||||||
|
@ -87,22 +86,15 @@ public class UpdateSnapshotViewAction implements Runnable {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
try {
|
try {
|
||||||
// TODO(b/32377148): Remove the legacySql view when migration complete.
|
SqlTemplate sqlTemplate =
|
||||||
SqlTemplate legacyTemplate =
|
|
||||||
SqlTemplate.create(
|
|
||||||
"#legacySQL\nSELECT * FROM [%PROJECT%:%SOURCE_DATASET%.%SOURCE_TABLE%]");
|
|
||||||
updateSnapshotView(
|
|
||||||
datasetId, tableId, kindName, LEGACY_LATEST_SNAPSHOT_DATASET, legacyTemplate, true);
|
|
||||||
|
|
||||||
SqlTemplate standardTemplate =
|
|
||||||
SqlTemplate.create(
|
SqlTemplate.create(
|
||||||
"#standardSQL\nSELECT * FROM `%PROJECT%.%SOURCE_DATASET%.%SOURCE_TABLE%`");
|
"#standardSQL\nSELECT * FROM `%PROJECT%.%SOURCE_DATASET%.%SOURCE_TABLE%`");
|
||||||
updateSnapshotView(
|
updateSnapshotView(datasetId, tableId, kindName, TARGET_DATASET_NAME, sqlTemplate);
|
||||||
datasetId, tableId, kindName, STANDARD_LATEST_SNAPSHOT_DATASET, standardTemplate, false);
|
|
||||||
|
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
throw new InternalServerErrorException(
|
throw new InternalServerErrorException(
|
||||||
String.format("Could not update snapshot view for table %s", tableId), e);
|
String.format(
|
||||||
|
"Could not update snapshot view %s for table %s", TARGET_DATASET_NAME, tableId),
|
||||||
|
e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -111,8 +103,7 @@ public class UpdateSnapshotViewAction implements Runnable {
|
||||||
String sourceTableId,
|
String sourceTableId,
|
||||||
String kindName,
|
String kindName,
|
||||||
String viewDataset,
|
String viewDataset,
|
||||||
SqlTemplate viewQueryTemplate,
|
SqlTemplate viewQueryTemplate)
|
||||||
boolean useLegacySql)
|
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
||||||
Bigquery bigquery = bigqueryFactory.create(projectId, viewDataset);
|
Bigquery bigquery = bigqueryFactory.create(projectId, viewDataset);
|
||||||
|
@ -126,7 +117,7 @@ public class UpdateSnapshotViewAction implements Runnable {
|
||||||
.setTableId(kindName))
|
.setTableId(kindName))
|
||||||
.setView(
|
.setView(
|
||||||
new ViewDefinition()
|
new ViewDefinition()
|
||||||
.setUseLegacySql(useLegacySql)
|
.setUseLegacySql(false)
|
||||||
.setQuery(
|
.setQuery(
|
||||||
viewQueryTemplate
|
viewQueryTemplate
|
||||||
.put("PROJECT", projectId)
|
.put("PROJECT", projectId)
|
||||||
|
|
|
@ -102,21 +102,18 @@ public class UpdateSnapshotViewActionTest {
|
||||||
InOrder factoryOrder = inOrder(bigqueryFactory);
|
InOrder factoryOrder = inOrder(bigqueryFactory);
|
||||||
// Check that the BigQuery factory was called in such a way that the dataset would be created
|
// Check that the BigQuery factory was called in such a way that the dataset would be created
|
||||||
// if it didn't already exist.
|
// if it didn't already exist.
|
||||||
factoryOrder.verify(bigqueryFactory).create("myproject", "latest_snapshot");
|
|
||||||
factoryOrder.verify(bigqueryFactory).create("myproject", "latest_datastore_export");
|
factoryOrder.verify(bigqueryFactory).create("myproject", "latest_datastore_export");
|
||||||
|
|
||||||
// Check that we updated both views
|
// Check that we updated both views
|
||||||
InOrder tableOrder = inOrder(bigqueryTables);
|
InOrder tableOrder = inOrder(bigqueryTables);
|
||||||
ArgumentCaptor<Table> tableArg = ArgumentCaptor.forClass(Table.class);
|
ArgumentCaptor<Table> tableArg = ArgumentCaptor.forClass(Table.class);
|
||||||
tableOrder.verify(bigqueryTables)
|
tableOrder
|
||||||
.update(eq("myproject"), eq("latest_snapshot"), eq("fookind"), tableArg.capture());
|
.verify(bigqueryTables)
|
||||||
tableOrder.verify(bigqueryTables)
|
|
||||||
.update(eq("myproject"), eq("latest_datastore_export"), eq("fookind"), tableArg.capture());
|
.update(eq("myproject"), eq("latest_datastore_export"), eq("fookind"), tableArg.capture());
|
||||||
Iterable<String> actualQueries =
|
Iterable<String> actualQueries =
|
||||||
Iterables.transform(tableArg.getAllValues(), table -> table.getView().getQuery());
|
Iterables.transform(tableArg.getAllValues(), table -> table.getView().getQuery());
|
||||||
assertThat(actualQueries).containsExactly(
|
assertThat(actualQueries)
|
||||||
"#legacySQL\nSELECT * FROM [myproject:some_dataset.12345_fookind]",
|
.containsExactly("#standardSQL\nSELECT * FROM `myproject.some_dataset.12345_fookind`");
|
||||||
"#standardSQL\nSELECT * FROM `myproject.some_dataset.12345_fookind`");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -125,7 +122,10 @@ public class UpdateSnapshotViewActionTest {
|
||||||
.thenThrow(new IOException("I'm sorry Dave, I can't let you do that"));
|
.thenThrow(new IOException("I'm sorry Dave, I can't let you do that"));
|
||||||
InternalServerErrorException thrown =
|
InternalServerErrorException thrown =
|
||||||
assertThrows(InternalServerErrorException.class, action::run);
|
assertThrows(InternalServerErrorException.class, action::run);
|
||||||
assertThat(thrown).hasMessageThat().contains("Could not update snapshot view for table");
|
assertThat(thrown)
|
||||||
|
.hasMessageThat()
|
||||||
|
.isEqualTo(
|
||||||
|
"Could not update snapshot view latest_datastore_export for table 12345_fookind");
|
||||||
assertThat(thrown).hasCauseThat().hasMessageThat().contains("I'm sorry Dave");
|
assertThat(thrown).hasCauseThat().hasMessageThat().contains("I'm sorry Dave");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue