zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-13 07:57:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add standardSQL views to Bigquery Datastore snapshots

Browse source 
-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=163124895
This commit is contained in:
larryruili 2017-07-25 14:38:47 -07:00 • committed by Ben McIlwain
parent 8869814e96
commit d2cd576796
2 changed files with 94 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

103
java/google/registry/export/UpdateSnapshotViewAction.java
View file

@ -35,36 +35,48 @@ import java.io.IOException;
import javax.inject.Inject;
/** Update a well-known view to point at a certain Datastore snapshot table in BigQuery. */
@Action(
path = UpdateSnapshotViewAction.PATH,
method = POST,
auth = Auth.AUTH_INTERNAL_ONLY
)
@Action(path = UpdateSnapshotViewAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_ONLY)
public class UpdateSnapshotViewAction implements Runnable {
/** Headers for passing parameters into the servlet. */
static final String UPDATE_SNAPSHOT_DATASET_ID_PARAM = "dataset";
static final String UPDATE_SNAPSHOT_TABLE_ID_PARAM = "table";
static final String UPDATE_SNAPSHOT_KIND_PARAM = "kind";
static final String LATEST_SNAPSHOT_DATASET = "latest_snapshot";
static final String LEGACY_LATEST_SNAPSHOT_DATASET = "latest_snapshot";
static final String STANDARD_LATEST_SNAPSHOT_DATASET = "latest_datastore_export";
/** Servlet-specific details needed for enqueuing tasks against itself. */
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
static final String PATH = "/_dr/task/updateSnapshotView"; // See web.xml.
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
static final String PATH = "/_dr/task/updateSnapshotView"; // See web.xml.
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject @Parameter(UPDATE_SNAPSHOT_DATASET_ID_PARAM) String datasetId;
@Inject @Parameter(UPDATE_SNAPSHOT_TABLE_ID_PARAM) String tableId;
@Inject @Parameter(UPDATE_SNAPSHOT_KIND_PARAM) String kindName;
@Inject @Config("projectId") String projectId;
@Inject BigqueryFactory bigqueryFactory;
@Inject UpdateSnapshotViewAction() {}
@Inject
@Parameter(UPDATE_SNAPSHOT_DATASET_ID_PARAM)
String datasetId;
/** Create a task for updating a snapshot view. */
public static TaskOptions createViewUpdateTask(
String datasetId, String tableId, String kindName) {
@Inject
@Parameter(UPDATE_SNAPSHOT_TABLE_ID_PARAM)
String tableId;
@Inject
@Parameter(UPDATE_SNAPSHOT_KIND_PARAM)
String kindName;
@Inject
@Config("projectId")
String projectId;
@Inject BigqueryFactory bigqueryFactory;
@Inject
UpdateSnapshotViewAction() {}
/** Create a task for updating a snapshot view. */
static TaskOptions createViewUpdateTask(String datasetId, String tableId, String kindName) {
return TaskOptions.Builder.withUrl(PATH)
.method(Method.POST)
.param(UPDATE_SNAPSHOT_DATASET_ID_PARAM, datasetId)
@ -75,39 +87,62 @@ public class UpdateSnapshotViewAction implements Runnable {
@Override
public void run() {
try {
updateSnapshotView(datasetId, tableId, kindName);
// TODO(b/32377148): Remove the legacySql view when migration complete.
SqlTemplate legacyTemplate =
SqlTemplate.create(
"#legacySQL\nSELECT * FROM [%PROJECT%:%SOURCE_DATASET%.%SOURCE_TABLE%]");
updateSnapshotView(
datasetId, tableId, kindName, LEGACY_LATEST_SNAPSHOT_DATASET, legacyTemplate);
SqlTemplate standardTemplate =
SqlTemplate.create(
"#standardSQL\nSELECT * FROM `%PROJECT%.%SOURCE_DATASET%.%SOURCE_TABLE%`");
updateSnapshotView(
datasetId, tableId, kindName, STANDARD_LATEST_SNAPSHOT_DATASET, standardTemplate);
} catch (Throwable e) {
logger.severefmt(e, "Could not update snapshot view for table %s", tableId);
throw new InternalServerErrorException("Error in update snapshot view action");
}
}
private void updateSnapshotView(String datasetId, String tableId, String kindName)
private void updateSnapshotView(
String sourceDatasetId,
String sourceTableId,
String kindName,
String viewDataset,
SqlTemplate viewQueryTemplate)
throws IOException {
Bigquery bigquery = bigqueryFactory.create(projectId, LATEST_SNAPSHOT_DATASET);
updateTable(bigquery, new Table()
.setTableReference(new TableReference()
.setProjectId(projectId)
.setDatasetId(LATEST_SNAPSHOT_DATASET)
.setTableId(kindName))
.setView(new ViewDefinition().setQuery(
SqlTemplate.create("SELECT * FROM [%PROJECT%:%DATASET%.%TABLE%]")
.put("PROJECT", projectId)
.put("DATASET", datasetId)
.put("TABLE", tableId)
.build())));
Bigquery bigquery = bigqueryFactory.create(projectId, viewDataset);
updateTable(
bigquery,
new Table()
.setTableReference(
new TableReference()
.setProjectId(projectId)
.setDatasetId(viewDataset)
.setTableId(kindName))
.setView(
new ViewDefinition()
.setQuery(
viewQueryTemplate
.put("PROJECT", projectId)
.put("SOURCE_DATASET", sourceDatasetId)
.put("SOURCE_TABLE", sourceTableId)
.build())));
logger.infofmt(
"Updated view %s to point at snapshot table %s.",
String.format("[%s:%s.%s]", projectId, LATEST_SNAPSHOT_DATASET, kindName),
String.format("[%s:%s.%s]", projectId, datasetId, tableId));
String.format("[%s:%s.%s]", projectId, viewDataset, kindName),
String.format("[%s:%s.%s]", projectId, sourceDatasetId, sourceTableId));
}
private static void updateTable(Bigquery bigquery, Table table) throws IOException {
TableReference ref = table.getTableReference();
try {
bigquery.tables()
bigquery
.tables()
.update(ref.getProjectId(), ref.getDatasetId(), ref.getTableId(), table)
.execute();
} catch (GoogleJsonResponseException e) {