zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-06-27 06:44:51 +02:00
Code Issues Projects Releases Packages Wiki Activity

Backup Datastore using the Admin REST API

Browse source 
Add server end points to backup Datastore using managed-export mechanism.
A cron job is defined in Alpha to run daily exports using this implementation.

Existing backup is left running. The new backups are saved to a new set of
locations:
- GCS bucket: gs://PROJECT-ID-datastore-backups
- Big Query data set: datastore_backups
- Big Query latest back up view name: latest_datastore_backup
Also, the names of Bigquery tables now use the export timestamp
assigned by Datastore. E.g., 2018_12_05T23_56_18_50532_ContactResource,

After the new import mechanism is implemented and the back-restore flow is
tested, we will stop the existing backup runs and deploy the new
implementation to all environments.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=224932957
This commit is contained in:
weiminyu 2018-12-10 20:26:53 -08:00 • committed by jianglai
parent ea154a8378
commit 9c706e79fd
27 changed files with 1179 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

22
java/google/registry/export/UpdateSnapshotViewAction.java
View file

@ -43,10 +43,12 @@ public class UpdateSnapshotViewAction implements Runnable {
static final String UPDATE_SNAPSHOT_TABLE_ID_PARAM = "table";
static final String UPDATE_SNAPSHOT_KIND_PARAM = "kind";
private static final String TARGET_DATASET_NAME = "latest_datastore_export";
static final String UPDATE_SNAPSHOT_VIEWNAME_PARAM = "viewname";
/** Servlet-specific details needed for enqueuing tasks against itself. */
// For now this queue is shared by the backup workflows started by both ExportSnapshotAction
// and BackupDatastoreAction.
// TODO(weiminyu): update queue name (snapshot->backup) after ExportSnapshot flow is removed.
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
static final String PATH = "/_dr/task/updateSnapshotView"; // See web.xml.
@ -65,6 +67,10 @@ public class UpdateSnapshotViewAction implements Runnable {
@Parameter(UPDATE_SNAPSHOT_KIND_PARAM)
String kindName;
@Inject
@Parameter(UPDATE_SNAPSHOT_VIEWNAME_PARAM)
String viewName;
@Inject
@Config("projectId")
String projectId;
@ -75,12 +81,14 @@ public class UpdateSnapshotViewAction implements Runnable {
UpdateSnapshotViewAction() {}
/** Create a task for updating a snapshot view. */
static TaskOptions createViewUpdateTask(String datasetId, String tableId, String kindName) {
static TaskOptions createViewUpdateTask(
String datasetId, String tableId, String kindName, String viewName) {
return TaskOptions.Builder.withUrl(PATH)
.method(Method.POST)
.param(UPDATE_SNAPSHOT_DATASET_ID_PARAM, datasetId)
.param(UPDATE_SNAPSHOT_TABLE_ID_PARAM, tableId)
.param(UPDATE_SNAPSHOT_KIND_PARAM, kindName);
.param(UPDATE_SNAPSHOT_KIND_PARAM, kindName)
.param(UPDATE_SNAPSHOT_VIEWNAME_PARAM, viewName);
}
@Override
@ -89,12 +97,10 @@ public class UpdateSnapshotViewAction implements Runnable {
SqlTemplate sqlTemplate =
SqlTemplate.create(
"#standardSQL\nSELECT * FROM `%PROJECT%.%SOURCE_DATASET%.%SOURCE_TABLE%`");
updateSnapshotView(datasetId, tableId, kindName, TARGET_DATASET_NAME, sqlTemplate);
updateSnapshotView(datasetId, tableId, kindName, viewName, sqlTemplate);
} catch (Throwable e) {
throw new InternalServerErrorException(
String.format(
"Could not update snapshot view %s for table %s", TARGET_DATASET_NAME, tableId),
e);
String.format("Could not update snapshot view %s for table %s", viewName, tableId), e);
}
}