Backup Datastore using the Admin REST API

Add server end points to backup Datastore using managed-export mechanism.
A cron job is defined in Alpha to run daily exports using this implementation.

Existing backup is left running. The new backups are saved to a new set of
locations:
- GCS bucket: gs://PROJECT-ID-datastore-backups
- Big Query data set: datastore_backups
- Big Query latest back up view name: latest_datastore_backup
Also, the names of Bigquery tables now use the export timestamp
assigned by Datastore. E.g., 2018_12_05T23_56_18_50532_ContactResource,

After the new import mechanism is implemented and the back-restore flow is
tested, we will stop the existing backup runs and deploy the new
implementation to all environments.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=224932957
This commit is contained in:
weiminyu 2018-12-10 20:26:53 -08:00 committed by jianglai
parent ea154a8378
commit 9c706e79fd
27 changed files with 1179 additions and 37 deletions

View file

@ -18,6 +18,7 @@ java_library(
"//java/google/registry/dns/writer/clouddns",
"//java/google/registry/dns/writer/dnsupdate",
"//java/google/registry/export",
"//java/google/registry/export/datastore",
"//java/google/registry/export/sheet",
"//java/google/registry/flows",
"//java/google/registry/gcs",

View file

@ -22,6 +22,7 @@ import google.registry.config.CredentialModule;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.dns.writer.VoidDnsWriterModule;
import google.registry.export.DriveModule;
import google.registry.export.datastore.DatastoreAdminModule;
import google.registry.export.sheet.SheetsServiceModule;
import google.registry.gcs.GcsServiceModule;
import google.registry.groups.DirectoryModule;
@ -56,6 +57,7 @@ import javax.inject.Singleton;
BigqueryModule.class,
ConfigModule.class,
CredentialModule.class,
DatastoreAdminModule.class,
DatastoreServiceModule.class,
DirectoryModule.class,
DummyKeyringModule.class,

View file

@ -39,7 +39,9 @@ import google.registry.dns.writer.VoidDnsWriterModule;
import google.registry.dns.writer.clouddns.CloudDnsWriterModule;
import google.registry.dns.writer.dnsupdate.DnsUpdateConfigModule;
import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule;
import google.registry.export.BackupDatastoreAction;
import google.registry.export.BigqueryPollJobAction;
import google.registry.export.CheckBackupAction;
import google.registry.export.CheckSnapshotAction;
import google.registry.export.ExportDomainListsAction;
import google.registry.export.ExportPremiumTermsAction;
@ -49,6 +51,7 @@ import google.registry.export.ExportSnapshotAction;
import google.registry.export.LoadSnapshotAction;
import google.registry.export.SyncGroupMembersAction;
import google.registry.export.UpdateSnapshotViewAction;
import google.registry.export.UploadDatastoreBackupAction;
import google.registry.export.sheet.SheetModule;
import google.registry.export.sheet.SyncRegistrarsSheetAction;
import google.registry.flows.async.AsyncFlowsModule;
@ -114,8 +117,10 @@ import google.registry.tmch.TmchSmdrlAction;
WhiteboxModule.class,
})
interface BackendRequestComponent {
BackupDatastoreAction backupDatastoreAction();
BigqueryPollJobAction bigqueryPollJobAction();
BrdaCopyAction brdaCopyAction();
CheckBackupAction checkBackupAction();
CheckSnapshotAction checkSnapshotAction();
CommitLogCheckpointAction commitLogCheckpointAction();
CommitLogFanoutAction commitLogFanoutAction();
@ -158,6 +163,7 @@ interface BackendRequestComponent {
TmchCrlAction tmchCrlAction();
TmchDnlAction tmchDnlAction();
TmchSmdrlAction tmchSmdrlAction();
UploadDatastoreBackupAction uploadDatastoreBackupAction();
UpdateSnapshotViewAction updateSnapshotViewAction();
PublishInvoicesAction uploadInvoicesAction();