Backup Datastore using the Admin REST API

Add server end points to backup Datastore using managed-export mechanism.
A cron job is defined in Alpha to run daily exports using this implementation.

Existing backup is left running. The new backups are saved to a new set of
locations:
- GCS bucket: gs://PROJECT-ID-datastore-backups
- Big Query data set: datastore_backups
- Big Query latest back up view name: latest_datastore_backup
Also, the names of Bigquery tables now use the export timestamp
assigned by Datastore. E.g., 2018_12_05T23_56_18_50532_ContactResource,

After the new import mechanism is implemented and the back-restore flow is
tested, we will stop the existing backup runs and deploy the new
implementation to all environments.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=224932957
This commit is contained in:
weiminyu 2018-12-10 20:26:53 -08:00 committed by jianglai
parent ea154a8378
commit 9c706e79fd
27 changed files with 1179 additions and 37 deletions

View file

@ -10,13 +10,16 @@ load("//java/com/google/testing/builddefs:GenTestRules.bzl", "GenTestRules")
java_library(
name = "export",
srcs = glob(["*.java"]),
resources = [
resources = glob([
"testdata/*.json",
"backup_kinds.txt",
"reporting_kinds.txt",
],
]),
deps = [
"//java/google/registry/bigquery",
"//java/google/registry/config",
"//java/google/registry/export",
"//java/google/registry/export/datastore",
"//java/google/registry/groups",
"//java/google/registry/model",
"//java/google/registry/request",
@ -25,6 +28,7 @@ java_library(
"//javatests/google/registry/testing",
"//javatests/google/registry/testing/mapreduce",
"//third_party/objectify:objectify-v4_1",
"@com_google_api_client",
"@com_google_apis_google_api_services_bigquery",
"@com_google_apis_google_api_services_drive",
"@com_google_appengine_api_1_0_sdk",
@ -35,6 +39,7 @@ java_library(
"@com_google_flogger_system_backend",
"@com_google_guava",
"@com_google_http_client",
"@com_google_http_client_jackson2",
"@com_google_re2j",
"@com_google_truth",
"@com_google_truth_extensions_truth_java8_extension",

View file

@ -0,0 +1,85 @@
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.export.CheckBackupAction.CHECK_BACKUP_KINDS_TO_LOAD_PARAM;
import static google.registry.export.CheckBackupAction.CHECK_BACKUP_NAME_PARAM;
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
import static org.mockito.Mockito.when;
import com.google.common.base.Joiner;
import google.registry.export.datastore.DatastoreAdmin;
import google.registry.export.datastore.DatastoreAdmin.Export;
import google.registry.export.datastore.Operation;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeResponse;
import google.registry.testing.MockitoJUnitRule;
import google.registry.testing.TaskQueueHelper.TaskMatcher;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.mockito.Mock;
/** Unit tests for {@link BackupDatastoreAction}. */
@RunWith(JUnit4.class)
public class BackupDatastoreActionTest {
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
@Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create();
@Mock private DatastoreAdmin datastoreAdmin;
@Mock private Export exportRequest;
@Mock private Operation backupOperation;
private final FakeResponse response = new FakeResponse();
private final BackupDatastoreAction action = new BackupDatastoreAction();
@Before
public void before() throws Exception {
action.datastoreAdmin = datastoreAdmin;
action.response = response;
when(datastoreAdmin.export(
"gs://registry-project-id-datastore-backups", ExportConstants.getBackupKinds()))
.thenReturn(exportRequest);
when(exportRequest.execute()).thenReturn(backupOperation);
when(backupOperation.getName())
.thenReturn("projects/registry-project-id/operations/ASA1ODYwNjc");
when(backupOperation.getExportFolderUrl())
.thenReturn("gs://registry-project-id-datastore-backups/some-id");
}
@Test
public void testBackup_enqueuesPollTask() {
action.run();
assertTasksEnqueued(
CheckBackupAction.QUEUE,
new TaskMatcher()
.url(CheckBackupAction.PATH)
.param(CHECK_BACKUP_NAME_PARAM, "projects/registry-project-id/operations/ASA1ODYwNjc")
.param(
CHECK_BACKUP_KINDS_TO_LOAD_PARAM,
Joiner.on(",").join(ExportConstants.getReportingKinds()))
.method("POST"));
assertThat(response.getPayload())
.isEqualTo(
"Datastore backup started with name: "
+ "projects/registry-project-id/operations/ASA1ODYwNjc\n"
+ "Saving to gs://registry-project-id-datastore-backups/some-id");
}
}

View file

@ -0,0 +1,222 @@
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.export.CheckBackupAction.CHECK_BACKUP_KINDS_TO_LOAD_PARAM;
import static google.registry.export.CheckBackupAction.CHECK_BACKUP_NAME_PARAM;
import static google.registry.testing.JUnitBackports.assertThrows;
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.when;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.common.collect.ImmutableSet;
import google.registry.export.datastore.DatastoreAdmin;
import google.registry.export.datastore.DatastoreAdmin.Get;
import google.registry.export.datastore.Operation;
import google.registry.request.Action.Method;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.HttpException.NoContentException;
import google.registry.request.HttpException.NotModifiedException;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeResponse;
import google.registry.testing.MockitoJUnitRule;
import google.registry.testing.TaskQueueHelper.TaskMatcher;
import google.registry.testing.TestDataHelper;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.mockito.Mock;
/** Unit tests for {@link CheckBackupAction}. */
@RunWith(JUnit4.class)
public class CheckBackupActionTest {
static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
static final DateTime COMPLETE_TIME = START_TIME.plus(Duration.standardMinutes(30));
static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
@Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create();
@Mock private DatastoreAdmin datastoreAdmin;
@Mock private Get getNotFoundBackupProgressRequest;
@Mock private Get getBackupProgressRequest;
private Operation backupOperation;
private final FakeResponse response = new FakeResponse();
private final FakeClock clock = new FakeClock(COMPLETE_TIME.plusMillis(1000));
private final CheckBackupAction action = new CheckBackupAction();
@Before
public void before() throws Exception {
action.requestMethod = Method.POST;
action.datastoreAdmin = datastoreAdmin;
action.clock = clock;
action.backupName = "some_backup";
action.kindsToLoadParam = "one,two";
action.response = response;
when(datastoreAdmin.get(anyString())).thenReturn(getBackupProgressRequest);
when(getBackupProgressRequest.execute()).thenAnswer(arg -> backupOperation);
}
private void setPendingBackup() throws Exception {
backupOperation =
JSON_FACTORY.fromString(
TestDataHelper.loadFile(
CheckBackupActionTest.class, "backup_operation_in_progress.json"),
Operation.class);
}
private void setCompleteBackup() throws Exception {
backupOperation =
JSON_FACTORY.fromString(
TestDataHelper.loadFile(CheckBackupActionTest.class, "backup_operation_success.json"),
Operation.class);
}
private void setBackupNotFound() throws Exception {
when(datastoreAdmin.get(anyString())).thenReturn(getNotFoundBackupProgressRequest);
when(getNotFoundBackupProgressRequest.execute())
.thenThrow(
new GoogleJsonResponseException(
new GoogleJsonResponseException.Builder(404, "NOT_FOUND", new HttpHeaders())
.setMessage("No backup found"),
null));
}
private static void assertLoadTaskEnqueued(String id, String folder, String kinds) {
assertTasksEnqueued(
"export-snapshot",
new TaskMatcher()
.url("/_dr/task/uploadDatastoreBackup")
.method("POST")
.param("id", id)
.param("folder", folder)
.param("kinds", kinds));
}
@Test
public void testSuccess_enqueuePollTask() {
CheckBackupAction.enqueuePollTask("some_backup_name", ImmutableSet.of("one", "two", "three"));
assertTasksEnqueued(
CheckBackupAction.QUEUE,
new TaskMatcher()
.url(CheckBackupAction.PATH)
.param(CHECK_BACKUP_NAME_PARAM, "some_backup_name")
.param(CHECK_BACKUP_KINDS_TO_LOAD_PARAM, "one,two,three")
.method("POST"));
}
@Test
public void testPost_forPendingBackup_returnsNotModified() throws Exception {
setPendingBackup();
NotModifiedException thrown = assertThrows(NotModifiedException.class, action::run);
assertThat(thrown)
.hasMessageThat()
.contains("Datastore backup some_backup still in progress: Progress: N/A");
}
@Test
public void testPost_forStalePendingBackupBackup_returnsNoContent() throws Exception {
setPendingBackup();
clock.setTo(
START_TIME
.plus(Duration.standardHours(20))
.plus(Duration.standardMinutes(3))
.plus(Duration.millis(1234)));
NoContentException thrown = assertThrows(NoContentException.class, action::run);
assertThat(thrown)
.hasMessageThat()
.contains(
"Datastore backup some_backup abandoned - "
+ "not complete after 20 hours, 3 minutes and 1 second. Progress: Progress: N/A");
}
@Test
public void testPost_forCompleteBackup_enqueuesLoadTask() throws Exception {
setCompleteBackup();
action.run();
assertLoadTaskEnqueued(
"2014-08-01T01:02:03_99364",
"gs://registry-project-id-datastore-export-test/2014-08-01T01:02:03_99364",
"one,two");
}
@Test
public void testPost_forCompleteBackup_withExtraKindsToLoad_enqueuesLoadTask() throws Exception {
setCompleteBackup();
action.kindsToLoadParam = "one,foo";
action.run();
assertLoadTaskEnqueued(
"2014-08-01T01:02:03_99364",
"gs://registry-project-id-datastore-export-test/2014-08-01T01:02:03_99364",
"one");
}
@Test
public void testPost_forCompleteBackup_withEmptyKindsToLoad_skipsLoadTask() throws Exception {
setCompleteBackup();
action.kindsToLoadParam = "";
action.run();
assertNoTasksEnqueued("export-snapshot");
}
@Test
public void testPost_forBadBackup_returnsBadRequest() throws Exception {
setBackupNotFound();
BadRequestException thrown = assertThrows(BadRequestException.class, action::run);
assertThat(thrown).hasMessageThat().contains("Bad backup name some_backup: No backup found");
}
@Test
public void testGet_returnsInformation() throws Exception {
setCompleteBackup();
action.requestMethod = Method.GET;
action.run();
assertThat(response.getPayload())
.isEqualTo(
TestDataHelper.loadFile(
CheckBackupActionTest.class, "pretty_printed_success_backup_operation.json")
.trim());
}
@Test
public void testGet_forBadBackup_returnsError() throws Exception {
setBackupNotFound();
action.requestMethod = Method.GET;
BadRequestException thrown = assertThrows(BadRequestException.class, action::run);
assertThat(thrown).hasMessageThat().contains("Bad backup name some_backup: No backup found");
}
}

View file

@ -16,6 +16,7 @@ package google.registry.export;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.export.LoadSnapshotAction.LATEST_SNAPSHOT_VIEW_NAME;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
@ -159,24 +160,30 @@ public class LoadSnapshotActionTest {
verify(bigqueryJobsInsert, times(3)).execute();
// Check that the poll tasks for each load job were enqueued.
verify(bigqueryPollEnqueuer).enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-snapshot-id12345-one-1391096117045"),
UpdateSnapshotViewAction.createViewUpdateTask("snapshots", "id12345_one", "one"),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
verify(bigqueryPollEnqueuer).enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-snapshot-id12345-two-1391096117045"),
UpdateSnapshotViewAction.createViewUpdateTask("snapshots", "id12345_two", "two"),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
verify(bigqueryPollEnqueuer).enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-snapshot-id12345-three-1391096117045"),
UpdateSnapshotViewAction.createViewUpdateTask("snapshots", "id12345_three", "three"),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
verify(bigqueryPollEnqueuer)
.enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-snapshot-id12345-one-1391096117045"),
UpdateSnapshotViewAction.createViewUpdateTask(
"snapshots", "id12345_one", "one", LATEST_SNAPSHOT_VIEW_NAME),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
verify(bigqueryPollEnqueuer)
.enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-snapshot-id12345-two-1391096117045"),
UpdateSnapshotViewAction.createViewUpdateTask(
"snapshots", "id12345_two", "two", LATEST_SNAPSHOT_VIEW_NAME),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
verify(bigqueryPollEnqueuer)
.enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-snapshot-id12345-three-1391096117045"),
UpdateSnapshotViewAction.createViewUpdateTask(
"snapshots", "id12345_three", "three", LATEST_SNAPSHOT_VIEW_NAME),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
}
@Test

View file

@ -20,6 +20,7 @@ import static google.registry.export.UpdateSnapshotViewAction.QUEUE;
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM;
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM;
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM;
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM;
import static google.registry.export.UpdateSnapshotViewAction.createViewUpdateTask;
import static google.registry.testing.JUnitBackports.assertThrows;
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
@ -79,20 +80,26 @@ public class UpdateSnapshotViewActionTest {
action.checkedBigquery = checkedBigquery;
action.datasetId = "some_dataset";
action.kindName = "fookind";
action.viewName = "latest_datastore_export";
action.projectId = "myproject";
action.tableId = "12345_fookind";
}
@Test
public void testSuccess_createViewUpdateTask() {
getQueue(QUEUE).add(createViewUpdateTask("some_dataset", "12345_fookind", "fookind"));
assertTasksEnqueued(QUEUE,
getQueue(QUEUE)
.add(
createViewUpdateTask(
"some_dataset", "12345_fookind", "fookind", "latest_datastore_export"));
assertTasksEnqueued(
QUEUE,
new TaskMatcher()
.url(UpdateSnapshotViewAction.PATH)
.method("POST")
.param(UPDATE_SNAPSHOT_DATASET_ID_PARAM, "some_dataset")
.param(UPDATE_SNAPSHOT_TABLE_ID_PARAM, "12345_fookind")
.param(UPDATE_SNAPSHOT_KIND_PARAM, "fookind"));
.param(UPDATE_SNAPSHOT_KIND_PARAM, "fookind")
.param(UPDATE_SNAPSHOT_VIEWNAME_PARAM, "latest_datastore_export"));
}
@Test

View file

@ -0,0 +1,209 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.export.UploadDatastoreBackupAction.BACKUP_DATASET;
import static google.registry.export.UploadDatastoreBackupAction.LATEST_BACKUP_VIEW_NAME;
import static google.registry.export.UploadDatastoreBackupAction.PATH;
import static google.registry.export.UploadDatastoreBackupAction.QUEUE;
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_FOLDER_PARAM;
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_ID_PARAM;
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_KINDS_PARAM;
import static google.registry.export.UploadDatastoreBackupAction.enqueueUploadBackupTask;
import static google.registry.export.UploadDatastoreBackupAction.getBackupInfoFileForKind;
import static google.registry.testing.JUnitBackports.assertThrows;
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.bigquery.model.Dataset;
import com.google.api.services.bigquery.model.Job;
import com.google.api.services.bigquery.model.JobConfigurationLoad;
import com.google.api.services.bigquery.model.JobReference;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import google.registry.bigquery.CheckedBigquery;
import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
import google.registry.request.HttpException.InternalServerErrorException;
import google.registry.testing.AppEngineRule;
import google.registry.testing.TaskQueueHelper.TaskMatcher;
import java.io.IOException;
import java.util.List;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.mockito.ArgumentCaptor;
/** Unit tests for {@link UploadDatastoreBackupAction}. */
@RunWith(JUnit4.class)
public class UploadDatastoreBackupActionTest {
@Rule
public final AppEngineRule appEngine = AppEngineRule.builder()
.withTaskQueue()
.build();
private final CheckedBigquery checkedBigquery = mock(CheckedBigquery.class);
private final Bigquery bigquery = mock(Bigquery.class);
private final Bigquery.Jobs bigqueryJobs = mock(Bigquery.Jobs.class);
private final Bigquery.Jobs.Insert bigqueryJobsInsert = mock(Bigquery.Jobs.Insert.class);
private final Bigquery.Datasets bigqueryDatasets = mock(Bigquery.Datasets.class);
private final Bigquery.Datasets.Insert bigqueryDatasetsInsert =
mock(Bigquery.Datasets.Insert.class);
private final BigqueryPollJobEnqueuer bigqueryPollEnqueuer = mock(BigqueryPollJobEnqueuer.class);
private UploadDatastoreBackupAction action;
@Before
public void before() throws Exception {
when(checkedBigquery.ensureDataSetExists("Project-Id", BACKUP_DATASET)).thenReturn(bigquery);
when(bigquery.jobs()).thenReturn(bigqueryJobs);
when(bigqueryJobs.insert(eq("Project-Id"), any(Job.class))).thenReturn(bigqueryJobsInsert);
when(bigquery.datasets()).thenReturn(bigqueryDatasets);
when(bigqueryDatasets.insert(eq("Project-Id"), any(Dataset.class)))
.thenReturn(bigqueryDatasetsInsert);
action = new UploadDatastoreBackupAction();
action.checkedBigquery = checkedBigquery;
action.bigqueryPollEnqueuer = bigqueryPollEnqueuer;
action.projectId = "Project-Id";
action.backupFolderUrl = "gs://bucket/path";
action.backupId = "2018-12-05T17:46:39_92612";
action.backupKinds = "one,two,three";
}
@Test
public void testSuccess_enqueueLoadTask() {
enqueueUploadBackupTask("id12345", "gs://bucket/path", ImmutableSet.of("one", "two", "three"));
assertTasksEnqueued(
QUEUE,
new TaskMatcher()
.url(PATH)
.method("POST")
.param(UPLOAD_BACKUP_ID_PARAM, "id12345")
.param(UPLOAD_BACKUP_FOLDER_PARAM, "gs://bucket/path")
.param(UPLOAD_BACKUP_KINDS_PARAM, "one,two,three"));
}
@Test
public void testSuccess_doPost() throws Exception {
action.run();
// Verify that checkedBigquery was called in a way that would create the dataset if it didn't
// already exist.
verify(checkedBigquery).ensureDataSetExists("Project-Id", BACKUP_DATASET);
// Capture the load jobs we inserted to do additional checking on them.
ArgumentCaptor<Job> jobArgument = ArgumentCaptor.forClass(Job.class);
verify(bigqueryJobs, times(3)).insert(eq("Project-Id"), jobArgument.capture());
List<Job> jobs = jobArgument.getAllValues();
assertThat(jobs).hasSize(3);
// Check properties that should be common to all load jobs.
for (Job job : jobs) {
assertThat(job.getJobReference().getProjectId()).isEqualTo("Project-Id");
JobConfigurationLoad config = job.getConfiguration().getLoad();
assertThat(config.getSourceFormat()).isEqualTo("DATASTORE_BACKUP");
assertThat(config.getDestinationTable().getProjectId()).isEqualTo("Project-Id");
assertThat(config.getDestinationTable().getDatasetId()).isEqualTo(BACKUP_DATASET);
}
// Check the job IDs for each load job.
assertThat(transform(jobs, job -> job.getJobReference().getJobId()))
.containsExactly(
"load-backup-2018_12_05T17_46_39_92612-one",
"load-backup-2018_12_05T17_46_39_92612-two",
"load-backup-2018_12_05T17_46_39_92612-three");
// Check the source URI for each load job.
assertThat(
transform(
jobs,
job -> Iterables.getOnlyElement(job.getConfiguration().getLoad().getSourceUris())))
.containsExactly(
"gs://bucket/path/all_namespaces/kind_one/all_namespaces_kind_one.export_metadata",
"gs://bucket/path/all_namespaces/kind_two/all_namespaces_kind_two.export_metadata",
"gs://bucket/path/all_namespaces/kind_three/all_namespaces_kind_three.export_metadata");
// Check the destination table ID for each load job.
assertThat(
transform(
jobs, job -> job.getConfiguration().getLoad().getDestinationTable().getTableId()))
.containsExactly(
"2018_12_05T17_46_39_92612_one",
"2018_12_05T17_46_39_92612_two",
"2018_12_05T17_46_39_92612_three");
// Check that we executed the inserted jobs.
verify(bigqueryJobsInsert, times(3)).execute();
// Check that the poll tasks for each load job were enqueued.
verify(bigqueryPollEnqueuer)
.enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-backup-2018_12_05T17_46_39_92612-one"),
UpdateSnapshotViewAction.createViewUpdateTask(
BACKUP_DATASET, "2018_12_05T17_46_39_92612_one", "one", LATEST_BACKUP_VIEW_NAME),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
verify(bigqueryPollEnqueuer)
.enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-backup-2018_12_05T17_46_39_92612-two"),
UpdateSnapshotViewAction.createViewUpdateTask(
BACKUP_DATASET, "2018_12_05T17_46_39_92612_two", "two", LATEST_BACKUP_VIEW_NAME),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
verify(bigqueryPollEnqueuer)
.enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-backup-2018_12_05T17_46_39_92612-three"),
UpdateSnapshotViewAction.createViewUpdateTask(
BACKUP_DATASET,
"2018_12_05T17_46_39_92612_three",
"three",
LATEST_BACKUP_VIEW_NAME),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
}
@Test
public void testFailure_doPost_bigqueryThrowsException() throws Exception {
when(bigqueryJobsInsert.execute()).thenThrow(new IOException("The Internet has gone missing"));
InternalServerErrorException thrown =
assertThrows(InternalServerErrorException.class, action::run);
assertThat(thrown)
.hasMessageThat()
.contains("Error loading backup: The Internet has gone missing");
}
@Test
public void testgetBackupInfoFileForKind() {
assertThat(
getBackupInfoFileForKind(
"gs://BucketName/2018-11-11T00:00:00_12345", "AllocationToken"))
.isEqualTo(
"gs://BucketName/2018-11-11T00:00:00_12345/"
+ "all_namespaces/kind_AllocationToken/"
+ "all_namespaces_kind_AllocationToken.export_metadata");
}
}

View file

@ -0,0 +1,18 @@
{
"name": "projects/registry-project-id/operations/ASAzNjMwOTEyNjUJ",
"metadata": {
"@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata",
"common": {
"startTime": "2014-08-01T01:02:03Z",
"operationType": "EXPORT_ENTITIES",
"state": "PROCESSING"
},
"entityFilter": {
"kinds": [
"one",
"two"
]
},
"outputUrlPrefix": "gs://registry-project-id-datastore-export-test/2014-08-01T01:02:03_99364"
}
}

View file

@ -0,0 +1,20 @@
{
"name": "projects/registry-project-id/operations/ASAzNjMwOTEyNjUJ",
"metadata": {
"@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata",
"common": {
"startTime": "2014-08-01T01:02:03Z",
"endTime": "2014-08-01T01:32:03Z",
"operationType": "EXPORT_ENTITIES",
"state": "SUCCESSFUL"
},
"entityFilter": {
"kinds": [
"one",
"two"
]
},
"outputUrlPrefix": "gs://registry-project-id-datastore-export-test/2014-08-01T01:02:03_99364"
},
"done": true
}

View file

@ -0,0 +1,17 @@
{
"done" : true,
"metadata" : {
"common" : {
"endTime" : "2014-08-01T01:32:03Z",
"operationType" : "EXPORT_ENTITIES",
"startTime" : "2014-08-01T01:02:03Z",
"state" : "SUCCESSFUL"
},
"entityFilter" : {
"kinds" : [ "one", "two" ]
},
"outputUrlPrefix" : "gs://registry-project-id-datastore-export-test/2014-08-01T01:02:03_99364",
"@type" : "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata"
},
"name" : "projects/registry-project-id/operations/ASAzNjMwOTEyNjUJ"
}

View file

@ -4,7 +4,9 @@ PATH CLASS METHOD
/_dr/cron/fanout TldFanoutAction GET y INTERNAL APP IGNORED
/_dr/cron/readDnsQueue ReadDnsQueueAction GET y INTERNAL APP IGNORED
/_dr/dnsRefresh RefreshDnsAction GET y INTERNAL APP IGNORED
/_dr/task/backupDatastore BackupDatastoreAction POST y INTERNAL APP IGNORED
/_dr/task/brdaCopy BrdaCopyAction POST y INTERNAL APP IGNORED
/_dr/task/checkDatastoreBackup CheckBackupAction POST,GET y INTERNAL APP IGNORED
/_dr/task/checkSnapshot CheckSnapshotAction POST,GET y INTERNAL APP IGNORED
/_dr/task/copyDetailReports CopyDetailReportsAction POST n INTERNAL,API APP ADMIN
/_dr/task/deleteContactsAndHosts DeleteContactsAndHostsAction GET n INTERNAL APP IGNORED
@ -44,3 +46,4 @@ PATH CLASS METHOD
/_dr/task/tmchDnl TmchDnlAction POST y INTERNAL APP IGNORED
/_dr/task/tmchSmdrl TmchSmdrlAction POST y INTERNAL APP IGNORED
/_dr/task/updateSnapshotView UpdateSnapshotViewAction POST n INTERNAL APP IGNORED
/_dr/task/uploadDatastoreBackup UploadDatastoreBackupAction POST n INTERNAL APP IGNORED