mirror of
https://github.com/google/nomulus.git
synced 2025-05-08 15:58:21 +02:00
Remove deprecated Datastore backup code
Removed three Action classes and the CheckSnapshot command. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=230545631
This commit is contained in:
parent
701ebc6a28
commit
acbd23fa64
20 changed files with 14 additions and 1527 deletions
|
@ -1362,15 +1362,6 @@ public final class RegistryConfig {
|
||||||
return CONFIG_SETTINGS.get().appEngine.projectId;
|
return CONFIG_SETTINGS.get().appEngine.projectId;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns the Google Cloud Storage bucket for storing backup snapshots.
|
|
||||||
*
|
|
||||||
* @see google.registry.export.ExportSnapshotAction
|
|
||||||
*/
|
|
||||||
public static String getSnapshotsBucket() {
|
|
||||||
return getProjectId() + "-snapshots";
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the Google Cloud Storage bucket for storing Datastore backups.
|
* Returns the Google Cloud Storage bucket for storing Datastore backups.
|
||||||
*
|
*
|
||||||
|
|
|
@ -192,24 +192,6 @@
|
||||||
<url-pattern>/_dr/task/uploadDatastoreBackup</url-pattern>
|
<url-pattern>/_dr/task/uploadDatastoreBackup</url-pattern>
|
||||||
</servlet-mapping>
|
</servlet-mapping>
|
||||||
|
|
||||||
<!-- Exports a Datastore backup snapshot to GCS. -->
|
|
||||||
<servlet-mapping>
|
|
||||||
<servlet-name>backend-servlet</servlet-name>
|
|
||||||
<url-pattern>/_dr/task/exportSnapshot</url-pattern>
|
|
||||||
</servlet-mapping>
|
|
||||||
|
|
||||||
<!-- Checks the completion of a Datastore backup snapshot. -->
|
|
||||||
<servlet-mapping>
|
|
||||||
<servlet-name>backend-servlet</servlet-name>
|
|
||||||
<url-pattern>/_dr/task/checkSnapshot</url-pattern>
|
|
||||||
</servlet-mapping>
|
|
||||||
|
|
||||||
<!-- Loads a Datastore backup snapshot into BigQuery. -->
|
|
||||||
<servlet-mapping>
|
|
||||||
<servlet-name>backend-servlet</servlet-name>
|
|
||||||
<url-pattern>/_dr/task/loadSnapshot</url-pattern>
|
|
||||||
</servlet-mapping>
|
|
||||||
|
|
||||||
<!-- Updates a view to point at a certain snapshot in BigQuery. -->
|
<!-- Updates a view to point at a certain snapshot in BigQuery. -->
|
||||||
<servlet-mapping>
|
<servlet-mapping>
|
||||||
<servlet-name>backend-servlet</servlet-name>
|
<servlet-name>backend-servlet</servlet-name>
|
||||||
|
|
|
@ -28,9 +28,7 @@ import google.registry.request.auth.Auth;
|
||||||
import javax.inject.Inject;
|
import javax.inject.Inject;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Action to trigger a Datastore backup job that writes a snapshot to Google Cloud Storage. This
|
* Action to trigger a Datastore backup job that writes a snapshot to Google Cloud Storage.
|
||||||
* class is introduced as an experimental feature, and will eventually replace {@link
|
|
||||||
* ExportSnapshotAction}.
|
|
||||||
*
|
*
|
||||||
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
|
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
|
||||||
* the next upon successful completion:
|
* the next upon successful completion:
|
||||||
|
|
|
@ -1,163 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.common.collect.Sets.intersection;
|
|
||||||
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
|
|
||||||
import static google.registry.request.Action.Method.GET;
|
|
||||||
import static google.registry.request.Action.Method.POST;
|
|
||||||
|
|
||||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
|
||||||
import com.google.common.base.Joiner;
|
|
||||||
import com.google.common.base.Splitter;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import com.google.common.collect.Sets;
|
|
||||||
import com.google.common.flogger.FluentLogger;
|
|
||||||
import google.registry.export.DatastoreBackupInfo.BackupStatus;
|
|
||||||
import google.registry.request.Action;
|
|
||||||
import google.registry.request.HttpException.BadRequestException;
|
|
||||||
import google.registry.request.HttpException.NoContentException;
|
|
||||||
import google.registry.request.HttpException.NotModifiedException;
|
|
||||||
import google.registry.request.Parameter;
|
|
||||||
import google.registry.request.RequestMethod;
|
|
||||||
import google.registry.request.Response;
|
|
||||||
import google.registry.request.auth.Auth;
|
|
||||||
import java.util.Set;
|
|
||||||
import javax.inject.Inject;
|
|
||||||
import org.joda.time.Duration;
|
|
||||||
import org.joda.time.PeriodType;
|
|
||||||
import org.joda.time.format.PeriodFormat;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Action that checks the status of a snapshot, and if complete, trigger loading it into BigQuery.
|
|
||||||
*/
|
|
||||||
@Action(
|
|
||||||
service = Action.Service.BACKEND,
|
|
||||||
path = CheckSnapshotAction.PATH,
|
|
||||||
method = {POST, GET},
|
|
||||||
automaticallyPrintOk = true,
|
|
||||||
auth = Auth.AUTH_INTERNAL_ONLY)
|
|
||||||
public class CheckSnapshotAction implements Runnable {
|
|
||||||
|
|
||||||
/** Parameter names for passing parameters into this action. */
|
|
||||||
static final String CHECK_SNAPSHOT_NAME_PARAM = "name";
|
|
||||||
|
|
||||||
static final String CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";
|
|
||||||
|
|
||||||
/** Action-specific details needed for enqueuing tasks against itself. */
|
|
||||||
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
|
|
||||||
|
|
||||||
static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml.
|
|
||||||
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
|
|
||||||
|
|
||||||
/** The maximum amount of time we allow a backup to run before abandoning it. */
|
|
||||||
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
|
|
||||||
|
|
||||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
|
||||||
|
|
||||||
@Inject Response response;
|
|
||||||
@Inject @RequestMethod Action.Method requestMethod;
|
|
||||||
@Inject DatastoreBackupService backupService;
|
|
||||||
@Inject @Parameter(CHECK_SNAPSHOT_NAME_PARAM) String snapshotName;
|
|
||||||
@Inject @Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM) String kindsToLoadParam;
|
|
||||||
@Inject CheckSnapshotAction() {}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
if (requestMethod == POST) {
|
|
||||||
checkAndLoadSnapshotIfComplete();
|
|
||||||
} else {
|
|
||||||
// This is a GET request.
|
|
||||||
response.setPayload(getBackup().getInformation());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private DatastoreBackupInfo getBackup() {
|
|
||||||
try {
|
|
||||||
return backupService.findByName(snapshotName);
|
|
||||||
} catch (IllegalArgumentException e) {
|
|
||||||
String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
|
|
||||||
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be
|
|
||||||
// retried but we might abandon backups that start late and haven't yet written to
|
|
||||||
// Datastore. We could fix that by replacing this with a two-phase polling strategy.
|
|
||||||
throw new BadRequestException(message, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void checkAndLoadSnapshotIfComplete() {
|
|
||||||
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
|
|
||||||
DatastoreBackupInfo backup = getBackup();
|
|
||||||
// Stop now if the backup is not complete.
|
|
||||||
if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
|
|
||||||
Duration runningTime = backup.getRunningTime();
|
|
||||||
if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
|
|
||||||
// Backup might still be running, so send a 304 to have the task retry.
|
|
||||||
throw new NotModifiedException(
|
|
||||||
String.format("Datastore backup %s still pending", snapshotName));
|
|
||||||
} else {
|
|
||||||
// Declare the backup a lost cause, and send 204 No Content so the task will
|
|
||||||
// not be retried.
|
|
||||||
String message =
|
|
||||||
String.format(
|
|
||||||
"Datastore backup %s abandoned - not complete after %s",
|
|
||||||
snapshotName,
|
|
||||||
PeriodFormat.getDefault()
|
|
||||||
.print(
|
|
||||||
runningTime
|
|
||||||
.toPeriod()
|
|
||||||
.normalizedStandard(PeriodType.dayTime().withMillisRemoved())));
|
|
||||||
throw new NoContentException(message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
|
|
||||||
// suffix out of the snapshot name and falling back to the start time as a string.
|
|
||||||
String snapshotId =
|
|
||||||
snapshotName.startsWith(ExportSnapshotAction.SNAPSHOT_PREFIX)
|
|
||||||
? snapshotName.substring(ExportSnapshotAction.SNAPSHOT_PREFIX.length())
|
|
||||||
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
|
|
||||||
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
|
|
||||||
if (!backup.getKinds().containsAll(kindsToLoad)) {
|
|
||||||
logger.atWarning().log(
|
|
||||||
"Kinds to load included non-exported kinds: %s",
|
|
||||||
Sets.difference(kindsToLoad, backup.getKinds()));
|
|
||||||
}
|
|
||||||
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
|
|
||||||
ImmutableSet<String> exportedKindsToLoad =
|
|
||||||
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
|
|
||||||
String message = String.format("Datastore backup %s complete - ", snapshotName);
|
|
||||||
if (exportedKindsToLoad.isEmpty()) {
|
|
||||||
message += "no kinds to load into BigQuery";
|
|
||||||
} else {
|
|
||||||
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
|
|
||||||
message += "BigQuery load task enqueued";
|
|
||||||
}
|
|
||||||
logger.atInfo().log(message);
|
|
||||||
response.setPayload(message);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Enqueue a poll task to monitor the named snapshot for completion. */
|
|
||||||
static TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
|
|
||||||
return QueueFactory.getQueue(QUEUE)
|
|
||||||
.add(
|
|
||||||
TaskOptions.Builder.withUrl(PATH)
|
|
||||||
.method(Method.POST)
|
|
||||||
.countdownMillis(POLL_COUNTDOWN.getMillis())
|
|
||||||
.param(CHECK_SNAPSHOT_NAME_PARAM, snapshotName)
|
|
||||||
.param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,149 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.common.base.Preconditions.checkArgument;
|
|
||||||
import static com.google.common.base.Preconditions.checkNotNull;
|
|
||||||
import static org.joda.time.DateTimeZone.UTC;
|
|
||||||
|
|
||||||
import com.google.appengine.api.datastore.Entity;
|
|
||||||
import com.google.appengine.api.datastore.Text;
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
|
||||||
import com.google.common.base.Ascii;
|
|
||||||
import com.google.common.base.Joiner;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import google.registry.util.Clock;
|
|
||||||
import google.registry.util.NonFinalForTesting;
|
|
||||||
import google.registry.util.SystemClock;
|
|
||||||
import java.util.Date;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Optional;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
import org.joda.time.Duration;
|
|
||||||
|
|
||||||
/** Container for information about a Datastore backup. */
|
|
||||||
public class DatastoreBackupInfo {
|
|
||||||
|
|
||||||
@NonFinalForTesting
|
|
||||||
private static Clock clock = new SystemClock();
|
|
||||||
|
|
||||||
/** The possible status values for a Datastore backup. */
|
|
||||||
public enum BackupStatus { PENDING, COMPLETE }
|
|
||||||
|
|
||||||
/** The name of the Datastore backup. */
|
|
||||||
private final String backupName;
|
|
||||||
|
|
||||||
/** The entity kinds included in this Datastore backup. */
|
|
||||||
private final ImmutableSet<String> kinds;
|
|
||||||
|
|
||||||
/** The start time of the Datastore backup. */
|
|
||||||
private final DateTime startTime;
|
|
||||||
|
|
||||||
/** The completion time of the Datastore backup, present if it has completed. */
|
|
||||||
private final Optional<DateTime> completeTime;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The GCS filename to which the backup's top-level .backup_info manifest file has been written,
|
|
||||||
* present if the backup has completed.
|
|
||||||
*/
|
|
||||||
private final Optional<String> gcsFilename;
|
|
||||||
|
|
||||||
/** DatastoreBackupInfo instances should only be obtained via DatastoreBackupService. */
|
|
||||||
DatastoreBackupInfo(Entity backupEntity) {
|
|
||||||
backupName = (String) checkNotNull(backupEntity.getProperty("name"), "name");
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
List<String> rawKinds = (List<String>) checkNotNull(backupEntity.getProperty("kinds"), "kinds");
|
|
||||||
Date rawStartTime = (Date) checkNotNull(backupEntity.getProperty("start_time"), "start_time");
|
|
||||||
Date rawCompleteTime = (Date) backupEntity.getProperty("complete_time");
|
|
||||||
Text rawGcsFilename = (Text) backupEntity.getProperty("gs_handle");
|
|
||||||
|
|
||||||
kinds = ImmutableSet.copyOf(rawKinds);
|
|
||||||
startTime = new DateTime(rawStartTime).withZone(UTC);
|
|
||||||
completeTime = Optional.ofNullable(
|
|
||||||
rawCompleteTime == null ? null : new DateTime(rawCompleteTime).withZone(UTC));
|
|
||||||
gcsFilename = Optional.ofNullable(
|
|
||||||
rawGcsFilename == null ? null : gcsPathToUri(rawGcsFilename.getValue()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/** This constructor is only exposed for test purposes. */
|
|
||||||
@VisibleForTesting
|
|
||||||
DatastoreBackupInfo(
|
|
||||||
String backupName,
|
|
||||||
DateTime startTime,
|
|
||||||
Optional<DateTime> completeTime,
|
|
||||||
ImmutableSet<String> kinds,
|
|
||||||
Optional<String> gcsFilename) {
|
|
||||||
this.backupName = backupName;
|
|
||||||
this.startTime = startTime;
|
|
||||||
this.completeTime = completeTime;
|
|
||||||
this.kinds = kinds;
|
|
||||||
this.gcsFilename = gcsFilename;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Rewrite a GCS path as stored by Datastore Admin (with a "/gs/" prefix) to the more standard
|
|
||||||
* URI format that uses a "gs://" scheme prefix.
|
|
||||||
*/
|
|
||||||
private static String gcsPathToUri(String backupGcsPath) {
|
|
||||||
checkArgument(backupGcsPath.startsWith("/gs/"), "GCS path not in expected format");
|
|
||||||
return backupGcsPath.replaceFirst("/gs/", "gs://");
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getName() {
|
|
||||||
return backupName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public ImmutableSet<String> getKinds() {
|
|
||||||
return kinds;
|
|
||||||
}
|
|
||||||
|
|
||||||
public BackupStatus getStatus() {
|
|
||||||
return completeTime.isPresent() ? BackupStatus.COMPLETE : BackupStatus.PENDING;
|
|
||||||
}
|
|
||||||
|
|
||||||
public DateTime getStartTime() {
|
|
||||||
return startTime;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Optional<DateTime> getCompleteTime() {
|
|
||||||
return completeTime;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns the length of time the backup ran for (if completed) or the length of time since the
|
|
||||||
* backup started (if it has not completed).
|
|
||||||
*/
|
|
||||||
public Duration getRunningTime() {
|
|
||||||
return new Duration(startTime, completeTime.orElse(clock.nowUtc()));
|
|
||||||
}
|
|
||||||
|
|
||||||
public Optional<String> getGcsFilename() {
|
|
||||||
return gcsFilename;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Returns a string version of key information about the backup. */
|
|
||||||
public String getInformation() {
|
|
||||||
return Joiner.on('\n')
|
|
||||||
.join(
|
|
||||||
"Backup name: " + backupName,
|
|
||||||
"Status: " + getStatus(),
|
|
||||||
"Started: " + startTime,
|
|
||||||
"Ended: " + completeTime.orElse(null),
|
|
||||||
"Duration: " + Ascii.toLowerCase(getRunningTime().toPeriod().toString().substring(2)),
|
|
||||||
"GCS: " + gcsFilename.orElse(null),
|
|
||||||
"Kinds: " + kinds,
|
|
||||||
"");
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,103 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
|
|
||||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
|
||||||
import static com.google.common.base.Strings.nullToEmpty;
|
|
||||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
|
||||||
|
|
||||||
import com.google.appengine.api.datastore.Query;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import com.google.common.collect.Iterables;
|
|
||||||
import com.google.common.collect.Streams;
|
|
||||||
import google.registry.util.AppEngineServiceUtils;
|
|
||||||
import java.util.NoSuchElementException;
|
|
||||||
import javax.inject.Inject;
|
|
||||||
|
|
||||||
/** An object providing methods for starting and querying Datastore backups. */
|
|
||||||
public class DatastoreBackupService {
|
|
||||||
|
|
||||||
/** The internal kind name used for entities storing information about Datastore backups. */
|
|
||||||
static final String BACKUP_INFO_KIND = "_AE_Backup_Information";
|
|
||||||
|
|
||||||
/** The name of the app version used for hosting the Datastore Admin functionality. */
|
|
||||||
static final String DATASTORE_ADMIN_VERSION_NAME = "ah-builtin-python-bundle";
|
|
||||||
|
|
||||||
private final AppEngineServiceUtils appEngineServiceUtils;
|
|
||||||
|
|
||||||
@Inject
|
|
||||||
public DatastoreBackupService(AppEngineServiceUtils appEngineServiceUtils) {
|
|
||||||
this.appEngineServiceUtils = appEngineServiceUtils;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generates the TaskOptions needed to trigger an AppEngine Datastore backup job.
|
|
||||||
*
|
|
||||||
* @see <a href="https://developers.google.com/appengine/articles/scheduled_backups">Scheduled Backups</a>
|
|
||||||
*/
|
|
||||||
private TaskOptions makeTaskOptions(
|
|
||||||
String queue, String name, String gcsBucket, ImmutableSet<String> kinds) {
|
|
||||||
String hostname =
|
|
||||||
appEngineServiceUtils.getVersionHostname("default", DATASTORE_ADMIN_VERSION_NAME);
|
|
||||||
TaskOptions options = TaskOptions.Builder.withUrl("/_ah/datastore_admin/backup.create")
|
|
||||||
.header("Host", hostname)
|
|
||||||
.method(Method.GET)
|
|
||||||
.param("name", name + "_") // Add underscore since the name will be used as a prefix.
|
|
||||||
.param("filesystem", "gs")
|
|
||||||
.param("gs_bucket_name", gcsBucket)
|
|
||||||
.param("queue", queue);
|
|
||||||
for (String kind : kinds) {
|
|
||||||
options.param("kind", kind);
|
|
||||||
}
|
|
||||||
return options;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Launches a new Datastore backup with the given name, GCS bucket, and set of kinds by
|
|
||||||
* submitting a task to the given task queue, and returns a handle to that task.
|
|
||||||
*/
|
|
||||||
public TaskHandle launchNewBackup(
|
|
||||||
String queue, String name, String gcsBucket, ImmutableSet<String> kinds) {
|
|
||||||
return getQueue(queue).add(makeTaskOptions(queue, name, gcsBucket, kinds));
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Return an iterable of all Datastore backups whose names have the given string prefix. */
|
|
||||||
public Iterable<DatastoreBackupInfo> findAllByNamePrefix(final String namePrefix) {
|
|
||||||
// Need the raw DatastoreService to access the internal _AE_Backup_Information entities.
|
|
||||||
// TODO(b/19081037): make an Objectify entity class for these raw Datastore entities instead.
|
|
||||||
return Streams.stream(getDatastoreService().prepare(new Query(BACKUP_INFO_KIND)).asIterable())
|
|
||||||
.filter(entity -> nullToEmpty((String) entity.getProperty("name")).startsWith(namePrefix))
|
|
||||||
.map(DatastoreBackupInfo::new)
|
|
||||||
.collect(toImmutableList());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Return a single DatastoreBackup that uniquely matches this name prefix. Throws an IAE
|
|
||||||
* if no backups match or if more than one backup matches.
|
|
||||||
*/
|
|
||||||
public DatastoreBackupInfo findByName(final String namePrefix) {
|
|
||||||
try {
|
|
||||||
return Iterables.getOnlyElement(findAllByNamePrefix(namePrefix));
|
|
||||||
} catch (IllegalArgumentException e) {
|
|
||||||
throw new IllegalArgumentException("More than one backup with name prefix " + namePrefix, e);
|
|
||||||
} catch (NoSuchElementException e) {
|
|
||||||
throw new IllegalArgumentException("No backup found with name prefix " + namePrefix, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -17,16 +17,15 @@ package google.registry.export;
|
||||||
import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
|
import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
|
||||||
import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
|
import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
|
||||||
import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
|
import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
|
||||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
import static google.registry.export.CheckBackupAction.CHECK_BACKUP_KINDS_TO_LOAD_PARAM;
|
||||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
|
import static google.registry.export.CheckBackupAction.CHECK_BACKUP_NAME_PARAM;
|
||||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
|
|
||||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
|
|
||||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
|
|
||||||
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM;
|
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM;
|
||||||
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM;
|
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM;
|
||||||
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM;
|
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM;
|
||||||
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM;
|
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM;
|
||||||
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_FOLDER_PARAM;
|
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_FOLDER_PARAM;
|
||||||
|
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_ID_PARAM;
|
||||||
|
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_KINDS_PARAM;
|
||||||
import static google.registry.request.RequestParameters.extractRequiredHeader;
|
import static google.registry.request.RequestParameters.extractRequiredHeader;
|
||||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||||
|
|
||||||
|
@ -64,12 +63,6 @@ public final class ExportRequestModule {
|
||||||
return extractRequiredParameter(req, UPDATE_SNAPSHOT_VIEWNAME_PARAM);
|
return extractRequiredParameter(req, UPDATE_SNAPSHOT_VIEWNAME_PARAM);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Provides
|
|
||||||
@Parameter(LOAD_SNAPSHOT_FILE_PARAM)
|
|
||||||
static String provideLoadSnapshotFile(HttpServletRequest req) {
|
|
||||||
return extractRequiredParameter(req, LOAD_SNAPSHOT_FILE_PARAM);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter(UPLOAD_BACKUP_FOLDER_PARAM)
|
@Parameter(UPLOAD_BACKUP_FOLDER_PARAM)
|
||||||
static String provideSnapshotUrlPrefix(HttpServletRequest req) {
|
static String provideSnapshotUrlPrefix(HttpServletRequest req) {
|
||||||
|
@ -77,27 +70,27 @@ public final class ExportRequestModule {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter(LOAD_SNAPSHOT_ID_PARAM)
|
@Parameter(UPLOAD_BACKUP_ID_PARAM)
|
||||||
static String provideLoadSnapshotId(HttpServletRequest req) {
|
static String provideLoadSnapshotId(HttpServletRequest req) {
|
||||||
return extractRequiredParameter(req, LOAD_SNAPSHOT_ID_PARAM);
|
return extractRequiredParameter(req, UPLOAD_BACKUP_ID_PARAM);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter(LOAD_SNAPSHOT_KINDS_PARAM)
|
@Parameter(UPLOAD_BACKUP_KINDS_PARAM)
|
||||||
static String provideLoadSnapshotKinds(HttpServletRequest req) {
|
static String provideLoadSnapshotKinds(HttpServletRequest req) {
|
||||||
return extractRequiredParameter(req, LOAD_SNAPSHOT_KINDS_PARAM);
|
return extractRequiredParameter(req, UPLOAD_BACKUP_KINDS_PARAM);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter(CHECK_SNAPSHOT_NAME_PARAM)
|
@Parameter(CHECK_BACKUP_NAME_PARAM)
|
||||||
static String provideCheckSnapshotName(HttpServletRequest req) {
|
static String provideCheckSnapshotName(HttpServletRequest req) {
|
||||||
return extractRequiredParameter(req, CHECK_SNAPSHOT_NAME_PARAM);
|
return extractRequiredParameter(req, CHECK_BACKUP_NAME_PARAM);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM)
|
@Parameter(CHECK_BACKUP_KINDS_TO_LOAD_PARAM)
|
||||||
static String provideCheckSnapshotKindsToLoad(HttpServletRequest req) {
|
static String provideCheckSnapshotKindsToLoad(HttpServletRequest req) {
|
||||||
return extractRequiredParameter(req, CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM);
|
return extractRequiredParameter(req, CHECK_BACKUP_KINDS_TO_LOAD_PARAM);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
|
|
|
@ -1,78 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static google.registry.export.CheckSnapshotAction.enqueuePollTask;
|
|
||||||
import static google.registry.request.Action.Method.POST;
|
|
||||||
|
|
||||||
import com.google.common.flogger.FluentLogger;
|
|
||||||
import google.registry.config.RegistryConfig;
|
|
||||||
import google.registry.request.Action;
|
|
||||||
import google.registry.request.Response;
|
|
||||||
import google.registry.request.auth.Auth;
|
|
||||||
import google.registry.util.Clock;
|
|
||||||
import javax.inject.Inject;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Action to trigger a Datastore backup job that writes a snapshot to Google Cloud Storage.
|
|
||||||
*
|
|
||||||
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
|
|
||||||
* the next upon successful completion:
|
|
||||||
*
|
|
||||||
* <ol>
|
|
||||||
* <li>The snapshot is exported to Google Cloud Storage (this action).
|
|
||||||
* <li>The {@link CheckSnapshotAction} polls until the export is completed.
|
|
||||||
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
|
|
||||||
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_datastore_export.
|
|
||||||
* </ol>
|
|
||||||
*/
|
|
||||||
@Action(
|
|
||||||
service = Action.Service.BACKEND,
|
|
||||||
path = ExportSnapshotAction.PATH,
|
|
||||||
method = POST,
|
|
||||||
automaticallyPrintOk = true,
|
|
||||||
auth = Auth.AUTH_INTERNAL_ONLY)
|
|
||||||
public class ExportSnapshotAction implements Runnable {
|
|
||||||
|
|
||||||
/** Queue to use for enqueuing the task that will actually launch the backup. */
|
|
||||||
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
|
||||||
|
|
||||||
static final String PATH = "/_dr/task/exportSnapshot"; // See web.xml.
|
|
||||||
|
|
||||||
/** Prefix to use for naming all snapshots that are started by this servlet. */
|
|
||||||
static final String SNAPSHOT_PREFIX = "auto_snapshot_";
|
|
||||||
|
|
||||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
|
||||||
|
|
||||||
@Inject Clock clock;
|
|
||||||
@Inject DatastoreBackupService backupService;
|
|
||||||
@Inject Response response;
|
|
||||||
|
|
||||||
@Inject
|
|
||||||
ExportSnapshotAction() {}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
// Use a unique name for the snapshot so we can explicitly check its completion later.
|
|
||||||
String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss");
|
|
||||||
backupService.launchNewBackup(
|
|
||||||
QUEUE, snapshotName, RegistryConfig.getSnapshotsBucket(), ExportConstants.getBackupKinds());
|
|
||||||
// Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery.
|
|
||||||
enqueuePollTask(snapshotName, ExportConstants.getReportingKinds());
|
|
||||||
String message = "Datastore backup started with name: " + snapshotName;
|
|
||||||
logger.atInfo().log(message);
|
|
||||||
response.setPayload(message);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,166 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
|
||||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
|
||||||
import static com.google.common.base.Preconditions.checkArgument;
|
|
||||||
import static google.registry.export.UpdateSnapshotViewAction.createViewUpdateTask;
|
|
||||||
import static google.registry.request.Action.Method.POST;
|
|
||||||
|
|
||||||
import com.google.api.services.bigquery.Bigquery;
|
|
||||||
import com.google.api.services.bigquery.model.Job;
|
|
||||||
import com.google.api.services.bigquery.model.JobConfiguration;
|
|
||||||
import com.google.api.services.bigquery.model.JobConfigurationLoad;
|
|
||||||
import com.google.api.services.bigquery.model.JobReference;
|
|
||||||
import com.google.api.services.bigquery.model.TableReference;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
|
||||||
import com.google.common.base.Joiner;
|
|
||||||
import com.google.common.base.Splitter;
|
|
||||||
import com.google.common.collect.ImmutableList;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import com.google.common.flogger.FluentLogger;
|
|
||||||
import google.registry.bigquery.BigqueryUtils.SourceFormat;
|
|
||||||
import google.registry.bigquery.BigqueryUtils.WriteDisposition;
|
|
||||||
import google.registry.bigquery.CheckedBigquery;
|
|
||||||
import google.registry.config.RegistryConfig.Config;
|
|
||||||
import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
|
|
||||||
import google.registry.request.Action;
|
|
||||||
import google.registry.request.HttpException.BadRequestException;
|
|
||||||
import google.registry.request.HttpException.InternalServerErrorException;
|
|
||||||
import google.registry.request.Parameter;
|
|
||||||
import google.registry.request.auth.Auth;
|
|
||||||
import google.registry.util.Clock;
|
|
||||||
import java.io.IOException;
|
|
||||||
import javax.inject.Inject;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
|
|
||||||
/** Action to load a Datastore snapshot from Google Cloud Storage into BigQuery. */
|
|
||||||
@Action(
|
|
||||||
service = Action.Service.BACKEND,
|
|
||||||
path = LoadSnapshotAction.PATH,
|
|
||||||
method = POST,
|
|
||||||
auth = Auth.AUTH_INTERNAL_ONLY)
|
|
||||||
public class LoadSnapshotAction implements Runnable {
|
|
||||||
|
|
||||||
/** Parameter names for passing parameters into the servlet. */
|
|
||||||
static final String LOAD_SNAPSHOT_ID_PARAM = "id";
|
|
||||||
static final String LOAD_SNAPSHOT_FILE_PARAM = "file";
|
|
||||||
static final String LOAD_SNAPSHOT_KINDS_PARAM = "kinds";
|
|
||||||
|
|
||||||
static final String SNAPSHOTS_DATASET = "snapshots";
|
|
||||||
|
|
||||||
static final String LATEST_SNAPSHOT_VIEW_NAME = "latest_datastore_export";
|
|
||||||
|
|
||||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
|
||||||
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
|
||||||
static final String PATH = "/_dr/task/loadSnapshot"; // See web.xml.
|
|
||||||
|
|
||||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
|
||||||
|
|
||||||
@Inject CheckedBigquery checkedBigquery;
|
|
||||||
@Inject BigqueryPollJobEnqueuer bigqueryPollEnqueuer;
|
|
||||||
@Inject Clock clock;
|
|
||||||
@Inject @Config("projectId") String projectId;
|
|
||||||
@Inject @Parameter(LOAD_SNAPSHOT_FILE_PARAM) String snapshotFile;
|
|
||||||
@Inject @Parameter(LOAD_SNAPSHOT_ID_PARAM) String snapshotId;
|
|
||||||
@Inject @Parameter(LOAD_SNAPSHOT_KINDS_PARAM) String snapshotKinds;
|
|
||||||
@Inject LoadSnapshotAction() {}
|
|
||||||
|
|
||||||
/** Enqueue a task for starting a backup load. */
|
|
||||||
public static TaskHandle enqueueLoadSnapshotTask(
|
|
||||||
String snapshotId, String gcsFile, ImmutableSet<String> kinds) {
|
|
||||||
return getQueue(QUEUE).add(
|
|
||||||
TaskOptions.Builder.withUrl(PATH)
|
|
||||||
.method(Method.POST)
|
|
||||||
.param(LOAD_SNAPSHOT_ID_PARAM, snapshotId)
|
|
||||||
.param(LOAD_SNAPSHOT_FILE_PARAM, gcsFile)
|
|
||||||
.param(LOAD_SNAPSHOT_KINDS_PARAM, Joiner.on(',').join(kinds)));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
try {
|
|
||||||
String message =
|
|
||||||
loadSnapshot(snapshotId, snapshotFile, Splitter.on(',').split(snapshotKinds));
|
|
||||||
logger.atInfo().log("Loaded snapshot successfully: %s", message);
|
|
||||||
} catch (Throwable e) {
|
|
||||||
logger.atSevere().withCause(e).log("Error loading snapshot");
|
|
||||||
if (e instanceof IllegalArgumentException) {
|
|
||||||
throw new BadRequestException("Error calling load snapshot: " + e.getMessage(), e);
|
|
||||||
} else {
|
|
||||||
throw new InternalServerErrorException(
|
|
||||||
"Error loading snapshot: " + firstNonNull(e.getMessage(), e.toString()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String loadSnapshot(String snapshotId, String gcsFilename, Iterable<String> kinds)
|
|
||||||
throws IOException {
|
|
||||||
Bigquery bigquery = checkedBigquery.ensureDataSetExists(projectId, SNAPSHOTS_DATASET);
|
|
||||||
DateTime now = clock.nowUtc();
|
|
||||||
String loadMessage =
|
|
||||||
String.format("Loading Datastore snapshot %s from %s...", snapshotId, gcsFilename);
|
|
||||||
logger.atInfo().log(loadMessage);
|
|
||||||
StringBuilder builder = new StringBuilder(loadMessage + "\n");
|
|
||||||
builder.append("Load jobs:\n");
|
|
||||||
|
|
||||||
for (String kindName : kinds) {
|
|
||||||
String jobId = String.format("load-snapshot-%s-%s-%d", snapshotId, kindName, now.getMillis());
|
|
||||||
JobReference jobRef = new JobReference().setProjectId(projectId).setJobId(jobId);
|
|
||||||
String sourceUri = getBackupInfoFileForKind(gcsFilename, kindName);
|
|
||||||
String tableId = String.format("%s_%s", snapshotId, kindName);
|
|
||||||
|
|
||||||
// Launch the load job.
|
|
||||||
Job job = makeLoadJob(jobRef, sourceUri, tableId);
|
|
||||||
bigquery.jobs().insert(projectId, job).execute();
|
|
||||||
|
|
||||||
// Enqueue a task to check on the load job's completion, and if it succeeds, to update a
|
|
||||||
// well-known view in BigQuery to point at the newly loaded snapshot table for this kind.
|
|
||||||
bigqueryPollEnqueuer.enqueuePollTask(
|
|
||||||
jobRef,
|
|
||||||
createViewUpdateTask(SNAPSHOTS_DATASET, tableId, kindName, LATEST_SNAPSHOT_VIEW_NAME),
|
|
||||||
getQueue(UpdateSnapshotViewAction.QUEUE));
|
|
||||||
|
|
||||||
builder.append(String.format(" - %s:%s\n", projectId, jobId));
|
|
||||||
logger.atInfo().log("Submitted load job %s:%s", projectId, jobId);
|
|
||||||
}
|
|
||||||
return builder.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String getBackupInfoFileForKind(String backupInfoFile, String kindName) {
|
|
||||||
String extension = ".backup_info";
|
|
||||||
checkArgument(backupInfoFile.endsWith(extension), "backup info file extension missing");
|
|
||||||
String prefix = backupInfoFile.substring(0, backupInfoFile.length() - extension.length());
|
|
||||||
return Joiner.on('.').join(prefix, kindName, extension.substring(1));
|
|
||||||
}
|
|
||||||
|
|
||||||
private Job makeLoadJob(JobReference jobRef, String sourceUri, String tableId) {
|
|
||||||
TableReference tableReference = new TableReference()
|
|
||||||
.setProjectId(jobRef.getProjectId())
|
|
||||||
.setDatasetId(SNAPSHOTS_DATASET)
|
|
||||||
.setTableId(tableId);
|
|
||||||
return new Job()
|
|
||||||
.setJobReference(jobRef)
|
|
||||||
.setConfiguration(new JobConfiguration()
|
|
||||||
.setLoad(new JobConfigurationLoad()
|
|
||||||
.setWriteDisposition(WriteDisposition.WRITE_EMPTY.toString())
|
|
||||||
.setSourceFormat(SourceFormat.DATASTORE_BACKUP.toString())
|
|
||||||
.setSourceUris(ImmutableList.of(sourceUri))
|
|
||||||
.setDestinationTable(tableReference)));
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -50,8 +50,7 @@ public class UpdateSnapshotViewAction implements Runnable {
|
||||||
static final String UPDATE_SNAPSHOT_VIEWNAME_PARAM = "viewname";
|
static final String UPDATE_SNAPSHOT_VIEWNAME_PARAM = "viewname";
|
||||||
|
|
||||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
||||||
// For now this queue is shared by the backup workflows started by both ExportSnapshotAction
|
// For now this queue is shared by the backup workflows started by BackupDatastoreAction.
|
||||||
// and BackupDatastoreAction.
|
|
||||||
// TODO(weiminyu): update queue name (snapshot->backup) after ExportSnapshot flow is removed.
|
// TODO(weiminyu): update queue name (snapshot->backup) after ExportSnapshot flow is removed.
|
||||||
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
|
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
|
||||||
|
|
||||||
|
|
|
@ -42,13 +42,10 @@ import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule;
|
||||||
import google.registry.export.BackupDatastoreAction;
|
import google.registry.export.BackupDatastoreAction;
|
||||||
import google.registry.export.BigqueryPollJobAction;
|
import google.registry.export.BigqueryPollJobAction;
|
||||||
import google.registry.export.CheckBackupAction;
|
import google.registry.export.CheckBackupAction;
|
||||||
import google.registry.export.CheckSnapshotAction;
|
|
||||||
import google.registry.export.ExportDomainListsAction;
|
import google.registry.export.ExportDomainListsAction;
|
||||||
import google.registry.export.ExportPremiumTermsAction;
|
import google.registry.export.ExportPremiumTermsAction;
|
||||||
import google.registry.export.ExportRequestModule;
|
import google.registry.export.ExportRequestModule;
|
||||||
import google.registry.export.ExportReservedTermsAction;
|
import google.registry.export.ExportReservedTermsAction;
|
||||||
import google.registry.export.ExportSnapshotAction;
|
|
||||||
import google.registry.export.LoadSnapshotAction;
|
|
||||||
import google.registry.export.SyncGroupMembersAction;
|
import google.registry.export.SyncGroupMembersAction;
|
||||||
import google.registry.export.UpdateSnapshotViewAction;
|
import google.registry.export.UpdateSnapshotViewAction;
|
||||||
import google.registry.export.UploadDatastoreBackupAction;
|
import google.registry.export.UploadDatastoreBackupAction;
|
||||||
|
@ -121,7 +118,6 @@ interface BackendRequestComponent {
|
||||||
BigqueryPollJobAction bigqueryPollJobAction();
|
BigqueryPollJobAction bigqueryPollJobAction();
|
||||||
BrdaCopyAction brdaCopyAction();
|
BrdaCopyAction brdaCopyAction();
|
||||||
CheckBackupAction checkBackupAction();
|
CheckBackupAction checkBackupAction();
|
||||||
CheckSnapshotAction checkSnapshotAction();
|
|
||||||
CommitLogCheckpointAction commitLogCheckpointAction();
|
CommitLogCheckpointAction commitLogCheckpointAction();
|
||||||
CommitLogFanoutAction commitLogFanoutAction();
|
CommitLogFanoutAction commitLogFanoutAction();
|
||||||
CopyDetailReportsAction copyDetailReportAction();
|
CopyDetailReportsAction copyDetailReportAction();
|
||||||
|
@ -134,12 +130,10 @@ interface BackendRequestComponent {
|
||||||
ExportDomainListsAction exportDomainListsAction();
|
ExportDomainListsAction exportDomainListsAction();
|
||||||
ExportPremiumTermsAction exportPremiumTermsAction();
|
ExportPremiumTermsAction exportPremiumTermsAction();
|
||||||
ExportReservedTermsAction exportReservedTermsAction();
|
ExportReservedTermsAction exportReservedTermsAction();
|
||||||
ExportSnapshotAction exportSnapshotAction();
|
|
||||||
GenerateInvoicesAction generateInvoicesAction();
|
GenerateInvoicesAction generateInvoicesAction();
|
||||||
GenerateSpec11ReportAction generateSpec11ReportAction();
|
GenerateSpec11ReportAction generateSpec11ReportAction();
|
||||||
IcannReportingStagingAction icannReportingStagingAction();
|
IcannReportingStagingAction icannReportingStagingAction();
|
||||||
IcannReportingUploadAction icannReportingUploadAction();
|
IcannReportingUploadAction icannReportingUploadAction();
|
||||||
LoadSnapshotAction loadSnapshotAction();
|
|
||||||
NordnUploadAction nordnUploadAction();
|
NordnUploadAction nordnUploadAction();
|
||||||
NordnVerifyAction nordnVerifyAction();
|
NordnVerifyAction nordnVerifyAction();
|
||||||
PublishDnsUpdatesAction publishDnsUpdatesAction();
|
PublishDnsUpdatesAction publishDnsUpdatesAction();
|
||||||
|
|
|
@ -1,49 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.tools;
|
|
||||||
|
|
||||||
import com.beust.jcommander.Parameter;
|
|
||||||
import com.beust.jcommander.Parameters;
|
|
||||||
import com.google.common.collect.Iterables;
|
|
||||||
import com.google.common.collect.Streams;
|
|
||||||
import google.registry.export.DatastoreBackupInfo;
|
|
||||||
import google.registry.export.DatastoreBackupService;
|
|
||||||
import javax.inject.Inject;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Command to check the status of a Datastore backup, or "snapshot".
|
|
||||||
*/
|
|
||||||
@Parameters(separators = " =", commandDescription = "Check the status of a Datastore snapshot")
|
|
||||||
public class CheckSnapshotCommand implements CommandWithRemoteApi {
|
|
||||||
|
|
||||||
@Parameter(
|
|
||||||
names = {"-s", "--snapshot"},
|
|
||||||
description = "Unique prefix of the snapshot to check",
|
|
||||||
required = true)
|
|
||||||
private String snapshotName;
|
|
||||||
|
|
||||||
@Inject DatastoreBackupService datastoreBackupService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
Iterable<DatastoreBackupInfo> backups =
|
|
||||||
datastoreBackupService.findAllByNamePrefix(snapshotName);
|
|
||||||
if (Iterables.isEmpty(backups)) {
|
|
||||||
System.err.println("No snapshot found with name: " + snapshotName);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
Streams.stream(backups).map(DatastoreBackupInfo::getInformation).forEach(System.out::println);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -32,7 +32,6 @@ public final class RegistryTool {
|
||||||
.put("canonicalize_labels", CanonicalizeLabelsCommand.class)
|
.put("canonicalize_labels", CanonicalizeLabelsCommand.class)
|
||||||
.put("check_domain", CheckDomainCommand.class)
|
.put("check_domain", CheckDomainCommand.class)
|
||||||
.put("check_domain_claims", CheckDomainClaimsCommand.class)
|
.put("check_domain_claims", CheckDomainClaimsCommand.class)
|
||||||
.put("check_snapshot", CheckSnapshotCommand.class)
|
|
||||||
.put("convert_idn", ConvertIdnCommand.class)
|
.put("convert_idn", ConvertIdnCommand.class)
|
||||||
.put("count_domains", CountDomainsCommand.class)
|
.put("count_domains", CountDomainsCommand.class)
|
||||||
.put("create_anchor_tenant", CreateAnchorTenantCommand.class)
|
.put("create_anchor_tenant", CreateAnchorTenantCommand.class)
|
||||||
|
|
|
@ -75,7 +75,6 @@ import javax.inject.Singleton;
|
||||||
interface RegistryToolComponent {
|
interface RegistryToolComponent {
|
||||||
void inject(CheckDomainClaimsCommand command);
|
void inject(CheckDomainClaimsCommand command);
|
||||||
void inject(CheckDomainCommand command);
|
void inject(CheckDomainCommand command);
|
||||||
void inject(CheckSnapshotCommand command);
|
|
||||||
void inject(CountDomainsCommand command);
|
void inject(CountDomainsCommand command);
|
||||||
void inject(CreateAnchorTenantCommand command);
|
void inject(CreateAnchorTenantCommand command);
|
||||||
void inject(CreateCdnsTld command);
|
void inject(CreateCdnsTld command);
|
||||||
|
|
|
@ -1,221 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.common.truth.Truth.assertThat;
|
|
||||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
|
||||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
|
|
||||||
import static google.registry.testing.JUnitBackports.assertThrows;
|
|
||||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
|
||||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import com.google.common.base.Joiner;
|
|
||||||
import com.google.common.collect.ImmutableList;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import google.registry.request.Action.Method;
|
|
||||||
import google.registry.request.HttpException.BadRequestException;
|
|
||||||
import google.registry.request.HttpException.NoContentException;
|
|
||||||
import google.registry.request.HttpException.NotModifiedException;
|
|
||||||
import google.registry.testing.AppEngineRule;
|
|
||||||
import google.registry.testing.FakeClock;
|
|
||||||
import google.registry.testing.FakeResponse;
|
|
||||||
import google.registry.testing.InjectRule;
|
|
||||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
|
||||||
import java.util.Optional;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
import org.joda.time.Duration;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.runner.RunWith;
|
|
||||||
import org.junit.runners.JUnit4;
|
|
||||||
|
|
||||||
/** Unit tests for {@link CheckSnapshotAction}. */
|
|
||||||
@RunWith(JUnit4.class)
|
|
||||||
public class CheckSnapshotActionTest {
|
|
||||||
|
|
||||||
static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
|
|
||||||
static final DateTime COMPLETE_TIME = START_TIME.plus(Duration.standardMinutes(30));
|
|
||||||
|
|
||||||
@Rule public final InjectRule inject = new InjectRule();
|
|
||||||
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
|
|
||||||
private final DatastoreBackupService backupService = mock(DatastoreBackupService.class);
|
|
||||||
|
|
||||||
private DatastoreBackupInfo backupInfo;
|
|
||||||
|
|
||||||
private final FakeResponse response = new FakeResponse();
|
|
||||||
private final FakeClock clock = new FakeClock(COMPLETE_TIME.plusMillis(1000));
|
|
||||||
private final CheckSnapshotAction action = new CheckSnapshotAction();
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void before() {
|
|
||||||
inject.setStaticField(DatastoreBackupInfo.class, "clock", clock);
|
|
||||||
action.requestMethod = Method.POST;
|
|
||||||
action.snapshotName = "some_backup";
|
|
||||||
action.kindsToLoadParam = "one,two";
|
|
||||||
action.response = response;
|
|
||||||
action.backupService = backupService;
|
|
||||||
|
|
||||||
backupInfo =
|
|
||||||
new DatastoreBackupInfo(
|
|
||||||
"some_backup",
|
|
||||||
START_TIME,
|
|
||||||
Optional.of(COMPLETE_TIME),
|
|
||||||
ImmutableSet.of("one", "two", "three"),
|
|
||||||
Optional.of("gs://somebucket/some_backup_20140801.backup_info"));
|
|
||||||
|
|
||||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void setPendingBackup() {
|
|
||||||
backupInfo =
|
|
||||||
new DatastoreBackupInfo(
|
|
||||||
backupInfo.getName(),
|
|
||||||
backupInfo.getStartTime(),
|
|
||||||
Optional.empty(),
|
|
||||||
backupInfo.getKinds(),
|
|
||||||
backupInfo.getGcsFilename());
|
|
||||||
|
|
||||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void assertLoadTaskEnqueued(String id, String file, String kinds) {
|
|
||||||
assertTasksEnqueued(
|
|
||||||
"export-snapshot",
|
|
||||||
new TaskMatcher()
|
|
||||||
.url("/_dr/task/loadSnapshot")
|
|
||||||
.method("POST")
|
|
||||||
.param("id", id)
|
|
||||||
.param("file", file)
|
|
||||||
.param("kinds", kinds));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_enqueuePollTask() {
|
|
||||||
CheckSnapshotAction.enqueuePollTask(
|
|
||||||
"some_snapshot_name", ImmutableSet.of("one", "two", "three"));
|
|
||||||
assertTasksEnqueued(
|
|
||||||
CheckSnapshotAction.QUEUE,
|
|
||||||
new TaskMatcher()
|
|
||||||
.url(CheckSnapshotAction.PATH)
|
|
||||||
.param(CHECK_SNAPSHOT_NAME_PARAM, "some_snapshot_name")
|
|
||||||
.param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, "one,two,three")
|
|
||||||
.method("POST"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forPendingBackup_returnsNotModified() {
|
|
||||||
setPendingBackup();
|
|
||||||
|
|
||||||
NotModifiedException thrown = assertThrows(NotModifiedException.class, action::run);
|
|
||||||
assertThat(thrown).hasMessageThat().contains("Datastore backup some_backup still pending");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forStalePendingBackupBackup_returnsNoContent() {
|
|
||||||
setPendingBackup();
|
|
||||||
|
|
||||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
|
||||||
|
|
||||||
clock.setTo(
|
|
||||||
START_TIME
|
|
||||||
.plus(Duration.standardHours(20))
|
|
||||||
.plus(Duration.standardMinutes(3))
|
|
||||||
.plus(Duration.millis(1234)));
|
|
||||||
|
|
||||||
NoContentException thrown = assertThrows(NoContentException.class, action::run);
|
|
||||||
assertThat(thrown)
|
|
||||||
.hasMessageThat()
|
|
||||||
.contains(
|
|
||||||
"Datastore backup some_backup abandoned - "
|
|
||||||
+ "not complete after 20 hours, 3 minutes and 1 second");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forCompleteBackup_enqueuesLoadTask() {
|
|
||||||
|
|
||||||
action.run();
|
|
||||||
assertLoadTaskEnqueued(
|
|
||||||
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forCompleteAutoBackup_enqueuesLoadTask_usingBackupName() {
|
|
||||||
action.snapshotName = "auto_snapshot_somestring";
|
|
||||||
when(backupService.findByName("auto_snapshot_somestring")).thenReturn(backupInfo);
|
|
||||||
|
|
||||||
action.run();
|
|
||||||
assertLoadTaskEnqueued(
|
|
||||||
"somestring", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forCompleteBackup_withExtraKindsToLoad_enqueuesLoadTask() {
|
|
||||||
action.kindsToLoadParam = "one,foo";
|
|
||||||
|
|
||||||
action.run();
|
|
||||||
assertLoadTaskEnqueued(
|
|
||||||
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forCompleteBackup_withEmptyKindsToLoad_skipsLoadTask() {
|
|
||||||
action.kindsToLoadParam = "";
|
|
||||||
|
|
||||||
action.run();
|
|
||||||
assertNoTasksEnqueued("export-snapshot");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forBadBackup_returnsBadRequest() {
|
|
||||||
when(backupService.findByName("some_backup"))
|
|
||||||
.thenThrow(new IllegalArgumentException("No backup found"));
|
|
||||||
|
|
||||||
BadRequestException thrown = assertThrows(BadRequestException.class, action::run);
|
|
||||||
assertThat(thrown).hasMessageThat().contains("Bad backup name some_backup: No backup found");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testGet_returnsInformation() {
|
|
||||||
action.requestMethod = Method.GET;
|
|
||||||
|
|
||||||
action.run();
|
|
||||||
assertThat(response.getPayload())
|
|
||||||
.isEqualTo(
|
|
||||||
Joiner.on("\n")
|
|
||||||
.join(
|
|
||||||
ImmutableList.of(
|
|
||||||
"Backup name: some_backup",
|
|
||||||
"Status: COMPLETE",
|
|
||||||
"Started: 2014-08-01T01:02:03.000Z",
|
|
||||||
"Ended: 2014-08-01T01:32:03.000Z",
|
|
||||||
"Duration: 30m",
|
|
||||||
"GCS: gs://somebucket/some_backup_20140801.backup_info",
|
|
||||||
"Kinds: [one, two, three]",
|
|
||||||
"")));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testGet_forBadBackup_returnsError() {
|
|
||||||
action.requestMethod = Method.GET;
|
|
||||||
when(backupService.findByName("some_backup"))
|
|
||||||
.thenThrow(new IllegalArgumentException("No backup found"));
|
|
||||||
|
|
||||||
BadRequestException thrown = assertThrows(BadRequestException.class, action::run);
|
|
||||||
assertThat(thrown).hasMessageThat().contains("Bad backup name some_backup: No backup found");
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,130 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
|
|
||||||
import static com.google.common.truth.Truth.assertThat;
|
|
||||||
import static com.google.common.truth.Truth8.assertThat;
|
|
||||||
import static google.registry.testing.JUnitBackports.assertThrows;
|
|
||||||
|
|
||||||
import com.google.appengine.api.datastore.Entity;
|
|
||||||
import com.google.appengine.api.datastore.EntityNotFoundException;
|
|
||||||
import com.google.appengine.api.datastore.Text;
|
|
||||||
import com.google.common.collect.ImmutableList;
|
|
||||||
import google.registry.export.DatastoreBackupInfo.BackupStatus;
|
|
||||||
import google.registry.testing.AppEngineRule;
|
|
||||||
import google.registry.testing.FakeClock;
|
|
||||||
import google.registry.testing.InjectRule;
|
|
||||||
import java.util.Date;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
import org.joda.time.Duration;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.runner.RunWith;
|
|
||||||
import org.junit.runners.JUnit4;
|
|
||||||
|
|
||||||
/** Unit tests for {@link DatastoreBackupInfo}. */
|
|
||||||
@RunWith(JUnit4.class)
|
|
||||||
public class DatastoreBackupInfoTest {
|
|
||||||
@Rule
|
|
||||||
public final InjectRule inject = new InjectRule();
|
|
||||||
|
|
||||||
@Rule
|
|
||||||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
|
||||||
.withDatastore()
|
|
||||||
.build();
|
|
||||||
|
|
||||||
private FakeClock clock = new FakeClock();
|
|
||||||
|
|
||||||
private DateTime startTime = DateTime.parse("2014-08-01T01:02:03Z");
|
|
||||||
|
|
||||||
private Entity backupEntity; // Can't initialize until AppEngineRule has set up Datastore.
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void before() {
|
|
||||||
inject.setStaticField(DatastoreBackupInfo.class, "clock", clock);
|
|
||||||
backupEntity = new Entity("_unused_");
|
|
||||||
backupEntity.setProperty("name", "backup1");
|
|
||||||
backupEntity.setProperty("kinds", ImmutableList.of("one", "two", "three"));
|
|
||||||
backupEntity.setProperty("start_time", new Date(startTime.getMillis()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Force a roundtrip to Datastore to ensure that we're simulating a freshly fetched entity. */
|
|
||||||
private static Entity persistEntity(Entity entity) throws EntityNotFoundException {
|
|
||||||
return getDatastoreService().get(getDatastoreService().put(entity));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_pendingBackup() throws Exception {
|
|
||||||
DatastoreBackupInfo backup = new DatastoreBackupInfo(persistEntity(backupEntity));
|
|
||||||
|
|
||||||
assertThat(backup.getName()).isEqualTo("backup1");
|
|
||||||
assertThat(backup.getKinds()).containsExactly("one", "two", "three");
|
|
||||||
assertThat(backup.getStartTime()).isEqualTo(startTime);
|
|
||||||
assertThat(backup.getCompleteTime()).isEmpty();
|
|
||||||
assertThat(backup.getGcsFilename()).isEmpty();
|
|
||||||
assertThat(backup.getStatus()).isEqualTo(BackupStatus.PENDING);
|
|
||||||
|
|
||||||
clock.setTo(startTime.plusMinutes(1));
|
|
||||||
assertThat(backup.getRunningTime()).isEqualTo(Duration.standardMinutes(1));
|
|
||||||
clock.setTo(startTime.plusMinutes(1).plusSeconds(42));
|
|
||||||
assertThat(backup.getRunningTime()).isEqualTo(Duration.standardSeconds(102));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_completeBackup() throws Exception {
|
|
||||||
DateTime completeTime = startTime.plusMinutes(1).plusSeconds(42);
|
|
||||||
backupEntity.setProperty("complete_time", new Date(completeTime.getMillis()));
|
|
||||||
backupEntity.setProperty("gs_handle", new Text("/gs/somebucket/timestamp.backup_info"));
|
|
||||||
DatastoreBackupInfo backup = new DatastoreBackupInfo(persistEntity(backupEntity));
|
|
||||||
|
|
||||||
assertThat(backup.getName()).isEqualTo("backup1");
|
|
||||||
assertThat(backup.getKinds()).containsExactly("one", "two", "three");
|
|
||||||
assertThat(backup.getStartTime()).isEqualTo(startTime);
|
|
||||||
assertThat(backup.getCompleteTime().get()).isEqualTo(completeTime);
|
|
||||||
assertThat(backup.getGcsFilename()).hasValue("gs://somebucket/timestamp.backup_info");
|
|
||||||
assertThat(backup.getStatus()).isEqualTo(BackupStatus.COMPLETE);
|
|
||||||
assertThat(backup.getRunningTime()).isEqualTo(Duration.standardSeconds(102));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testFailure_missingName() {
|
|
||||||
backupEntity.removeProperty("name");
|
|
||||||
assertThrows(
|
|
||||||
NullPointerException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity)));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testFailure_missingKinds() {
|
|
||||||
backupEntity.removeProperty("kinds");
|
|
||||||
assertThrows(
|
|
||||||
NullPointerException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity)));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testFailure_missingStartTime() {
|
|
||||||
backupEntity.removeProperty("start_time");
|
|
||||||
assertThrows(
|
|
||||||
NullPointerException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity)));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testFailure_badGcsFilenameFormat() {
|
|
||||||
backupEntity.setProperty("gs_handle", new Text("foo"));
|
|
||||||
assertThrows(
|
|
||||||
IllegalArgumentException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity)));
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,123 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
|
|
||||||
import static com.google.common.collect.Iterables.transform;
|
|
||||||
import static com.google.common.truth.Truth.assertThat;
|
|
||||||
import static google.registry.testing.JUnitBackports.assertThrows;
|
|
||||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import com.google.appengine.api.datastore.Entity;
|
|
||||||
import com.google.common.collect.ImmutableList;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import google.registry.testing.AppEngineRule;
|
|
||||||
import google.registry.testing.MockitoJUnitRule;
|
|
||||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
|
||||||
import google.registry.util.AppEngineServiceUtils;
|
|
||||||
import java.util.Date;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.runner.RunWith;
|
|
||||||
import org.junit.runners.JUnit4;
|
|
||||||
import org.mockito.Mock;
|
|
||||||
|
|
||||||
/** Unit tests for {@link DatastoreBackupService}. */
|
|
||||||
@RunWith(JUnit4.class)
|
|
||||||
public class DatastoreBackupServiceTest {
|
|
||||||
|
|
||||||
@Rule
|
|
||||||
public final AppEngineRule appEngine =
|
|
||||||
AppEngineRule.builder().withDatastore().withTaskQueue().build();
|
|
||||||
|
|
||||||
@Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create();
|
|
||||||
|
|
||||||
@Mock private AppEngineServiceUtils appEngineServiceUtils;
|
|
||||||
|
|
||||||
private static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
|
|
||||||
|
|
||||||
private DatastoreBackupService backupService;
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void before() {
|
|
||||||
backupService = new DatastoreBackupService(appEngineServiceUtils);
|
|
||||||
when(appEngineServiceUtils.getVersionHostname("default", "ah-builtin-python-bundle"))
|
|
||||||
.thenReturn("ah-builtin-python-bundle.default.localhost");
|
|
||||||
|
|
||||||
persistBackupEntityWithName("backupA1");
|
|
||||||
persistBackupEntityWithName("backupA2");
|
|
||||||
persistBackupEntityWithName("backupA3");
|
|
||||||
persistBackupEntityWithName("backupB1");
|
|
||||||
persistBackupEntityWithName("backupB42");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void persistBackupEntityWithName(String name) {
|
|
||||||
Entity entity = new Entity(DatastoreBackupService.BACKUP_INFO_KIND);
|
|
||||||
entity.setProperty("name", name);
|
|
||||||
entity.setProperty("kinds", ImmutableList.of("one", "two", "three"));
|
|
||||||
entity.setProperty("start_time", new Date(START_TIME.getMillis()));
|
|
||||||
getDatastoreService().put(entity);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_launchBackup() {
|
|
||||||
backupService.launchNewBackup(
|
|
||||||
"export-snapshot", "backup1", "somebucket", ImmutableSet.of("foo", "bar"));
|
|
||||||
assertTasksEnqueued("export-snapshot",
|
|
||||||
new TaskMatcher()
|
|
||||||
.url("/_ah/datastore_admin/backup.create")
|
|
||||||
.header("Host", "ah-builtin-python-bundle.default.localhost")
|
|
||||||
.method("GET")
|
|
||||||
.param("name", "backup1_")
|
|
||||||
.param("filesystem", "gs")
|
|
||||||
.param("gs_bucket_name", "somebucket")
|
|
||||||
.param("queue", "export-snapshot")
|
|
||||||
.param("kind", "foo")
|
|
||||||
.param("kind", "bar"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_findAllByNamePrefix() {
|
|
||||||
assertThat(
|
|
||||||
transform(backupService.findAllByNamePrefix("backupA"), DatastoreBackupInfo::getName))
|
|
||||||
.containsExactly("backupA1", "backupA2", "backupA3");
|
|
||||||
assertThat(
|
|
||||||
transform(backupService.findAllByNamePrefix("backupB"), DatastoreBackupInfo::getName))
|
|
||||||
.containsExactly("backupB1", "backupB42");
|
|
||||||
assertThat(
|
|
||||||
transform(backupService.findAllByNamePrefix("backupB4"), DatastoreBackupInfo::getName))
|
|
||||||
.containsExactly("backupB42");
|
|
||||||
assertThat(backupService.findAllByNamePrefix("backupX")).isEmpty();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_findByName() {
|
|
||||||
assertThat(backupService.findByName("backupA1").getName()).isEqualTo("backupA1");
|
|
||||||
assertThat(backupService.findByName("backupB4").getName()).isEqualTo("backupB42");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testFailure_findByName_multipleMatchingBackups() {
|
|
||||||
assertThrows(IllegalArgumentException.class, () -> backupService.findByName("backupA"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testFailure_findByName_noMatchingBackups() {
|
|
||||||
assertThrows(IllegalArgumentException.class, () -> backupService.findByName("backupX"));
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,76 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.common.truth.Truth.assertThat;
|
|
||||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
|
||||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
|
|
||||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
|
|
||||||
import com.google.common.base.Joiner;
|
|
||||||
import google.registry.testing.AppEngineRule;
|
|
||||||
import google.registry.testing.FakeClock;
|
|
||||||
import google.registry.testing.FakeResponse;
|
|
||||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.runner.RunWith;
|
|
||||||
import org.junit.runners.JUnit4;
|
|
||||||
|
|
||||||
/** Unit tests for {@link ExportSnapshotAction}. */
|
|
||||||
@RunWith(JUnit4.class)
|
|
||||||
public class ExportSnapshotActionTest {
|
|
||||||
|
|
||||||
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
|
|
||||||
|
|
||||||
private final DatastoreBackupService backupService = mock(DatastoreBackupService.class);
|
|
||||||
|
|
||||||
private final FakeResponse response = new FakeResponse();
|
|
||||||
private final FakeClock clock = new FakeClock(DateTime.parse("2014-08-01T01:02:03Z"));
|
|
||||||
private final ExportSnapshotAction action = new ExportSnapshotAction();
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void before() {
|
|
||||||
action.clock = clock;
|
|
||||||
action.backupService = backupService;
|
|
||||||
action.response = response;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_launchesBackup_andEnqueuesPollTask() {
|
|
||||||
action.run();
|
|
||||||
verify(backupService)
|
|
||||||
.launchNewBackup(
|
|
||||||
ExportSnapshotAction.QUEUE,
|
|
||||||
"auto_snapshot_20140801_010203",
|
|
||||||
"registry-project-id-snapshots",
|
|
||||||
ExportConstants.getBackupKinds());
|
|
||||||
assertTasksEnqueued(
|
|
||||||
CheckSnapshotAction.QUEUE,
|
|
||||||
new TaskMatcher()
|
|
||||||
.url(CheckSnapshotAction.PATH)
|
|
||||||
.param(CHECK_SNAPSHOT_NAME_PARAM, "auto_snapshot_20140801_010203")
|
|
||||||
.param(
|
|
||||||
CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM,
|
|
||||||
Joiner.on(",").join(ExportConstants.getReportingKinds()))
|
|
||||||
.method("POST"));
|
|
||||||
assertThat(response.getPayload())
|
|
||||||
.isEqualTo("Datastore backup started with name: auto_snapshot_20140801_010203");
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,207 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.common.collect.Iterables.transform;
|
|
||||||
import static com.google.common.truth.Truth.assertThat;
|
|
||||||
import static google.registry.export.LoadSnapshotAction.LATEST_SNAPSHOT_VIEW_NAME;
|
|
||||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
|
|
||||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
|
|
||||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
|
|
||||||
import static google.registry.export.LoadSnapshotAction.PATH;
|
|
||||||
import static google.registry.export.LoadSnapshotAction.QUEUE;
|
|
||||||
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
|
|
||||||
import static google.registry.testing.JUnitBackports.assertThrows;
|
|
||||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
|
||||||
import static org.joda.time.DateTimeZone.UTC;
|
|
||||||
import static org.mockito.Matchers.any;
|
|
||||||
import static org.mockito.Matchers.eq;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.times;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import com.google.api.services.bigquery.Bigquery;
|
|
||||||
import com.google.api.services.bigquery.model.Dataset;
|
|
||||||
import com.google.api.services.bigquery.model.Job;
|
|
||||||
import com.google.api.services.bigquery.model.JobConfigurationLoad;
|
|
||||||
import com.google.api.services.bigquery.model.JobReference;
|
|
||||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import com.google.common.collect.Iterables;
|
|
||||||
import google.registry.bigquery.CheckedBigquery;
|
|
||||||
import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
|
|
||||||
import google.registry.request.HttpException.BadRequestException;
|
|
||||||
import google.registry.request.HttpException.InternalServerErrorException;
|
|
||||||
import google.registry.testing.AppEngineRule;
|
|
||||||
import google.registry.testing.FakeClock;
|
|
||||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.List;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.runner.RunWith;
|
|
||||||
import org.junit.runners.JUnit4;
|
|
||||||
import org.mockito.ArgumentCaptor;
|
|
||||||
|
|
||||||
/** Unit tests for {@link LoadSnapshotAction}. */
|
|
||||||
@RunWith(JUnit4.class)
|
|
||||||
public class LoadSnapshotActionTest {
|
|
||||||
|
|
||||||
@Rule
|
|
||||||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
|
||||||
.withTaskQueue()
|
|
||||||
.build();
|
|
||||||
private final CheckedBigquery checkedBigquery = mock(CheckedBigquery.class);
|
|
||||||
private final Bigquery bigquery = mock(Bigquery.class);
|
|
||||||
private final Bigquery.Jobs bigqueryJobs = mock(Bigquery.Jobs.class);
|
|
||||||
private final Bigquery.Jobs.Insert bigqueryJobsInsert = mock(Bigquery.Jobs.Insert.class);
|
|
||||||
private final Bigquery.Datasets bigqueryDatasets = mock(Bigquery.Datasets.class);
|
|
||||||
private final Bigquery.Datasets.Insert bigqueryDatasetsInsert =
|
|
||||||
mock(Bigquery.Datasets.Insert.class);
|
|
||||||
private final BigqueryPollJobEnqueuer bigqueryPollEnqueuer = mock(BigqueryPollJobEnqueuer.class);
|
|
||||||
|
|
||||||
private FakeClock clock = new FakeClock(new DateTime(1391096117045L, UTC));
|
|
||||||
private LoadSnapshotAction action;
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void before() throws Exception {
|
|
||||||
when(checkedBigquery.ensureDataSetExists("Project-Id", "snapshots")).thenReturn(bigquery);
|
|
||||||
when(bigquery.jobs()).thenReturn(bigqueryJobs);
|
|
||||||
when(bigqueryJobs.insert(eq("Project-Id"), any(Job.class))).thenReturn(bigqueryJobsInsert);
|
|
||||||
when(bigquery.datasets()).thenReturn(bigqueryDatasets);
|
|
||||||
when(bigqueryDatasets.insert(eq("Project-Id"), any(Dataset.class)))
|
|
||||||
.thenReturn(bigqueryDatasetsInsert);
|
|
||||||
action = new LoadSnapshotAction();
|
|
||||||
action.checkedBigquery = checkedBigquery;
|
|
||||||
action.bigqueryPollEnqueuer = bigqueryPollEnqueuer;
|
|
||||||
action.clock = clock;
|
|
||||||
action.projectId = "Project-Id";
|
|
||||||
action.snapshotFile = "gs://bucket/snapshot.backup_info";
|
|
||||||
action.snapshotId = "id12345";
|
|
||||||
action.snapshotKinds = "one,two,three";
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_enqueueLoadTask() {
|
|
||||||
enqueueLoadSnapshotTask(
|
|
||||||
"id12345", "gs://bucket/snapshot.backup_info", ImmutableSet.of("one", "two", "three"));
|
|
||||||
assertTasksEnqueued(
|
|
||||||
QUEUE,
|
|
||||||
new TaskMatcher()
|
|
||||||
.url(PATH)
|
|
||||||
.method("POST")
|
|
||||||
.param(LOAD_SNAPSHOT_ID_PARAM, "id12345")
|
|
||||||
.param(LOAD_SNAPSHOT_FILE_PARAM, "gs://bucket/snapshot.backup_info")
|
|
||||||
.param(LOAD_SNAPSHOT_KINDS_PARAM, "one,two,three"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_doPost() throws Exception {
|
|
||||||
action.run();
|
|
||||||
|
|
||||||
// Verify that checkedBigquery was called in a way that would create the dataset if it didn't
|
|
||||||
// already exist.
|
|
||||||
verify(checkedBigquery).ensureDataSetExists("Project-Id", "snapshots");
|
|
||||||
|
|
||||||
// Capture the load jobs we inserted to do additional checking on them.
|
|
||||||
ArgumentCaptor<Job> jobArgument = ArgumentCaptor.forClass(Job.class);
|
|
||||||
verify(bigqueryJobs, times(3)).insert(eq("Project-Id"), jobArgument.capture());
|
|
||||||
List<Job> jobs = jobArgument.getAllValues();
|
|
||||||
assertThat(jobs).hasSize(3);
|
|
||||||
|
|
||||||
// Check properties that should be common to all load jobs.
|
|
||||||
for (Job job : jobs) {
|
|
||||||
assertThat(job.getJobReference().getProjectId()).isEqualTo("Project-Id");
|
|
||||||
JobConfigurationLoad config = job.getConfiguration().getLoad();
|
|
||||||
assertThat(config.getSourceFormat()).isEqualTo("DATASTORE_BACKUP");
|
|
||||||
assertThat(config.getDestinationTable().getProjectId()).isEqualTo("Project-Id");
|
|
||||||
assertThat(config.getDestinationTable().getDatasetId()).isEqualTo("snapshots");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check the job IDs for each load job.
|
|
||||||
assertThat(transform(jobs, job -> job.getJobReference().getJobId()))
|
|
||||||
.containsExactly(
|
|
||||||
"load-snapshot-id12345-one-1391096117045",
|
|
||||||
"load-snapshot-id12345-two-1391096117045",
|
|
||||||
"load-snapshot-id12345-three-1391096117045");
|
|
||||||
|
|
||||||
// Check the source URI for each load job.
|
|
||||||
assertThat(
|
|
||||||
transform(
|
|
||||||
jobs,
|
|
||||||
job -> Iterables.getOnlyElement(job.getConfiguration().getLoad().getSourceUris())))
|
|
||||||
.containsExactly(
|
|
||||||
"gs://bucket/snapshot.one.backup_info",
|
|
||||||
"gs://bucket/snapshot.two.backup_info",
|
|
||||||
"gs://bucket/snapshot.three.backup_info");
|
|
||||||
|
|
||||||
// Check the destination table ID for each load job.
|
|
||||||
assertThat(
|
|
||||||
transform(
|
|
||||||
jobs, job -> job.getConfiguration().getLoad().getDestinationTable().getTableId()))
|
|
||||||
.containsExactly("id12345_one", "id12345_two", "id12345_three");
|
|
||||||
|
|
||||||
// Check that we executed the inserted jobs.
|
|
||||||
verify(bigqueryJobsInsert, times(3)).execute();
|
|
||||||
|
|
||||||
// Check that the poll tasks for each load job were enqueued.
|
|
||||||
verify(bigqueryPollEnqueuer)
|
|
||||||
.enqueuePollTask(
|
|
||||||
new JobReference()
|
|
||||||
.setProjectId("Project-Id")
|
|
||||||
.setJobId("load-snapshot-id12345-one-1391096117045"),
|
|
||||||
UpdateSnapshotViewAction.createViewUpdateTask(
|
|
||||||
"snapshots", "id12345_one", "one", LATEST_SNAPSHOT_VIEW_NAME),
|
|
||||||
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
|
|
||||||
verify(bigqueryPollEnqueuer)
|
|
||||||
.enqueuePollTask(
|
|
||||||
new JobReference()
|
|
||||||
.setProjectId("Project-Id")
|
|
||||||
.setJobId("load-snapshot-id12345-two-1391096117045"),
|
|
||||||
UpdateSnapshotViewAction.createViewUpdateTask(
|
|
||||||
"snapshots", "id12345_two", "two", LATEST_SNAPSHOT_VIEW_NAME),
|
|
||||||
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
|
|
||||||
verify(bigqueryPollEnqueuer)
|
|
||||||
.enqueuePollTask(
|
|
||||||
new JobReference()
|
|
||||||
.setProjectId("Project-Id")
|
|
||||||
.setJobId("load-snapshot-id12345-three-1391096117045"),
|
|
||||||
UpdateSnapshotViewAction.createViewUpdateTask(
|
|
||||||
"snapshots", "id12345_three", "three", LATEST_SNAPSHOT_VIEW_NAME),
|
|
||||||
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testFailure_doPost_badGcsFilename() {
|
|
||||||
action.snapshotFile = "gs://bucket/snapshot";
|
|
||||||
BadRequestException thrown = assertThrows(BadRequestException.class, action::run);
|
|
||||||
assertThat(thrown)
|
|
||||||
.hasMessageThat()
|
|
||||||
.contains("Error calling load snapshot: backup info file extension missing");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testFailure_doPost_bigqueryThrowsException() throws Exception {
|
|
||||||
when(bigqueryJobsInsert.execute()).thenThrow(new IOException("The Internet has gone missing"));
|
|
||||||
InternalServerErrorException thrown =
|
|
||||||
assertThrows(InternalServerErrorException.class, action::run);
|
|
||||||
assertThat(thrown)
|
|
||||||
.hasMessageThat()
|
|
||||||
.contains("Error loading snapshot: The Internet has gone missing");
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -7,7 +7,6 @@ PATH CLASS METHOD
|
||||||
/_dr/task/backupDatastore BackupDatastoreAction POST y INTERNAL APP IGNORED
|
/_dr/task/backupDatastore BackupDatastoreAction POST y INTERNAL APP IGNORED
|
||||||
/_dr/task/brdaCopy BrdaCopyAction POST y INTERNAL APP IGNORED
|
/_dr/task/brdaCopy BrdaCopyAction POST y INTERNAL APP IGNORED
|
||||||
/_dr/task/checkDatastoreBackup CheckBackupAction POST,GET y INTERNAL APP IGNORED
|
/_dr/task/checkDatastoreBackup CheckBackupAction POST,GET y INTERNAL APP IGNORED
|
||||||
/_dr/task/checkSnapshot CheckSnapshotAction POST,GET y INTERNAL APP IGNORED
|
|
||||||
/_dr/task/copyDetailReports CopyDetailReportsAction POST n INTERNAL,API APP ADMIN
|
/_dr/task/copyDetailReports CopyDetailReportsAction POST n INTERNAL,API APP ADMIN
|
||||||
/_dr/task/deleteContactsAndHosts DeleteContactsAndHostsAction GET n INTERNAL APP IGNORED
|
/_dr/task/deleteContactsAndHosts DeleteContactsAndHostsAction GET n INTERNAL APP IGNORED
|
||||||
/_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n INTERNAL APP IGNORED
|
/_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n INTERNAL APP IGNORED
|
||||||
|
@ -18,7 +17,6 @@ PATH CLASS METHOD
|
||||||
/_dr/task/exportDomainLists ExportDomainListsAction POST n INTERNAL APP IGNORED
|
/_dr/task/exportDomainLists ExportDomainListsAction POST n INTERNAL APP IGNORED
|
||||||
/_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n INTERNAL APP IGNORED
|
/_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n INTERNAL APP IGNORED
|
||||||
/_dr/task/exportReservedTerms ExportReservedTermsAction POST n INTERNAL APP IGNORED
|
/_dr/task/exportReservedTerms ExportReservedTermsAction POST n INTERNAL APP IGNORED
|
||||||
/_dr/task/exportSnapshot ExportSnapshotAction POST y INTERNAL APP IGNORED
|
|
||||||
/_dr/task/generateInvoices GenerateInvoicesAction POST n INTERNAL APP IGNORED
|
/_dr/task/generateInvoices GenerateInvoicesAction POST n INTERNAL APP IGNORED
|
||||||
/_dr/task/generateSpec11 GenerateSpec11ReportAction POST n INTERNAL APP IGNORED
|
/_dr/task/generateSpec11 GenerateSpec11ReportAction POST n INTERNAL APP IGNORED
|
||||||
/_dr/task/icannReportingStaging IcannReportingStagingAction POST n INTERNAL APP IGNORED
|
/_dr/task/icannReportingStaging IcannReportingStagingAction POST n INTERNAL APP IGNORED
|
||||||
|
@ -27,7 +25,6 @@ PATH CLASS METHOD
|
||||||
/_dr/task/importRdeDomains RdeDomainImportAction GET n INTERNAL APP IGNORED
|
/_dr/task/importRdeDomains RdeDomainImportAction GET n INTERNAL APP IGNORED
|
||||||
/_dr/task/importRdeHosts RdeHostImportAction GET n INTERNAL APP IGNORED
|
/_dr/task/importRdeHosts RdeHostImportAction GET n INTERNAL APP IGNORED
|
||||||
/_dr/task/linkRdeHosts RdeHostLinkAction GET n INTERNAL APP IGNORED
|
/_dr/task/linkRdeHosts RdeHostLinkAction GET n INTERNAL APP IGNORED
|
||||||
/_dr/task/loadSnapshot LoadSnapshotAction POST n INTERNAL APP IGNORED
|
|
||||||
/_dr/task/nordnUpload NordnUploadAction POST y INTERNAL APP IGNORED
|
/_dr/task/nordnUpload NordnUploadAction POST y INTERNAL APP IGNORED
|
||||||
/_dr/task/nordnVerify NordnVerifyAction POST y INTERNAL APP IGNORED
|
/_dr/task/nordnVerify NordnVerifyAction POST y INTERNAL APP IGNORED
|
||||||
/_dr/task/pollBigqueryJob BigqueryPollJobAction GET,POST y INTERNAL APP IGNORED
|
/_dr/task/pollBigqueryJob BigqueryPollJobAction GET,POST y INTERNAL APP IGNORED
|
||||||
|
|
Loading…
Add table
Reference in a new issue