zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-08 15:58:21 +02:00
Code Issues Projects Releases Packages Wiki Activity

Remove deprecated Datastore backup code

Browse source 
Removed three Action classes and the CheckSnapshot command.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=230545631
This commit is contained in:
weiminyu 2019-01-23 09:39:26 -08:00 • committed by jianglai
parent 701ebc6a28
commit acbd23fa64
20 changed files with 14 additions and 1527 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
java/google/registry/config/RegistryConfig.java
View file

@ -1362,15 +1362,6 @@ public final class RegistryConfig {
return CONFIG_SETTINGS.get().appEngine.projectId; return CONFIG_SETTINGS.get().appEngine.projectId;
} }
/**
* Returns the Google Cloud Storage bucket for storing backup snapshots.
*
* @see google.registry.export.ExportSnapshotAction
*/
public static String getSnapshotsBucket() {
return getProjectId() + "-snapshots";
}
/** /**
* Returns the Google Cloud Storage bucket for storing Datastore backups. * Returns the Google Cloud Storage bucket for storing Datastore backups.
* *

18
java/google/registry/env/common/backend/WEB-INF/web.xml vendored
View file

@ -192,24 +192,6 @@
<url-pattern>/_dr/task/uploadDatastoreBackup</url-pattern> <url-pattern>/_dr/task/uploadDatastoreBackup</url-pattern>
</servlet-mapping> </servlet-mapping>
<!-- Exports a Datastore backup snapshot to GCS. -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>
<url-pattern>/_dr/task/exportSnapshot</url-pattern>
</servlet-mapping>
<!-- Checks the completion of a Datastore backup snapshot. -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>
<url-pattern>/_dr/task/checkSnapshot</url-pattern>
</servlet-mapping>
<!-- Loads a Datastore backup snapshot into BigQuery. -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>
<url-pattern>/_dr/task/loadSnapshot</url-pattern>
</servlet-mapping>
<!-- Updates a view to point at a certain snapshot in BigQuery. --> <!-- Updates a view to point at a certain snapshot in BigQuery. -->
<servlet-mapping> <servlet-mapping>
<servlet-name>backend-servlet</servlet-name> <servlet-name>backend-servlet</servlet-name>

4
java/google/registry/export/BackupDatastoreAction.java
View file

@ -28,9 +28,7 @@ import google.registry.request.auth.Auth;
import javax.inject.Inject; import javax.inject.Inject;
/** /**
* Action to trigger a Datastore backup job that writes a snapshot to Google Cloud Storage. This * Action to trigger a Datastore backup job that writes a snapshot to Google Cloud Storage.
* class is introduced as an experimental feature, and will eventually replace {@link
* ExportSnapshotAction}.
* *
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling * <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
* the next upon successful completion: * the next upon successful completion:

163
java/google/registry/export/CheckSnapshotAction.java
View file

@ -1,163 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.common.collect.Sets.intersection;
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
import static google.registry.request.Action.Method.GET;
import static google.registry.request.Action.Method.POST;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.flogger.FluentLogger;
import google.registry.export.DatastoreBackupInfo.BackupStatus;
import google.registry.request.Action;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.HttpException.NoContentException;
import google.registry.request.HttpException.NotModifiedException;
import google.registry.request.Parameter;
import google.registry.request.RequestMethod;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import java.util.Set;
import javax.inject.Inject;
import org.joda.time.Duration;
import org.joda.time.PeriodType;
import org.joda.time.format.PeriodFormat;
/**
* Action that checks the status of a snapshot, and if complete, trigger loading it into BigQuery.
*/
@Action(
service = Action.Service.BACKEND,
path = CheckSnapshotAction.PATH,
method = {POST, GET},
automaticallyPrintOk = true,
auth = Auth.AUTH_INTERNAL_ONLY)
public class CheckSnapshotAction implements Runnable {
/** Parameter names for passing parameters into this action. */
static final String CHECK_SNAPSHOT_NAME_PARAM = "name";
static final String CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";
/** Action-specific details needed for enqueuing tasks against itself. */
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml.
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
/** The maximum amount of time we allow a backup to run before abandoning it. */
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@Inject Response response;
@Inject @RequestMethod Action.Method requestMethod;
@Inject DatastoreBackupService backupService;
@Inject @Parameter(CHECK_SNAPSHOT_NAME_PARAM) String snapshotName;
@Inject @Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM) String kindsToLoadParam;
@Inject CheckSnapshotAction() {}
@Override
public void run() {
if (requestMethod == POST) {
checkAndLoadSnapshotIfComplete();
} else {
// This is a GET request.
response.setPayload(getBackup().getInformation());
}
}
private DatastoreBackupInfo getBackup() {
try {
return backupService.findByName(snapshotName);
} catch (IllegalArgumentException e) {
String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be
// retried but we might abandon backups that start late and haven't yet written to
// Datastore. We could fix that by replacing this with a two-phase polling strategy.
throw new BadRequestException(message, e);
}
}
private void checkAndLoadSnapshotIfComplete() {
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
DatastoreBackupInfo backup = getBackup();
// Stop now if the backup is not complete.
if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
Duration runningTime = backup.getRunningTime();
if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
// Backup might still be running, so send a 304 to have the task retry.
throw new NotModifiedException(
String.format("Datastore backup %s still pending", snapshotName));
} else {
// Declare the backup a lost cause, and send 204 No Content so the task will
// not be retried.
String message =
String.format(
"Datastore backup %s abandoned - not complete after %s",
snapshotName,
PeriodFormat.getDefault()
.print(
runningTime
.toPeriod()
.normalizedStandard(PeriodType.dayTime().withMillisRemoved())));
throw new NoContentException(message);
}
}
// Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
// suffix out of the snapshot name and falling back to the start time as a string.
String snapshotId =
snapshotName.startsWith(ExportSnapshotAction.SNAPSHOT_PREFIX)
? snapshotName.substring(ExportSnapshotAction.SNAPSHOT_PREFIX.length())
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
if (!backup.getKinds().containsAll(kindsToLoad)) {
logger.atWarning().log(
"Kinds to load included non-exported kinds: %s",
Sets.difference(kindsToLoad, backup.getKinds()));
}
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
ImmutableSet<String> exportedKindsToLoad =
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
String message = String.format("Datastore backup %s complete - ", snapshotName);
if (exportedKindsToLoad.isEmpty()) {
message += "no kinds to load into BigQuery";
} else {
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
message += "BigQuery load task enqueued";
}
logger.atInfo().log(message);
response.setPayload(message);
}
/** Enqueue a poll task to monitor the named snapshot for completion. */
static TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
return QueueFactory.getQueue(QUEUE)
.add(
TaskOptions.Builder.withUrl(PATH)
.method(Method.POST)
.countdownMillis(POLL_COUNTDOWN.getMillis())
.param(CHECK_SNAPSHOT_NAME_PARAM, snapshotName)
.param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
}
}

149
java/google/registry/export/DatastoreBackupInfo.java
View file

@ -1,149 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.joda.time.DateTimeZone.UTC;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.Text;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Ascii;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
import google.registry.util.Clock;
import google.registry.util.NonFinalForTesting;
import google.registry.util.SystemClock;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import org.joda.time.DateTime;
import org.joda.time.Duration;
/** Container for information about a Datastore backup. */
public class DatastoreBackupInfo {
@NonFinalForTesting
private static Clock clock = new SystemClock();
/** The possible status values for a Datastore backup. */
public enum BackupStatus { PENDING, COMPLETE }
/** The name of the Datastore backup. */
private final String backupName;
/** The entity kinds included in this Datastore backup. */
private final ImmutableSet<String> kinds;
/** The start time of the Datastore backup. */
private final DateTime startTime;
/** The completion time of the Datastore backup, present if it has completed. */
private final Optional<DateTime> completeTime;
/**
* The GCS filename to which the backup's top-level .backup_info manifest file has been written,
* present if the backup has completed.
*/
private final Optional<String> gcsFilename;
/** DatastoreBackupInfo instances should only be obtained via DatastoreBackupService. */
DatastoreBackupInfo(Entity backupEntity) {
backupName = (String) checkNotNull(backupEntity.getProperty("name"), "name");
@SuppressWarnings("unchecked")
List<String> rawKinds = (List<String>) checkNotNull(backupEntity.getProperty("kinds"), "kinds");
Date rawStartTime = (Date) checkNotNull(backupEntity.getProperty("start_time"), "start_time");
Date rawCompleteTime = (Date) backupEntity.getProperty("complete_time");
Text rawGcsFilename = (Text) backupEntity.getProperty("gs_handle");
kinds = ImmutableSet.copyOf(rawKinds);
startTime = new DateTime(rawStartTime).withZone(UTC);
completeTime = Optional.ofNullable(
rawCompleteTime == null ? null : new DateTime(rawCompleteTime).withZone(UTC));
gcsFilename = Optional.ofNullable(
rawGcsFilename == null ? null : gcsPathToUri(rawGcsFilename.getValue()));
}
/** This constructor is only exposed for test purposes. */
@VisibleForTesting
DatastoreBackupInfo(
String backupName,
DateTime startTime,
Optional<DateTime> completeTime,
ImmutableSet<String> kinds,
Optional<String> gcsFilename) {
this.backupName = backupName;
this.startTime = startTime;
this.completeTime = completeTime;
this.kinds = kinds;
this.gcsFilename = gcsFilename;
}
/**
* Rewrite a GCS path as stored by Datastore Admin (with a "/gs/" prefix) to the more standard
* URI format that uses a "gs://" scheme prefix.
*/
private static String gcsPathToUri(String backupGcsPath) {
checkArgument(backupGcsPath.startsWith("/gs/"), "GCS path not in expected format");
return backupGcsPath.replaceFirst("/gs/", "gs://");
}
public String getName() {
return backupName;
}
public ImmutableSet<String> getKinds() {
return kinds;
}
public BackupStatus getStatus() {
return completeTime.isPresent() ? BackupStatus.COMPLETE : BackupStatus.PENDING;
}
public DateTime getStartTime() {
return startTime;
}
public Optional<DateTime> getCompleteTime() {
return completeTime;
}
/**
* Returns the length of time the backup ran for (if completed) or the length of time since the
* backup started (if it has not completed).
*/
public Duration getRunningTime() {
return new Duration(startTime, completeTime.orElse(clock.nowUtc()));
}
public Optional<String> getGcsFilename() {
return gcsFilename;
}
/** Returns a string version of key information about the backup. */
public String getInformation() {
return Joiner.on('\n')
.join(
"Backup name: " + backupName,
"Status: " + getStatus(),
"Started: " + startTime,
"Ended: " + completeTime.orElse(null),
"Duration: " + Ascii.toLowerCase(getRunningTime().toPeriod().toString().substring(2)),
"GCS: " + gcsFilename.orElse(null),
"Kinds: " + kinds,
"");
}
}

103
java/google/registry/export/DatastoreBackupService.java
View file

@ -1,103 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.common.base.Strings.nullToEmpty;
import static com.google.common.collect.ImmutableList.toImmutableList;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Streams;
import google.registry.util.AppEngineServiceUtils;
import java.util.NoSuchElementException;
import javax.inject.Inject;
/** An object providing methods for starting and querying Datastore backups. */
public class DatastoreBackupService {
/** The internal kind name used for entities storing information about Datastore backups. */
static final String BACKUP_INFO_KIND = "_AE_Backup_Information";
/** The name of the app version used for hosting the Datastore Admin functionality. */
static final String DATASTORE_ADMIN_VERSION_NAME = "ah-builtin-python-bundle";
private final AppEngineServiceUtils appEngineServiceUtils;
@Inject
public DatastoreBackupService(AppEngineServiceUtils appEngineServiceUtils) {
this.appEngineServiceUtils = appEngineServiceUtils;
}
/**
* Generates the TaskOptions needed to trigger an AppEngine Datastore backup job.
*
* @see <a href="https://developers.google.com/appengine/articles/scheduled_backups">Scheduled Backups</a>
*/
private TaskOptions makeTaskOptions(
String queue, String name, String gcsBucket, ImmutableSet<String> kinds) {
String hostname =
appEngineServiceUtils.getVersionHostname("default", DATASTORE_ADMIN_VERSION_NAME);
TaskOptions options = TaskOptions.Builder.withUrl("/_ah/datastore_admin/backup.create")
.header("Host", hostname)
.method(Method.GET)
.param("name", name + "_") // Add underscore since the name will be used as a prefix.
.param("filesystem", "gs")
.param("gs_bucket_name", gcsBucket)
.param("queue", queue);
for (String kind : kinds) {
options.param("kind", kind);
}
return options;
}
/**
* Launches a new Datastore backup with the given name, GCS bucket, and set of kinds by
* submitting a task to the given task queue, and returns a handle to that task.
*/
public TaskHandle launchNewBackup(
String queue, String name, String gcsBucket, ImmutableSet<String> kinds) {
return getQueue(queue).add(makeTaskOptions(queue, name, gcsBucket, kinds));
}
/** Return an iterable of all Datastore backups whose names have the given string prefix. */
public Iterable<DatastoreBackupInfo> findAllByNamePrefix(final String namePrefix) {
// Need the raw DatastoreService to access the internal _AE_Backup_Information entities.
// TODO(b/19081037): make an Objectify entity class for these raw Datastore entities instead.
return Streams.stream(getDatastoreService().prepare(new Query(BACKUP_INFO_KIND)).asIterable())
.filter(entity -> nullToEmpty((String) entity.getProperty("name")).startsWith(namePrefix))
.map(DatastoreBackupInfo::new)
.collect(toImmutableList());
}
/**
* Return a single DatastoreBackup that uniquely matches this name prefix. Throws an IAE
* if no backups match or if more than one backup matches.
*/
public DatastoreBackupInfo findByName(final String namePrefix) {
try {
return Iterables.getOnlyElement(findAllByNamePrefix(namePrefix));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("More than one backup with name prefix " + namePrefix, e);
} catch (NoSuchElementException e) {
throw new IllegalArgumentException("No backup found with name prefix " + namePrefix, e);
}
}
}

31
java/google/registry/export/ExportRequestModule.java
View file

@ -17,16 +17,15 @@ package google.registry.export;
import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER; import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER; import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER; import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM; import static google.registry.export.CheckBackupAction.CHECK_BACKUP_KINDS_TO_LOAD_PARAM;
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM; import static google.registry.export.CheckBackupAction.CHECK_BACKUP_NAME_PARAM;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM;
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM;
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM;
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM;
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_FOLDER_PARAM; import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_FOLDER_PARAM;
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_ID_PARAM;
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_KINDS_PARAM;
import static google.registry.request.RequestParameters.extractRequiredHeader; import static google.registry.request.RequestParameters.extractRequiredHeader;
import static google.registry.request.RequestParameters.extractRequiredParameter; import static google.registry.request.RequestParameters.extractRequiredParameter;
@ -64,12 +63,6 @@ public final class ExportRequestModule {
return extractRequiredParameter(req, UPDATE_SNAPSHOT_VIEWNAME_PARAM); return extractRequiredParameter(req, UPDATE_SNAPSHOT_VIEWNAME_PARAM);
} }
@Provides
@Parameter(LOAD_SNAPSHOT_FILE_PARAM)
static String provideLoadSnapshotFile(HttpServletRequest req) {
return extractRequiredParameter(req, LOAD_SNAPSHOT_FILE_PARAM);
}
@Provides @Provides
@Parameter(UPLOAD_BACKUP_FOLDER_PARAM) @Parameter(UPLOAD_BACKUP_FOLDER_PARAM)
static String provideSnapshotUrlPrefix(HttpServletRequest req) { static String provideSnapshotUrlPrefix(HttpServletRequest req) {
@ -77,27 +70,27 @@ public final class ExportRequestModule {
} }
@Provides @Provides
@Parameter(LOAD_SNAPSHOT_ID_PARAM) @Parameter(UPLOAD_BACKUP_ID_PARAM)
static String provideLoadSnapshotId(HttpServletRequest req) { static String provideLoadSnapshotId(HttpServletRequest req) {
return extractRequiredParameter(req, LOAD_SNAPSHOT_ID_PARAM); return extractRequiredParameter(req, UPLOAD_BACKUP_ID_PARAM);
} }
@Provides @Provides
@Parameter(LOAD_SNAPSHOT_KINDS_PARAM) @Parameter(UPLOAD_BACKUP_KINDS_PARAM)
static String provideLoadSnapshotKinds(HttpServletRequest req) { static String provideLoadSnapshotKinds(HttpServletRequest req) {
return extractRequiredParameter(req, LOAD_SNAPSHOT_KINDS_PARAM); return extractRequiredParameter(req, UPLOAD_BACKUP_KINDS_PARAM);
} }
@Provides @Provides
@Parameter(CHECK_SNAPSHOT_NAME_PARAM) @Parameter(CHECK_BACKUP_NAME_PARAM)
static String provideCheckSnapshotName(HttpServletRequest req) { static String provideCheckSnapshotName(HttpServletRequest req) {
return extractRequiredParameter(req, CHECK_SNAPSHOT_NAME_PARAM); return extractRequiredParameter(req, CHECK_BACKUP_NAME_PARAM);
} }
@Provides @Provides
@Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM) @Parameter(CHECK_BACKUP_KINDS_TO_LOAD_PARAM)
static String provideCheckSnapshotKindsToLoad(HttpServletRequest req) { static String provideCheckSnapshotKindsToLoad(HttpServletRequest req) {
return extractRequiredParameter(req, CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM); return extractRequiredParameter(req, CHECK_BACKUP_KINDS_TO_LOAD_PARAM);
} }
@Provides @Provides

78
java/google/registry/export/ExportSnapshotAction.java
View file

@ -1,78 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static google.registry.export.CheckSnapshotAction.enqueuePollTask;
import static google.registry.request.Action.Method.POST;
import com.google.common.flogger.FluentLogger;
import google.registry.config.RegistryConfig;
import google.registry.request.Action;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.Clock;
import javax.inject.Inject;
/**
* Action to trigger a Datastore backup job that writes a snapshot to Google Cloud Storage.
*
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
* the next upon successful completion:
*
* <ol>
* <li>The snapshot is exported to Google Cloud Storage (this action).
* <li>The {@link CheckSnapshotAction} polls until the export is completed.
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_datastore_export.
* </ol>
*/
@Action(
service = Action.Service.BACKEND,
path = ExportSnapshotAction.PATH,
method = POST,
automaticallyPrintOk = true,
auth = Auth.AUTH_INTERNAL_ONLY)
public class ExportSnapshotAction implements Runnable {
/** Queue to use for enqueuing the task that will actually launch the backup. */
static final String QUEUE = "export-snapshot"; // See queue.xml.
static final String PATH = "/_dr/task/exportSnapshot"; // See web.xml.
/** Prefix to use for naming all snapshots that are started by this servlet. */
static final String SNAPSHOT_PREFIX = "auto_snapshot_";
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@Inject Clock clock;
@Inject DatastoreBackupService backupService;
@Inject Response response;
@Inject
ExportSnapshotAction() {}
@Override
public void run() {
// Use a unique name for the snapshot so we can explicitly check its completion later.
String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss");
backupService.launchNewBackup(
QUEUE, snapshotName, RegistryConfig.getSnapshotsBucket(), ExportConstants.getBackupKinds());
// Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery.
enqueuePollTask(snapshotName, ExportConstants.getReportingKinds());
String message = "Datastore backup started with name: " + snapshotName;
logger.atInfo().log(message);
response.setPayload(message);
}
}

166
java/google/registry/export/LoadSnapshotAction.java
View file

@ -1,166 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.export.UpdateSnapshotViewAction.createViewUpdateTask;
import static google.registry.request.Action.Method.POST;
import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.bigquery.model.Job;
import com.google.api.services.bigquery.model.JobConfiguration;
import com.google.api.services.bigquery.model.JobConfigurationLoad;
import com.google.api.services.bigquery.model.JobReference;
import com.google.api.services.bigquery.model.TableReference;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.flogger.FluentLogger;
import google.registry.bigquery.BigqueryUtils.SourceFormat;
import google.registry.bigquery.BigqueryUtils.WriteDisposition;
import google.registry.bigquery.CheckedBigquery;
import google.registry.config.RegistryConfig.Config;
import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
import google.registry.request.Action;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.HttpException.InternalServerErrorException;
import google.registry.request.Parameter;
import google.registry.request.auth.Auth;
import google.registry.util.Clock;
import java.io.IOException;
import javax.inject.Inject;
import org.joda.time.DateTime;
/** Action to load a Datastore snapshot from Google Cloud Storage into BigQuery. */
@Action(
service = Action.Service.BACKEND,
path = LoadSnapshotAction.PATH,
method = POST,
auth = Auth.AUTH_INTERNAL_ONLY)
public class LoadSnapshotAction implements Runnable {
/** Parameter names for passing parameters into the servlet. */
static final String LOAD_SNAPSHOT_ID_PARAM = "id";
static final String LOAD_SNAPSHOT_FILE_PARAM = "file";
static final String LOAD_SNAPSHOT_KINDS_PARAM = "kinds";
static final String SNAPSHOTS_DATASET = "snapshots";
static final String LATEST_SNAPSHOT_VIEW_NAME = "latest_datastore_export";
/** Servlet-specific details needed for enqueuing tasks against itself. */
static final String QUEUE = "export-snapshot"; // See queue.xml.
static final String PATH = "/_dr/task/loadSnapshot"; // See web.xml.
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@Inject CheckedBigquery checkedBigquery;
@Inject BigqueryPollJobEnqueuer bigqueryPollEnqueuer;
@Inject Clock clock;
@Inject @Config("projectId") String projectId;
@Inject @Parameter(LOAD_SNAPSHOT_FILE_PARAM) String snapshotFile;
@Inject @Parameter(LOAD_SNAPSHOT_ID_PARAM) String snapshotId;
@Inject @Parameter(LOAD_SNAPSHOT_KINDS_PARAM) String snapshotKinds;
@Inject LoadSnapshotAction() {}
/** Enqueue a task for starting a backup load. */
public static TaskHandle enqueueLoadSnapshotTask(
String snapshotId, String gcsFile, ImmutableSet<String> kinds) {
return getQueue(QUEUE).add(
TaskOptions.Builder.withUrl(PATH)
.method(Method.POST)
.param(LOAD_SNAPSHOT_ID_PARAM, snapshotId)
.param(LOAD_SNAPSHOT_FILE_PARAM, gcsFile)
.param(LOAD_SNAPSHOT_KINDS_PARAM, Joiner.on(',').join(kinds)));
}
@Override
public void run() {
try {
String message =
loadSnapshot(snapshotId, snapshotFile, Splitter.on(',').split(snapshotKinds));
logger.atInfo().log("Loaded snapshot successfully: %s", message);
} catch (Throwable e) {
logger.atSevere().withCause(e).log("Error loading snapshot");
if (e instanceof IllegalArgumentException) {
throw new BadRequestException("Error calling load snapshot: " + e.getMessage(), e);
} else {
throw new InternalServerErrorException(
"Error loading snapshot: " + firstNonNull(e.getMessage(), e.toString()));
}
}
}
private String loadSnapshot(String snapshotId, String gcsFilename, Iterable<String> kinds)
throws IOException {
Bigquery bigquery = checkedBigquery.ensureDataSetExists(projectId, SNAPSHOTS_DATASET);
DateTime now = clock.nowUtc();
String loadMessage =
String.format("Loading Datastore snapshot %s from %s...", snapshotId, gcsFilename);
logger.atInfo().log(loadMessage);
StringBuilder builder = new StringBuilder(loadMessage + "\n");
builder.append("Load jobs:\n");
for (String kindName : kinds) {
String jobId = String.format("load-snapshot-%s-%s-%d", snapshotId, kindName, now.getMillis());
JobReference jobRef = new JobReference().setProjectId(projectId).setJobId(jobId);
String sourceUri = getBackupInfoFileForKind(gcsFilename, kindName);
String tableId = String.format("%s_%s", snapshotId, kindName);
// Launch the load job.
Job job = makeLoadJob(jobRef, sourceUri, tableId);
bigquery.jobs().insert(projectId, job).execute();
// Enqueue a task to check on the load job's completion, and if it succeeds, to update a
// well-known view in BigQuery to point at the newly loaded snapshot table for this kind.
bigqueryPollEnqueuer.enqueuePollTask(
jobRef,
createViewUpdateTask(SNAPSHOTS_DATASET, tableId, kindName, LATEST_SNAPSHOT_VIEW_NAME),
getQueue(UpdateSnapshotViewAction.QUEUE));
builder.append(String.format(" - %s:%s\n", projectId, jobId));
logger.atInfo().log("Submitted load job %s:%s", projectId, jobId);
}
return builder.toString();
}
private static String getBackupInfoFileForKind(String backupInfoFile, String kindName) {
String extension = ".backup_info";
checkArgument(backupInfoFile.endsWith(extension), "backup info file extension missing");
String prefix = backupInfoFile.substring(0, backupInfoFile.length() - extension.length());
return Joiner.on('.').join(prefix, kindName, extension.substring(1));
}
private Job makeLoadJob(JobReference jobRef, String sourceUri, String tableId) {
TableReference tableReference = new TableReference()
.setProjectId(jobRef.getProjectId())
.setDatasetId(SNAPSHOTS_DATASET)
.setTableId(tableId);
return new Job()
.setJobReference(jobRef)
.setConfiguration(new JobConfiguration()
.setLoad(new JobConfigurationLoad()
.setWriteDisposition(WriteDisposition.WRITE_EMPTY.toString())
.setSourceFormat(SourceFormat.DATASTORE_BACKUP.toString())
.setSourceUris(ImmutableList.of(sourceUri))
.setDestinationTable(tableReference)));
}
}

3
java/google/registry/export/UpdateSnapshotViewAction.java
View file

@ -50,8 +50,7 @@ public class UpdateSnapshotViewAction implements Runnable {
static final String UPDATE_SNAPSHOT_VIEWNAME_PARAM = "viewname"; static final String UPDATE_SNAPSHOT_VIEWNAME_PARAM = "viewname";
/** Servlet-specific details needed for enqueuing tasks against itself. */ /** Servlet-specific details needed for enqueuing tasks against itself. */
// For now this queue is shared by the backup workflows started by both ExportSnapshotAction // For now this queue is shared by the backup workflows started by BackupDatastoreAction.
// and BackupDatastoreAction.
// TODO(weiminyu): update queue name (snapshot->backup) after ExportSnapshot flow is removed. // TODO(weiminyu): update queue name (snapshot->backup) after ExportSnapshot flow is removed.
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml. static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.

6
java/google/registry/module/backend/BackendRequestComponent.java
View file

@ -42,13 +42,10 @@ import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule;
import google.registry.export.BackupDatastoreAction; import google.registry.export.BackupDatastoreAction;
import google.registry.export.BigqueryPollJobAction; import google.registry.export.BigqueryPollJobAction;
import google.registry.export.CheckBackupAction; import google.registry.export.CheckBackupAction;
import google.registry.export.CheckSnapshotAction;
import google.registry.export.ExportDomainListsAction; import google.registry.export.ExportDomainListsAction;
import google.registry.export.ExportPremiumTermsAction; import google.registry.export.ExportPremiumTermsAction;
import google.registry.export.ExportRequestModule; import google.registry.export.ExportRequestModule;
import google.registry.export.ExportReservedTermsAction; import google.registry.export.ExportReservedTermsAction;
import google.registry.export.ExportSnapshotAction;
import google.registry.export.LoadSnapshotAction;
import google.registry.export.SyncGroupMembersAction; import google.registry.export.SyncGroupMembersAction;
import google.registry.export.UpdateSnapshotViewAction; import google.registry.export.UpdateSnapshotViewAction;
import google.registry.export.UploadDatastoreBackupAction; import google.registry.export.UploadDatastoreBackupAction;
@ -121,7 +118,6 @@ interface BackendRequestComponent {
BigqueryPollJobAction bigqueryPollJobAction(); BigqueryPollJobAction bigqueryPollJobAction();
BrdaCopyAction brdaCopyAction(); BrdaCopyAction brdaCopyAction();
CheckBackupAction checkBackupAction(); CheckBackupAction checkBackupAction();
CheckSnapshotAction checkSnapshotAction();
CommitLogCheckpointAction commitLogCheckpointAction(); CommitLogCheckpointAction commitLogCheckpointAction();
CommitLogFanoutAction commitLogFanoutAction(); CommitLogFanoutAction commitLogFanoutAction();
CopyDetailReportsAction copyDetailReportAction(); CopyDetailReportsAction copyDetailReportAction();
@ -134,12 +130,10 @@ interface BackendRequestComponent {
ExportDomainListsAction exportDomainListsAction(); ExportDomainListsAction exportDomainListsAction();
ExportPremiumTermsAction exportPremiumTermsAction(); ExportPremiumTermsAction exportPremiumTermsAction();
ExportReservedTermsAction exportReservedTermsAction(); ExportReservedTermsAction exportReservedTermsAction();
ExportSnapshotAction exportSnapshotAction();
GenerateInvoicesAction generateInvoicesAction(); GenerateInvoicesAction generateInvoicesAction();
GenerateSpec11ReportAction generateSpec11ReportAction(); GenerateSpec11ReportAction generateSpec11ReportAction();
IcannReportingStagingAction icannReportingStagingAction(); IcannReportingStagingAction icannReportingStagingAction();
IcannReportingUploadAction icannReportingUploadAction(); IcannReportingUploadAction icannReportingUploadAction();
LoadSnapshotAction loadSnapshotAction();
NordnUploadAction nordnUploadAction(); NordnUploadAction nordnUploadAction();
NordnVerifyAction nordnVerifyAction(); NordnVerifyAction nordnVerifyAction();
PublishDnsUpdatesAction publishDnsUpdatesAction(); PublishDnsUpdatesAction publishDnsUpdatesAction();

49
java/google/registry/tools/CheckSnapshotCommand.java
View file

@ -1,49 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.collect.Iterables;
import com.google.common.collect.Streams;
import google.registry.export.DatastoreBackupInfo;
import google.registry.export.DatastoreBackupService;
import javax.inject.Inject;
/**
* Command to check the status of a Datastore backup, or "snapshot".
*/
@Parameters(separators = " =", commandDescription = "Check the status of a Datastore snapshot")
public class CheckSnapshotCommand implements CommandWithRemoteApi {
@Parameter(
names = {"-s", "--snapshot"},
description = "Unique prefix of the snapshot to check",
required = true)
private String snapshotName;
@Inject DatastoreBackupService datastoreBackupService;
@Override
public void run() {
Iterable<DatastoreBackupInfo> backups =
datastoreBackupService.findAllByNamePrefix(snapshotName);
if (Iterables.isEmpty(backups)) {
System.err.println("No snapshot found with name: " + snapshotName);
return;
}
Streams.stream(backups).map(DatastoreBackupInfo::getInformation).forEach(System.out::println);
}
}

1
java/google/registry/tools/RegistryTool.java
View file

@ -32,7 +32,6 @@ public final class RegistryTool {
.put("canonicalize_labels", CanonicalizeLabelsCommand.class) .put("canonicalize_labels", CanonicalizeLabelsCommand.class)
.put("check_domain", CheckDomainCommand.class) .put("check_domain", CheckDomainCommand.class)
.put("check_domain_claims", CheckDomainClaimsCommand.class) .put("check_domain_claims", CheckDomainClaimsCommand.class)
.put("check_snapshot", CheckSnapshotCommand.class)
.put("convert_idn", ConvertIdnCommand.class) .put("convert_idn", ConvertIdnCommand.class)
.put("count_domains", CountDomainsCommand.class) .put("count_domains", CountDomainsCommand.class)
.put("create_anchor_tenant", CreateAnchorTenantCommand.class) .put("create_anchor_tenant", CreateAnchorTenantCommand.class)

1
java/google/registry/tools/RegistryToolComponent.java
View file

@ -75,7 +75,6 @@ import javax.inject.Singleton;
interface RegistryToolComponent { interface RegistryToolComponent {
void inject(CheckDomainClaimsCommand command); void inject(CheckDomainClaimsCommand command);
void inject(CheckDomainCommand command); void inject(CheckDomainCommand command);
void inject(CheckSnapshotCommand command);
void inject(CountDomainsCommand command); void inject(CountDomainsCommand command);
void inject(CreateAnchorTenantCommand command); void inject(CreateAnchorTenantCommand command);
void inject(CreateCdnsTld command); void inject(CreateCdnsTld command);

221
javatests/google/registry/export/CheckSnapshotActionTest.java
View file

@ -1,221 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
import static google.registry.testing.JUnitBackports.assertThrows;
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import google.registry.request.Action.Method;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.HttpException.NoContentException;
import google.registry.request.HttpException.NotModifiedException;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeResponse;
import google.registry.testing.InjectRule;
import google.registry.testing.TaskQueueHelper.TaskMatcher;
import java.util.Optional;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link CheckSnapshotAction}. */
@RunWith(JUnit4.class)
public class CheckSnapshotActionTest {
static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
static final DateTime COMPLETE_TIME = START_TIME.plus(Duration.standardMinutes(30));
@Rule public final InjectRule inject = new InjectRule();
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
private final DatastoreBackupService backupService = mock(DatastoreBackupService.class);
private DatastoreBackupInfo backupInfo;
private final FakeResponse response = new FakeResponse();
private final FakeClock clock = new FakeClock(COMPLETE_TIME.plusMillis(1000));
private final CheckSnapshotAction action = new CheckSnapshotAction();
@Before
public void before() {
inject.setStaticField(DatastoreBackupInfo.class, "clock", clock);
action.requestMethod = Method.POST;
action.snapshotName = "some_backup";
action.kindsToLoadParam = "one,two";
action.response = response;
action.backupService = backupService;
backupInfo =
new DatastoreBackupInfo(
"some_backup",
START_TIME,
Optional.of(COMPLETE_TIME),
ImmutableSet.of("one", "two", "three"),
Optional.of("gs://somebucket/some_backup_20140801.backup_info"));
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
}
private void setPendingBackup() {
backupInfo =
new DatastoreBackupInfo(
backupInfo.getName(),
backupInfo.getStartTime(),
Optional.empty(),
backupInfo.getKinds(),
backupInfo.getGcsFilename());
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
}
private static void assertLoadTaskEnqueued(String id, String file, String kinds) {
assertTasksEnqueued(
"export-snapshot",
new TaskMatcher()
.url("/_dr/task/loadSnapshot")
.method("POST")
.param("id", id)
.param("file", file)
.param("kinds", kinds));
}
@Test
public void testSuccess_enqueuePollTask() {
CheckSnapshotAction.enqueuePollTask(
"some_snapshot_name", ImmutableSet.of("one", "two", "three"));
assertTasksEnqueued(
CheckSnapshotAction.QUEUE,
new TaskMatcher()
.url(CheckSnapshotAction.PATH)
.param(CHECK_SNAPSHOT_NAME_PARAM, "some_snapshot_name")
.param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, "one,two,three")
.method("POST"));
}
@Test
public void testPost_forPendingBackup_returnsNotModified() {
setPendingBackup();
NotModifiedException thrown = assertThrows(NotModifiedException.class, action::run);
assertThat(thrown).hasMessageThat().contains("Datastore backup some_backup still pending");
}
@Test
public void testPost_forStalePendingBackupBackup_returnsNoContent() {
setPendingBackup();
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
clock.setTo(
START_TIME
.plus(Duration.standardHours(20))
.plus(Duration.standardMinutes(3))
.plus(Duration.millis(1234)));
NoContentException thrown = assertThrows(NoContentException.class, action::run);
assertThat(thrown)
.hasMessageThat()
.contains(
"Datastore backup some_backup abandoned - "
+ "not complete after 20 hours, 3 minutes and 1 second");
}
@Test
public void testPost_forCompleteBackup_enqueuesLoadTask() {
action.run();
assertLoadTaskEnqueued(
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
}
@Test
public void testPost_forCompleteAutoBackup_enqueuesLoadTask_usingBackupName() {
action.snapshotName = "auto_snapshot_somestring";
when(backupService.findByName("auto_snapshot_somestring")).thenReturn(backupInfo);
action.run();
assertLoadTaskEnqueued(
"somestring", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
}
@Test
public void testPost_forCompleteBackup_withExtraKindsToLoad_enqueuesLoadTask() {
action.kindsToLoadParam = "one,foo";
action.run();
assertLoadTaskEnqueued(
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one");
}
@Test
public void testPost_forCompleteBackup_withEmptyKindsToLoad_skipsLoadTask() {
action.kindsToLoadParam = "";
action.run();
assertNoTasksEnqueued("export-snapshot");
}
@Test
public void testPost_forBadBackup_returnsBadRequest() {
when(backupService.findByName("some_backup"))
.thenThrow(new IllegalArgumentException("No backup found"));
BadRequestException thrown = assertThrows(BadRequestException.class, action::run);
assertThat(thrown).hasMessageThat().contains("Bad backup name some_backup: No backup found");
}
@Test
public void testGet_returnsInformation() {
action.requestMethod = Method.GET;
action.run();
assertThat(response.getPayload())
.isEqualTo(
Joiner.on("\n")
.join(
ImmutableList.of(
"Backup name: some_backup",
"Status: COMPLETE",
"Started: 2014-08-01T01:02:03.000Z",
"Ended: 2014-08-01T01:32:03.000Z",
"Duration: 30m",
"GCS: gs://somebucket/some_backup_20140801.backup_info",
"Kinds: [one, two, three]",
"")));
}
@Test
public void testGet_forBadBackup_returnsError() {
action.requestMethod = Method.GET;
when(backupService.findByName("some_backup"))
.thenThrow(new IllegalArgumentException("No backup found"));
BadRequestException thrown = assertThrows(BadRequestException.class, action::run);
assertThat(thrown).hasMessageThat().contains("Bad backup name some_backup: No backup found");
}
}

130
javatests/google/registry/export/DatastoreBackupInfoTest.java
View file

@ -1,130 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static google.registry.testing.JUnitBackports.assertThrows;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.EntityNotFoundException;
import com.google.appengine.api.datastore.Text;
import com.google.common.collect.ImmutableList;
import google.registry.export.DatastoreBackupInfo.BackupStatus;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock;
import google.registry.testing.InjectRule;
import java.util.Date;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link DatastoreBackupInfo}. */
@RunWith(JUnit4.class)
public class DatastoreBackupInfoTest {
@Rule
public final InjectRule inject = new InjectRule();
@Rule
public final AppEngineRule appEngine = AppEngineRule.builder()
.withDatastore()
.build();
private FakeClock clock = new FakeClock();
private DateTime startTime = DateTime.parse("2014-08-01T01:02:03Z");
private Entity backupEntity; // Can't initialize until AppEngineRule has set up Datastore.
@Before
public void before() {
inject.setStaticField(DatastoreBackupInfo.class, "clock", clock);
backupEntity = new Entity("_unused_");
backupEntity.setProperty("name", "backup1");
backupEntity.setProperty("kinds", ImmutableList.of("one", "two", "three"));
backupEntity.setProperty("start_time", new Date(startTime.getMillis()));
}
/** Force a roundtrip to Datastore to ensure that we're simulating a freshly fetched entity. */
private static Entity persistEntity(Entity entity) throws EntityNotFoundException {
return getDatastoreService().get(getDatastoreService().put(entity));
}
@Test
public void testSuccess_pendingBackup() throws Exception {
DatastoreBackupInfo backup = new DatastoreBackupInfo(persistEntity(backupEntity));
assertThat(backup.getName()).isEqualTo("backup1");
assertThat(backup.getKinds()).containsExactly("one", "two", "three");
assertThat(backup.getStartTime()).isEqualTo(startTime);
assertThat(backup.getCompleteTime()).isEmpty();
assertThat(backup.getGcsFilename()).isEmpty();
assertThat(backup.getStatus()).isEqualTo(BackupStatus.PENDING);
clock.setTo(startTime.plusMinutes(1));
assertThat(backup.getRunningTime()).isEqualTo(Duration.standardMinutes(1));
clock.setTo(startTime.plusMinutes(1).plusSeconds(42));
assertThat(backup.getRunningTime()).isEqualTo(Duration.standardSeconds(102));
}
@Test
public void testSuccess_completeBackup() throws Exception {
DateTime completeTime = startTime.plusMinutes(1).plusSeconds(42);
backupEntity.setProperty("complete_time", new Date(completeTime.getMillis()));
backupEntity.setProperty("gs_handle", new Text("/gs/somebucket/timestamp.backup_info"));
DatastoreBackupInfo backup = new DatastoreBackupInfo(persistEntity(backupEntity));
assertThat(backup.getName()).isEqualTo("backup1");
assertThat(backup.getKinds()).containsExactly("one", "two", "three");
assertThat(backup.getStartTime()).isEqualTo(startTime);
assertThat(backup.getCompleteTime().get()).isEqualTo(completeTime);
assertThat(backup.getGcsFilename()).hasValue("gs://somebucket/timestamp.backup_info");
assertThat(backup.getStatus()).isEqualTo(BackupStatus.COMPLETE);
assertThat(backup.getRunningTime()).isEqualTo(Duration.standardSeconds(102));
}
@Test
public void testFailure_missingName() {
backupEntity.removeProperty("name");
assertThrows(
NullPointerException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity)));
}
@Test
public void testFailure_missingKinds() {
backupEntity.removeProperty("kinds");
assertThrows(
NullPointerException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity)));
}
@Test
public void testFailure_missingStartTime() {
backupEntity.removeProperty("start_time");
assertThrows(
NullPointerException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity)));
}
@Test
public void testFailure_badGcsFilenameFormat() {
backupEntity.setProperty("gs_handle", new Text("foo"));
assertThrows(
IllegalArgumentException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity)));
}
}

123
javatests/google/registry/export/DatastoreBackupServiceTest.java
View file

@ -1,123 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.testing.JUnitBackports.assertThrows;
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
import static org.mockito.Mockito.when;
import com.google.appengine.api.datastore.Entity;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import google.registry.testing.AppEngineRule;
import google.registry.testing.MockitoJUnitRule;
import google.registry.testing.TaskQueueHelper.TaskMatcher;
import google.registry.util.AppEngineServiceUtils;
import java.util.Date;
import org.joda.time.DateTime;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.mockito.Mock;
/** Unit tests for {@link DatastoreBackupService}. */
@RunWith(JUnit4.class)
public class DatastoreBackupServiceTest {
@Rule
public final AppEngineRule appEngine =
AppEngineRule.builder().withDatastore().withTaskQueue().build();
@Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create();
@Mock private AppEngineServiceUtils appEngineServiceUtils;
private static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
private DatastoreBackupService backupService;
@Before
public void before() {
backupService = new DatastoreBackupService(appEngineServiceUtils);
when(appEngineServiceUtils.getVersionHostname("default", "ah-builtin-python-bundle"))
.thenReturn("ah-builtin-python-bundle.default.localhost");
persistBackupEntityWithName("backupA1");
persistBackupEntityWithName("backupA2");
persistBackupEntityWithName("backupA3");
persistBackupEntityWithName("backupB1");
persistBackupEntityWithName("backupB42");
}
private static void persistBackupEntityWithName(String name) {
Entity entity = new Entity(DatastoreBackupService.BACKUP_INFO_KIND);
entity.setProperty("name", name);
entity.setProperty("kinds", ImmutableList.of("one", "two", "three"));
entity.setProperty("start_time", new Date(START_TIME.getMillis()));
getDatastoreService().put(entity);
}
@Test
public void testSuccess_launchBackup() {
backupService.launchNewBackup(
"export-snapshot", "backup1", "somebucket", ImmutableSet.of("foo", "bar"));
assertTasksEnqueued("export-snapshot",
new TaskMatcher()
.url("/_ah/datastore_admin/backup.create")
.header("Host", "ah-builtin-python-bundle.default.localhost")
.method("GET")
.param("name", "backup1_")
.param("filesystem", "gs")
.param("gs_bucket_name", "somebucket")
.param("queue", "export-snapshot")
.param("kind", "foo")
.param("kind", "bar"));
}
@Test
public void testSuccess_findAllByNamePrefix() {
assertThat(
transform(backupService.findAllByNamePrefix("backupA"), DatastoreBackupInfo::getName))
.containsExactly("backupA1", "backupA2", "backupA3");
assertThat(
transform(backupService.findAllByNamePrefix("backupB"), DatastoreBackupInfo::getName))
.containsExactly("backupB1", "backupB42");
assertThat(
transform(backupService.findAllByNamePrefix("backupB4"), DatastoreBackupInfo::getName))
.containsExactly("backupB42");
assertThat(backupService.findAllByNamePrefix("backupX")).isEmpty();
}
@Test
public void testSuccess_findByName() {
assertThat(backupService.findByName("backupA1").getName()).isEqualTo("backupA1");
assertThat(backupService.findByName("backupB4").getName()).isEqualTo("backupB42");
}
@Test
public void testFailure_findByName_multipleMatchingBackups() {
assertThrows(IllegalArgumentException.class, () -> backupService.findByName("backupA"));
}
@Test
public void testFailure_findByName_noMatchingBackups() {
assertThrows(IllegalArgumentException.class, () -> backupService.findByName("backupX"));
}
}

76
javatests/google/registry/export/ExportSnapshotActionTest.java
View file

@ -1,76 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import com.google.common.base.Joiner;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeResponse;
import google.registry.testing.TaskQueueHelper.TaskMatcher;
import org.joda.time.DateTime;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link ExportSnapshotAction}. */
@RunWith(JUnit4.class)
public class ExportSnapshotActionTest {
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
private final DatastoreBackupService backupService = mock(DatastoreBackupService.class);
private final FakeResponse response = new FakeResponse();
private final FakeClock clock = new FakeClock(DateTime.parse("2014-08-01T01:02:03Z"));
private final ExportSnapshotAction action = new ExportSnapshotAction();
@Before
public void before() {
action.clock = clock;
action.backupService = backupService;
action.response = response;
}
@Test
public void testPost_launchesBackup_andEnqueuesPollTask() {
action.run();
verify(backupService)
.launchNewBackup(
ExportSnapshotAction.QUEUE,
"auto_snapshot_20140801_010203",
"registry-project-id-snapshots",
ExportConstants.getBackupKinds());
assertTasksEnqueued(
CheckSnapshotAction.QUEUE,
new TaskMatcher()
.url(CheckSnapshotAction.PATH)
.param(CHECK_SNAPSHOT_NAME_PARAM, "auto_snapshot_20140801_010203")
.param(
CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM,
Joiner.on(",").join(ExportConstants.getReportingKinds()))
.method("POST"));
assertThat(response.getPayload())
.isEqualTo("Datastore backup started with name: auto_snapshot_20140801_010203");
}
}

207
javatests/google/registry/export/LoadSnapshotActionTest.java
View file

@ -1,207 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.export.LoadSnapshotAction.LATEST_SNAPSHOT_VIEW_NAME;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
import static google.registry.export.LoadSnapshotAction.PATH;
import static google.registry.export.LoadSnapshotAction.QUEUE;
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
import static google.registry.testing.JUnitBackports.assertThrows;
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
import static org.joda.time.DateTimeZone.UTC;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.bigquery.model.Dataset;
import com.google.api.services.bigquery.model.Job;
import com.google.api.services.bigquery.model.JobConfigurationLoad;
import com.google.api.services.bigquery.model.JobReference;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import google.registry.bigquery.CheckedBigquery;
import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.HttpException.InternalServerErrorException;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock;
import google.registry.testing.TaskQueueHelper.TaskMatcher;
import java.io.IOException;
import java.util.List;
import org.joda.time.DateTime;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.mockito.ArgumentCaptor;
/** Unit tests for {@link LoadSnapshotAction}. */
@RunWith(JUnit4.class)
public class LoadSnapshotActionTest {
@Rule
public final AppEngineRule appEngine = AppEngineRule.builder()
.withTaskQueue()
.build();
private final CheckedBigquery checkedBigquery = mock(CheckedBigquery.class);
private final Bigquery bigquery = mock(Bigquery.class);
private final Bigquery.Jobs bigqueryJobs = mock(Bigquery.Jobs.class);
private final Bigquery.Jobs.Insert bigqueryJobsInsert = mock(Bigquery.Jobs.Insert.class);
private final Bigquery.Datasets bigqueryDatasets = mock(Bigquery.Datasets.class);
private final Bigquery.Datasets.Insert bigqueryDatasetsInsert =
mock(Bigquery.Datasets.Insert.class);
private final BigqueryPollJobEnqueuer bigqueryPollEnqueuer = mock(BigqueryPollJobEnqueuer.class);
private FakeClock clock = new FakeClock(new DateTime(1391096117045L, UTC));
private LoadSnapshotAction action;
@Before
public void before() throws Exception {
when(checkedBigquery.ensureDataSetExists("Project-Id", "snapshots")).thenReturn(bigquery);
when(bigquery.jobs()).thenReturn(bigqueryJobs);
when(bigqueryJobs.insert(eq("Project-Id"), any(Job.class))).thenReturn(bigqueryJobsInsert);
when(bigquery.datasets()).thenReturn(bigqueryDatasets);
when(bigqueryDatasets.insert(eq("Project-Id"), any(Dataset.class)))
.thenReturn(bigqueryDatasetsInsert);
action = new LoadSnapshotAction();
action.checkedBigquery = checkedBigquery;
action.bigqueryPollEnqueuer = bigqueryPollEnqueuer;
action.clock = clock;
action.projectId = "Project-Id";
action.snapshotFile = "gs://bucket/snapshot.backup_info";
action.snapshotId = "id12345";
action.snapshotKinds = "one,two,three";
}
@Test
public void testSuccess_enqueueLoadTask() {
enqueueLoadSnapshotTask(
"id12345", "gs://bucket/snapshot.backup_info", ImmutableSet.of("one", "two", "three"));
assertTasksEnqueued(
QUEUE,
new TaskMatcher()
.url(PATH)
.method("POST")
.param(LOAD_SNAPSHOT_ID_PARAM, "id12345")
.param(LOAD_SNAPSHOT_FILE_PARAM, "gs://bucket/snapshot.backup_info")
.param(LOAD_SNAPSHOT_KINDS_PARAM, "one,two,three"));
}
@Test
public void testSuccess_doPost() throws Exception {
action.run();
// Verify that checkedBigquery was called in a way that would create the dataset if it didn't
// already exist.
verify(checkedBigquery).ensureDataSetExists("Project-Id", "snapshots");
// Capture the load jobs we inserted to do additional checking on them.
ArgumentCaptor<Job> jobArgument = ArgumentCaptor.forClass(Job.class);
verify(bigqueryJobs, times(3)).insert(eq("Project-Id"), jobArgument.capture());
List<Job> jobs = jobArgument.getAllValues();
assertThat(jobs).hasSize(3);
// Check properties that should be common to all load jobs.
for (Job job : jobs) {
assertThat(job.getJobReference().getProjectId()).isEqualTo("Project-Id");
JobConfigurationLoad config = job.getConfiguration().getLoad();
assertThat(config.getSourceFormat()).isEqualTo("DATASTORE_BACKUP");
assertThat(config.getDestinationTable().getProjectId()).isEqualTo("Project-Id");
assertThat(config.getDestinationTable().getDatasetId()).isEqualTo("snapshots");
}
// Check the job IDs for each load job.
assertThat(transform(jobs, job -> job.getJobReference().getJobId()))
.containsExactly(
"load-snapshot-id12345-one-1391096117045",
"load-snapshot-id12345-two-1391096117045",
"load-snapshot-id12345-three-1391096117045");
// Check the source URI for each load job.
assertThat(
transform(
jobs,
job -> Iterables.getOnlyElement(job.getConfiguration().getLoad().getSourceUris())))
.containsExactly(
"gs://bucket/snapshot.one.backup_info",
"gs://bucket/snapshot.two.backup_info",
"gs://bucket/snapshot.three.backup_info");
// Check the destination table ID for each load job.
assertThat(
transform(
jobs, job -> job.getConfiguration().getLoad().getDestinationTable().getTableId()))
.containsExactly("id12345_one", "id12345_two", "id12345_three");
// Check that we executed the inserted jobs.
verify(bigqueryJobsInsert, times(3)).execute();
// Check that the poll tasks for each load job were enqueued.
verify(bigqueryPollEnqueuer)
.enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-snapshot-id12345-one-1391096117045"),
UpdateSnapshotViewAction.createViewUpdateTask(
"snapshots", "id12345_one", "one", LATEST_SNAPSHOT_VIEW_NAME),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
verify(bigqueryPollEnqueuer)
.enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-snapshot-id12345-two-1391096117045"),
UpdateSnapshotViewAction.createViewUpdateTask(
"snapshots", "id12345_two", "two", LATEST_SNAPSHOT_VIEW_NAME),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
verify(bigqueryPollEnqueuer)
.enqueuePollTask(
new JobReference()
.setProjectId("Project-Id")
.setJobId("load-snapshot-id12345-three-1391096117045"),
UpdateSnapshotViewAction.createViewUpdateTask(
"snapshots", "id12345_three", "three", LATEST_SNAPSHOT_VIEW_NAME),
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
}
@Test
public void testFailure_doPost_badGcsFilename() {
action.snapshotFile = "gs://bucket/snapshot";
BadRequestException thrown = assertThrows(BadRequestException.class, action::run);
assertThat(thrown)
.hasMessageThat()
.contains("Error calling load snapshot: backup info file extension missing");
}
@Test
public void testFailure_doPost_bigqueryThrowsException() throws Exception {
when(bigqueryJobsInsert.execute()).thenThrow(new IOException("The Internet has gone missing"));
InternalServerErrorException thrown =
assertThrows(InternalServerErrorException.class, action::run);
assertThat(thrown)
.hasMessageThat()
.contains("Error loading snapshot: The Internet has gone missing");
}
}

3
javatests/google/registry/module/backend/testdata/backend_routing.txt vendored
View file

@ -7,7 +7,6 @@ PATH CLASS METHOD
/_dr/task/backupDatastore BackupDatastoreAction POST y INTERNAL APP IGNORED /_dr/task/backupDatastore BackupDatastoreAction POST y INTERNAL APP IGNORED
/_dr/task/brdaCopy BrdaCopyAction POST y INTERNAL APP IGNORED /_dr/task/brdaCopy BrdaCopyAction POST y INTERNAL APP IGNORED
/_dr/task/checkDatastoreBackup CheckBackupAction POST,GET y INTERNAL APP IGNORED /_dr/task/checkDatastoreBackup CheckBackupAction POST,GET y INTERNAL APP IGNORED
/_dr/task/checkSnapshot CheckSnapshotAction POST,GET y INTERNAL APP IGNORED
/_dr/task/copyDetailReports CopyDetailReportsAction POST n INTERNAL,API APP ADMIN /_dr/task/copyDetailReports CopyDetailReportsAction POST n INTERNAL,API APP ADMIN
/_dr/task/deleteContactsAndHosts DeleteContactsAndHostsAction GET n INTERNAL APP IGNORED /_dr/task/deleteContactsAndHosts DeleteContactsAndHostsAction GET n INTERNAL APP IGNORED
/_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n INTERNAL APP IGNORED /_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n INTERNAL APP IGNORED
@ -18,7 +17,6 @@ PATH CLASS METHOD
/_dr/task/exportDomainLists ExportDomainListsAction POST n INTERNAL APP IGNORED /_dr/task/exportDomainLists ExportDomainListsAction POST n INTERNAL APP IGNORED
/_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n INTERNAL APP IGNORED /_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n INTERNAL APP IGNORED
/_dr/task/exportReservedTerms ExportReservedTermsAction POST n INTERNAL APP IGNORED /_dr/task/exportReservedTerms ExportReservedTermsAction POST n INTERNAL APP IGNORED
/_dr/task/exportSnapshot ExportSnapshotAction POST y INTERNAL APP IGNORED
/_dr/task/generateInvoices GenerateInvoicesAction POST n INTERNAL APP IGNORED /_dr/task/generateInvoices GenerateInvoicesAction POST n INTERNAL APP IGNORED
/_dr/task/generateSpec11 GenerateSpec11ReportAction POST n INTERNAL APP IGNORED /_dr/task/generateSpec11 GenerateSpec11ReportAction POST n INTERNAL APP IGNORED
/_dr/task/icannReportingStaging IcannReportingStagingAction POST n INTERNAL APP IGNORED /_dr/task/icannReportingStaging IcannReportingStagingAction POST n INTERNAL APP IGNORED
@ -27,7 +25,6 @@ PATH CLASS METHOD
/_dr/task/importRdeDomains RdeDomainImportAction GET n INTERNAL APP IGNORED /_dr/task/importRdeDomains RdeDomainImportAction GET n INTERNAL APP IGNORED
/_dr/task/importRdeHosts RdeHostImportAction GET n INTERNAL APP IGNORED /_dr/task/importRdeHosts RdeHostImportAction GET n INTERNAL APP IGNORED
/_dr/task/linkRdeHosts RdeHostLinkAction GET n INTERNAL APP IGNORED /_dr/task/linkRdeHosts RdeHostLinkAction GET n INTERNAL APP IGNORED
/_dr/task/loadSnapshot LoadSnapshotAction POST n INTERNAL APP IGNORED
/_dr/task/nordnUpload NordnUploadAction POST y INTERNAL APP IGNORED /_dr/task/nordnUpload NordnUploadAction POST y INTERNAL APP IGNORED
/_dr/task/nordnVerify NordnVerifyAction POST y INTERNAL APP IGNORED /_dr/task/nordnVerify NordnVerifyAction POST y INTERNAL APP IGNORED
/_dr/task/pollBigqueryJob BigqueryPollJobAction GET,POST y INTERNAL APP IGNORED /_dr/task/pollBigqueryJob BigqueryPollJobAction GET,POST y INTERNAL APP IGNORED