mirror of
https://github.com/google/nomulus.git
synced 2025-05-13 07:57:13 +02:00
Daggerize ExportSnapshotServlet and CheckSnapshotServlet
Eradicate the last remnants of un-injectable servlets! ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=145598002
This commit is contained in:
jianglai
Ben McIlwain
parent
a8aeff96f6
commit
4fed3a9ae6
15 changed files with 577 additions and 642 deletions
|
|
@ -110,9 +110,9 @@ Here are the task queues in use by the system. All are push queues unless
|
||||||
explicitly marked as otherwise.
|
explicitly marked as otherwise.
|
||||||
|
|
||||||
* `async-delete-pull` and `async-host-rename-pull` -- Pull queues for tasks to
|
* `async-delete-pull` and `async-host-rename-pull` -- Pull queues for tasks to
|
||||||
asynchronously delete contacts/hosts and to asynchronously refresh DNS
|
asynchronously delete contacts/hosts and to asynchronously refresh DNS for
|
||||||
for renamed hosts, respectively. Tasks are enqueued during EPP
|
renamed hosts, respectively. Tasks are enqueued during EPP flows and then
|
||||||
flows and then handled in batches by the regularly running cron tasks
|
handled in batches by the regularly running cron tasks
|
||||||
`DeleteContactsAndHostsAction` and `RefreshDnsOnHostRenameAction`.
|
`DeleteContactsAndHostsAction` and `RefreshDnsOnHostRenameAction`.
|
||||||
* `bigquery-streaming-metrics` -- Queue for metrics that are asynchronously
|
* `bigquery-streaming-metrics` -- Queue for metrics that are asynchronously
|
||||||
streamed to BigQuery in the `Metrics` class. Tasks are enqueued during EPP
|
streamed to BigQuery in the `Metrics` class. Tasks are enqueued during EPP
|
||||||
|
|
@ -140,14 +140,14 @@ explicitly marked as otherwise.
|
||||||
cron) and executed by `ExportCommitLogDiffAction`.
|
cron) and executed by `ExportCommitLogDiffAction`.
|
||||||
* `export-snapshot` -- Cron and push queue for tasks to load a Datastore
|
* `export-snapshot` -- Cron and push queue for tasks to load a Datastore
|
||||||
snapshot that was stored in Google Cloud Storage and export it to BigQuery.
|
snapshot that was stored in Google Cloud Storage and export it to BigQuery.
|
||||||
Tasks are enqueued by both cron and `CheckSnapshotServlet` and are executed
|
Tasks are enqueued by both cron and `CheckSnapshotAction` and are executed
|
||||||
by both `ExportSnapshotServlet` and `LoadSnapshotAction`.
|
by both `ExportSnapshotAction` and `LoadSnapshotAction`.
|
||||||
* `export-snapshot-poll` -- Queue for tasks to check that a Datastore snapshot
|
* `export-snapshot-poll` -- Queue for tasks to check that a Datastore snapshot
|
||||||
has been successfully uploaded to Google Cloud Storage (this is an
|
has been successfully uploaded to Google Cloud Storage (this is an
|
||||||
asynchronous background operation that can take an indeterminate amount of
|
asynchronous background operation that can take an indeterminate amount of
|
||||||
time). Once the snapshot is successfully uploaded, it is imported into
|
time). Once the snapshot is successfully uploaded, it is imported into
|
||||||
BigQuery. Tasks are enqueued by `ExportSnapshotServlet` and executed by
|
BigQuery. Tasks are enqueued by `ExportSnapshotAction` and executed by
|
||||||
`CheckSnapshotServlet`.
|
`CheckSnapshotAction`.
|
||||||
* `export-snapshot-update-view` -- Queue for tasks to update the BigQuery
|
* `export-snapshot-update-view` -- Queue for tasks to update the BigQuery
|
||||||
views to point to the most recently uploaded snapshot. Tasks are enqueued by
|
views to point to the most recently uploaded snapshot. Tasks are enqueued by
|
||||||
`LoadSnapshotAction` and executed by `UpdateSnapshotViewAction`.
|
`LoadSnapshotAction` and executed by `UpdateSnapshotViewAction`.
|
||||||
|
|
|
||||||
|
|
@ -1102,7 +1102,7 @@ public final class RegistryConfig {
|
||||||
/**
|
/**
|
||||||
* Returns the Google Cloud Storage bucket for storing backup snapshots.
|
* Returns the Google Cloud Storage bucket for storing backup snapshots.
|
||||||
*
|
*
|
||||||
* @see google.registry.export.ExportSnapshotServlet
|
* @see google.registry.export.ExportSnapshotAction
|
||||||
*/
|
*/
|
||||||
public static String getSnapshotsBucket() {
|
public static String getSnapshotsBucket() {
|
||||||
return getProjectId() + "-snapshots";
|
return getProjectId() + "-snapshots";
|
||||||
|
|
|
||||||
|
|
@ -76,7 +76,7 @@ public abstract class DnsModule {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Parameter("name")
|
@Parameter("domainOrHostName")
|
||||||
static String provideName(HttpServletRequest req) {
|
static String provideName(HttpServletRequest req) {
|
||||||
return extractRequiredParameter(req, "name");
|
return extractRequiredParameter(req, "name");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,7 @@ public final class RefreshDnsAction implements Runnable {
|
||||||
|
|
||||||
@Inject Clock clock;
|
@Inject Clock clock;
|
||||||
@Inject DnsQueue dnsQueue;
|
@Inject DnsQueue dnsQueue;
|
||||||
@Inject @Parameter("name") String domainOrHostName;
|
@Inject @Parameter("domainOrHostName") String domainOrHostName;
|
||||||
@Inject @Parameter("type") TargetType type;
|
@Inject @Parameter("type") TargetType type;
|
||||||
@Inject RefreshDnsAction() {}
|
@Inject RefreshDnsAction() {}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<web-app xmlns="http://java.sun.com/xml/ns/javaee" version="2.5"
|
<web-app xmlns="http://java.sun.com/xml/ns/javaee" version="2.5"
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
|
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
|
||||||
http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd">
|
http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd">
|
||||||
<!-- Servlets -->
|
<!-- Servlets -->
|
||||||
|
|
||||||
|
|
@ -113,25 +113,15 @@
|
||||||
<url-pattern>/_dr/task/verifyEntityIntegrity</url-pattern>
|
<url-pattern>/_dr/task/verifyEntityIntegrity</url-pattern>
|
||||||
</servlet-mapping>
|
</servlet-mapping>
|
||||||
|
|
||||||
<servlet>
|
<!-- Exports a datastore backup snapshot to GCS. -->
|
||||||
<description>Exports a datastore backup snapshot to GCS.</description>
|
|
||||||
<display-name>Export snapshot to GCS</display-name>
|
|
||||||
<servlet-name>exportSnapshot</servlet-name>
|
|
||||||
<servlet-class>google.registry.export.ExportSnapshotServlet</servlet-class>
|
|
||||||
</servlet>
|
|
||||||
<servlet-mapping>
|
<servlet-mapping>
|
||||||
<servlet-name>exportSnapshot</servlet-name>
|
<servlet-name>backend-servlet</servlet-name>
|
||||||
<url-pattern>/_dr/task/exportSnapshot</url-pattern>
|
<url-pattern>/_dr/task/exportSnapshot</url-pattern>
|
||||||
</servlet-mapping>
|
</servlet-mapping>
|
||||||
|
|
||||||
<servlet>
|
<!-- Checks the completion of a datastore backup snapshot. -->
|
||||||
<description>Checks the completion of a datastore backup snapshot.</description>
|
|
||||||
<display-name>Check on snapshot status</display-name>
|
|
||||||
<servlet-name>checkSnapshot</servlet-name>
|
|
||||||
<servlet-class>google.registry.export.CheckSnapshotServlet</servlet-class>
|
|
||||||
</servlet>
|
|
||||||
<servlet-mapping>
|
<servlet-mapping>
|
||||||
<servlet-name>checkSnapshot</servlet-name>
|
<servlet-name>backend-servlet</servlet-name>
|
||||||
<url-pattern>/_dr/task/checkSnapshot</url-pattern>
|
<url-pattern>/_dr/task/checkSnapshot</url-pattern>
|
||||||
</servlet-mapping>
|
</servlet-mapping>
|
||||||
|
|
||||||
|
|
@ -290,10 +280,10 @@
|
||||||
<web-resource-collection>
|
<web-resource-collection>
|
||||||
<web-resource-name>Internal</web-resource-name>
|
<web-resource-name>Internal</web-resource-name>
|
||||||
<description>
|
<description>
|
||||||
Admin-only internal section. Requests for paths covered by the URL patterns below will be
|
Admin-only internal section. Requests for paths covered by the URL patterns below will be
|
||||||
checked for a logged-in user account that's allowed to access the AppEngine admin console
|
checked for a logged-in user account that's allowed to access the AppEngine admin console
|
||||||
(NOTE: this includes Editor/Viewer permissions in addition to Owner and the new IAM
|
(NOTE: this includes Editor/Viewer permissions in addition to Owner and the new IAM
|
||||||
App Engine Admin role. See https://cloud.google.com/appengine/docs/java/access-control
|
App Engine Admin role. See https://cloud.google.com/appengine/docs/java/access-control
|
||||||
specifically the "Access handlers that have a login:admin restriction" line.)
|
specifically the "Access handlers that have a login:admin restriction" line.)
|
||||||
|
|
||||||
TODO(b/28219927): lift some of these restrictions so that we can allow OAuth authentication
|
TODO(b/28219927): lift some of these restrictions so that we can allow OAuth authentication
|
||||||
|
|
|
||||||
162
java/google/registry/export/CheckSnapshotAction.java
Normal file
162
java/google/registry/export/CheckSnapshotAction.java
Normal file
|
|
@ -0,0 +1,162 @@
|
||||||
|
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package google.registry.export;
|
||||||
|
|
||||||
|
import static com.google.common.collect.Sets.intersection;
|
||||||
|
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
|
||||||
|
import static google.registry.request.Action.Method.GET;
|
||||||
|
import static google.registry.request.Action.Method.POST;
|
||||||
|
import static google.registry.util.FormattingLogger.getLoggerForCallerClass;
|
||||||
|
|
||||||
|
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||||
|
import com.google.appengine.api.taskqueue.TaskHandle;
|
||||||
|
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||||
|
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||||
|
import com.google.common.base.Joiner;
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
import com.google.common.collect.ImmutableSet;
|
||||||
|
import com.google.common.collect.Sets;
|
||||||
|
import google.registry.export.DatastoreBackupInfo.BackupStatus;
|
||||||
|
import google.registry.request.Action;
|
||||||
|
import google.registry.request.HttpException.BadRequestException;
|
||||||
|
import google.registry.request.HttpException.NoContentException;
|
||||||
|
import google.registry.request.HttpException.NotModifiedException;
|
||||||
|
import google.registry.request.Parameter;
|
||||||
|
import google.registry.request.RequestMethod;
|
||||||
|
import google.registry.request.Response;
|
||||||
|
import google.registry.util.FormattingLogger;
|
||||||
|
import java.util.Set;
|
||||||
|
import javax.inject.Inject;
|
||||||
|
import org.joda.time.Duration;
|
||||||
|
import org.joda.time.PeriodType;
|
||||||
|
import org.joda.time.format.PeriodFormat;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Action that checks the status of a snapshot, and if complete, trigger loading it into BigQuery.
|
||||||
|
*/
|
||||||
|
@Action(
|
||||||
|
path = CheckSnapshotAction.PATH,
|
||||||
|
method = {POST, GET},
|
||||||
|
automaticallyPrintOk = true
|
||||||
|
)
|
||||||
|
public class CheckSnapshotAction implements Runnable {
|
||||||
|
|
||||||
|
/** Parameter names for passing parameters into this action. */
|
||||||
|
static final String CHECK_SNAPSHOT_NAME_PARAM = "name";
|
||||||
|
|
||||||
|
static final String CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";
|
||||||
|
|
||||||
|
/** Action-specific details needed for enqueuing tasks against itself. */
|
||||||
|
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
|
||||||
|
|
||||||
|
static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml.
|
||||||
|
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
|
||||||
|
|
||||||
|
/** The maximum amount of time we allow a backup to run before abandoning it. */
|
||||||
|
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
|
||||||
|
|
||||||
|
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||||
|
|
||||||
|
@Inject Response response;
|
||||||
|
@Inject @RequestMethod Action.Method requestMethod;
|
||||||
|
@Inject DatastoreBackupService backupService;
|
||||||
|
@Inject @Parameter(CHECK_SNAPSHOT_NAME_PARAM) String snapshotName;
|
||||||
|
@Inject @Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM) String kindsToLoadParam;
|
||||||
|
@Inject CheckSnapshotAction() {}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
if (requestMethod == POST) {
|
||||||
|
checkAndLoadSnapshotIfComplete();
|
||||||
|
} else {
|
||||||
|
// This is a GET request.
|
||||||
|
response.setPayload(getBackup().getInformation());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatastoreBackupInfo getBackup() {
|
||||||
|
try {
|
||||||
|
return backupService.findByName(snapshotName);
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
|
||||||
|
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be
|
||||||
|
// retried but we might abandon backups that start late and haven't yet written to
|
||||||
|
// datastore. We could fix that by replacing this with a two-phase polling strategy.
|
||||||
|
throw new BadRequestException(message, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkAndLoadSnapshotIfComplete() {
|
||||||
|
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
|
||||||
|
DatastoreBackupInfo backup = getBackup();
|
||||||
|
// Stop now if the backup is not complete.
|
||||||
|
if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
|
||||||
|
Duration runningTime = backup.getRunningTime();
|
||||||
|
if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
|
||||||
|
// Backup might still be running, so send a 304 to have the task retry.
|
||||||
|
throw new NotModifiedException(
|
||||||
|
String.format("Datastore backup %s still pending", snapshotName));
|
||||||
|
} else {
|
||||||
|
// Declare the backup a lost cause, and send 204 No Content so the task will
|
||||||
|
// not be retried.
|
||||||
|
String message =
|
||||||
|
String.format(
|
||||||
|
"Datastore backup %s abandoned - not complete after %s",
|
||||||
|
snapshotName,
|
||||||
|
PeriodFormat.getDefault()
|
||||||
|
.print(
|
||||||
|
runningTime
|
||||||
|
.toPeriod()
|
||||||
|
.normalizedStandard(PeriodType.dayTime().withMillisRemoved())));
|
||||||
|
throw new NoContentException(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
|
||||||
|
// suffix out of the snapshot name and falling back to the start time as a string.
|
||||||
|
String snapshotId =
|
||||||
|
snapshotName.startsWith(ExportSnapshotAction.SNAPSHOT_PREFIX)
|
||||||
|
? snapshotName.substring(ExportSnapshotAction.SNAPSHOT_PREFIX.length())
|
||||||
|
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
|
||||||
|
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
|
||||||
|
if (!backup.getKinds().containsAll(kindsToLoad)) {
|
||||||
|
logger.warningfmt(
|
||||||
|
"Kinds to load included non-exported kinds: %s",
|
||||||
|
Sets.difference(kindsToLoad, backup.getKinds()));
|
||||||
|
}
|
||||||
|
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
|
||||||
|
ImmutableSet<String> exportedKindsToLoad =
|
||||||
|
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
|
||||||
|
String message = String.format("Datastore backup %s complete - ", snapshotName);
|
||||||
|
if (exportedKindsToLoad.isEmpty()) {
|
||||||
|
message += "no kinds to load into BigQuery";
|
||||||
|
} else {
|
||||||
|
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
|
||||||
|
message += "BigQuery load task enqueued";
|
||||||
|
}
|
||||||
|
logger.info(message);
|
||||||
|
response.setPayload(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Enqueue a poll task to monitor the named snapshot for completion. */
|
||||||
|
static TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
|
||||||
|
return QueueFactory.getQueue(QUEUE)
|
||||||
|
.add(
|
||||||
|
TaskOptions.Builder.withUrl(PATH)
|
||||||
|
.method(Method.POST)
|
||||||
|
.countdownMillis(POLL_COUNTDOWN.getMillis())
|
||||||
|
.param(CHECK_SNAPSHOT_NAME_PARAM, snapshotName)
|
||||||
|
.param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,176 +0,0 @@
|
||||||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
|
||||||
import static com.google.common.collect.Sets.intersection;
|
|
||||||
import static com.google.common.html.HtmlEscapers.htmlEscaper;
|
|
||||||
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
|
|
||||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
|
||||||
import static google.registry.util.FormattingLogger.getLoggerForCallerClass;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
|
||||||
|
|
||||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
|
||||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
|
||||||
import com.google.common.base.Joiner;
|
|
||||||
import com.google.common.base.Splitter;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import com.google.common.collect.Sets;
|
|
||||||
import com.google.common.net.MediaType;
|
|
||||||
import google.registry.export.DatastoreBackupInfo.BackupStatus;
|
|
||||||
import google.registry.request.HttpException.BadRequestException;
|
|
||||||
import google.registry.util.FormattingLogger;
|
|
||||||
import google.registry.util.NonFinalForTesting;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.Set;
|
|
||||||
import javax.servlet.http.HttpServlet;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
import org.joda.time.Duration;
|
|
||||||
import org.joda.time.PeriodType;
|
|
||||||
import org.joda.time.format.PeriodFormat;
|
|
||||||
|
|
||||||
/** Check the status of a snapshot, and if complete, trigger loading it into BigQuery. */
|
|
||||||
public class CheckSnapshotServlet extends HttpServlet {
|
|
||||||
|
|
||||||
/** Parameter names for passing parameters into this servlet. */
|
|
||||||
static final String SNAPSHOT_NAME_PARAM = "name";
|
|
||||||
static final String SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";
|
|
||||||
|
|
||||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
|
||||||
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
|
|
||||||
static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml.
|
|
||||||
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
|
|
||||||
|
|
||||||
/** The maximum amount of time we allow a backup to run before abandoning it. */
|
|
||||||
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
|
|
||||||
|
|
||||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
|
||||||
|
|
||||||
@NonFinalForTesting
|
|
||||||
private static DatastoreBackupService backupService = DatastoreBackupService.get();
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
|
||||||
try {
|
|
||||||
rsp.setStatus(SC_OK);
|
|
||||||
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
|
|
||||||
rsp.getWriter().write("OK\n\n");
|
|
||||||
super.service(req, rsp);
|
|
||||||
} catch (Throwable e) {
|
|
||||||
logger.severe(e, e.toString());
|
|
||||||
rsp.sendError(
|
|
||||||
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
|
|
||||||
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void doGet(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
|
||||||
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
|
|
||||||
try {
|
|
||||||
String snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
|
|
||||||
rsp.getWriter().write(backupService.findByName(snapshotName).getInformation());
|
|
||||||
} catch (BadRequestException e) {
|
|
||||||
throw new IllegalArgumentException(e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
|
||||||
String snapshotName;
|
|
||||||
String kindsToLoadParam;
|
|
||||||
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
|
|
||||||
try {
|
|
||||||
snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
|
|
||||||
kindsToLoadParam = extractRequiredParameter(req, SNAPSHOT_KINDS_TO_LOAD_PARAM);
|
|
||||||
} catch (BadRequestException e) {
|
|
||||||
throw new IllegalArgumentException(e.getMessage());
|
|
||||||
}
|
|
||||||
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
|
|
||||||
|
|
||||||
// Look up the backup by the provided name, stopping if we can't find it.
|
|
||||||
DatastoreBackupInfo backup;
|
|
||||||
try {
|
|
||||||
backup = backupService.findByName(snapshotName);
|
|
||||||
} catch (IllegalArgumentException e) {
|
|
||||||
String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
|
|
||||||
logger.severe(e, message);
|
|
||||||
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be retried,
|
|
||||||
// but we might abandon backups that start late and haven't yet written to datastore.
|
|
||||||
// We could fix that by replacing this with a two-phase polling strategy.
|
|
||||||
rsp.sendError(SC_BAD_REQUEST, htmlEscaper().escape(message));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Stop now if the backup is not complete.
|
|
||||||
if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
|
|
||||||
Duration runningTime = backup.getRunningTime();
|
|
||||||
if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
|
|
||||||
// Backup might still be running, so send a 304 to have the task retry.
|
|
||||||
rsp.sendError(SC_NOT_MODIFIED,
|
|
||||||
htmlEscaper().escape(String.format("Datastore backup %s still pending", snapshotName)));
|
|
||||||
} else {
|
|
||||||
// Declare the backup a lost cause, and send 202 Accepted so the task will not be retried.
|
|
||||||
String message = String.format("Datastore backup %s abandoned - not complete after %s",
|
|
||||||
snapshotName,
|
|
||||||
PeriodFormat.getDefault().print(
|
|
||||||
runningTime.toPeriod().normalizedStandard(
|
|
||||||
PeriodType.dayTime().withMillisRemoved())));
|
|
||||||
logger.severe(message);
|
|
||||||
rsp.sendError(SC_ACCEPTED, htmlEscaper().escape(message));
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
|
|
||||||
// suffix out of the snapshot name and falling back to the start time as a string.
|
|
||||||
String snapshotId = snapshotName.startsWith(ExportSnapshotServlet.SNAPSHOT_PREFIX)
|
|
||||||
? snapshotName.substring(ExportSnapshotServlet.SNAPSHOT_PREFIX.length())
|
|
||||||
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
|
|
||||||
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
|
|
||||||
if (!backup.getKinds().containsAll(kindsToLoad)) {
|
|
||||||
logger.warningfmt(
|
|
||||||
"Kinds to load included non-exported kinds: %s",
|
|
||||||
Sets.difference(kindsToLoad, backup.getKinds()));
|
|
||||||
}
|
|
||||||
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
|
|
||||||
ImmutableSet<String> exportedKindsToLoad =
|
|
||||||
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
|
|
||||||
String message = String.format("Datastore backup %s complete - ", snapshotName);
|
|
||||||
if (exportedKindsToLoad.isEmpty()) {
|
|
||||||
message += "no kinds to load into BigQuery";
|
|
||||||
} else {
|
|
||||||
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
|
|
||||||
message += "BigQuery load task enqueued";
|
|
||||||
}
|
|
||||||
logger.info(message);
|
|
||||||
rsp.getWriter().write(message);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Enqueue a poll task to monitor the named snapshot for completion. */
|
|
||||||
TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
|
|
||||||
return QueueFactory.getQueue(QUEUE).add(
|
|
||||||
TaskOptions.Builder.withUrl(PATH)
|
|
||||||
.method(Method.POST)
|
|
||||||
.countdownMillis(POLL_COUNTDOWN.getMillis())
|
|
||||||
.param(SNAPSHOT_NAME_PARAM, snapshotName)
|
|
||||||
.param(SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -17,6 +17,8 @@ package google.registry.export;
|
||||||
import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
|
import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
|
||||||
import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
|
import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
|
||||||
import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
|
import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
|
||||||
|
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
||||||
|
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
|
||||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
|
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
|
||||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
|
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
|
||||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
|
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
|
||||||
|
|
@ -72,6 +74,18 @@ public final class ExportRequestModule {
|
||||||
return extractRequiredParameter(req, LOAD_SNAPSHOT_KINDS_PARAM);
|
return extractRequiredParameter(req, LOAD_SNAPSHOT_KINDS_PARAM);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Provides
|
||||||
|
@Parameter(CHECK_SNAPSHOT_NAME_PARAM)
|
||||||
|
static String provideCheckSnapshotName(HttpServletRequest req) {
|
||||||
|
return extractRequiredParameter(req, CHECK_SNAPSHOT_NAME_PARAM);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Provides
|
||||||
|
@Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM)
|
||||||
|
static String provideCheckSnapshotKindsToLoad(HttpServletRequest req) {
|
||||||
|
return extractRequiredParameter(req, CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM);
|
||||||
|
}
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Header(CHAINED_TASK_QUEUE_HEADER)
|
@Header(CHAINED_TASK_QUEUE_HEADER)
|
||||||
static String provideChainedTaskQueue(HttpServletRequest req) {
|
static String provideChainedTaskQueue(HttpServletRequest req) {
|
||||||
|
|
@ -89,4 +103,9 @@ public final class ExportRequestModule {
|
||||||
static String provideProjectId(HttpServletRequest req) {
|
static String provideProjectId(HttpServletRequest req) {
|
||||||
return extractRequiredHeader(req, PROJECT_ID_HEADER);
|
return extractRequiredHeader(req, PROJECT_ID_HEADER);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Provides
|
||||||
|
static DatastoreBackupService provideDatastoreBackupService() {
|
||||||
|
return DatastoreBackupService.get();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
72
java/google/registry/export/ExportSnapshotAction.java
Normal file
72
java/google/registry/export/ExportSnapshotAction.java
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package google.registry.export;
|
||||||
|
|
||||||
|
import static google.registry.export.CheckSnapshotAction.enqueuePollTask;
|
||||||
|
import static google.registry.request.Action.Method.POST;
|
||||||
|
|
||||||
|
import google.registry.config.RegistryConfig;
|
||||||
|
import google.registry.request.Action;
|
||||||
|
import google.registry.request.Response;
|
||||||
|
import google.registry.util.Clock;
|
||||||
|
import google.registry.util.FormattingLogger;
|
||||||
|
import javax.inject.Inject;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Action to trigger a datastore backup job that writes a snapshot to Google Cloud Storage.
|
||||||
|
*
|
||||||
|
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
|
||||||
|
* the next upon successful completion:
|
||||||
|
*
|
||||||
|
* <ol>
|
||||||
|
* <li>The snapshot is exported to Google Cloud Storage (this action).
|
||||||
|
* <li>The {@link CheckSnapshotAction} polls until the export is completed.
|
||||||
|
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
|
||||||
|
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
|
||||||
|
* </ol>
|
||||||
|
*/
|
||||||
|
@Action(path = ExportSnapshotAction.PATH, method = POST, automaticallyPrintOk = true)
|
||||||
|
public class ExportSnapshotAction implements Runnable {
|
||||||
|
|
||||||
|
/** Queue to use for enqueuing the task that will actually launch the backup. */
|
||||||
|
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
||||||
|
|
||||||
|
static final String PATH = "/_dr/task/exportSnapshot"; // See web.xml.
|
||||||
|
|
||||||
|
/** Prefix to use for naming all snapshots that are started by this servlet. */
|
||||||
|
static final String SNAPSHOT_PREFIX = "auto_snapshot_";
|
||||||
|
|
||||||
|
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||||
|
|
||||||
|
@Inject Clock clock;
|
||||||
|
@Inject DatastoreBackupService backupService;
|
||||||
|
@Inject Response response;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
ExportSnapshotAction() {}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
// Use a unique name for the snapshot so we can explicitly check its completion later.
|
||||||
|
String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss");
|
||||||
|
backupService.launchNewBackup(
|
||||||
|
QUEUE, snapshotName, RegistryConfig.getSnapshotsBucket(), ExportConstants.getBackupKinds());
|
||||||
|
// Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery.
|
||||||
|
enqueuePollTask(snapshotName, ExportConstants.getReportingKinds());
|
||||||
|
String message = "Datastore backup started with name: " + snapshotName;
|
||||||
|
logger.info(message);
|
||||||
|
response.setPayload(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,88 +0,0 @@
|
||||||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
|
||||||
import static com.google.common.html.HtmlEscapers.htmlEscaper;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
|
||||||
|
|
||||||
import com.google.common.net.MediaType;
|
|
||||||
import google.registry.config.RegistryConfig;
|
|
||||||
import google.registry.util.Clock;
|
|
||||||
import google.registry.util.FormattingLogger;
|
|
||||||
import google.registry.util.NonFinalForTesting;
|
|
||||||
import google.registry.util.SystemClock;
|
|
||||||
import java.io.IOException;
|
|
||||||
import javax.servlet.http.HttpServlet;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Trigger a backup-as-a-service job that writes a snapshot to Google Cloud Storage.
|
|
||||||
*
|
|
||||||
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
|
|
||||||
* the next upon successful completion:<ol>
|
|
||||||
* <li>The snapshot is exported to Google Cloud Storage (this servlet).
|
|
||||||
* <li>The {@link CheckSnapshotServlet} polls until the export is completed.
|
|
||||||
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
|
|
||||||
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
|
|
||||||
* </ol>
|
|
||||||
*/
|
|
||||||
public class ExportSnapshotServlet extends HttpServlet {
|
|
||||||
|
|
||||||
/** Queue to use for enqueuing the task that will actually launch the backup. */
|
|
||||||
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
|
||||||
|
|
||||||
/** Prefix to use for naming all snapshots that are started by this servlet. */
|
|
||||||
static final String SNAPSHOT_PREFIX = "auto_snapshot_";
|
|
||||||
|
|
||||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
|
||||||
|
|
||||||
@NonFinalForTesting
|
|
||||||
private static Clock clock = new SystemClock();
|
|
||||||
|
|
||||||
@NonFinalForTesting
|
|
||||||
private static DatastoreBackupService backupService = DatastoreBackupService.get();
|
|
||||||
|
|
||||||
@NonFinalForTesting
|
|
||||||
private static CheckSnapshotServlet checkSnapshotServlet = new CheckSnapshotServlet();
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
|
||||||
try {
|
|
||||||
// Use a unique name for the snapshot so we can explicitly check its completion later.
|
|
||||||
String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss");
|
|
||||||
backupService.launchNewBackup(
|
|
||||||
QUEUE,
|
|
||||||
snapshotName,
|
|
||||||
RegistryConfig.getSnapshotsBucket(),
|
|
||||||
ExportConstants.getBackupKinds());
|
|
||||||
// Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery.
|
|
||||||
checkSnapshotServlet.enqueuePollTask(snapshotName, ExportConstants.getReportingKinds());
|
|
||||||
String message = "Datastore backup started with name: " + snapshotName;
|
|
||||||
logger.info(message);
|
|
||||||
rsp.setStatus(SC_OK);
|
|
||||||
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
|
|
||||||
rsp.getWriter().write("OK\n\n" + message);
|
|
||||||
} catch (Throwable e) {
|
|
||||||
logger.severe(e, e.toString());
|
|
||||||
rsp.sendError(
|
|
||||||
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
|
|
||||||
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -38,9 +38,11 @@ import google.registry.dns.writer.clouddns.CloudDnsWriterModule;
|
||||||
import google.registry.dns.writer.dnsupdate.DnsUpdateConfigModule;
|
import google.registry.dns.writer.dnsupdate.DnsUpdateConfigModule;
|
||||||
import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule;
|
import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule;
|
||||||
import google.registry.export.BigqueryPollJobAction;
|
import google.registry.export.BigqueryPollJobAction;
|
||||||
|
import google.registry.export.CheckSnapshotAction;
|
||||||
import google.registry.export.ExportDomainListsAction;
|
import google.registry.export.ExportDomainListsAction;
|
||||||
import google.registry.export.ExportRequestModule;
|
import google.registry.export.ExportRequestModule;
|
||||||
import google.registry.export.ExportReservedTermsAction;
|
import google.registry.export.ExportReservedTermsAction;
|
||||||
|
import google.registry.export.ExportSnapshotAction;
|
||||||
import google.registry.export.LoadSnapshotAction;
|
import google.registry.export.LoadSnapshotAction;
|
||||||
import google.registry.export.SyncGroupMembersAction;
|
import google.registry.export.SyncGroupMembersAction;
|
||||||
import google.registry.export.UpdateSnapshotViewAction;
|
import google.registry.export.UpdateSnapshotViewAction;
|
||||||
|
|
@ -94,6 +96,7 @@ import google.registry.tmch.TmchSmdrlAction;
|
||||||
interface BackendRequestComponent {
|
interface BackendRequestComponent {
|
||||||
BigqueryPollJobAction bigqueryPollJobAction();
|
BigqueryPollJobAction bigqueryPollJobAction();
|
||||||
BrdaCopyAction brdaCopyAction();
|
BrdaCopyAction brdaCopyAction();
|
||||||
|
CheckSnapshotAction checkSnapshotAction();
|
||||||
CommitLogCheckpointAction commitLogCheckpointAction();
|
CommitLogCheckpointAction commitLogCheckpointAction();
|
||||||
CommitLogFanoutAction commitLogFanoutAction();
|
CommitLogFanoutAction commitLogFanoutAction();
|
||||||
DeleteContactsAndHostsAction deleteContactsAndHostsAction();
|
DeleteContactsAndHostsAction deleteContactsAndHostsAction();
|
||||||
|
|
@ -103,6 +106,7 @@ interface BackendRequestComponent {
|
||||||
ExportCommitLogDiffAction exportCommitLogDiffAction();
|
ExportCommitLogDiffAction exportCommitLogDiffAction();
|
||||||
ExportDomainListsAction exportDomainListsAction();
|
ExportDomainListsAction exportDomainListsAction();
|
||||||
ExportReservedTermsAction exportReservedTermsAction();
|
ExportReservedTermsAction exportReservedTermsAction();
|
||||||
|
ExportSnapshotAction exportSnapshotAction();
|
||||||
LoadSnapshotAction loadSnapshotAction();
|
LoadSnapshotAction loadSnapshotAction();
|
||||||
MetricsExportAction metricsExportAction();
|
MetricsExportAction metricsExportAction();
|
||||||
NordnUploadAction nordnUploadAction();
|
NordnUploadAction nordnUploadAction();
|
||||||
|
|
|
||||||
226
javatests/google/registry/export/CheckSnapshotActionTest.java
Normal file
226
javatests/google/registry/export/CheckSnapshotActionTest.java
Normal file
|
|
@ -0,0 +1,226 @@
|
||||||
|
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package google.registry.export;
|
||||||
|
|
||||||
|
import static com.google.common.truth.Truth.assertThat;
|
||||||
|
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
||||||
|
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
|
||||||
|
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||||
|
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
import com.google.common.base.Joiner;
|
||||||
|
import com.google.common.base.Optional;
|
||||||
|
import com.google.common.collect.ImmutableList;
|
||||||
|
import com.google.common.collect.ImmutableSet;
|
||||||
|
import google.registry.request.Action.Method;
|
||||||
|
import google.registry.request.HttpException.BadRequestException;
|
||||||
|
import google.registry.request.HttpException.NoContentException;
|
||||||
|
import google.registry.request.HttpException.NotModifiedException;
|
||||||
|
import google.registry.testing.AppEngineRule;
|
||||||
|
import google.registry.testing.ExceptionRule;
|
||||||
|
import google.registry.testing.FakeClock;
|
||||||
|
import google.registry.testing.FakeResponse;
|
||||||
|
import google.registry.testing.InjectRule;
|
||||||
|
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||||
|
import org.joda.time.DateTime;
|
||||||
|
import org.joda.time.Duration;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.runner.RunWith;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.runners.MockitoJUnitRunner;
|
||||||
|
|
||||||
|
/** Unit tests for {@link CheckSnapshotAction}. */
|
||||||
|
@RunWith(MockitoJUnitRunner.class)
|
||||||
|
public class CheckSnapshotActionTest {
|
||||||
|
|
||||||
|
static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
|
||||||
|
static final DateTime COMPLETE_TIME = START_TIME.plus(Duration.standardMinutes(30));
|
||||||
|
|
||||||
|
@Rule public final InjectRule inject = new InjectRule();
|
||||||
|
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
|
||||||
|
@Rule public final ExceptionRule thrown = new ExceptionRule();
|
||||||
|
|
||||||
|
@Mock private DatastoreBackupService backupService;
|
||||||
|
|
||||||
|
private DatastoreBackupInfo backupInfo;
|
||||||
|
|
||||||
|
private final FakeResponse response = new FakeResponse();
|
||||||
|
private final FakeClock clock = new FakeClock(COMPLETE_TIME.plusMillis(1000));
|
||||||
|
private final CheckSnapshotAction action = new CheckSnapshotAction();
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void before() throws Exception {
|
||||||
|
inject.setStaticField(DatastoreBackupInfo.class, "clock", clock);
|
||||||
|
action.requestMethod = Method.POST;
|
||||||
|
action.snapshotName = "some_backup";
|
||||||
|
action.kindsToLoadParam = "one,two";
|
||||||
|
action.response = response;
|
||||||
|
action.backupService = backupService;
|
||||||
|
|
||||||
|
backupInfo =
|
||||||
|
new DatastoreBackupInfo(
|
||||||
|
"some_backup",
|
||||||
|
START_TIME,
|
||||||
|
Optional.of(COMPLETE_TIME),
|
||||||
|
ImmutableSet.of("one", "two", "three"),
|
||||||
|
Optional.of("gs://somebucket/some_backup_20140801.backup_info"));
|
||||||
|
|
||||||
|
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void setPendingBackup() {
|
||||||
|
backupInfo =
|
||||||
|
new DatastoreBackupInfo(
|
||||||
|
backupInfo.getName(),
|
||||||
|
backupInfo.getStartTime(),
|
||||||
|
Optional.<DateTime>absent(),
|
||||||
|
backupInfo.getKinds(),
|
||||||
|
backupInfo.getGcsFilename());
|
||||||
|
|
||||||
|
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void assertLoadTaskEnqueued(String id, String file, String kinds)
|
||||||
|
throws Exception {
|
||||||
|
assertTasksEnqueued(
|
||||||
|
"export-snapshot",
|
||||||
|
new TaskMatcher()
|
||||||
|
.url("/_dr/task/loadSnapshot")
|
||||||
|
.method("POST")
|
||||||
|
.param("id", id)
|
||||||
|
.param("file", file)
|
||||||
|
.param("kinds", kinds));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSuccess_enqueuePollTask() throws Exception {
|
||||||
|
CheckSnapshotAction.enqueuePollTask(
|
||||||
|
"some_snapshot_name", ImmutableSet.of("one", "two", "three"));
|
||||||
|
assertTasksEnqueued(
|
||||||
|
CheckSnapshotAction.QUEUE,
|
||||||
|
new TaskMatcher()
|
||||||
|
.url(CheckSnapshotAction.PATH)
|
||||||
|
.param(CHECK_SNAPSHOT_NAME_PARAM, "some_snapshot_name")
|
||||||
|
.param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, "one,two,three")
|
||||||
|
.method("POST"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPost_forPendingBackup_returnsNotModified() throws Exception {
|
||||||
|
setPendingBackup();
|
||||||
|
|
||||||
|
thrown.expect(NotModifiedException.class, "Datastore backup some_backup still pending");
|
||||||
|
action.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPost_forStalePendingBackupBackup_returnsNoContent() throws Exception {
|
||||||
|
setPendingBackup();
|
||||||
|
|
||||||
|
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||||
|
|
||||||
|
clock.setTo(
|
||||||
|
START_TIME
|
||||||
|
.plus(Duration.standardHours(20))
|
||||||
|
.plus(Duration.standardMinutes(3))
|
||||||
|
.plus(Duration.millis(1234)));
|
||||||
|
|
||||||
|
thrown.expect(
|
||||||
|
NoContentException.class,
|
||||||
|
"Datastore backup some_backup abandoned - "
|
||||||
|
+ "not complete after 20 hours, 3 minutes and 1 second");
|
||||||
|
|
||||||
|
action.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPost_forCompleteBackup_enqueuesLoadTask() throws Exception {
|
||||||
|
|
||||||
|
action.run();
|
||||||
|
assertLoadTaskEnqueued(
|
||||||
|
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPost_forCompleteAutoBackup_enqueuesLoadTask_usingBackupName() throws Exception {
|
||||||
|
action.snapshotName = "auto_snapshot_somestring";
|
||||||
|
when(backupService.findByName("auto_snapshot_somestring")).thenReturn(backupInfo);
|
||||||
|
|
||||||
|
action.run();
|
||||||
|
assertLoadTaskEnqueued(
|
||||||
|
"somestring", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPost_forCompleteBackup_withExtraKindsToLoad_enqueuesLoadTask() throws Exception {
|
||||||
|
action.kindsToLoadParam = "one,foo";
|
||||||
|
|
||||||
|
action.run();
|
||||||
|
assertLoadTaskEnqueued(
|
||||||
|
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPost_forCompleteBackup_withEmptyKindsToLoad_skipsLoadTask() throws Exception {
|
||||||
|
action.kindsToLoadParam = "";
|
||||||
|
|
||||||
|
action.run();
|
||||||
|
assertNoTasksEnqueued("export-snapshot");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPost_forBadBackup_returnsBadRequest() throws Exception {
|
||||||
|
when(backupService.findByName("some_backup"))
|
||||||
|
.thenThrow(new IllegalArgumentException("No backup found"));
|
||||||
|
|
||||||
|
thrown.expect(BadRequestException.class, "Bad backup name some_backup: No backup found");
|
||||||
|
|
||||||
|
action.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGet_returnsInformation() throws Exception {
|
||||||
|
action.requestMethod = Method.GET;
|
||||||
|
|
||||||
|
action.run();
|
||||||
|
assertThat(response.getPayload())
|
||||||
|
.isEqualTo(
|
||||||
|
Joiner.on("\n")
|
||||||
|
.join(
|
||||||
|
ImmutableList.of(
|
||||||
|
"Backup name: some_backup",
|
||||||
|
"Status: COMPLETE",
|
||||||
|
"Started: 2014-08-01T01:02:03.000Z",
|
||||||
|
"Ended: 2014-08-01T01:32:03.000Z",
|
||||||
|
"Duration: 30m",
|
||||||
|
"GCS: gs://somebucket/some_backup_20140801.backup_info",
|
||||||
|
"Kinds: [one, two, three]",
|
||||||
|
"")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGet_forBadBackup_returnsError() throws Exception {
|
||||||
|
action.requestMethod = Method.GET;
|
||||||
|
when(backupService.findByName("some_backup"))
|
||||||
|
.thenThrow(new IllegalArgumentException("No backup found"));
|
||||||
|
|
||||||
|
thrown.expect(BadRequestException.class, "Bad backup name some_backup: No backup found");
|
||||||
|
|
||||||
|
action.run();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,263 +0,0 @@
|
||||||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static com.google.common.truth.Truth.assertThat;
|
|
||||||
import static google.registry.export.CheckSnapshotServlet.SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
|
||||||
import static google.registry.export.CheckSnapshotServlet.SNAPSHOT_NAME_PARAM;
|
|
||||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
|
||||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED;
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import com.google.common.base.Joiner;
|
|
||||||
import com.google.common.base.Optional;
|
|
||||||
import com.google.common.collect.ImmutableList;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
|
||||||
import google.registry.testing.AppEngineRule;
|
|
||||||
import google.registry.testing.FakeClock;
|
|
||||||
import google.registry.testing.InjectRule;
|
|
||||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
|
||||||
import java.io.PrintWriter;
|
|
||||||
import java.io.StringWriter;
|
|
||||||
import javax.servlet.ServletConfig;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
import org.joda.time.Duration;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.runner.RunWith;
|
|
||||||
import org.mockito.Mock;
|
|
||||||
import org.mockito.runners.MockitoJUnitRunner;
|
|
||||||
|
|
||||||
/** Unit tests for {@link CheckSnapshotServlet}. */
|
|
||||||
@RunWith(MockitoJUnitRunner.class)
|
|
||||||
public class CheckSnapshotServletTest {
|
|
||||||
|
|
||||||
static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
|
|
||||||
static final DateTime COMPLETE_TIME = START_TIME.plus(Duration.standardMinutes(30));
|
|
||||||
|
|
||||||
@Rule
|
|
||||||
public final InjectRule inject = new InjectRule();
|
|
||||||
|
|
||||||
@Rule
|
|
||||||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
|
||||||
.withTaskQueue()
|
|
||||||
.build();
|
|
||||||
|
|
||||||
@Mock
|
|
||||||
private HttpServletRequest req;
|
|
||||||
|
|
||||||
@Mock
|
|
||||||
private HttpServletResponse rsp;
|
|
||||||
|
|
||||||
private DatastoreBackupInfo backupInfo;
|
|
||||||
|
|
||||||
@Mock
|
|
||||||
private DatastoreBackupService backupService;
|
|
||||||
|
|
||||||
private final FakeClock clock = new FakeClock(COMPLETE_TIME.plusMillis(1000));
|
|
||||||
private final StringWriter httpOutput = new StringWriter();
|
|
||||||
private final CheckSnapshotServlet servlet = new CheckSnapshotServlet();
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void before() throws Exception {
|
|
||||||
inject.setStaticField(CheckSnapshotServlet.class, "backupService", backupService);
|
|
||||||
inject.setStaticField(DatastoreBackupInfo.class, "clock", clock);
|
|
||||||
|
|
||||||
when(rsp.getWriter()).thenReturn(new PrintWriter(httpOutput));
|
|
||||||
|
|
||||||
servlet.init(mock(ServletConfig.class));
|
|
||||||
when(req.getMethod()).thenReturn("POST");
|
|
||||||
|
|
||||||
backupInfo = new DatastoreBackupInfo(
|
|
||||||
"some_backup",
|
|
||||||
START_TIME,
|
|
||||||
Optional.of(COMPLETE_TIME),
|
|
||||||
ImmutableSet.of("one", "two", "three"),
|
|
||||||
Optional.of("gs://somebucket/some_backup_20140801.backup_info"));
|
|
||||||
}
|
|
||||||
|
|
||||||
private void setPendingBackup() {
|
|
||||||
backupInfo = new DatastoreBackupInfo(
|
|
||||||
backupInfo.getName(),
|
|
||||||
backupInfo.getStartTime(),
|
|
||||||
Optional.<DateTime>absent(),
|
|
||||||
backupInfo.getKinds(),
|
|
||||||
backupInfo.getGcsFilename());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void assertLoadTaskEnqueued(String id, String file, String kinds)
|
|
||||||
throws Exception {
|
|
||||||
assertTasksEnqueued(
|
|
||||||
"export-snapshot",
|
|
||||||
new TaskMatcher()
|
|
||||||
.url("/_dr/task/loadSnapshot")
|
|
||||||
.method("POST")
|
|
||||||
.param("id", id)
|
|
||||||
.param("file", file)
|
|
||||||
.param("kinds", kinds));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_enqueuePollTask() throws Exception {
|
|
||||||
servlet.enqueuePollTask("some_snapshot_name", ImmutableSet.of("one", "two", "three"));
|
|
||||||
assertTasksEnqueued(CheckSnapshotServlet.QUEUE,
|
|
||||||
new TaskMatcher()
|
|
||||||
.url(CheckSnapshotServlet.PATH)
|
|
||||||
.param(SNAPSHOT_NAME_PARAM, "some_snapshot_name")
|
|
||||||
.param(SNAPSHOT_KINDS_TO_LOAD_PARAM, "one,two,three")
|
|
||||||
.method("POST"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forPendingBackup_returnsNotModified() throws Exception {
|
|
||||||
setPendingBackup();
|
|
||||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
|
||||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
|
||||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
|
||||||
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).sendError(SC_NOT_MODIFIED, "Datastore backup some_backup still pending");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forStalePendingBackupBackup_returnsAccepted() throws Exception {
|
|
||||||
setPendingBackup();
|
|
||||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
|
||||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
|
||||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
|
||||||
clock.setTo(START_TIME
|
|
||||||
.plus(Duration.standardHours(20))
|
|
||||||
.plus(Duration.standardMinutes(3))
|
|
||||||
.plus(Duration.millis(1234)));
|
|
||||||
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).sendError(SC_ACCEPTED,
|
|
||||||
"Datastore backup some_backup abandoned - "
|
|
||||||
+ "not complete after 20 hours, 3 minutes and 1 second");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forCompleteBackup_enqueuesLoadTask() throws Exception {
|
|
||||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
|
||||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
|
||||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
|
||||||
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).setStatus(SC_OK);
|
|
||||||
assertLoadTaskEnqueued(
|
|
||||||
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forCompleteAutoBackup_enqueuesLoadTask_usingBackupName() throws Exception {
|
|
||||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("auto_snapshot_somestring");
|
|
||||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
|
||||||
when(backupService.findByName("auto_snapshot_somestring")).thenReturn(backupInfo);
|
|
||||||
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).setStatus(SC_OK);
|
|
||||||
assertLoadTaskEnqueued(
|
|
||||||
"somestring", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forCompleteBackup_withExtraKindsToLoad_enqueuesLoadTask() throws Exception {
|
|
||||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
|
||||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,foo");
|
|
||||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
|
||||||
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).setStatus(SC_OK);
|
|
||||||
assertLoadTaskEnqueued(
|
|
||||||
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forCompleteBackup_withEmptyKindsToLoad_skipsLoadTask() throws Exception {
|
|
||||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
|
||||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("");
|
|
||||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
|
||||||
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).setStatus(SC_OK);
|
|
||||||
assertNoTasksEnqueued("export-snapshot");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_forBadBackup_returnsBadRequest() throws Exception {
|
|
||||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
|
||||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
|
||||||
when(backupService.findByName("some_backup"))
|
|
||||||
.thenThrow(new IllegalArgumentException("No backup found"));
|
|
||||||
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).sendError(SC_BAD_REQUEST, "Bad backup name some_backup: No backup found");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_noBackupSpecified_returnsError() throws Exception {
|
|
||||||
when(req.getMethod()).thenReturn("POST");
|
|
||||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn(null);
|
|
||||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).sendError(SC_BAD_REQUEST, "Missing parameter: name");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testGet_returnsInformation() throws Exception {
|
|
||||||
when(req.getMethod()).thenReturn("GET");
|
|
||||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
|
||||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
|
||||||
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).setStatus(SC_OK);
|
|
||||||
assertThat(httpOutput.toString()).isEqualTo("OK\n\n" + Joiner.on("\n").join(ImmutableList.of(
|
|
||||||
"Backup name: some_backup",
|
|
||||||
"Status: COMPLETE",
|
|
||||||
"Started: 2014-08-01T01:02:03.000Z",
|
|
||||||
"Ended: 2014-08-01T01:32:03.000Z",
|
|
||||||
"Duration: 30m",
|
|
||||||
"GCS: gs://somebucket/some_backup_20140801.backup_info",
|
|
||||||
"Kinds: [one, two, three]",
|
|
||||||
"")));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testGet_forBadBackup_returnsError() throws Exception {
|
|
||||||
when(req.getMethod()).thenReturn("GET");
|
|
||||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
|
||||||
when(backupService.findByName("some_backup")).thenThrow(
|
|
||||||
new IllegalArgumentException("No backup found"));
|
|
||||||
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).sendError(SC_BAD_REQUEST, "No backup found");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testGet_noBackupSpecified_returnsError() throws Exception {
|
|
||||||
when(req.getMethod()).thenReturn("GET");
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).sendError(SC_BAD_REQUEST, "Missing parameter: name");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -0,0 +1,76 @@
|
||||||
|
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package google.registry.export;
|
||||||
|
|
||||||
|
import static com.google.common.truth.Truth.assertThat;
|
||||||
|
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
||||||
|
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
|
||||||
|
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
|
||||||
|
import com.google.common.base.Joiner;
|
||||||
|
import google.registry.testing.AppEngineRule;
|
||||||
|
import google.registry.testing.FakeClock;
|
||||||
|
import google.registry.testing.FakeResponse;
|
||||||
|
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||||
|
import org.joda.time.DateTime;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.runner.RunWith;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.runners.MockitoJUnitRunner;
|
||||||
|
|
||||||
|
/** Unit tests for {@link ExportSnapshotAction}. */
|
||||||
|
@RunWith(MockitoJUnitRunner.class)
|
||||||
|
public class ExportSnapshotActionTest {
|
||||||
|
|
||||||
|
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
|
||||||
|
|
||||||
|
@Mock private DatastoreBackupService backupService;
|
||||||
|
|
||||||
|
private final FakeResponse response = new FakeResponse();
|
||||||
|
private final FakeClock clock = new FakeClock(DateTime.parse("2014-08-01T01:02:03Z"));
|
||||||
|
private final ExportSnapshotAction action = new ExportSnapshotAction();
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void before() throws Exception {
|
||||||
|
action.clock = clock;
|
||||||
|
action.backupService = backupService;
|
||||||
|
action.response = response;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPost_launchesBackup_andEnqueuesPollTask() throws Exception {
|
||||||
|
action.run();
|
||||||
|
verify(backupService)
|
||||||
|
.launchNewBackup(
|
||||||
|
ExportSnapshotAction.QUEUE,
|
||||||
|
"auto_snapshot_20140801_010203",
|
||||||
|
"domain-registry-snapshots",
|
||||||
|
ExportConstants.getBackupKinds());
|
||||||
|
assertTasksEnqueued(
|
||||||
|
CheckSnapshotAction.QUEUE,
|
||||||
|
new TaskMatcher()
|
||||||
|
.url(CheckSnapshotAction.PATH)
|
||||||
|
.param(CHECK_SNAPSHOT_NAME_PARAM, "auto_snapshot_20140801_010203")
|
||||||
|
.param(
|
||||||
|
CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM,
|
||||||
|
Joiner.on(",").join(ExportConstants.getReportingKinds()))
|
||||||
|
.method("POST"));
|
||||||
|
assertThat(response.getPayload())
|
||||||
|
.isEqualTo("Datastore backup started with name: auto_snapshot_20140801_010203");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,87 +0,0 @@
|
||||||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.export;
|
|
||||||
|
|
||||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import google.registry.testing.FakeClock;
|
|
||||||
import google.registry.testing.InjectRule;
|
|
||||||
import java.io.PrintWriter;
|
|
||||||
import java.io.StringWriter;
|
|
||||||
import javax.servlet.ServletConfig;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
import org.joda.time.DateTime;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.runner.RunWith;
|
|
||||||
import org.mockito.Mock;
|
|
||||||
import org.mockito.runners.MockitoJUnitRunner;
|
|
||||||
|
|
||||||
/** Unit tests for {@link ExportSnapshotServlet}. */
|
|
||||||
@RunWith(MockitoJUnitRunner.class)
|
|
||||||
public class ExportSnapshotServletTest {
|
|
||||||
|
|
||||||
@Rule
|
|
||||||
public final InjectRule inject = new InjectRule();
|
|
||||||
|
|
||||||
@Mock
|
|
||||||
private HttpServletRequest req;
|
|
||||||
|
|
||||||
@Mock
|
|
||||||
private HttpServletResponse rsp;
|
|
||||||
|
|
||||||
@Mock
|
|
||||||
private DatastoreBackupService backupService;
|
|
||||||
|
|
||||||
@Mock
|
|
||||||
private CheckSnapshotServlet checkSnapshotServlet;
|
|
||||||
|
|
||||||
private final FakeClock clock = new FakeClock();
|
|
||||||
private final StringWriter httpOutput = new StringWriter();
|
|
||||||
private final ExportSnapshotServlet servlet = new ExportSnapshotServlet();
|
|
||||||
|
|
||||||
private static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void before() throws Exception {
|
|
||||||
clock.setTo(START_TIME);
|
|
||||||
inject.setStaticField(ExportSnapshotServlet.class, "clock", clock);
|
|
||||||
inject.setStaticField(ExportSnapshotServlet.class, "backupService", backupService);
|
|
||||||
inject.setStaticField(
|
|
||||||
ExportSnapshotServlet.class, "checkSnapshotServlet", checkSnapshotServlet);
|
|
||||||
when(rsp.getWriter()).thenReturn(new PrintWriter(httpOutput));
|
|
||||||
|
|
||||||
servlet.init(mock(ServletConfig.class));
|
|
||||||
when(req.getMethod()).thenReturn("POST");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testPost_launchesBackup_andEnqueuesPollTask() throws Exception {
|
|
||||||
servlet.service(req, rsp);
|
|
||||||
verify(rsp).setStatus(SC_OK);
|
|
||||||
verify(backupService).launchNewBackup(
|
|
||||||
ExportSnapshotServlet.QUEUE,
|
|
||||||
"auto_snapshot_20140801_010203",
|
|
||||||
"domain-registry-snapshots",
|
|
||||||
ExportConstants.getBackupKinds());
|
|
||||||
verify(checkSnapshotServlet)
|
|
||||||
.enqueuePollTask("auto_snapshot_20140801_010203", ExportConstants.getReportingKinds());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue