mirror of
https://github.com/google/nomulus.git
synced 2025-05-14 00:17:20 +02:00
Daggerize ExportSnapshotServlet and CheckSnapshotServlet
Eradicate the last remnants of un-injectable servlets! ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=145598002
This commit is contained in:
jianglai
Ben McIlwain
parent
a8aeff96f6
commit
4fed3a9ae6
15 changed files with 577 additions and 642 deletions
|
|
@ -110,9 +110,9 @@ Here are the task queues in use by the system. All are push queues unless
|
|||
explicitly marked as otherwise.
|
||||
|
||||
* `async-delete-pull` and `async-host-rename-pull` -- Pull queues for tasks to
|
||||
asynchronously delete contacts/hosts and to asynchronously refresh DNS
|
||||
for renamed hosts, respectively. Tasks are enqueued during EPP
|
||||
flows and then handled in batches by the regularly running cron tasks
|
||||
asynchronously delete contacts/hosts and to asynchronously refresh DNS for
|
||||
renamed hosts, respectively. Tasks are enqueued during EPP flows and then
|
||||
handled in batches by the regularly running cron tasks
|
||||
`DeleteContactsAndHostsAction` and `RefreshDnsOnHostRenameAction`.
|
||||
* `bigquery-streaming-metrics` -- Queue for metrics that are asynchronously
|
||||
streamed to BigQuery in the `Metrics` class. Tasks are enqueued during EPP
|
||||
|
|
@ -140,14 +140,14 @@ explicitly marked as otherwise.
|
|||
cron) and executed by `ExportCommitLogDiffAction`.
|
||||
* `export-snapshot` -- Cron and push queue for tasks to load a Datastore
|
||||
snapshot that was stored in Google Cloud Storage and export it to BigQuery.
|
||||
Tasks are enqueued by both cron and `CheckSnapshotServlet` and are executed
|
||||
by both `ExportSnapshotServlet` and `LoadSnapshotAction`.
|
||||
Tasks are enqueued by both cron and `CheckSnapshotAction` and are executed
|
||||
by both `ExportSnapshotAction` and `LoadSnapshotAction`.
|
||||
* `export-snapshot-poll` -- Queue for tasks to check that a Datastore snapshot
|
||||
has been successfully uploaded to Google Cloud Storage (this is an
|
||||
asynchronous background operation that can take an indeterminate amount of
|
||||
time). Once the snapshot is successfully uploaded, it is imported into
|
||||
BigQuery. Tasks are enqueued by `ExportSnapshotServlet` and executed by
|
||||
`CheckSnapshotServlet`.
|
||||
BigQuery. Tasks are enqueued by `ExportSnapshotAction` and executed by
|
||||
`CheckSnapshotAction`.
|
||||
* `export-snapshot-update-view` -- Queue for tasks to update the BigQuery
|
||||
views to point to the most recently uploaded snapshot. Tasks are enqueued by
|
||||
`LoadSnapshotAction` and executed by `UpdateSnapshotViewAction`.
|
||||
|
|
|
|||
|
|
@ -1102,7 +1102,7 @@ public final class RegistryConfig {
|
|||
/**
|
||||
* Returns the Google Cloud Storage bucket for storing backup snapshots.
|
||||
*
|
||||
* @see google.registry.export.ExportSnapshotServlet
|
||||
* @see google.registry.export.ExportSnapshotAction
|
||||
*/
|
||||
public static String getSnapshotsBucket() {
|
||||
return getProjectId() + "-snapshots";
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ public abstract class DnsModule {
|
|||
}
|
||||
|
||||
@Provides
|
||||
@Parameter("name")
|
||||
@Parameter("domainOrHostName")
|
||||
static String provideName(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, "name");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ public final class RefreshDnsAction implements Runnable {
|
|||
|
||||
@Inject Clock clock;
|
||||
@Inject DnsQueue dnsQueue;
|
||||
@Inject @Parameter("name") String domainOrHostName;
|
||||
@Inject @Parameter("domainOrHostName") String domainOrHostName;
|
||||
@Inject @Parameter("type") TargetType type;
|
||||
@Inject RefreshDnsAction() {}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<web-app xmlns="http://java.sun.com/xml/ns/javaee" version="2.5"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
|
||||
http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd">
|
||||
<!-- Servlets -->
|
||||
|
||||
|
|
@ -113,25 +113,15 @@
|
|||
<url-pattern>/_dr/task/verifyEntityIntegrity</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<servlet>
|
||||
<description>Exports a datastore backup snapshot to GCS.</description>
|
||||
<display-name>Export snapshot to GCS</display-name>
|
||||
<servlet-name>exportSnapshot</servlet-name>
|
||||
<servlet-class>google.registry.export.ExportSnapshotServlet</servlet-class>
|
||||
</servlet>
|
||||
<!-- Exports a datastore backup snapshot to GCS. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>exportSnapshot</servlet-name>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/exportSnapshot</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<servlet>
|
||||
<description>Checks the completion of a datastore backup snapshot.</description>
|
||||
<display-name>Check on snapshot status</display-name>
|
||||
<servlet-name>checkSnapshot</servlet-name>
|
||||
<servlet-class>google.registry.export.CheckSnapshotServlet</servlet-class>
|
||||
</servlet>
|
||||
<!-- Checks the completion of a datastore backup snapshot. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>checkSnapshot</servlet-name>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/checkSnapshot</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
|
|
@ -290,10 +280,10 @@
|
|||
<web-resource-collection>
|
||||
<web-resource-name>Internal</web-resource-name>
|
||||
<description>
|
||||
Admin-only internal section. Requests for paths covered by the URL patterns below will be
|
||||
Admin-only internal section. Requests for paths covered by the URL patterns below will be
|
||||
checked for a logged-in user account that's allowed to access the AppEngine admin console
|
||||
(NOTE: this includes Editor/Viewer permissions in addition to Owner and the new IAM
|
||||
App Engine Admin role. See https://cloud.google.com/appengine/docs/java/access-control
|
||||
App Engine Admin role. See https://cloud.google.com/appengine/docs/java/access-control
|
||||
specifically the "Access handlers that have a login:admin restriction" line.)
|
||||
|
||||
TODO(b/28219927): lift some of these restrictions so that we can allow OAuth authentication
|
||||
|
|
|
|||
162
java/google/registry/export/CheckSnapshotAction.java
Normal file
162
java/google/registry/export/CheckSnapshotAction.java
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.collect.Sets.intersection;
|
||||
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
|
||||
import static google.registry.request.Action.Method.GET;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static google.registry.util.FormattingLogger.getLoggerForCallerClass;
|
||||
|
||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Sets;
|
||||
import google.registry.export.DatastoreBackupInfo.BackupStatus;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.HttpException.NoContentException;
|
||||
import google.registry.request.HttpException.NotModifiedException;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.RequestMethod;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import java.util.Set;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.Duration;
|
||||
import org.joda.time.PeriodType;
|
||||
import org.joda.time.format.PeriodFormat;
|
||||
|
||||
/**
|
||||
* Action that checks the status of a snapshot, and if complete, trigger loading it into BigQuery.
|
||||
*/
|
||||
@Action(
|
||||
path = CheckSnapshotAction.PATH,
|
||||
method = {POST, GET},
|
||||
automaticallyPrintOk = true
|
||||
)
|
||||
public class CheckSnapshotAction implements Runnable {
|
||||
|
||||
/** Parameter names for passing parameters into this action. */
|
||||
static final String CHECK_SNAPSHOT_NAME_PARAM = "name";
|
||||
|
||||
static final String CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";
|
||||
|
||||
/** Action-specific details needed for enqueuing tasks against itself. */
|
||||
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
|
||||
|
||||
static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml.
|
||||
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
|
||||
|
||||
/** The maximum amount of time we allow a backup to run before abandoning it. */
|
||||
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
|
||||
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
|
||||
@Inject Response response;
|
||||
@Inject @RequestMethod Action.Method requestMethod;
|
||||
@Inject DatastoreBackupService backupService;
|
||||
@Inject @Parameter(CHECK_SNAPSHOT_NAME_PARAM) String snapshotName;
|
||||
@Inject @Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM) String kindsToLoadParam;
|
||||
@Inject CheckSnapshotAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
if (requestMethod == POST) {
|
||||
checkAndLoadSnapshotIfComplete();
|
||||
} else {
|
||||
// This is a GET request.
|
||||
response.setPayload(getBackup().getInformation());
|
||||
}
|
||||
}
|
||||
|
||||
private DatastoreBackupInfo getBackup() {
|
||||
try {
|
||||
return backupService.findByName(snapshotName);
|
||||
} catch (IllegalArgumentException e) {
|
||||
String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
|
||||
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be
|
||||
// retried but we might abandon backups that start late and haven't yet written to
|
||||
// datastore. We could fix that by replacing this with a two-phase polling strategy.
|
||||
throw new BadRequestException(message, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkAndLoadSnapshotIfComplete() {
|
||||
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
|
||||
DatastoreBackupInfo backup = getBackup();
|
||||
// Stop now if the backup is not complete.
|
||||
if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
|
||||
Duration runningTime = backup.getRunningTime();
|
||||
if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
|
||||
// Backup might still be running, so send a 304 to have the task retry.
|
||||
throw new NotModifiedException(
|
||||
String.format("Datastore backup %s still pending", snapshotName));
|
||||
} else {
|
||||
// Declare the backup a lost cause, and send 204 No Content so the task will
|
||||
// not be retried.
|
||||
String message =
|
||||
String.format(
|
||||
"Datastore backup %s abandoned - not complete after %s",
|
||||
snapshotName,
|
||||
PeriodFormat.getDefault()
|
||||
.print(
|
||||
runningTime
|
||||
.toPeriod()
|
||||
.normalizedStandard(PeriodType.dayTime().withMillisRemoved())));
|
||||
throw new NoContentException(message);
|
||||
}
|
||||
}
|
||||
// Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
|
||||
// suffix out of the snapshot name and falling back to the start time as a string.
|
||||
String snapshotId =
|
||||
snapshotName.startsWith(ExportSnapshotAction.SNAPSHOT_PREFIX)
|
||||
? snapshotName.substring(ExportSnapshotAction.SNAPSHOT_PREFIX.length())
|
||||
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
|
||||
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
|
||||
if (!backup.getKinds().containsAll(kindsToLoad)) {
|
||||
logger.warningfmt(
|
||||
"Kinds to load included non-exported kinds: %s",
|
||||
Sets.difference(kindsToLoad, backup.getKinds()));
|
||||
}
|
||||
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
|
||||
ImmutableSet<String> exportedKindsToLoad =
|
||||
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
|
||||
String message = String.format("Datastore backup %s complete - ", snapshotName);
|
||||
if (exportedKindsToLoad.isEmpty()) {
|
||||
message += "no kinds to load into BigQuery";
|
||||
} else {
|
||||
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
|
||||
message += "BigQuery load task enqueued";
|
||||
}
|
||||
logger.info(message);
|
||||
response.setPayload(message);
|
||||
}
|
||||
|
||||
/** Enqueue a poll task to monitor the named snapshot for completion. */
|
||||
static TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
|
||||
return QueueFactory.getQueue(QUEUE)
|
||||
.add(
|
||||
TaskOptions.Builder.withUrl(PATH)
|
||||
.method(Method.POST)
|
||||
.countdownMillis(POLL_COUNTDOWN.getMillis())
|
||||
.param(CHECK_SNAPSHOT_NAME_PARAM, snapshotName)
|
||||
.param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,176 +0,0 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.collect.Sets.intersection;
|
||||
import static com.google.common.html.HtmlEscapers.htmlEscaper;
|
||||
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
|
||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||
import static google.registry.util.FormattingLogger.getLoggerForCallerClass;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.export.DatastoreBackupInfo.BackupStatus;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import org.joda.time.Duration;
|
||||
import org.joda.time.PeriodType;
|
||||
import org.joda.time.format.PeriodFormat;
|
||||
|
||||
/** Check the status of a snapshot, and if complete, trigger loading it into BigQuery. */
|
||||
public class CheckSnapshotServlet extends HttpServlet {
|
||||
|
||||
/** Parameter names for passing parameters into this servlet. */
|
||||
static final String SNAPSHOT_NAME_PARAM = "name";
|
||||
static final String SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";
|
||||
|
||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
||||
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
|
||||
static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml.
|
||||
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
|
||||
|
||||
/** The maximum amount of time we allow a backup to run before abandoning it. */
|
||||
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
|
||||
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static DatastoreBackupService backupService = DatastoreBackupService.get();
|
||||
|
||||
@Override
|
||||
public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
try {
|
||||
rsp.setStatus(SC_OK);
|
||||
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
|
||||
rsp.getWriter().write("OK\n\n");
|
||||
super.service(req, rsp);
|
||||
} catch (Throwable e) {
|
||||
logger.severe(e, e.toString());
|
||||
rsp.sendError(
|
||||
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
|
||||
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doGet(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
|
||||
try {
|
||||
String snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
|
||||
rsp.getWriter().write(backupService.findByName(snapshotName).getInformation());
|
||||
} catch (BadRequestException e) {
|
||||
throw new IllegalArgumentException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
String snapshotName;
|
||||
String kindsToLoadParam;
|
||||
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
|
||||
try {
|
||||
snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
|
||||
kindsToLoadParam = extractRequiredParameter(req, SNAPSHOT_KINDS_TO_LOAD_PARAM);
|
||||
} catch (BadRequestException e) {
|
||||
throw new IllegalArgumentException(e.getMessage());
|
||||
}
|
||||
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
|
||||
|
||||
// Look up the backup by the provided name, stopping if we can't find it.
|
||||
DatastoreBackupInfo backup;
|
||||
try {
|
||||
backup = backupService.findByName(snapshotName);
|
||||
} catch (IllegalArgumentException e) {
|
||||
String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
|
||||
logger.severe(e, message);
|
||||
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be retried,
|
||||
// but we might abandon backups that start late and haven't yet written to datastore.
|
||||
// We could fix that by replacing this with a two-phase polling strategy.
|
||||
rsp.sendError(SC_BAD_REQUEST, htmlEscaper().escape(message));
|
||||
return;
|
||||
}
|
||||
// Stop now if the backup is not complete.
|
||||
if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
|
||||
Duration runningTime = backup.getRunningTime();
|
||||
if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
|
||||
// Backup might still be running, so send a 304 to have the task retry.
|
||||
rsp.sendError(SC_NOT_MODIFIED,
|
||||
htmlEscaper().escape(String.format("Datastore backup %s still pending", snapshotName)));
|
||||
} else {
|
||||
// Declare the backup a lost cause, and send 202 Accepted so the task will not be retried.
|
||||
String message = String.format("Datastore backup %s abandoned - not complete after %s",
|
||||
snapshotName,
|
||||
PeriodFormat.getDefault().print(
|
||||
runningTime.toPeriod().normalizedStandard(
|
||||
PeriodType.dayTime().withMillisRemoved())));
|
||||
logger.severe(message);
|
||||
rsp.sendError(SC_ACCEPTED, htmlEscaper().escape(message));
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
|
||||
// suffix out of the snapshot name and falling back to the start time as a string.
|
||||
String snapshotId = snapshotName.startsWith(ExportSnapshotServlet.SNAPSHOT_PREFIX)
|
||||
? snapshotName.substring(ExportSnapshotServlet.SNAPSHOT_PREFIX.length())
|
||||
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
|
||||
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
|
||||
if (!backup.getKinds().containsAll(kindsToLoad)) {
|
||||
logger.warningfmt(
|
||||
"Kinds to load included non-exported kinds: %s",
|
||||
Sets.difference(kindsToLoad, backup.getKinds()));
|
||||
}
|
||||
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
|
||||
ImmutableSet<String> exportedKindsToLoad =
|
||||
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
|
||||
String message = String.format("Datastore backup %s complete - ", snapshotName);
|
||||
if (exportedKindsToLoad.isEmpty()) {
|
||||
message += "no kinds to load into BigQuery";
|
||||
} else {
|
||||
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
|
||||
message += "BigQuery load task enqueued";
|
||||
}
|
||||
logger.info(message);
|
||||
rsp.getWriter().write(message);
|
||||
}
|
||||
|
||||
/** Enqueue a poll task to monitor the named snapshot for completion. */
|
||||
TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
|
||||
return QueueFactory.getQueue(QUEUE).add(
|
||||
TaskOptions.Builder.withUrl(PATH)
|
||||
.method(Method.POST)
|
||||
.countdownMillis(POLL_COUNTDOWN.getMillis())
|
||||
.param(SNAPSHOT_NAME_PARAM, snapshotName)
|
||||
.param(SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
|
||||
}
|
||||
}
|
||||
|
|
@ -17,6 +17,8 @@ package google.registry.export;
|
|||
import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
|
||||
import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
|
||||
import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
|
||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
|
||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
|
||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
|
||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
|
||||
|
|
@ -72,6 +74,18 @@ public final class ExportRequestModule {
|
|||
return extractRequiredParameter(req, LOAD_SNAPSHOT_KINDS_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(CHECK_SNAPSHOT_NAME_PARAM)
|
||||
static String provideCheckSnapshotName(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, CHECK_SNAPSHOT_NAME_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM)
|
||||
static String provideCheckSnapshotKindsToLoad(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Header(CHAINED_TASK_QUEUE_HEADER)
|
||||
static String provideChainedTaskQueue(HttpServletRequest req) {
|
||||
|
|
@ -89,4 +103,9 @@ public final class ExportRequestModule {
|
|||
static String provideProjectId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, PROJECT_ID_HEADER);
|
||||
}
|
||||
|
||||
@Provides
|
||||
static DatastoreBackupService provideDatastoreBackupService() {
|
||||
return DatastoreBackupService.get();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
72
java/google/registry/export/ExportSnapshotAction.java
Normal file
72
java/google/registry/export/ExportSnapshotAction.java
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static google.registry.export.CheckSnapshotAction.enqueuePollTask;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* Action to trigger a datastore backup job that writes a snapshot to Google Cloud Storage.
|
||||
*
|
||||
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
|
||||
* the next upon successful completion:
|
||||
*
|
||||
* <ol>
|
||||
* <li>The snapshot is exported to Google Cloud Storage (this action).
|
||||
* <li>The {@link CheckSnapshotAction} polls until the export is completed.
|
||||
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
|
||||
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
|
||||
* </ol>
|
||||
*/
|
||||
@Action(path = ExportSnapshotAction.PATH, method = POST, automaticallyPrintOk = true)
|
||||
public class ExportSnapshotAction implements Runnable {
|
||||
|
||||
/** Queue to use for enqueuing the task that will actually launch the backup. */
|
||||
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
||||
|
||||
static final String PATH = "/_dr/task/exportSnapshot"; // See web.xml.
|
||||
|
||||
/** Prefix to use for naming all snapshots that are started by this servlet. */
|
||||
static final String SNAPSHOT_PREFIX = "auto_snapshot_";
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@Inject Clock clock;
|
||||
@Inject DatastoreBackupService backupService;
|
||||
@Inject Response response;
|
||||
|
||||
@Inject
|
||||
ExportSnapshotAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
// Use a unique name for the snapshot so we can explicitly check its completion later.
|
||||
String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss");
|
||||
backupService.launchNewBackup(
|
||||
QUEUE, snapshotName, RegistryConfig.getSnapshotsBucket(), ExportConstants.getBackupKinds());
|
||||
// Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery.
|
||||
enqueuePollTask(snapshotName, ExportConstants.getReportingKinds());
|
||||
String message = "Datastore backup started with name: " + snapshotName;
|
||||
logger.info(message);
|
||||
response.setPayload(message);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.html.HtmlEscapers.htmlEscaper;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import google.registry.util.SystemClock;
|
||||
import java.io.IOException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
/**
|
||||
* Trigger a backup-as-a-service job that writes a snapshot to Google Cloud Storage.
|
||||
*
|
||||
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
|
||||
* the next upon successful completion:<ol>
|
||||
* <li>The snapshot is exported to Google Cloud Storage (this servlet).
|
||||
* <li>The {@link CheckSnapshotServlet} polls until the export is completed.
|
||||
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
|
||||
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
|
||||
* </ol>
|
||||
*/
|
||||
public class ExportSnapshotServlet extends HttpServlet {
|
||||
|
||||
/** Queue to use for enqueuing the task that will actually launch the backup. */
|
||||
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
||||
|
||||
/** Prefix to use for naming all snapshots that are started by this servlet. */
|
||||
static final String SNAPSHOT_PREFIX = "auto_snapshot_";
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static Clock clock = new SystemClock();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static DatastoreBackupService backupService = DatastoreBackupService.get();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static CheckSnapshotServlet checkSnapshotServlet = new CheckSnapshotServlet();
|
||||
|
||||
@Override
|
||||
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
try {
|
||||
// Use a unique name for the snapshot so we can explicitly check its completion later.
|
||||
String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss");
|
||||
backupService.launchNewBackup(
|
||||
QUEUE,
|
||||
snapshotName,
|
||||
RegistryConfig.getSnapshotsBucket(),
|
||||
ExportConstants.getBackupKinds());
|
||||
// Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery.
|
||||
checkSnapshotServlet.enqueuePollTask(snapshotName, ExportConstants.getReportingKinds());
|
||||
String message = "Datastore backup started with name: " + snapshotName;
|
||||
logger.info(message);
|
||||
rsp.setStatus(SC_OK);
|
||||
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
|
||||
rsp.getWriter().write("OK\n\n" + message);
|
||||
} catch (Throwable e) {
|
||||
logger.severe(e, e.toString());
|
||||
rsp.sendError(
|
||||
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
|
||||
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -38,9 +38,11 @@ import google.registry.dns.writer.clouddns.CloudDnsWriterModule;
|
|||
import google.registry.dns.writer.dnsupdate.DnsUpdateConfigModule;
|
||||
import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule;
|
||||
import google.registry.export.BigqueryPollJobAction;
|
||||
import google.registry.export.CheckSnapshotAction;
|
||||
import google.registry.export.ExportDomainListsAction;
|
||||
import google.registry.export.ExportRequestModule;
|
||||
import google.registry.export.ExportReservedTermsAction;
|
||||
import google.registry.export.ExportSnapshotAction;
|
||||
import google.registry.export.LoadSnapshotAction;
|
||||
import google.registry.export.SyncGroupMembersAction;
|
||||
import google.registry.export.UpdateSnapshotViewAction;
|
||||
|
|
@ -94,6 +96,7 @@ import google.registry.tmch.TmchSmdrlAction;
|
|||
interface BackendRequestComponent {
|
||||
BigqueryPollJobAction bigqueryPollJobAction();
|
||||
BrdaCopyAction brdaCopyAction();
|
||||
CheckSnapshotAction checkSnapshotAction();
|
||||
CommitLogCheckpointAction commitLogCheckpointAction();
|
||||
CommitLogFanoutAction commitLogFanoutAction();
|
||||
DeleteContactsAndHostsAction deleteContactsAndHostsAction();
|
||||
|
|
@ -103,6 +106,7 @@ interface BackendRequestComponent {
|
|||
ExportCommitLogDiffAction exportCommitLogDiffAction();
|
||||
ExportDomainListsAction exportDomainListsAction();
|
||||
ExportReservedTermsAction exportReservedTermsAction();
|
||||
ExportSnapshotAction exportSnapshotAction();
|
||||
LoadSnapshotAction loadSnapshotAction();
|
||||
MetricsExportAction metricsExportAction();
|
||||
NordnUploadAction nordnUploadAction();
|
||||
|
|
|
|||
226
javatests/google/registry/export/CheckSnapshotActionTest.java
Normal file
226
javatests/google/registry/export/CheckSnapshotActionTest.java
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
|
||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Optional;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.request.Action.Method;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.HttpException.NoContentException;
|
||||
import google.registry.request.HttpException.NotModifiedException;
|
||||
import google.registry.testing.AppEngineRule;
|
||||
import google.registry.testing.ExceptionRule;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import google.registry.testing.InjectRule;
|
||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
/** Unit tests for {@link CheckSnapshotAction}. */
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class CheckSnapshotActionTest {
|
||||
|
||||
static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
|
||||
static final DateTime COMPLETE_TIME = START_TIME.plus(Duration.standardMinutes(30));
|
||||
|
||||
@Rule public final InjectRule inject = new InjectRule();
|
||||
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
|
||||
@Rule public final ExceptionRule thrown = new ExceptionRule();
|
||||
|
||||
@Mock private DatastoreBackupService backupService;
|
||||
|
||||
private DatastoreBackupInfo backupInfo;
|
||||
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
private final FakeClock clock = new FakeClock(COMPLETE_TIME.plusMillis(1000));
|
||||
private final CheckSnapshotAction action = new CheckSnapshotAction();
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
inject.setStaticField(DatastoreBackupInfo.class, "clock", clock);
|
||||
action.requestMethod = Method.POST;
|
||||
action.snapshotName = "some_backup";
|
||||
action.kindsToLoadParam = "one,two";
|
||||
action.response = response;
|
||||
action.backupService = backupService;
|
||||
|
||||
backupInfo =
|
||||
new DatastoreBackupInfo(
|
||||
"some_backup",
|
||||
START_TIME,
|
||||
Optional.of(COMPLETE_TIME),
|
||||
ImmutableSet.of("one", "two", "three"),
|
||||
Optional.of("gs://somebucket/some_backup_20140801.backup_info"));
|
||||
|
||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||
}
|
||||
|
||||
private void setPendingBackup() {
|
||||
backupInfo =
|
||||
new DatastoreBackupInfo(
|
||||
backupInfo.getName(),
|
||||
backupInfo.getStartTime(),
|
||||
Optional.<DateTime>absent(),
|
||||
backupInfo.getKinds(),
|
||||
backupInfo.getGcsFilename());
|
||||
|
||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||
}
|
||||
|
||||
private static void assertLoadTaskEnqueued(String id, String file, String kinds)
|
||||
throws Exception {
|
||||
assertTasksEnqueued(
|
||||
"export-snapshot",
|
||||
new TaskMatcher()
|
||||
.url("/_dr/task/loadSnapshot")
|
||||
.method("POST")
|
||||
.param("id", id)
|
||||
.param("file", file)
|
||||
.param("kinds", kinds));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSuccess_enqueuePollTask() throws Exception {
|
||||
CheckSnapshotAction.enqueuePollTask(
|
||||
"some_snapshot_name", ImmutableSet.of("one", "two", "three"));
|
||||
assertTasksEnqueued(
|
||||
CheckSnapshotAction.QUEUE,
|
||||
new TaskMatcher()
|
||||
.url(CheckSnapshotAction.PATH)
|
||||
.param(CHECK_SNAPSHOT_NAME_PARAM, "some_snapshot_name")
|
||||
.param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, "one,two,three")
|
||||
.method("POST"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forPendingBackup_returnsNotModified() throws Exception {
|
||||
setPendingBackup();
|
||||
|
||||
thrown.expect(NotModifiedException.class, "Datastore backup some_backup still pending");
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forStalePendingBackupBackup_returnsNoContent() throws Exception {
|
||||
setPendingBackup();
|
||||
|
||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||
|
||||
clock.setTo(
|
||||
START_TIME
|
||||
.plus(Duration.standardHours(20))
|
||||
.plus(Duration.standardMinutes(3))
|
||||
.plus(Duration.millis(1234)));
|
||||
|
||||
thrown.expect(
|
||||
NoContentException.class,
|
||||
"Datastore backup some_backup abandoned - "
|
||||
+ "not complete after 20 hours, 3 minutes and 1 second");
|
||||
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forCompleteBackup_enqueuesLoadTask() throws Exception {
|
||||
|
||||
action.run();
|
||||
assertLoadTaskEnqueued(
|
||||
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forCompleteAutoBackup_enqueuesLoadTask_usingBackupName() throws Exception {
|
||||
action.snapshotName = "auto_snapshot_somestring";
|
||||
when(backupService.findByName("auto_snapshot_somestring")).thenReturn(backupInfo);
|
||||
|
||||
action.run();
|
||||
assertLoadTaskEnqueued(
|
||||
"somestring", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forCompleteBackup_withExtraKindsToLoad_enqueuesLoadTask() throws Exception {
|
||||
action.kindsToLoadParam = "one,foo";
|
||||
|
||||
action.run();
|
||||
assertLoadTaskEnqueued(
|
||||
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forCompleteBackup_withEmptyKindsToLoad_skipsLoadTask() throws Exception {
|
||||
action.kindsToLoadParam = "";
|
||||
|
||||
action.run();
|
||||
assertNoTasksEnqueued("export-snapshot");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forBadBackup_returnsBadRequest() throws Exception {
|
||||
when(backupService.findByName("some_backup"))
|
||||
.thenThrow(new IllegalArgumentException("No backup found"));
|
||||
|
||||
thrown.expect(BadRequestException.class, "Bad backup name some_backup: No backup found");
|
||||
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGet_returnsInformation() throws Exception {
|
||||
action.requestMethod = Method.GET;
|
||||
|
||||
action.run();
|
||||
assertThat(response.getPayload())
|
||||
.isEqualTo(
|
||||
Joiner.on("\n")
|
||||
.join(
|
||||
ImmutableList.of(
|
||||
"Backup name: some_backup",
|
||||
"Status: COMPLETE",
|
||||
"Started: 2014-08-01T01:02:03.000Z",
|
||||
"Ended: 2014-08-01T01:32:03.000Z",
|
||||
"Duration: 30m",
|
||||
"GCS: gs://somebucket/some_backup_20140801.backup_info",
|
||||
"Kinds: [one, two, three]",
|
||||
"")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGet_forBadBackup_returnsError() throws Exception {
|
||||
action.requestMethod = Method.GET;
|
||||
when(backupService.findByName("some_backup"))
|
||||
.thenThrow(new IllegalArgumentException("No backup found"));
|
||||
|
||||
thrown.expect(BadRequestException.class, "Bad backup name some_backup: No backup found");
|
||||
|
||||
action.run();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,263 +0,0 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.export.CheckSnapshotServlet.SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
||||
import static google.registry.export.CheckSnapshotServlet.SNAPSHOT_NAME_PARAM;
|
||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Optional;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.testing.AppEngineRule;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.InjectRule;
|
||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
/** Unit tests for {@link CheckSnapshotServlet}. */
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class CheckSnapshotServletTest {
|
||||
|
||||
static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
|
||||
static final DateTime COMPLETE_TIME = START_TIME.plus(Duration.standardMinutes(30));
|
||||
|
||||
@Rule
|
||||
public final InjectRule inject = new InjectRule();
|
||||
|
||||
@Rule
|
||||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
||||
.withTaskQueue()
|
||||
.build();
|
||||
|
||||
@Mock
|
||||
private HttpServletRequest req;
|
||||
|
||||
@Mock
|
||||
private HttpServletResponse rsp;
|
||||
|
||||
private DatastoreBackupInfo backupInfo;
|
||||
|
||||
@Mock
|
||||
private DatastoreBackupService backupService;
|
||||
|
||||
private final FakeClock clock = new FakeClock(COMPLETE_TIME.plusMillis(1000));
|
||||
private final StringWriter httpOutput = new StringWriter();
|
||||
private final CheckSnapshotServlet servlet = new CheckSnapshotServlet();
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
inject.setStaticField(CheckSnapshotServlet.class, "backupService", backupService);
|
||||
inject.setStaticField(DatastoreBackupInfo.class, "clock", clock);
|
||||
|
||||
when(rsp.getWriter()).thenReturn(new PrintWriter(httpOutput));
|
||||
|
||||
servlet.init(mock(ServletConfig.class));
|
||||
when(req.getMethod()).thenReturn("POST");
|
||||
|
||||
backupInfo = new DatastoreBackupInfo(
|
||||
"some_backup",
|
||||
START_TIME,
|
||||
Optional.of(COMPLETE_TIME),
|
||||
ImmutableSet.of("one", "two", "three"),
|
||||
Optional.of("gs://somebucket/some_backup_20140801.backup_info"));
|
||||
}
|
||||
|
||||
private void setPendingBackup() {
|
||||
backupInfo = new DatastoreBackupInfo(
|
||||
backupInfo.getName(),
|
||||
backupInfo.getStartTime(),
|
||||
Optional.<DateTime>absent(),
|
||||
backupInfo.getKinds(),
|
||||
backupInfo.getGcsFilename());
|
||||
}
|
||||
|
||||
private static void assertLoadTaskEnqueued(String id, String file, String kinds)
|
||||
throws Exception {
|
||||
assertTasksEnqueued(
|
||||
"export-snapshot",
|
||||
new TaskMatcher()
|
||||
.url("/_dr/task/loadSnapshot")
|
||||
.method("POST")
|
||||
.param("id", id)
|
||||
.param("file", file)
|
||||
.param("kinds", kinds));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSuccess_enqueuePollTask() throws Exception {
|
||||
servlet.enqueuePollTask("some_snapshot_name", ImmutableSet.of("one", "two", "three"));
|
||||
assertTasksEnqueued(CheckSnapshotServlet.QUEUE,
|
||||
new TaskMatcher()
|
||||
.url(CheckSnapshotServlet.PATH)
|
||||
.param(SNAPSHOT_NAME_PARAM, "some_snapshot_name")
|
||||
.param(SNAPSHOT_KINDS_TO_LOAD_PARAM, "one,two,three")
|
||||
.method("POST"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forPendingBackup_returnsNotModified() throws Exception {
|
||||
setPendingBackup();
|
||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).sendError(SC_NOT_MODIFIED, "Datastore backup some_backup still pending");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forStalePendingBackupBackup_returnsAccepted() throws Exception {
|
||||
setPendingBackup();
|
||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||
clock.setTo(START_TIME
|
||||
.plus(Duration.standardHours(20))
|
||||
.plus(Duration.standardMinutes(3))
|
||||
.plus(Duration.millis(1234)));
|
||||
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).sendError(SC_ACCEPTED,
|
||||
"Datastore backup some_backup abandoned - "
|
||||
+ "not complete after 20 hours, 3 minutes and 1 second");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forCompleteBackup_enqueuesLoadTask() throws Exception {
|
||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).setStatus(SC_OK);
|
||||
assertLoadTaskEnqueued(
|
||||
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forCompleteAutoBackup_enqueuesLoadTask_usingBackupName() throws Exception {
|
||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("auto_snapshot_somestring");
|
||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
||||
when(backupService.findByName("auto_snapshot_somestring")).thenReturn(backupInfo);
|
||||
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).setStatus(SC_OK);
|
||||
assertLoadTaskEnqueued(
|
||||
"somestring", "gs://somebucket/some_backup_20140801.backup_info", "one,two");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forCompleteBackup_withExtraKindsToLoad_enqueuesLoadTask() throws Exception {
|
||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,foo");
|
||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).setStatus(SC_OK);
|
||||
assertLoadTaskEnqueued(
|
||||
"20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forCompleteBackup_withEmptyKindsToLoad_skipsLoadTask() throws Exception {
|
||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("");
|
||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).setStatus(SC_OK);
|
||||
assertNoTasksEnqueued("export-snapshot");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_forBadBackup_returnsBadRequest() throws Exception {
|
||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
||||
when(backupService.findByName("some_backup"))
|
||||
.thenThrow(new IllegalArgumentException("No backup found"));
|
||||
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).sendError(SC_BAD_REQUEST, "Bad backup name some_backup: No backup found");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_noBackupSpecified_returnsError() throws Exception {
|
||||
when(req.getMethod()).thenReturn("POST");
|
||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn(null);
|
||||
when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two");
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).sendError(SC_BAD_REQUEST, "Missing parameter: name");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGet_returnsInformation() throws Exception {
|
||||
when(req.getMethod()).thenReturn("GET");
|
||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
||||
when(backupService.findByName("some_backup")).thenReturn(backupInfo);
|
||||
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).setStatus(SC_OK);
|
||||
assertThat(httpOutput.toString()).isEqualTo("OK\n\n" + Joiner.on("\n").join(ImmutableList.of(
|
||||
"Backup name: some_backup",
|
||||
"Status: COMPLETE",
|
||||
"Started: 2014-08-01T01:02:03.000Z",
|
||||
"Ended: 2014-08-01T01:32:03.000Z",
|
||||
"Duration: 30m",
|
||||
"GCS: gs://somebucket/some_backup_20140801.backup_info",
|
||||
"Kinds: [one, two, three]",
|
||||
"")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGet_forBadBackup_returnsError() throws Exception {
|
||||
when(req.getMethod()).thenReturn("GET");
|
||||
when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup");
|
||||
when(backupService.findByName("some_backup")).thenThrow(
|
||||
new IllegalArgumentException("No backup found"));
|
||||
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).sendError(SC_BAD_REQUEST, "No backup found");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGet_noBackupSpecified_returnsError() throws Exception {
|
||||
when(req.getMethod()).thenReturn("GET");
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).sendError(SC_BAD_REQUEST, "Missing parameter: name");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import google.registry.testing.AppEngineRule;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
/** Unit tests for {@link ExportSnapshotAction}. */
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class ExportSnapshotActionTest {
|
||||
|
||||
@Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
|
||||
|
||||
@Mock private DatastoreBackupService backupService;
|
||||
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
private final FakeClock clock = new FakeClock(DateTime.parse("2014-08-01T01:02:03Z"));
|
||||
private final ExportSnapshotAction action = new ExportSnapshotAction();
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
action.clock = clock;
|
||||
action.backupService = backupService;
|
||||
action.response = response;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_launchesBackup_andEnqueuesPollTask() throws Exception {
|
||||
action.run();
|
||||
verify(backupService)
|
||||
.launchNewBackup(
|
||||
ExportSnapshotAction.QUEUE,
|
||||
"auto_snapshot_20140801_010203",
|
||||
"domain-registry-snapshots",
|
||||
ExportConstants.getBackupKinds());
|
||||
assertTasksEnqueued(
|
||||
CheckSnapshotAction.QUEUE,
|
||||
new TaskMatcher()
|
||||
.url(CheckSnapshotAction.PATH)
|
||||
.param(CHECK_SNAPSHOT_NAME_PARAM, "auto_snapshot_20140801_010203")
|
||||
.param(
|
||||
CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM,
|
||||
Joiner.on(",").join(ExportConstants.getReportingKinds()))
|
||||
.method("POST"));
|
||||
assertThat(response.getPayload())
|
||||
.isEqualTo("Datastore backup started with name: auto_snapshot_20140801_010203");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.InjectRule;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
import javax.servlet.ServletConfig;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
/** Unit tests for {@link ExportSnapshotServlet}. */
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class ExportSnapshotServletTest {
|
||||
|
||||
@Rule
|
||||
public final InjectRule inject = new InjectRule();
|
||||
|
||||
@Mock
|
||||
private HttpServletRequest req;
|
||||
|
||||
@Mock
|
||||
private HttpServletResponse rsp;
|
||||
|
||||
@Mock
|
||||
private DatastoreBackupService backupService;
|
||||
|
||||
@Mock
|
||||
private CheckSnapshotServlet checkSnapshotServlet;
|
||||
|
||||
private final FakeClock clock = new FakeClock();
|
||||
private final StringWriter httpOutput = new StringWriter();
|
||||
private final ExportSnapshotServlet servlet = new ExportSnapshotServlet();
|
||||
|
||||
private static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
clock.setTo(START_TIME);
|
||||
inject.setStaticField(ExportSnapshotServlet.class, "clock", clock);
|
||||
inject.setStaticField(ExportSnapshotServlet.class, "backupService", backupService);
|
||||
inject.setStaticField(
|
||||
ExportSnapshotServlet.class, "checkSnapshotServlet", checkSnapshotServlet);
|
||||
when(rsp.getWriter()).thenReturn(new PrintWriter(httpOutput));
|
||||
|
||||
servlet.init(mock(ServletConfig.class));
|
||||
when(req.getMethod()).thenReturn("POST");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPost_launchesBackup_andEnqueuesPollTask() throws Exception {
|
||||
servlet.service(req, rsp);
|
||||
verify(rsp).setStatus(SC_OK);
|
||||
verify(backupService).launchNewBackup(
|
||||
ExportSnapshotServlet.QUEUE,
|
||||
"auto_snapshot_20140801_010203",
|
||||
"domain-registry-snapshots",
|
||||
ExportConstants.getBackupKinds());
|
||||
verify(checkSnapshotServlet)
|
||||
.enqueuePollTask("auto_snapshot_20140801_010203", ExportConstants.getReportingKinds());
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue