mirror of
https://github.com/google/nomulus.git
synced 2025-06-27 06:44:51 +02:00
Daggerize ExportSnapshotServlet and CheckSnapshotServlet
Eradicate the last remnants of un-injectable servlets! ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=145598002
This commit is contained in:
jianglai
Ben McIlwain
parent
a8aeff96f6
commit
4fed3a9ae6
15 changed files with 577 additions and 642 deletions
162
java/google/registry/export/CheckSnapshotAction.java
Normal file
162
java/google/registry/export/CheckSnapshotAction.java
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.collect.Sets.intersection;
|
||||
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
|
||||
import static google.registry.request.Action.Method.GET;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static google.registry.util.FormattingLogger.getLoggerForCallerClass;
|
||||
|
||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Sets;
|
||||
import google.registry.export.DatastoreBackupInfo.BackupStatus;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.HttpException.NoContentException;
|
||||
import google.registry.request.HttpException.NotModifiedException;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.RequestMethod;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import java.util.Set;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.Duration;
|
||||
import org.joda.time.PeriodType;
|
||||
import org.joda.time.format.PeriodFormat;
|
||||
|
||||
/**
|
||||
* Action that checks the status of a snapshot, and if complete, trigger loading it into BigQuery.
|
||||
*/
|
||||
@Action(
|
||||
path = CheckSnapshotAction.PATH,
|
||||
method = {POST, GET},
|
||||
automaticallyPrintOk = true
|
||||
)
|
||||
public class CheckSnapshotAction implements Runnable {
|
||||
|
||||
/** Parameter names for passing parameters into this action. */
|
||||
static final String CHECK_SNAPSHOT_NAME_PARAM = "name";
|
||||
|
||||
static final String CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";
|
||||
|
||||
/** Action-specific details needed for enqueuing tasks against itself. */
|
||||
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
|
||||
|
||||
static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml.
|
||||
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
|
||||
|
||||
/** The maximum amount of time we allow a backup to run before abandoning it. */
|
||||
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
|
||||
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
|
||||
@Inject Response response;
|
||||
@Inject @RequestMethod Action.Method requestMethod;
|
||||
@Inject DatastoreBackupService backupService;
|
||||
@Inject @Parameter(CHECK_SNAPSHOT_NAME_PARAM) String snapshotName;
|
||||
@Inject @Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM) String kindsToLoadParam;
|
||||
@Inject CheckSnapshotAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
if (requestMethod == POST) {
|
||||
checkAndLoadSnapshotIfComplete();
|
||||
} else {
|
||||
// This is a GET request.
|
||||
response.setPayload(getBackup().getInformation());
|
||||
}
|
||||
}
|
||||
|
||||
private DatastoreBackupInfo getBackup() {
|
||||
try {
|
||||
return backupService.findByName(snapshotName);
|
||||
} catch (IllegalArgumentException e) {
|
||||
String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
|
||||
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be
|
||||
// retried but we might abandon backups that start late and haven't yet written to
|
||||
// datastore. We could fix that by replacing this with a two-phase polling strategy.
|
||||
throw new BadRequestException(message, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkAndLoadSnapshotIfComplete() {
|
||||
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
|
||||
DatastoreBackupInfo backup = getBackup();
|
||||
// Stop now if the backup is not complete.
|
||||
if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
|
||||
Duration runningTime = backup.getRunningTime();
|
||||
if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
|
||||
// Backup might still be running, so send a 304 to have the task retry.
|
||||
throw new NotModifiedException(
|
||||
String.format("Datastore backup %s still pending", snapshotName));
|
||||
} else {
|
||||
// Declare the backup a lost cause, and send 204 No Content so the task will
|
||||
// not be retried.
|
||||
String message =
|
||||
String.format(
|
||||
"Datastore backup %s abandoned - not complete after %s",
|
||||
snapshotName,
|
||||
PeriodFormat.getDefault()
|
||||
.print(
|
||||
runningTime
|
||||
.toPeriod()
|
||||
.normalizedStandard(PeriodType.dayTime().withMillisRemoved())));
|
||||
throw new NoContentException(message);
|
||||
}
|
||||
}
|
||||
// Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
|
||||
// suffix out of the snapshot name and falling back to the start time as a string.
|
||||
String snapshotId =
|
||||
snapshotName.startsWith(ExportSnapshotAction.SNAPSHOT_PREFIX)
|
||||
? snapshotName.substring(ExportSnapshotAction.SNAPSHOT_PREFIX.length())
|
||||
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
|
||||
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
|
||||
if (!backup.getKinds().containsAll(kindsToLoad)) {
|
||||
logger.warningfmt(
|
||||
"Kinds to load included non-exported kinds: %s",
|
||||
Sets.difference(kindsToLoad, backup.getKinds()));
|
||||
}
|
||||
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
|
||||
ImmutableSet<String> exportedKindsToLoad =
|
||||
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
|
||||
String message = String.format("Datastore backup %s complete - ", snapshotName);
|
||||
if (exportedKindsToLoad.isEmpty()) {
|
||||
message += "no kinds to load into BigQuery";
|
||||
} else {
|
||||
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
|
||||
message += "BigQuery load task enqueued";
|
||||
}
|
||||
logger.info(message);
|
||||
response.setPayload(message);
|
||||
}
|
||||
|
||||
/** Enqueue a poll task to monitor the named snapshot for completion. */
|
||||
static TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
|
||||
return QueueFactory.getQueue(QUEUE)
|
||||
.add(
|
||||
TaskOptions.Builder.withUrl(PATH)
|
||||
.method(Method.POST)
|
||||
.countdownMillis(POLL_COUNTDOWN.getMillis())
|
||||
.param(CHECK_SNAPSHOT_NAME_PARAM, snapshotName)
|
||||
.param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,176 +0,0 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.collect.Sets.intersection;
|
||||
import static com.google.common.html.HtmlEscapers.htmlEscaper;
|
||||
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
|
||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||
import static google.registry.util.FormattingLogger.getLoggerForCallerClass;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.export.DatastoreBackupInfo.BackupStatus;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import org.joda.time.Duration;
|
||||
import org.joda.time.PeriodType;
|
||||
import org.joda.time.format.PeriodFormat;
|
||||
|
||||
/** Check the status of a snapshot, and if complete, trigger loading it into BigQuery. */
|
||||
public class CheckSnapshotServlet extends HttpServlet {
|
||||
|
||||
/** Parameter names for passing parameters into this servlet. */
|
||||
static final String SNAPSHOT_NAME_PARAM = "name";
|
||||
static final String SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";
|
||||
|
||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
||||
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
|
||||
static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml.
|
||||
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
|
||||
|
||||
/** The maximum amount of time we allow a backup to run before abandoning it. */
|
||||
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
|
||||
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static DatastoreBackupService backupService = DatastoreBackupService.get();
|
||||
|
||||
@Override
|
||||
public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
try {
|
||||
rsp.setStatus(SC_OK);
|
||||
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
|
||||
rsp.getWriter().write("OK\n\n");
|
||||
super.service(req, rsp);
|
||||
} catch (Throwable e) {
|
||||
logger.severe(e, e.toString());
|
||||
rsp.sendError(
|
||||
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
|
||||
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doGet(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
|
||||
try {
|
||||
String snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
|
||||
rsp.getWriter().write(backupService.findByName(snapshotName).getInformation());
|
||||
} catch (BadRequestException e) {
|
||||
throw new IllegalArgumentException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
String snapshotName;
|
||||
String kindsToLoadParam;
|
||||
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
|
||||
try {
|
||||
snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
|
||||
kindsToLoadParam = extractRequiredParameter(req, SNAPSHOT_KINDS_TO_LOAD_PARAM);
|
||||
} catch (BadRequestException e) {
|
||||
throw new IllegalArgumentException(e.getMessage());
|
||||
}
|
||||
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
|
||||
|
||||
// Look up the backup by the provided name, stopping if we can't find it.
|
||||
DatastoreBackupInfo backup;
|
||||
try {
|
||||
backup = backupService.findByName(snapshotName);
|
||||
} catch (IllegalArgumentException e) {
|
||||
String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
|
||||
logger.severe(e, message);
|
||||
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be retried,
|
||||
// but we might abandon backups that start late and haven't yet written to datastore.
|
||||
// We could fix that by replacing this with a two-phase polling strategy.
|
||||
rsp.sendError(SC_BAD_REQUEST, htmlEscaper().escape(message));
|
||||
return;
|
||||
}
|
||||
// Stop now if the backup is not complete.
|
||||
if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
|
||||
Duration runningTime = backup.getRunningTime();
|
||||
if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
|
||||
// Backup might still be running, so send a 304 to have the task retry.
|
||||
rsp.sendError(SC_NOT_MODIFIED,
|
||||
htmlEscaper().escape(String.format("Datastore backup %s still pending", snapshotName)));
|
||||
} else {
|
||||
// Declare the backup a lost cause, and send 202 Accepted so the task will not be retried.
|
||||
String message = String.format("Datastore backup %s abandoned - not complete after %s",
|
||||
snapshotName,
|
||||
PeriodFormat.getDefault().print(
|
||||
runningTime.toPeriod().normalizedStandard(
|
||||
PeriodType.dayTime().withMillisRemoved())));
|
||||
logger.severe(message);
|
||||
rsp.sendError(SC_ACCEPTED, htmlEscaper().escape(message));
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
|
||||
// suffix out of the snapshot name and falling back to the start time as a string.
|
||||
String snapshotId = snapshotName.startsWith(ExportSnapshotServlet.SNAPSHOT_PREFIX)
|
||||
? snapshotName.substring(ExportSnapshotServlet.SNAPSHOT_PREFIX.length())
|
||||
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
|
||||
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
|
||||
if (!backup.getKinds().containsAll(kindsToLoad)) {
|
||||
logger.warningfmt(
|
||||
"Kinds to load included non-exported kinds: %s",
|
||||
Sets.difference(kindsToLoad, backup.getKinds()));
|
||||
}
|
||||
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
|
||||
ImmutableSet<String> exportedKindsToLoad =
|
||||
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
|
||||
String message = String.format("Datastore backup %s complete - ", snapshotName);
|
||||
if (exportedKindsToLoad.isEmpty()) {
|
||||
message += "no kinds to load into BigQuery";
|
||||
} else {
|
||||
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
|
||||
message += "BigQuery load task enqueued";
|
||||
}
|
||||
logger.info(message);
|
||||
rsp.getWriter().write(message);
|
||||
}
|
||||
|
||||
/** Enqueue a poll task to monitor the named snapshot for completion. */
|
||||
TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
|
||||
return QueueFactory.getQueue(QUEUE).add(
|
||||
TaskOptions.Builder.withUrl(PATH)
|
||||
.method(Method.POST)
|
||||
.countdownMillis(POLL_COUNTDOWN.getMillis())
|
||||
.param(SNAPSHOT_NAME_PARAM, snapshotName)
|
||||
.param(SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
|
||||
}
|
||||
}
|
||||
|
|
@ -17,6 +17,8 @@ package google.registry.export;
|
|||
import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
|
||||
import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
|
||||
import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
|
||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
|
||||
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
|
||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
|
||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
|
||||
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
|
||||
|
|
@ -72,6 +74,18 @@ public final class ExportRequestModule {
|
|||
return extractRequiredParameter(req, LOAD_SNAPSHOT_KINDS_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(CHECK_SNAPSHOT_NAME_PARAM)
|
||||
static String provideCheckSnapshotName(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, CHECK_SNAPSHOT_NAME_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM)
|
||||
static String provideCheckSnapshotKindsToLoad(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Header(CHAINED_TASK_QUEUE_HEADER)
|
||||
static String provideChainedTaskQueue(HttpServletRequest req) {
|
||||
|
|
@ -89,4 +103,9 @@ public final class ExportRequestModule {
|
|||
static String provideProjectId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, PROJECT_ID_HEADER);
|
||||
}
|
||||
|
||||
@Provides
|
||||
static DatastoreBackupService provideDatastoreBackupService() {
|
||||
return DatastoreBackupService.get();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
72
java/google/registry/export/ExportSnapshotAction.java
Normal file
72
java/google/registry/export/ExportSnapshotAction.java
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static google.registry.export.CheckSnapshotAction.enqueuePollTask;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* Action to trigger a datastore backup job that writes a snapshot to Google Cloud Storage.
|
||||
*
|
||||
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
|
||||
* the next upon successful completion:
|
||||
*
|
||||
* <ol>
|
||||
* <li>The snapshot is exported to Google Cloud Storage (this action).
|
||||
* <li>The {@link CheckSnapshotAction} polls until the export is completed.
|
||||
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
|
||||
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
|
||||
* </ol>
|
||||
*/
|
||||
@Action(path = ExportSnapshotAction.PATH, method = POST, automaticallyPrintOk = true)
|
||||
public class ExportSnapshotAction implements Runnable {
|
||||
|
||||
/** Queue to use for enqueuing the task that will actually launch the backup. */
|
||||
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
||||
|
||||
static final String PATH = "/_dr/task/exportSnapshot"; // See web.xml.
|
||||
|
||||
/** Prefix to use for naming all snapshots that are started by this servlet. */
|
||||
static final String SNAPSHOT_PREFIX = "auto_snapshot_";
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@Inject Clock clock;
|
||||
@Inject DatastoreBackupService backupService;
|
||||
@Inject Response response;
|
||||
|
||||
@Inject
|
||||
ExportSnapshotAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
// Use a unique name for the snapshot so we can explicitly check its completion later.
|
||||
String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss");
|
||||
backupService.launchNewBackup(
|
||||
QUEUE, snapshotName, RegistryConfig.getSnapshotsBucket(), ExportConstants.getBackupKinds());
|
||||
// Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery.
|
||||
enqueuePollTask(snapshotName, ExportConstants.getReportingKinds());
|
||||
String message = "Datastore backup started with name: " + snapshotName;
|
||||
logger.info(message);
|
||||
response.setPayload(message);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.html.HtmlEscapers.htmlEscaper;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import google.registry.util.SystemClock;
|
||||
import java.io.IOException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
/**
|
||||
* Trigger a backup-as-a-service job that writes a snapshot to Google Cloud Storage.
|
||||
*
|
||||
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
|
||||
* the next upon successful completion:<ol>
|
||||
* <li>The snapshot is exported to Google Cloud Storage (this servlet).
|
||||
* <li>The {@link CheckSnapshotServlet} polls until the export is completed.
|
||||
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
|
||||
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
|
||||
* </ol>
|
||||
*/
|
||||
public class ExportSnapshotServlet extends HttpServlet {
|
||||
|
||||
/** Queue to use for enqueuing the task that will actually launch the backup. */
|
||||
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
||||
|
||||
/** Prefix to use for naming all snapshots that are started by this servlet. */
|
||||
static final String SNAPSHOT_PREFIX = "auto_snapshot_";
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static Clock clock = new SystemClock();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static DatastoreBackupService backupService = DatastoreBackupService.get();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static CheckSnapshotServlet checkSnapshotServlet = new CheckSnapshotServlet();
|
||||
|
||||
@Override
|
||||
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
try {
|
||||
// Use a unique name for the snapshot so we can explicitly check its completion later.
|
||||
String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss");
|
||||
backupService.launchNewBackup(
|
||||
QUEUE,
|
||||
snapshotName,
|
||||
RegistryConfig.getSnapshotsBucket(),
|
||||
ExportConstants.getBackupKinds());
|
||||
// Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery.
|
||||
checkSnapshotServlet.enqueuePollTask(snapshotName, ExportConstants.getReportingKinds());
|
||||
String message = "Datastore backup started with name: " + snapshotName;
|
||||
logger.info(message);
|
||||
rsp.setStatus(SC_OK);
|
||||
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
|
||||
rsp.getWriter().write("OK\n\n" + message);
|
||||
} catch (Throwable e) {
|
||||
logger.severe(e, e.toString());
|
||||
rsp.sendError(
|
||||
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
|
||||
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue