Daggerize ExportSnapshotServlet and CheckSnapshotServlet

Eradicate the last remnants of un-injectable servlets!

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=145598002
This commit is contained in:
jianglai 2017-01-25 14:36:42 -08:00 committed by Ben McIlwain
parent a8aeff96f6
commit 4fed3a9ae6
15 changed files with 577 additions and 642 deletions

View file

@ -1102,7 +1102,7 @@ public final class RegistryConfig {
/**
* Returns the Google Cloud Storage bucket for storing backup snapshots.
*
* @see google.registry.export.ExportSnapshotServlet
* @see google.registry.export.ExportSnapshotAction
*/
public static String getSnapshotsBucket() {
return getProjectId() + "-snapshots";

View file

@ -76,7 +76,7 @@ public abstract class DnsModule {
}
@Provides
@Parameter("name")
@Parameter("domainOrHostName")
static String provideName(HttpServletRequest req) {
return extractRequiredParameter(req, "name");
}

View file

@ -35,7 +35,7 @@ public final class RefreshDnsAction implements Runnable {
@Inject Clock clock;
@Inject DnsQueue dnsQueue;
@Inject @Parameter("name") String domainOrHostName;
@Inject @Parameter("domainOrHostName") String domainOrHostName;
@Inject @Parameter("type") TargetType type;
@Inject RefreshDnsAction() {}

View file

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns="http://java.sun.com/xml/ns/javaee" version="2.5"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd">
<!-- Servlets -->
@ -113,25 +113,15 @@
<url-pattern>/_dr/task/verifyEntityIntegrity</url-pattern>
</servlet-mapping>
<servlet>
<description>Exports a datastore backup snapshot to GCS.</description>
<display-name>Export snapshot to GCS</display-name>
<servlet-name>exportSnapshot</servlet-name>
<servlet-class>google.registry.export.ExportSnapshotServlet</servlet-class>
</servlet>
<!-- Exports a datastore backup snapshot to GCS. -->
<servlet-mapping>
<servlet-name>exportSnapshot</servlet-name>
<servlet-name>backend-servlet</servlet-name>
<url-pattern>/_dr/task/exportSnapshot</url-pattern>
</servlet-mapping>
<servlet>
<description>Checks the completion of a datastore backup snapshot.</description>
<display-name>Check on snapshot status</display-name>
<servlet-name>checkSnapshot</servlet-name>
<servlet-class>google.registry.export.CheckSnapshotServlet</servlet-class>
</servlet>
<!-- Checks the completion of a datastore backup snapshot. -->
<servlet-mapping>
<servlet-name>checkSnapshot</servlet-name>
<servlet-name>backend-servlet</servlet-name>
<url-pattern>/_dr/task/checkSnapshot</url-pattern>
</servlet-mapping>
@ -290,10 +280,10 @@
<web-resource-collection>
<web-resource-name>Internal</web-resource-name>
<description>
Admin-only internal section. Requests for paths covered by the URL patterns below will be
Admin-only internal section. Requests for paths covered by the URL patterns below will be
checked for a logged-in user account that's allowed to access the AppEngine admin console
(NOTE: this includes Editor/Viewer permissions in addition to Owner and the new IAM
App Engine Admin role. See https://cloud.google.com/appengine/docs/java/access-control
App Engine Admin role. See https://cloud.google.com/appengine/docs/java/access-control
specifically the "Access handlers that have a login:admin restriction" line.)
TODO(b/28219927): lift some of these restrictions so that we can allow OAuth authentication

View file

@ -0,0 +1,162 @@
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.common.collect.Sets.intersection;
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
import static google.registry.request.Action.Method.GET;
import static google.registry.request.Action.Method.POST;
import static google.registry.util.FormattingLogger.getLoggerForCallerClass;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import google.registry.export.DatastoreBackupInfo.BackupStatus;
import google.registry.request.Action;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.HttpException.NoContentException;
import google.registry.request.HttpException.NotModifiedException;
import google.registry.request.Parameter;
import google.registry.request.RequestMethod;
import google.registry.request.Response;
import google.registry.util.FormattingLogger;
import java.util.Set;
import javax.inject.Inject;
import org.joda.time.Duration;
import org.joda.time.PeriodType;
import org.joda.time.format.PeriodFormat;
/**
* Action that checks the status of a snapshot, and if complete, trigger loading it into BigQuery.
*/
@Action(
path = CheckSnapshotAction.PATH,
method = {POST, GET},
automaticallyPrintOk = true
)
public class CheckSnapshotAction implements Runnable {
/** Parameter names for passing parameters into this action. */
static final String CHECK_SNAPSHOT_NAME_PARAM = "name";
static final String CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";
/** Action-specific details needed for enqueuing tasks against itself. */
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml.
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
/** The maximum amount of time we allow a backup to run before abandoning it. */
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
private static final FormattingLogger logger = getLoggerForCallerClass();
@Inject Response response;
@Inject @RequestMethod Action.Method requestMethod;
@Inject DatastoreBackupService backupService;
@Inject @Parameter(CHECK_SNAPSHOT_NAME_PARAM) String snapshotName;
@Inject @Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM) String kindsToLoadParam;
@Inject CheckSnapshotAction() {}
@Override
public void run() {
if (requestMethod == POST) {
checkAndLoadSnapshotIfComplete();
} else {
// This is a GET request.
response.setPayload(getBackup().getInformation());
}
}
private DatastoreBackupInfo getBackup() {
try {
return backupService.findByName(snapshotName);
} catch (IllegalArgumentException e) {
String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be
// retried but we might abandon backups that start late and haven't yet written to
// datastore. We could fix that by replacing this with a two-phase polling strategy.
throw new BadRequestException(message, e);
}
}
private void checkAndLoadSnapshotIfComplete() {
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
DatastoreBackupInfo backup = getBackup();
// Stop now if the backup is not complete.
if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
Duration runningTime = backup.getRunningTime();
if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
// Backup might still be running, so send a 304 to have the task retry.
throw new NotModifiedException(
String.format("Datastore backup %s still pending", snapshotName));
} else {
// Declare the backup a lost cause, and send 204 No Content so the task will
// not be retried.
String message =
String.format(
"Datastore backup %s abandoned - not complete after %s",
snapshotName,
PeriodFormat.getDefault()
.print(
runningTime
.toPeriod()
.normalizedStandard(PeriodType.dayTime().withMillisRemoved())));
throw new NoContentException(message);
}
}
// Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
// suffix out of the snapshot name and falling back to the start time as a string.
String snapshotId =
snapshotName.startsWith(ExportSnapshotAction.SNAPSHOT_PREFIX)
? snapshotName.substring(ExportSnapshotAction.SNAPSHOT_PREFIX.length())
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
if (!backup.getKinds().containsAll(kindsToLoad)) {
logger.warningfmt(
"Kinds to load included non-exported kinds: %s",
Sets.difference(kindsToLoad, backup.getKinds()));
}
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
ImmutableSet<String> exportedKindsToLoad =
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
String message = String.format("Datastore backup %s complete - ", snapshotName);
if (exportedKindsToLoad.isEmpty()) {
message += "no kinds to load into BigQuery";
} else {
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
message += "BigQuery load task enqueued";
}
logger.info(message);
response.setPayload(message);
}
/** Enqueue a poll task to monitor the named snapshot for completion. */
static TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
return QueueFactory.getQueue(QUEUE)
.add(
TaskOptions.Builder.withUrl(PATH)
.method(Method.POST)
.countdownMillis(POLL_COUNTDOWN.getMillis())
.param(CHECK_SNAPSHOT_NAME_PARAM, snapshotName)
.param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
}
}

View file

@ -1,176 +0,0 @@
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.collect.Sets.intersection;
import static com.google.common.html.HtmlEscapers.htmlEscaper;
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
import static google.registry.request.RequestParameters.extractRequiredParameter;
import static google.registry.util.FormattingLogger.getLoggerForCallerClass;
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.net.MediaType;
import google.registry.export.DatastoreBackupInfo.BackupStatus;
import google.registry.request.HttpException.BadRequestException;
import google.registry.util.FormattingLogger;
import google.registry.util.NonFinalForTesting;
import java.io.IOException;
import java.util.Set;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.joda.time.Duration;
import org.joda.time.PeriodType;
import org.joda.time.format.PeriodFormat;
/** Check the status of a snapshot, and if complete, trigger loading it into BigQuery. */
public class CheckSnapshotServlet extends HttpServlet {
/** Parameter names for passing parameters into this servlet. */
static final String SNAPSHOT_NAME_PARAM = "name";
static final String SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";
/** Servlet-specific details needed for enqueuing tasks against itself. */
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml.
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
/** The maximum amount of time we allow a backup to run before abandoning it. */
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
private static final FormattingLogger logger = getLoggerForCallerClass();
@NonFinalForTesting
private static DatastoreBackupService backupService = DatastoreBackupService.get();
@Override
public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
try {
rsp.setStatus(SC_OK);
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
rsp.getWriter().write("OK\n\n");
super.service(req, rsp);
} catch (Throwable e) {
logger.severe(e, e.toString());
rsp.sendError(
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
}
}
@Override
public void doGet(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
try {
String snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
rsp.getWriter().write(backupService.findByName(snapshotName).getInformation());
} catch (BadRequestException e) {
throw new IllegalArgumentException(e.getMessage());
}
}
@Override
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
String snapshotName;
String kindsToLoadParam;
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
try {
snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
kindsToLoadParam = extractRequiredParameter(req, SNAPSHOT_KINDS_TO_LOAD_PARAM);
} catch (BadRequestException e) {
throw new IllegalArgumentException(e.getMessage());
}
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
// Look up the backup by the provided name, stopping if we can't find it.
DatastoreBackupInfo backup;
try {
backup = backupService.findByName(snapshotName);
} catch (IllegalArgumentException e) {
String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
logger.severe(e, message);
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be retried,
// but we might abandon backups that start late and haven't yet written to datastore.
// We could fix that by replacing this with a two-phase polling strategy.
rsp.sendError(SC_BAD_REQUEST, htmlEscaper().escape(message));
return;
}
// Stop now if the backup is not complete.
if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
Duration runningTime = backup.getRunningTime();
if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
// Backup might still be running, so send a 304 to have the task retry.
rsp.sendError(SC_NOT_MODIFIED,
htmlEscaper().escape(String.format("Datastore backup %s still pending", snapshotName)));
} else {
// Declare the backup a lost cause, and send 202 Accepted so the task will not be retried.
String message = String.format("Datastore backup %s abandoned - not complete after %s",
snapshotName,
PeriodFormat.getDefault().print(
runningTime.toPeriod().normalizedStandard(
PeriodType.dayTime().withMillisRemoved())));
logger.severe(message);
rsp.sendError(SC_ACCEPTED, htmlEscaper().escape(message));
}
return;
}
// Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
// suffix out of the snapshot name and falling back to the start time as a string.
String snapshotId = snapshotName.startsWith(ExportSnapshotServlet.SNAPSHOT_PREFIX)
? snapshotName.substring(ExportSnapshotServlet.SNAPSHOT_PREFIX.length())
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
if (!backup.getKinds().containsAll(kindsToLoad)) {
logger.warningfmt(
"Kinds to load included non-exported kinds: %s",
Sets.difference(kindsToLoad, backup.getKinds()));
}
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
ImmutableSet<String> exportedKindsToLoad =
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
String message = String.format("Datastore backup %s complete - ", snapshotName);
if (exportedKindsToLoad.isEmpty()) {
message += "no kinds to load into BigQuery";
} else {
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
message += "BigQuery load task enqueued";
}
logger.info(message);
rsp.getWriter().write(message);
}
/** Enqueue a poll task to monitor the named snapshot for completion. */
TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
return QueueFactory.getQueue(QUEUE).add(
TaskOptions.Builder.withUrl(PATH)
.method(Method.POST)
.countdownMillis(POLL_COUNTDOWN.getMillis())
.param(SNAPSHOT_NAME_PARAM, snapshotName)
.param(SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
}
}

View file

@ -17,6 +17,8 @@ package google.registry.export;
import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM;
import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
@ -72,6 +74,18 @@ public final class ExportRequestModule {
return extractRequiredParameter(req, LOAD_SNAPSHOT_KINDS_PARAM);
}
@Provides
@Parameter(CHECK_SNAPSHOT_NAME_PARAM)
static String provideCheckSnapshotName(HttpServletRequest req) {
return extractRequiredParameter(req, CHECK_SNAPSHOT_NAME_PARAM);
}
@Provides
@Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM)
static String provideCheckSnapshotKindsToLoad(HttpServletRequest req) {
return extractRequiredParameter(req, CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM);
}
@Provides
@Header(CHAINED_TASK_QUEUE_HEADER)
static String provideChainedTaskQueue(HttpServletRequest req) {
@ -89,4 +103,9 @@ public final class ExportRequestModule {
static String provideProjectId(HttpServletRequest req) {
return extractRequiredHeader(req, PROJECT_ID_HEADER);
}
@Provides
static DatastoreBackupService provideDatastoreBackupService() {
return DatastoreBackupService.get();
}
}

View file

@ -0,0 +1,72 @@
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static google.registry.export.CheckSnapshotAction.enqueuePollTask;
import static google.registry.request.Action.Method.POST;
import google.registry.config.RegistryConfig;
import google.registry.request.Action;
import google.registry.request.Response;
import google.registry.util.Clock;
import google.registry.util.FormattingLogger;
import javax.inject.Inject;
/**
* Action to trigger a datastore backup job that writes a snapshot to Google Cloud Storage.
*
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
* the next upon successful completion:
*
* <ol>
* <li>The snapshot is exported to Google Cloud Storage (this action).
* <li>The {@link CheckSnapshotAction} polls until the export is completed.
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
* </ol>
*/
@Action(path = ExportSnapshotAction.PATH, method = POST, automaticallyPrintOk = true)
public class ExportSnapshotAction implements Runnable {
/** Queue to use for enqueuing the task that will actually launch the backup. */
static final String QUEUE = "export-snapshot"; // See queue.xml.
static final String PATH = "/_dr/task/exportSnapshot"; // See web.xml.
/** Prefix to use for naming all snapshots that are started by this servlet. */
static final String SNAPSHOT_PREFIX = "auto_snapshot_";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject Clock clock;
@Inject DatastoreBackupService backupService;
@Inject Response response;
@Inject
ExportSnapshotAction() {}
@Override
public void run() {
// Use a unique name for the snapshot so we can explicitly check its completion later.
String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss");
backupService.launchNewBackup(
QUEUE, snapshotName, RegistryConfig.getSnapshotsBucket(), ExportConstants.getBackupKinds());
// Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery.
enqueuePollTask(snapshotName, ExportConstants.getReportingKinds());
String message = "Datastore backup started with name: " + snapshotName;
logger.info(message);
response.setPayload(message);
}
}

View file

@ -1,88 +0,0 @@
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.export;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.html.HtmlEscapers.htmlEscaper;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.common.net.MediaType;
import google.registry.config.RegistryConfig;
import google.registry.util.Clock;
import google.registry.util.FormattingLogger;
import google.registry.util.NonFinalForTesting;
import google.registry.util.SystemClock;
import java.io.IOException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/**
* Trigger a backup-as-a-service job that writes a snapshot to Google Cloud Storage.
*
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
* the next upon successful completion:<ol>
* <li>The snapshot is exported to Google Cloud Storage (this servlet).
* <li>The {@link CheckSnapshotServlet} polls until the export is completed.
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
* </ol>
*/
public class ExportSnapshotServlet extends HttpServlet {
/** Queue to use for enqueuing the task that will actually launch the backup. */
static final String QUEUE = "export-snapshot"; // See queue.xml.
/** Prefix to use for naming all snapshots that are started by this servlet. */
static final String SNAPSHOT_PREFIX = "auto_snapshot_";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@NonFinalForTesting
private static Clock clock = new SystemClock();
@NonFinalForTesting
private static DatastoreBackupService backupService = DatastoreBackupService.get();
@NonFinalForTesting
private static CheckSnapshotServlet checkSnapshotServlet = new CheckSnapshotServlet();
@Override
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
try {
// Use a unique name for the snapshot so we can explicitly check its completion later.
String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss");
backupService.launchNewBackup(
QUEUE,
snapshotName,
RegistryConfig.getSnapshotsBucket(),
ExportConstants.getBackupKinds());
// Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery.
checkSnapshotServlet.enqueuePollTask(snapshotName, ExportConstants.getReportingKinds());
String message = "Datastore backup started with name: " + snapshotName;
logger.info(message);
rsp.setStatus(SC_OK);
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
rsp.getWriter().write("OK\n\n" + message);
} catch (Throwable e) {
logger.severe(e, e.toString());
rsp.sendError(
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
}
}
}

View file

@ -38,9 +38,11 @@ import google.registry.dns.writer.clouddns.CloudDnsWriterModule;
import google.registry.dns.writer.dnsupdate.DnsUpdateConfigModule;
import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule;
import google.registry.export.BigqueryPollJobAction;
import google.registry.export.CheckSnapshotAction;
import google.registry.export.ExportDomainListsAction;
import google.registry.export.ExportRequestModule;
import google.registry.export.ExportReservedTermsAction;
import google.registry.export.ExportSnapshotAction;
import google.registry.export.LoadSnapshotAction;
import google.registry.export.SyncGroupMembersAction;
import google.registry.export.UpdateSnapshotViewAction;
@ -94,6 +96,7 @@ import google.registry.tmch.TmchSmdrlAction;
interface BackendRequestComponent {
BigqueryPollJobAction bigqueryPollJobAction();
BrdaCopyAction brdaCopyAction();
CheckSnapshotAction checkSnapshotAction();
CommitLogCheckpointAction commitLogCheckpointAction();
CommitLogFanoutAction commitLogFanoutAction();
DeleteContactsAndHostsAction deleteContactsAndHostsAction();
@ -103,6 +106,7 @@ interface BackendRequestComponent {
ExportCommitLogDiffAction exportCommitLogDiffAction();
ExportDomainListsAction exportDomainListsAction();
ExportReservedTermsAction exportReservedTermsAction();
ExportSnapshotAction exportSnapshotAction();
LoadSnapshotAction loadSnapshotAction();
MetricsExportAction metricsExportAction();
NordnUploadAction nordnUploadAction();