diff --git a/docs/app-engine-architecture.md b/docs/app-engine-architecture.md index ccf7f57fe..d9693af6e 100644 --- a/docs/app-engine-architecture.md +++ b/docs/app-engine-architecture.md @@ -110,9 +110,9 @@ Here are the task queues in use by the system. All are push queues unless explicitly marked as otherwise. * `async-delete-pull` and `async-host-rename-pull` -- Pull queues for tasks to - asynchronously delete contacts/hosts and to asynchronously refresh DNS - for renamed hosts, respectively. Tasks are enqueued during EPP - flows and then handled in batches by the regularly running cron tasks + asynchronously delete contacts/hosts and to asynchronously refresh DNS for + renamed hosts, respectively. Tasks are enqueued during EPP flows and then + handled in batches by the regularly running cron tasks `DeleteContactsAndHostsAction` and `RefreshDnsOnHostRenameAction`. * `bigquery-streaming-metrics` -- Queue for metrics that are asynchronously streamed to BigQuery in the `Metrics` class. Tasks are enqueued during EPP @@ -140,14 +140,14 @@ explicitly marked as otherwise. cron) and executed by `ExportCommitLogDiffAction`. * `export-snapshot` -- Cron and push queue for tasks to load a Datastore snapshot that was stored in Google Cloud Storage and export it to BigQuery. - Tasks are enqueued by both cron and `CheckSnapshotServlet` and are executed - by both `ExportSnapshotServlet` and `LoadSnapshotAction`. + Tasks are enqueued by both cron and `CheckSnapshotAction` and are executed + by both `ExportSnapshotAction` and `LoadSnapshotAction`. * `export-snapshot-poll` -- Queue for tasks to check that a Datastore snapshot has been successfully uploaded to Google Cloud Storage (this is an asynchronous background operation that can take an indeterminate amount of time). Once the snapshot is successfully uploaded, it is imported into - BigQuery. Tasks are enqueued by `ExportSnapshotServlet` and executed by - `CheckSnapshotServlet`. + BigQuery. Tasks are enqueued by `ExportSnapshotAction` and executed by + `CheckSnapshotAction`. * `export-snapshot-update-view` -- Queue for tasks to update the BigQuery views to point to the most recently uploaded snapshot. Tasks are enqueued by `LoadSnapshotAction` and executed by `UpdateSnapshotViewAction`. diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index dbd6f6a66..1d6a72593 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -1102,7 +1102,7 @@ public final class RegistryConfig { /** * Returns the Google Cloud Storage bucket for storing backup snapshots. * - * @see google.registry.export.ExportSnapshotServlet + * @see google.registry.export.ExportSnapshotAction */ public static String getSnapshotsBucket() { return getProjectId() + "-snapshots"; diff --git a/java/google/registry/dns/DnsModule.java b/java/google/registry/dns/DnsModule.java index 60d258b88..e289146d1 100644 --- a/java/google/registry/dns/DnsModule.java +++ b/java/google/registry/dns/DnsModule.java @@ -76,7 +76,7 @@ public abstract class DnsModule { } @Provides - @Parameter("name") + @Parameter("domainOrHostName") static String provideName(HttpServletRequest req) { return extractRequiredParameter(req, "name"); } diff --git a/java/google/registry/dns/RefreshDnsAction.java b/java/google/registry/dns/RefreshDnsAction.java index 25ffd87de..f68c31757 100644 --- a/java/google/registry/dns/RefreshDnsAction.java +++ b/java/google/registry/dns/RefreshDnsAction.java @@ -35,7 +35,7 @@ public final class RefreshDnsAction implements Runnable { @Inject Clock clock; @Inject DnsQueue dnsQueue; - @Inject @Parameter("name") String domainOrHostName; + @Inject @Parameter("domainOrHostName") String domainOrHostName; @Inject @Parameter("type") TargetType type; @Inject RefreshDnsAction() {} diff --git a/java/google/registry/env/common/backend/WEB-INF/web.xml b/java/google/registry/env/common/backend/WEB-INF/web.xml index 27041cda8..0f82e765b 100644 --- a/java/google/registry/env/common/backend/WEB-INF/web.xml +++ b/java/google/registry/env/common/backend/WEB-INF/web.xml @@ -1,7 +1,7 @@ @@ -113,25 +113,15 @@ /_dr/task/verifyEntityIntegrity - - Exports a datastore backup snapshot to GCS. - Export snapshot to GCS - exportSnapshot - google.registry.export.ExportSnapshotServlet - + - exportSnapshot + backend-servlet /_dr/task/exportSnapshot - - Checks the completion of a datastore backup snapshot. - Check on snapshot status - checkSnapshot - google.registry.export.CheckSnapshotServlet - + - checkSnapshot + backend-servlet /_dr/task/checkSnapshot @@ -290,10 +280,10 @@ Internal - Admin-only internal section. Requests for paths covered by the URL patterns below will be + Admin-only internal section. Requests for paths covered by the URL patterns below will be checked for a logged-in user account that's allowed to access the AppEngine admin console (NOTE: this includes Editor/Viewer permissions in addition to Owner and the new IAM - App Engine Admin role. See https://cloud.google.com/appengine/docs/java/access-control + App Engine Admin role. See https://cloud.google.com/appengine/docs/java/access-control specifically the "Access handlers that have a login:admin restriction" line.) TODO(b/28219927): lift some of these restrictions so that we can allow OAuth authentication diff --git a/java/google/registry/export/CheckSnapshotAction.java b/java/google/registry/export/CheckSnapshotAction.java new file mode 100644 index 000000000..401014bc3 --- /dev/null +++ b/java/google/registry/export/CheckSnapshotAction.java @@ -0,0 +1,162 @@ +// Copyright 2016 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export; + +import static com.google.common.collect.Sets.intersection; +import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask; +import static google.registry.request.Action.Method.GET; +import static google.registry.request.Action.Method.POST; +import static google.registry.util.FormattingLogger.getLoggerForCallerClass; + +import com.google.appengine.api.taskqueue.QueueFactory; +import com.google.appengine.api.taskqueue.TaskHandle; +import com.google.appengine.api.taskqueue.TaskOptions; +import com.google.appengine.api.taskqueue.TaskOptions.Method; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import google.registry.export.DatastoreBackupInfo.BackupStatus; +import google.registry.request.Action; +import google.registry.request.HttpException.BadRequestException; +import google.registry.request.HttpException.NoContentException; +import google.registry.request.HttpException.NotModifiedException; +import google.registry.request.Parameter; +import google.registry.request.RequestMethod; +import google.registry.request.Response; +import google.registry.util.FormattingLogger; +import java.util.Set; +import javax.inject.Inject; +import org.joda.time.Duration; +import org.joda.time.PeriodType; +import org.joda.time.format.PeriodFormat; + +/** + * Action that checks the status of a snapshot, and if complete, trigger loading it into BigQuery. + */ +@Action( + path = CheckSnapshotAction.PATH, + method = {POST, GET}, + automaticallyPrintOk = true +) +public class CheckSnapshotAction implements Runnable { + + /** Parameter names for passing parameters into this action. */ + static final String CHECK_SNAPSHOT_NAME_PARAM = "name"; + + static final String CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad"; + + /** Action-specific details needed for enqueuing tasks against itself. */ + static final String QUEUE = "export-snapshot-poll"; // See queue.xml. + + static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml. + static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2); + + /** The maximum amount of time we allow a backup to run before abandoning it. */ + static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20); + + private static final FormattingLogger logger = getLoggerForCallerClass(); + + @Inject Response response; + @Inject @RequestMethod Action.Method requestMethod; + @Inject DatastoreBackupService backupService; + @Inject @Parameter(CHECK_SNAPSHOT_NAME_PARAM) String snapshotName; + @Inject @Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM) String kindsToLoadParam; + @Inject CheckSnapshotAction() {} + + @Override + public void run() { + if (requestMethod == POST) { + checkAndLoadSnapshotIfComplete(); + } else { + // This is a GET request. + response.setPayload(getBackup().getInformation()); + } + } + + private DatastoreBackupInfo getBackup() { + try { + return backupService.findByName(snapshotName); + } catch (IllegalArgumentException e) { + String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage()); + // TODO(b/19081569): Ideally this would return a 2XX error so the task would not be + // retried but we might abandon backups that start late and haven't yet written to + // datastore. We could fix that by replacing this with a two-phase polling strategy. + throw new BadRequestException(message, e); + } + } + + private void checkAndLoadSnapshotIfComplete() { + Set kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam)); + DatastoreBackupInfo backup = getBackup(); + // Stop now if the backup is not complete. + if (!backup.getStatus().equals(BackupStatus.COMPLETE)) { + Duration runningTime = backup.getRunningTime(); + if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) { + // Backup might still be running, so send a 304 to have the task retry. + throw new NotModifiedException( + String.format("Datastore backup %s still pending", snapshotName)); + } else { + // Declare the backup a lost cause, and send 204 No Content so the task will + // not be retried. + String message = + String.format( + "Datastore backup %s abandoned - not complete after %s", + snapshotName, + PeriodFormat.getDefault() + .print( + runningTime + .toPeriod() + .normalizedStandard(PeriodType.dayTime().withMillisRemoved()))); + throw new NoContentException(message); + } + } + // Get a compact string to identify this snapshot in BigQuery by trying to parse the unique + // suffix out of the snapshot name and falling back to the start time as a string. + String snapshotId = + snapshotName.startsWith(ExportSnapshotAction.SNAPSHOT_PREFIX) + ? snapshotName.substring(ExportSnapshotAction.SNAPSHOT_PREFIX.length()) + : backup.getStartTime().toString("YYYYMMdd_HHmmss"); + // Log a warning if kindsToLoad is not a subset of the exported snapshot kinds. + if (!backup.getKinds().containsAll(kindsToLoad)) { + logger.warningfmt( + "Kinds to load included non-exported kinds: %s", + Sets.difference(kindsToLoad, backup.getKinds())); + } + // Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present). + ImmutableSet exportedKindsToLoad = + ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad)); + String message = String.format("Datastore backup %s complete - ", snapshotName); + if (exportedKindsToLoad.isEmpty()) { + message += "no kinds to load into BigQuery"; + } else { + enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad); + message += "BigQuery load task enqueued"; + } + logger.info(message); + response.setPayload(message); + } + + /** Enqueue a poll task to monitor the named snapshot for completion. */ + static TaskHandle enqueuePollTask(String snapshotName, ImmutableSet kindsToLoad) { + return QueueFactory.getQueue(QUEUE) + .add( + TaskOptions.Builder.withUrl(PATH) + .method(Method.POST) + .countdownMillis(POLL_COUNTDOWN.getMillis()) + .param(CHECK_SNAPSHOT_NAME_PARAM, snapshotName) + .param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad))); + } +} diff --git a/java/google/registry/export/CheckSnapshotServlet.java b/java/google/registry/export/CheckSnapshotServlet.java deleted file mode 100644 index 4adc45f35..000000000 --- a/java/google/registry/export/CheckSnapshotServlet.java +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2016 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.google.common.collect.Sets.intersection; -import static com.google.common.html.HtmlEscapers.htmlEscaper; -import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask; -import static google.registry.request.RequestParameters.extractRequiredParameter; -import static google.registry.util.FormattingLogger.getLoggerForCallerClass; -import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED; -import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; -import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; -import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED; -import static javax.servlet.http.HttpServletResponse.SC_OK; - -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TaskHandle; -import com.google.appengine.api.taskqueue.TaskOptions; -import com.google.appengine.api.taskqueue.TaskOptions.Method; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import com.google.common.net.MediaType; -import google.registry.export.DatastoreBackupInfo.BackupStatus; -import google.registry.request.HttpException.BadRequestException; -import google.registry.util.FormattingLogger; -import google.registry.util.NonFinalForTesting; -import java.io.IOException; -import java.util.Set; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.joda.time.Duration; -import org.joda.time.PeriodType; -import org.joda.time.format.PeriodFormat; - -/** Check the status of a snapshot, and if complete, trigger loading it into BigQuery. */ -public class CheckSnapshotServlet extends HttpServlet { - - /** Parameter names for passing parameters into this servlet. */ - static final String SNAPSHOT_NAME_PARAM = "name"; - static final String SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad"; - - /** Servlet-specific details needed for enqueuing tasks against itself. */ - static final String QUEUE = "export-snapshot-poll"; // See queue.xml. - static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml. - static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2); - - /** The maximum amount of time we allow a backup to run before abandoning it. */ - static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20); - - private static final FormattingLogger logger = getLoggerForCallerClass(); - - @NonFinalForTesting - private static DatastoreBackupService backupService = DatastoreBackupService.get(); - - @Override - public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - try { - rsp.setStatus(SC_OK); - rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString()); - rsp.getWriter().write("OK\n\n"); - super.service(req, rsp); - } catch (Throwable e) { - logger.severe(e, e.toString()); - rsp.sendError( - e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR, - htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString()))); - } - } - - @Override - public void doGet(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - // TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized. - try { - String snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM); - rsp.getWriter().write(backupService.findByName(snapshotName).getInformation()); - } catch (BadRequestException e) { - throw new IllegalArgumentException(e.getMessage()); - } - } - - @Override - public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - String snapshotName; - String kindsToLoadParam; - // TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized. - try { - snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM); - kindsToLoadParam = extractRequiredParameter(req, SNAPSHOT_KINDS_TO_LOAD_PARAM); - } catch (BadRequestException e) { - throw new IllegalArgumentException(e.getMessage()); - } - Set kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam)); - - // Look up the backup by the provided name, stopping if we can't find it. - DatastoreBackupInfo backup; - try { - backup = backupService.findByName(snapshotName); - } catch (IllegalArgumentException e) { - String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage()); - logger.severe(e, message); - // TODO(b/19081569): Ideally this would return a 2XX error so the task would not be retried, - // but we might abandon backups that start late and haven't yet written to datastore. - // We could fix that by replacing this with a two-phase polling strategy. - rsp.sendError(SC_BAD_REQUEST, htmlEscaper().escape(message)); - return; - } - // Stop now if the backup is not complete. - if (!backup.getStatus().equals(BackupStatus.COMPLETE)) { - Duration runningTime = backup.getRunningTime(); - if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) { - // Backup might still be running, so send a 304 to have the task retry. - rsp.sendError(SC_NOT_MODIFIED, - htmlEscaper().escape(String.format("Datastore backup %s still pending", snapshotName))); - } else { - // Declare the backup a lost cause, and send 202 Accepted so the task will not be retried. - String message = String.format("Datastore backup %s abandoned - not complete after %s", - snapshotName, - PeriodFormat.getDefault().print( - runningTime.toPeriod().normalizedStandard( - PeriodType.dayTime().withMillisRemoved()))); - logger.severe(message); - rsp.sendError(SC_ACCEPTED, htmlEscaper().escape(message)); - } - return; - } - // Get a compact string to identify this snapshot in BigQuery by trying to parse the unique - // suffix out of the snapshot name and falling back to the start time as a string. - String snapshotId = snapshotName.startsWith(ExportSnapshotServlet.SNAPSHOT_PREFIX) - ? snapshotName.substring(ExportSnapshotServlet.SNAPSHOT_PREFIX.length()) - : backup.getStartTime().toString("YYYYMMdd_HHmmss"); - // Log a warning if kindsToLoad is not a subset of the exported snapshot kinds. - if (!backup.getKinds().containsAll(kindsToLoad)) { - logger.warningfmt( - "Kinds to load included non-exported kinds: %s", - Sets.difference(kindsToLoad, backup.getKinds())); - } - // Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present). - ImmutableSet exportedKindsToLoad = - ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad)); - String message = String.format("Datastore backup %s complete - ", snapshotName); - if (exportedKindsToLoad.isEmpty()) { - message += "no kinds to load into BigQuery"; - } else { - enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad); - message += "BigQuery load task enqueued"; - } - logger.info(message); - rsp.getWriter().write(message); - } - - /** Enqueue a poll task to monitor the named snapshot for completion. */ - TaskHandle enqueuePollTask(String snapshotName, ImmutableSet kindsToLoad) { - return QueueFactory.getQueue(QUEUE).add( - TaskOptions.Builder.withUrl(PATH) - .method(Method.POST) - .countdownMillis(POLL_COUNTDOWN.getMillis()) - .param(SNAPSHOT_NAME_PARAM, snapshotName) - .param(SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad))); - } -} diff --git a/java/google/registry/export/ExportRequestModule.java b/java/google/registry/export/ExportRequestModule.java index b4327c792..6ad1a6fdb 100644 --- a/java/google/registry/export/ExportRequestModule.java +++ b/java/google/registry/export/ExportRequestModule.java @@ -17,6 +17,8 @@ package google.registry.export; import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER; import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER; import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER; +import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM; +import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM; import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM; import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM; import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM; @@ -72,6 +74,18 @@ public final class ExportRequestModule { return extractRequiredParameter(req, LOAD_SNAPSHOT_KINDS_PARAM); } + @Provides + @Parameter(CHECK_SNAPSHOT_NAME_PARAM) + static String provideCheckSnapshotName(HttpServletRequest req) { + return extractRequiredParameter(req, CHECK_SNAPSHOT_NAME_PARAM); + } + + @Provides + @Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM) + static String provideCheckSnapshotKindsToLoad(HttpServletRequest req) { + return extractRequiredParameter(req, CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM); + } + @Provides @Header(CHAINED_TASK_QUEUE_HEADER) static String provideChainedTaskQueue(HttpServletRequest req) { @@ -89,4 +103,9 @@ public final class ExportRequestModule { static String provideProjectId(HttpServletRequest req) { return extractRequiredHeader(req, PROJECT_ID_HEADER); } + + @Provides + static DatastoreBackupService provideDatastoreBackupService() { + return DatastoreBackupService.get(); + } } diff --git a/java/google/registry/export/ExportSnapshotAction.java b/java/google/registry/export/ExportSnapshotAction.java new file mode 100644 index 000000000..4a0dba041 --- /dev/null +++ b/java/google/registry/export/ExportSnapshotAction.java @@ -0,0 +1,72 @@ +// Copyright 2016 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export; + +import static google.registry.export.CheckSnapshotAction.enqueuePollTask; +import static google.registry.request.Action.Method.POST; + +import google.registry.config.RegistryConfig; +import google.registry.request.Action; +import google.registry.request.Response; +import google.registry.util.Clock; +import google.registry.util.FormattingLogger; +import javax.inject.Inject; + +/** + * Action to trigger a datastore backup job that writes a snapshot to Google Cloud Storage. + * + *

This is the first step of a four step workflow for exporting snapshots, with each step calling + * the next upon successful completion: + * + *

    + *
  1. The snapshot is exported to Google Cloud Storage (this action). + *
  2. The {@link CheckSnapshotAction} polls until the export is completed. + *
  3. The {@link LoadSnapshotAction} imports the data from GCS to BigQuery. + *
  4. The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot. + *
+ */ +@Action(path = ExportSnapshotAction.PATH, method = POST, automaticallyPrintOk = true) +public class ExportSnapshotAction implements Runnable { + + /** Queue to use for enqueuing the task that will actually launch the backup. */ + static final String QUEUE = "export-snapshot"; // See queue.xml. + + static final String PATH = "/_dr/task/exportSnapshot"; // See web.xml. + + /** Prefix to use for naming all snapshots that are started by this servlet. */ + static final String SNAPSHOT_PREFIX = "auto_snapshot_"; + + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); + + @Inject Clock clock; + @Inject DatastoreBackupService backupService; + @Inject Response response; + + @Inject + ExportSnapshotAction() {} + + @Override + public void run() { + // Use a unique name for the snapshot so we can explicitly check its completion later. + String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss"); + backupService.launchNewBackup( + QUEUE, snapshotName, RegistryConfig.getSnapshotsBucket(), ExportConstants.getBackupKinds()); + // Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery. + enqueuePollTask(snapshotName, ExportConstants.getReportingKinds()); + String message = "Datastore backup started with name: " + snapshotName; + logger.info(message); + response.setPayload(message); + } +} diff --git a/java/google/registry/export/ExportSnapshotServlet.java b/java/google/registry/export/ExportSnapshotServlet.java deleted file mode 100644 index 3212c3073..000000000 --- a/java/google/registry/export/ExportSnapshotServlet.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2016 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.google.common.html.HtmlEscapers.htmlEscaper; -import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; -import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; -import static javax.servlet.http.HttpServletResponse.SC_OK; - -import com.google.common.net.MediaType; -import google.registry.config.RegistryConfig; -import google.registry.util.Clock; -import google.registry.util.FormattingLogger; -import google.registry.util.NonFinalForTesting; -import google.registry.util.SystemClock; -import java.io.IOException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/** - * Trigger a backup-as-a-service job that writes a snapshot to Google Cloud Storage. - * - *

This is the first step of a four step workflow for exporting snapshots, with each step calling - * the next upon successful completion:

    - *
  1. The snapshot is exported to Google Cloud Storage (this servlet). - *
  2. The {@link CheckSnapshotServlet} polls until the export is completed. - *
  3. The {@link LoadSnapshotAction} imports the data from GCS to BigQuery. - *
  4. The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot. - *
- */ -public class ExportSnapshotServlet extends HttpServlet { - - /** Queue to use for enqueuing the task that will actually launch the backup. */ - static final String QUEUE = "export-snapshot"; // See queue.xml. - - /** Prefix to use for naming all snapshots that are started by this servlet. */ - static final String SNAPSHOT_PREFIX = "auto_snapshot_"; - - private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); - - @NonFinalForTesting - private static Clock clock = new SystemClock(); - - @NonFinalForTesting - private static DatastoreBackupService backupService = DatastoreBackupService.get(); - - @NonFinalForTesting - private static CheckSnapshotServlet checkSnapshotServlet = new CheckSnapshotServlet(); - - @Override - public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - try { - // Use a unique name for the snapshot so we can explicitly check its completion later. - String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss"); - backupService.launchNewBackup( - QUEUE, - snapshotName, - RegistryConfig.getSnapshotsBucket(), - ExportConstants.getBackupKinds()); - // Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery. - checkSnapshotServlet.enqueuePollTask(snapshotName, ExportConstants.getReportingKinds()); - String message = "Datastore backup started with name: " + snapshotName; - logger.info(message); - rsp.setStatus(SC_OK); - rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString()); - rsp.getWriter().write("OK\n\n" + message); - } catch (Throwable e) { - logger.severe(e, e.toString()); - rsp.sendError( - e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR, - htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString()))); - } - } -} diff --git a/java/google/registry/module/backend/BackendRequestComponent.java b/java/google/registry/module/backend/BackendRequestComponent.java index 4331d54c5..33da24553 100644 --- a/java/google/registry/module/backend/BackendRequestComponent.java +++ b/java/google/registry/module/backend/BackendRequestComponent.java @@ -38,9 +38,11 @@ import google.registry.dns.writer.clouddns.CloudDnsWriterModule; import google.registry.dns.writer.dnsupdate.DnsUpdateConfigModule; import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule; import google.registry.export.BigqueryPollJobAction; +import google.registry.export.CheckSnapshotAction; import google.registry.export.ExportDomainListsAction; import google.registry.export.ExportRequestModule; import google.registry.export.ExportReservedTermsAction; +import google.registry.export.ExportSnapshotAction; import google.registry.export.LoadSnapshotAction; import google.registry.export.SyncGroupMembersAction; import google.registry.export.UpdateSnapshotViewAction; @@ -94,6 +96,7 @@ import google.registry.tmch.TmchSmdrlAction; interface BackendRequestComponent { BigqueryPollJobAction bigqueryPollJobAction(); BrdaCopyAction brdaCopyAction(); + CheckSnapshotAction checkSnapshotAction(); CommitLogCheckpointAction commitLogCheckpointAction(); CommitLogFanoutAction commitLogFanoutAction(); DeleteContactsAndHostsAction deleteContactsAndHostsAction(); @@ -103,6 +106,7 @@ interface BackendRequestComponent { ExportCommitLogDiffAction exportCommitLogDiffAction(); ExportDomainListsAction exportDomainListsAction(); ExportReservedTermsAction exportReservedTermsAction(); + ExportSnapshotAction exportSnapshotAction(); LoadSnapshotAction loadSnapshotAction(); MetricsExportAction metricsExportAction(); NordnUploadAction nordnUploadAction(); diff --git a/javatests/google/registry/export/CheckSnapshotActionTest.java b/javatests/google/registry/export/CheckSnapshotActionTest.java new file mode 100644 index 000000000..39e444dda --- /dev/null +++ b/javatests/google/registry/export/CheckSnapshotActionTest.java @@ -0,0 +1,226 @@ +// Copyright 2016 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM; +import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM; +import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued; +import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; +import static org.mockito.Mockito.when; + +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import google.registry.request.Action.Method; +import google.registry.request.HttpException.BadRequestException; +import google.registry.request.HttpException.NoContentException; +import google.registry.request.HttpException.NotModifiedException; +import google.registry.testing.AppEngineRule; +import google.registry.testing.ExceptionRule; +import google.registry.testing.FakeClock; +import google.registry.testing.FakeResponse; +import google.registry.testing.InjectRule; +import google.registry.testing.TaskQueueHelper.TaskMatcher; +import org.joda.time.DateTime; +import org.joda.time.Duration; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +/** Unit tests for {@link CheckSnapshotAction}. */ +@RunWith(MockitoJUnitRunner.class) +public class CheckSnapshotActionTest { + + static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z"); + static final DateTime COMPLETE_TIME = START_TIME.plus(Duration.standardMinutes(30)); + + @Rule public final InjectRule inject = new InjectRule(); + @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build(); + @Rule public final ExceptionRule thrown = new ExceptionRule(); + + @Mock private DatastoreBackupService backupService; + + private DatastoreBackupInfo backupInfo; + + private final FakeResponse response = new FakeResponse(); + private final FakeClock clock = new FakeClock(COMPLETE_TIME.plusMillis(1000)); + private final CheckSnapshotAction action = new CheckSnapshotAction(); + + @Before + public void before() throws Exception { + inject.setStaticField(DatastoreBackupInfo.class, "clock", clock); + action.requestMethod = Method.POST; + action.snapshotName = "some_backup"; + action.kindsToLoadParam = "one,two"; + action.response = response; + action.backupService = backupService; + + backupInfo = + new DatastoreBackupInfo( + "some_backup", + START_TIME, + Optional.of(COMPLETE_TIME), + ImmutableSet.of("one", "two", "three"), + Optional.of("gs://somebucket/some_backup_20140801.backup_info")); + + when(backupService.findByName("some_backup")).thenReturn(backupInfo); + } + + private void setPendingBackup() { + backupInfo = + new DatastoreBackupInfo( + backupInfo.getName(), + backupInfo.getStartTime(), + Optional.absent(), + backupInfo.getKinds(), + backupInfo.getGcsFilename()); + + when(backupService.findByName("some_backup")).thenReturn(backupInfo); + } + + private static void assertLoadTaskEnqueued(String id, String file, String kinds) + throws Exception { + assertTasksEnqueued( + "export-snapshot", + new TaskMatcher() + .url("/_dr/task/loadSnapshot") + .method("POST") + .param("id", id) + .param("file", file) + .param("kinds", kinds)); + } + + @Test + public void testSuccess_enqueuePollTask() throws Exception { + CheckSnapshotAction.enqueuePollTask( + "some_snapshot_name", ImmutableSet.of("one", "two", "three")); + assertTasksEnqueued( + CheckSnapshotAction.QUEUE, + new TaskMatcher() + .url(CheckSnapshotAction.PATH) + .param(CHECK_SNAPSHOT_NAME_PARAM, "some_snapshot_name") + .param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, "one,two,three") + .method("POST")); + } + + @Test + public void testPost_forPendingBackup_returnsNotModified() throws Exception { + setPendingBackup(); + + thrown.expect(NotModifiedException.class, "Datastore backup some_backup still pending"); + action.run(); + } + + @Test + public void testPost_forStalePendingBackupBackup_returnsNoContent() throws Exception { + setPendingBackup(); + + when(backupService.findByName("some_backup")).thenReturn(backupInfo); + + clock.setTo( + START_TIME + .plus(Duration.standardHours(20)) + .plus(Duration.standardMinutes(3)) + .plus(Duration.millis(1234))); + + thrown.expect( + NoContentException.class, + "Datastore backup some_backup abandoned - " + + "not complete after 20 hours, 3 minutes and 1 second"); + + action.run(); + } + + @Test + public void testPost_forCompleteBackup_enqueuesLoadTask() throws Exception { + + action.run(); + assertLoadTaskEnqueued( + "20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one,two"); + } + + @Test + public void testPost_forCompleteAutoBackup_enqueuesLoadTask_usingBackupName() throws Exception { + action.snapshotName = "auto_snapshot_somestring"; + when(backupService.findByName("auto_snapshot_somestring")).thenReturn(backupInfo); + + action.run(); + assertLoadTaskEnqueued( + "somestring", "gs://somebucket/some_backup_20140801.backup_info", "one,two"); + } + + @Test + public void testPost_forCompleteBackup_withExtraKindsToLoad_enqueuesLoadTask() throws Exception { + action.kindsToLoadParam = "one,foo"; + + action.run(); + assertLoadTaskEnqueued( + "20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one"); + } + + @Test + public void testPost_forCompleteBackup_withEmptyKindsToLoad_skipsLoadTask() throws Exception { + action.kindsToLoadParam = ""; + + action.run(); + assertNoTasksEnqueued("export-snapshot"); + } + + @Test + public void testPost_forBadBackup_returnsBadRequest() throws Exception { + when(backupService.findByName("some_backup")) + .thenThrow(new IllegalArgumentException("No backup found")); + + thrown.expect(BadRequestException.class, "Bad backup name some_backup: No backup found"); + + action.run(); + } + + @Test + public void testGet_returnsInformation() throws Exception { + action.requestMethod = Method.GET; + + action.run(); + assertThat(response.getPayload()) + .isEqualTo( + Joiner.on("\n") + .join( + ImmutableList.of( + "Backup name: some_backup", + "Status: COMPLETE", + "Started: 2014-08-01T01:02:03.000Z", + "Ended: 2014-08-01T01:32:03.000Z", + "Duration: 30m", + "GCS: gs://somebucket/some_backup_20140801.backup_info", + "Kinds: [one, two, three]", + ""))); + } + + @Test + public void testGet_forBadBackup_returnsError() throws Exception { + action.requestMethod = Method.GET; + when(backupService.findByName("some_backup")) + .thenThrow(new IllegalArgumentException("No backup found")); + + thrown.expect(BadRequestException.class, "Bad backup name some_backup: No backup found"); + + action.run(); + } +} diff --git a/javatests/google/registry/export/CheckSnapshotServletTest.java b/javatests/google/registry/export/CheckSnapshotServletTest.java deleted file mode 100644 index a95be410f..000000000 --- a/javatests/google/registry/export/CheckSnapshotServletTest.java +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright 2016 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.export.CheckSnapshotServlet.SNAPSHOT_KINDS_TO_LOAD_PARAM; -import static google.registry.export.CheckSnapshotServlet.SNAPSHOT_NAME_PARAM; -import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued; -import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; -import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED; -import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; -import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED; -import static javax.servlet.http.HttpServletResponse.SC_OK; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.google.common.base.Joiner; -import com.google.common.base.Optional; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import google.registry.testing.AppEngineRule; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectRule; -import google.registry.testing.TaskQueueHelper.TaskMatcher; -import java.io.PrintWriter; -import java.io.StringWriter; -import javax.servlet.ServletConfig; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.joda.time.DateTime; -import org.joda.time.Duration; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; - -/** Unit tests for {@link CheckSnapshotServlet}. */ -@RunWith(MockitoJUnitRunner.class) -public class CheckSnapshotServletTest { - - static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z"); - static final DateTime COMPLETE_TIME = START_TIME.plus(Duration.standardMinutes(30)); - - @Rule - public final InjectRule inject = new InjectRule(); - - @Rule - public final AppEngineRule appEngine = AppEngineRule.builder() - .withTaskQueue() - .build(); - - @Mock - private HttpServletRequest req; - - @Mock - private HttpServletResponse rsp; - - private DatastoreBackupInfo backupInfo; - - @Mock - private DatastoreBackupService backupService; - - private final FakeClock clock = new FakeClock(COMPLETE_TIME.plusMillis(1000)); - private final StringWriter httpOutput = new StringWriter(); - private final CheckSnapshotServlet servlet = new CheckSnapshotServlet(); - - @Before - public void before() throws Exception { - inject.setStaticField(CheckSnapshotServlet.class, "backupService", backupService); - inject.setStaticField(DatastoreBackupInfo.class, "clock", clock); - - when(rsp.getWriter()).thenReturn(new PrintWriter(httpOutput)); - - servlet.init(mock(ServletConfig.class)); - when(req.getMethod()).thenReturn("POST"); - - backupInfo = new DatastoreBackupInfo( - "some_backup", - START_TIME, - Optional.of(COMPLETE_TIME), - ImmutableSet.of("one", "two", "three"), - Optional.of("gs://somebucket/some_backup_20140801.backup_info")); - } - - private void setPendingBackup() { - backupInfo = new DatastoreBackupInfo( - backupInfo.getName(), - backupInfo.getStartTime(), - Optional.absent(), - backupInfo.getKinds(), - backupInfo.getGcsFilename()); - } - - private static void assertLoadTaskEnqueued(String id, String file, String kinds) - throws Exception { - assertTasksEnqueued( - "export-snapshot", - new TaskMatcher() - .url("/_dr/task/loadSnapshot") - .method("POST") - .param("id", id) - .param("file", file) - .param("kinds", kinds)); - } - - @Test - public void testSuccess_enqueuePollTask() throws Exception { - servlet.enqueuePollTask("some_snapshot_name", ImmutableSet.of("one", "two", "three")); - assertTasksEnqueued(CheckSnapshotServlet.QUEUE, - new TaskMatcher() - .url(CheckSnapshotServlet.PATH) - .param(SNAPSHOT_NAME_PARAM, "some_snapshot_name") - .param(SNAPSHOT_KINDS_TO_LOAD_PARAM, "one,two,three") - .method("POST")); - } - - @Test - public void testPost_forPendingBackup_returnsNotModified() throws Exception { - setPendingBackup(); - when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup"); - when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two"); - when(backupService.findByName("some_backup")).thenReturn(backupInfo); - - servlet.service(req, rsp); - verify(rsp).sendError(SC_NOT_MODIFIED, "Datastore backup some_backup still pending"); - } - - @Test - public void testPost_forStalePendingBackupBackup_returnsAccepted() throws Exception { - setPendingBackup(); - when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup"); - when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two"); - when(backupService.findByName("some_backup")).thenReturn(backupInfo); - clock.setTo(START_TIME - .plus(Duration.standardHours(20)) - .plus(Duration.standardMinutes(3)) - .plus(Duration.millis(1234))); - - servlet.service(req, rsp); - verify(rsp).sendError(SC_ACCEPTED, - "Datastore backup some_backup abandoned - " - + "not complete after 20 hours, 3 minutes and 1 second"); - } - - @Test - public void testPost_forCompleteBackup_enqueuesLoadTask() throws Exception { - when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup"); - when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two"); - when(backupService.findByName("some_backup")).thenReturn(backupInfo); - - servlet.service(req, rsp); - verify(rsp).setStatus(SC_OK); - assertLoadTaskEnqueued( - "20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one,two"); - } - - @Test - public void testPost_forCompleteAutoBackup_enqueuesLoadTask_usingBackupName() throws Exception { - when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("auto_snapshot_somestring"); - when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two"); - when(backupService.findByName("auto_snapshot_somestring")).thenReturn(backupInfo); - - servlet.service(req, rsp); - verify(rsp).setStatus(SC_OK); - assertLoadTaskEnqueued( - "somestring", "gs://somebucket/some_backup_20140801.backup_info", "one,two"); - } - - @Test - public void testPost_forCompleteBackup_withExtraKindsToLoad_enqueuesLoadTask() throws Exception { - when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup"); - when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,foo"); - when(backupService.findByName("some_backup")).thenReturn(backupInfo); - - servlet.service(req, rsp); - verify(rsp).setStatus(SC_OK); - assertLoadTaskEnqueued( - "20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one"); - } - -@Test - public void testPost_forCompleteBackup_withEmptyKindsToLoad_skipsLoadTask() throws Exception { - when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup"); - when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn(""); - when(backupService.findByName("some_backup")).thenReturn(backupInfo); - - servlet.service(req, rsp); - verify(rsp).setStatus(SC_OK); - assertNoTasksEnqueued("export-snapshot"); - } - - @Test - public void testPost_forBadBackup_returnsBadRequest() throws Exception { - when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup"); - when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two"); - when(backupService.findByName("some_backup")) - .thenThrow(new IllegalArgumentException("No backup found")); - - servlet.service(req, rsp); - verify(rsp).sendError(SC_BAD_REQUEST, "Bad backup name some_backup: No backup found"); - } - - @Test - public void testPost_noBackupSpecified_returnsError() throws Exception { - when(req.getMethod()).thenReturn("POST"); - when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn(null); - when(req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM)).thenReturn("one,two"); - servlet.service(req, rsp); - verify(rsp).sendError(SC_BAD_REQUEST, "Missing parameter: name"); - } - - @Test - public void testGet_returnsInformation() throws Exception { - when(req.getMethod()).thenReturn("GET"); - when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup"); - when(backupService.findByName("some_backup")).thenReturn(backupInfo); - - servlet.service(req, rsp); - verify(rsp).setStatus(SC_OK); - assertThat(httpOutput.toString()).isEqualTo("OK\n\n" + Joiner.on("\n").join(ImmutableList.of( - "Backup name: some_backup", - "Status: COMPLETE", - "Started: 2014-08-01T01:02:03.000Z", - "Ended: 2014-08-01T01:32:03.000Z", - "Duration: 30m", - "GCS: gs://somebucket/some_backup_20140801.backup_info", - "Kinds: [one, two, three]", - ""))); - } - - @Test - public void testGet_forBadBackup_returnsError() throws Exception { - when(req.getMethod()).thenReturn("GET"); - when(req.getParameter(SNAPSHOT_NAME_PARAM)).thenReturn("some_backup"); - when(backupService.findByName("some_backup")).thenThrow( - new IllegalArgumentException("No backup found")); - - servlet.service(req, rsp); - verify(rsp).sendError(SC_BAD_REQUEST, "No backup found"); - } - - @Test - public void testGet_noBackupSpecified_returnsError() throws Exception { - when(req.getMethod()).thenReturn("GET"); - servlet.service(req, rsp); - verify(rsp).sendError(SC_BAD_REQUEST, "Missing parameter: name"); - } -} diff --git a/javatests/google/registry/export/ExportSnapshotActionTest.java b/javatests/google/registry/export/ExportSnapshotActionTest.java new file mode 100644 index 000000000..8d4499122 --- /dev/null +++ b/javatests/google/registry/export/ExportSnapshotActionTest.java @@ -0,0 +1,76 @@ +// Copyright 2016 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM; +import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM; +import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; +import static org.mockito.Mockito.verify; + +import com.google.common.base.Joiner; +import google.registry.testing.AppEngineRule; +import google.registry.testing.FakeClock; +import google.registry.testing.FakeResponse; +import google.registry.testing.TaskQueueHelper.TaskMatcher; +import org.joda.time.DateTime; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +/** Unit tests for {@link ExportSnapshotAction}. */ +@RunWith(MockitoJUnitRunner.class) +public class ExportSnapshotActionTest { + + @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build(); + + @Mock private DatastoreBackupService backupService; + + private final FakeResponse response = new FakeResponse(); + private final FakeClock clock = new FakeClock(DateTime.parse("2014-08-01T01:02:03Z")); + private final ExportSnapshotAction action = new ExportSnapshotAction(); + + @Before + public void before() throws Exception { + action.clock = clock; + action.backupService = backupService; + action.response = response; + } + + @Test + public void testPost_launchesBackup_andEnqueuesPollTask() throws Exception { + action.run(); + verify(backupService) + .launchNewBackup( + ExportSnapshotAction.QUEUE, + "auto_snapshot_20140801_010203", + "domain-registry-snapshots", + ExportConstants.getBackupKinds()); + assertTasksEnqueued( + CheckSnapshotAction.QUEUE, + new TaskMatcher() + .url(CheckSnapshotAction.PATH) + .param(CHECK_SNAPSHOT_NAME_PARAM, "auto_snapshot_20140801_010203") + .param( + CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, + Joiner.on(",").join(ExportConstants.getReportingKinds())) + .method("POST")); + assertThat(response.getPayload()) + .isEqualTo("Datastore backup started with name: auto_snapshot_20140801_010203"); + } +} diff --git a/javatests/google/registry/export/ExportSnapshotServletTest.java b/javatests/google/registry/export/ExportSnapshotServletTest.java deleted file mode 100644 index dbac0f174..000000000 --- a/javatests/google/registry/export/ExportSnapshotServletTest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2016 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static javax.servlet.http.HttpServletResponse.SC_OK; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import google.registry.testing.FakeClock; -import google.registry.testing.InjectRule; -import java.io.PrintWriter; -import java.io.StringWriter; -import javax.servlet.ServletConfig; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.joda.time.DateTime; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; - -/** Unit tests for {@link ExportSnapshotServlet}. */ -@RunWith(MockitoJUnitRunner.class) -public class ExportSnapshotServletTest { - - @Rule - public final InjectRule inject = new InjectRule(); - - @Mock - private HttpServletRequest req; - - @Mock - private HttpServletResponse rsp; - - @Mock - private DatastoreBackupService backupService; - - @Mock - private CheckSnapshotServlet checkSnapshotServlet; - - private final FakeClock clock = new FakeClock(); - private final StringWriter httpOutput = new StringWriter(); - private final ExportSnapshotServlet servlet = new ExportSnapshotServlet(); - - private static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z"); - - @Before - public void before() throws Exception { - clock.setTo(START_TIME); - inject.setStaticField(ExportSnapshotServlet.class, "clock", clock); - inject.setStaticField(ExportSnapshotServlet.class, "backupService", backupService); - inject.setStaticField( - ExportSnapshotServlet.class, "checkSnapshotServlet", checkSnapshotServlet); - when(rsp.getWriter()).thenReturn(new PrintWriter(httpOutput)); - - servlet.init(mock(ServletConfig.class)); - when(req.getMethod()).thenReturn("POST"); - } - - @Test - public void testPost_launchesBackup_andEnqueuesPollTask() throws Exception { - servlet.service(req, rsp); - verify(rsp).setStatus(SC_OK); - verify(backupService).launchNewBackup( - ExportSnapshotServlet.QUEUE, - "auto_snapshot_20140801_010203", - "domain-registry-snapshots", - ExportConstants.getBackupKinds()); - verify(checkSnapshotServlet) - .enqueuePollTask("auto_snapshot_20140801_010203", ExportConstants.getReportingKinds()); - } -}