google-nomulus/java/google/registry/export/CheckSnapshotServlet.java

// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package google.registry.export;

import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.collect.Sets.intersection;
import static com.google.common.html.HtmlEscapers.htmlEscaper;
import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
import static google.registry.request.RequestParameters.extractRequiredParameter;
import static google.registry.util.FormattingLogger.getLoggerForCallerClass;
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED;
import static javax.servlet.http.HttpServletResponse.SC_OK;

import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.net.MediaType;

import google.registry.export.DatastoreBackupInfo.BackupStatus;
import google.registry.request.HttpException.BadRequestException;
import google.registry.util.FormattingLogger;
import google.registry.util.NonFinalForTesting;

import org.joda.time.Duration;
import org.joda.time.PeriodType;
import org.joda.time.format.PeriodFormat;

import java.io.IOException;
import java.util.Set;

import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

/** Check the status of a snapshot, and if complete, trigger loading it into BigQuery. */
public class CheckSnapshotServlet extends HttpServlet {

  /** Parameter names for passing parameters into this servlet. */
  static final String SNAPSHOT_NAME_PARAM = "name";
  static final String SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";

  /** Servlet-specific details needed for enqueuing tasks against itself. */
  static final String QUEUE = "export-snapshot-poll";  // See queue.xml.
  static final String PATH = "/_dr/task/checkSnapshot";  // See web.xml.
  static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);

  /** The maximum amount of time we allow a backup to run before abandoning it. */
  static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);

  private static final FormattingLogger logger = getLoggerForCallerClass();

  @NonFinalForTesting
  private static DatastoreBackupService backupService = DatastoreBackupService.get();

  @Override
  public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
    try {
      rsp.setStatus(SC_OK);
      rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
      rsp.getWriter().write("OK\n\n");
      super.service(req, rsp);
    } catch (Throwable e) {
      logger.severe(e, e.toString());
      rsp.sendError(
          e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
          htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
    }
  }

  @Override
  public void doGet(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
    // TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
    try {
      String snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
      rsp.getWriter().write(backupService.findByName(snapshotName).getInformation());
    } catch (BadRequestException e) {
      throw new IllegalArgumentException(e.getMessage());
    }
  }

  @Override
  public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
    String snapshotName;
    String kindsToLoadParam;
    // TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
    try {
      snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
      kindsToLoadParam = extractRequiredParameter(req, SNAPSHOT_KINDS_TO_LOAD_PARAM);
    } catch (BadRequestException e) {
      throw new IllegalArgumentException(e.getMessage());
    }
    Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));

    // Look up the backup by the provided name, stopping if we can't find it.
    DatastoreBackupInfo backup;
    try {
      backup = backupService.findByName(snapshotName);
    } catch (IllegalArgumentException e) {
      String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
      logger.severe(e, message);
      // TODO(b/19081569): Ideally this would return a 2XX error so the task would not be retried,
      //   but we might abandon backups that start late and haven't yet written to datastore.
      //   We could fix that by replacing this with a two-phase polling strategy.
      rsp.sendError(SC_BAD_REQUEST, htmlEscaper().escape(message));
      return;
    }
    // Stop now if the backup is not complete.
    if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
      Duration runningTime = backup.getRunningTime();
      if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
        // Backup might still be running, so send a 304 to have the task retry.
        rsp.sendError(SC_NOT_MODIFIED,
            htmlEscaper().escape(String.format("Datastore backup %s still pending", snapshotName)));
      } else {
        // Declare the backup a lost cause, and send 202 Accepted so the task will not be retried.
        String message = String.format("Datastore backup %s abandoned - not complete after %s",
            snapshotName,
            PeriodFormat.getDefault().print(
                runningTime.toPeriod().normalizedStandard(
                    PeriodType.dayTime().withMillisRemoved())));
        logger.severe(message);
        rsp.sendError(SC_ACCEPTED, htmlEscaper().escape(message));
      }
      return;
    }
    // Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
    // suffix out of the snapshot name and falling back to the start time as a string.
    String snapshotId = snapshotName.startsWith(ExportSnapshotServlet.SNAPSHOT_PREFIX)
        ? snapshotName.substring(ExportSnapshotServlet.SNAPSHOT_PREFIX.length())
        : backup.getStartTime().toString("YYYYMMdd_HHmmss");
    // Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
    if (!backup.getKinds().containsAll(kindsToLoad)) {
      logger.warningfmt(
          "Kinds to load included non-exported kinds: %s",
          Sets.difference(kindsToLoad, backup.getKinds()));
    }
    // Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
    ImmutableSet<String> exportedKindsToLoad =
        ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
    String message = String.format("Datastore backup %s complete - ", snapshotName);
    if (exportedKindsToLoad.isEmpty()) {
      message += "no kinds to load into BigQuery";
    } else {
      enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
      message += "BigQuery load task enqueued";
    }
    logger.info(message);
    rsp.getWriter().write(message);
  }

  /** Enqueue a poll task to monitor the named snapshot for completion. */
  TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
    return QueueFactory.getQueue(QUEUE).add(
        TaskOptions.Builder.withUrl(PATH)
            .method(Method.POST)
            .countdownMillis(POLL_COUNTDOWN.getMillis())
            .param(SNAPSHOT_NAME_PARAM, snapshotName)
            .param(SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
  }
}