mirror of
https://github.com/google/nomulus.git
synced 2025-05-12 22:38:16 +02:00
Daggerize/Actionize the load snapshot servlet
This is needed to eliminate the last non-injected use of BigqueryFactory. It gets rid of the deprecated HttpServletUtils now that we don't have lots of separate servlets, and changes the check snapshot servlet's usage of such. It also fixes up some related minor style issues in the update snapshot action (and its injectable parameters). ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=120547998
This commit is contained in:
mcilwain
Justine Tunney
parent
9e7934684e
commit
d65bf2a714
13 changed files with 248 additions and 419 deletions
|
|
@ -59,7 +59,7 @@ public class BigqueryFactory {
|
|||
@VisibleForTesting
|
||||
Subfactory subfactory = new Subfactory();
|
||||
|
||||
@Inject public BigqueryFactory() {}
|
||||
@Inject BigqueryFactory() {}
|
||||
|
||||
/** This class is broken out solely so that it can be mocked inside of tests. */
|
||||
static class Subfactory {
|
||||
|
|
|
|||
|
|
@ -51,20 +51,6 @@ public interface RegistryConfig {
|
|||
*/
|
||||
public String getDomainListsBucket();
|
||||
|
||||
/**
|
||||
* Returns the BigQuery dataset for storing directly imported datastore snapshots.
|
||||
*
|
||||
* @see com.google.domain.registry.export.LoadSnapshotServlet
|
||||
*/
|
||||
public String getSnapshotsDataset();
|
||||
|
||||
/**
|
||||
* Returns the BigQuery dataset for storing views pointing to the latest datastore snapshot.
|
||||
*
|
||||
* @see com.google.domain.registry.export.UpdateSnapshotViewAction
|
||||
*/
|
||||
public String getLatestSnapshotDataset();
|
||||
|
||||
/**
|
||||
* Number of sharded commit log buckets.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -57,16 +57,6 @@ public class TestRegistryConfig implements RegistryConfig {
|
|||
return getProjectId() + "-domain-lists";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSnapshotsDataset() {
|
||||
return "snapshots";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLatestSnapshotDataset() {
|
||||
return "latest_snapshot";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCommitsBucket() {
|
||||
return getProjectId() + "-commits";
|
||||
|
|
|
|||
|
|
@ -15,8 +15,11 @@
|
|||
package com.google.domain.registry.export;
|
||||
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.collect.Sets.intersection;
|
||||
import static com.google.common.html.HtmlEscapers.htmlEscaper;
|
||||
import static com.google.domain.registry.util.HttpServletUtils.getRequiredParameterValue;
|
||||
import static com.google.domain.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
|
||||
import static com.google.domain.registry.request.RequestParameters.extractRequiredParameter;
|
||||
import static com.google.domain.registry.util.FormattingLogger.getLoggerForCallerClass;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
|
|
@ -28,12 +31,12 @@ import com.google.appengine.api.taskqueue.TaskHandle;
|
|||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Optional;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.net.MediaType;
|
||||
import com.google.domain.registry.export.DatastoreBackupInfo.BackupStatus;
|
||||
import com.google.domain.registry.request.HttpException.BadRequestException;
|
||||
import com.google.domain.registry.util.FormattingLogger;
|
||||
import com.google.domain.registry.util.NonFinalForTesting;
|
||||
|
||||
|
|
@ -42,6 +45,7 @@ import org.joda.time.PeriodType;
|
|||
import org.joda.time.format.PeriodFormat;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
|
@ -62,14 +66,11 @@ public class CheckSnapshotServlet extends HttpServlet {
|
|||
/** The maximum amount of time we allow a backup to run before abandoning it. */
|
||||
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static DatastoreBackupService backupService = DatastoreBackupService.get();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static LoadSnapshotServlet loadSnapshotServlet = new LoadSnapshotServlet();
|
||||
|
||||
@Override
|
||||
public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
try {
|
||||
|
|
@ -87,18 +88,27 @@ public class CheckSnapshotServlet extends HttpServlet {
|
|||
|
||||
@Override
|
||||
public void doGet(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
String snapshotName = getRequiredParameterValue(req, SNAPSHOT_NAME_PARAM);
|
||||
rsp.getWriter().write(backupService.findByName(snapshotName).getInformation());
|
||||
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
|
||||
try {
|
||||
String snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
|
||||
rsp.getWriter().write(backupService.findByName(snapshotName).getInformation());
|
||||
} catch (BadRequestException e) {
|
||||
throw new IllegalArgumentException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
String snapshotName = getRequiredParameterValue(req, SNAPSHOT_NAME_PARAM);
|
||||
// TODO(b/19237926): make this non-optional once all new tasks will have this parameter.
|
||||
String kindsToLoadParam = req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM);
|
||||
Optional<ImmutableSet<String>> kindsToLoad = Optional.fromNullable(
|
||||
kindsToLoadParam == null ? null
|
||||
: ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam)));
|
||||
String snapshotName;
|
||||
String kindsToLoadParam;
|
||||
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
|
||||
try {
|
||||
snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
|
||||
kindsToLoadParam = extractRequiredParameter(req, SNAPSHOT_KINDS_TO_LOAD_PARAM);
|
||||
} catch (BadRequestException e) {
|
||||
throw new IllegalArgumentException(e.getMessage());
|
||||
}
|
||||
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
|
||||
|
||||
// Look up the backup by the provided name, stopping if we can't find it.
|
||||
DatastoreBackupInfo backup;
|
||||
|
|
@ -137,21 +147,20 @@ public class CheckSnapshotServlet extends HttpServlet {
|
|||
String snapshotId = snapshotName.startsWith(ExportSnapshotServlet.SNAPSHOT_PREFIX)
|
||||
? snapshotName.substring(ExportSnapshotServlet.SNAPSHOT_PREFIX.length())
|
||||
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
|
||||
// Log a warning if kindsToLoad is specified and not a subset of the exported snapshot kinds.
|
||||
if (kindsToLoad.isPresent() && !backup.getKinds().containsAll(kindsToLoad.get())) {
|
||||
logger.warningfmt("Kinds to load included non-exported kinds: %s",
|
||||
Sets.difference(kindsToLoad.get(), backup.getKinds()));
|
||||
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
|
||||
if (!backup.getKinds().containsAll(kindsToLoad)) {
|
||||
logger.warningfmt(
|
||||
"Kinds to load included non-exported kinds: %s",
|
||||
Sets.difference(kindsToLoad, backup.getKinds()));
|
||||
}
|
||||
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
|
||||
ImmutableSet<String> exportedKindsToLoad = ImmutableSet.copyOf(kindsToLoad.isPresent()
|
||||
? Sets.intersection(backup.getKinds(), kindsToLoad.get())
|
||||
: backup.getKinds());
|
||||
ImmutableSet<String> exportedKindsToLoad =
|
||||
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
|
||||
String message = String.format("Datastore backup %s complete - ", snapshotName);
|
||||
if (exportedKindsToLoad.isEmpty()) {
|
||||
message += "no kinds to load into BigQuery";
|
||||
} else {
|
||||
loadSnapshotServlet.enqueueLoadTask(
|
||||
snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
|
||||
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
|
||||
message += "BigQuery load task enqueued";
|
||||
}
|
||||
logger.info(message);
|
||||
|
|
|
|||
|
|
@ -17,9 +17,12 @@ package com.google.domain.registry.export;
|
|||
import static com.google.domain.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
|
||||
import static com.google.domain.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
|
||||
import static com.google.domain.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.SNAPSHOT_DATASET_ID_PARAM;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.SNAPSHOT_KIND_PARAM;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.SNAPSHOT_TABLE_ID_PARAM;
|
||||
import static com.google.domain.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
|
||||
import static com.google.domain.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
|
||||
import static com.google.domain.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM;
|
||||
import static com.google.domain.registry.request.RequestParameters.extractRequiredHeader;
|
||||
|
||||
import com.google.domain.registry.request.Header;
|
||||
|
|
@ -35,21 +38,39 @@ import javax.servlet.http.HttpServletRequest;
|
|||
public final class ExportRequestModule {
|
||||
|
||||
@Provides
|
||||
@Parameter(SNAPSHOT_DATASET_ID_PARAM)
|
||||
static String provideDatasetId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, SNAPSHOT_DATASET_ID_PARAM);
|
||||
@Parameter(UPDATE_SNAPSHOT_DATASET_ID_PARAM)
|
||||
static String provideUpdateSnapshotDatasetId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, UPDATE_SNAPSHOT_DATASET_ID_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(SNAPSHOT_TABLE_ID_PARAM)
|
||||
static String provideTableId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, SNAPSHOT_TABLE_ID_PARAM);
|
||||
@Parameter(UPDATE_SNAPSHOT_TABLE_ID_PARAM)
|
||||
static String provideUpdateSnapshotTableId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, UPDATE_SNAPSHOT_TABLE_ID_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(SNAPSHOT_KIND_PARAM)
|
||||
static String provideKind(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, SNAPSHOT_KIND_PARAM);
|
||||
@Parameter(UPDATE_SNAPSHOT_KIND_PARAM)
|
||||
static String provideUpdateSnapshotKind(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, UPDATE_SNAPSHOT_KIND_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(LOAD_SNAPSHOT_FILE_PARAM)
|
||||
static String provideLoadSnapshotFile(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, LOAD_SNAPSHOT_FILE_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(LOAD_SNAPSHOT_ID_PARAM)
|
||||
static String provideLoadSnapshotId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, LOAD_SNAPSHOT_ID_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(LOAD_SNAPSHOT_KINDS_PARAM)
|
||||
static String provideLoadSnapshotKinds(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, LOAD_SNAPSHOT_KINDS_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
|
|
|
|||
|
|
@ -14,27 +14,19 @@
|
|||
|
||||
package com.google.domain.registry.export;
|
||||
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.html.HtmlEscapers.htmlEscaper;
|
||||
import static com.google.domain.registry.util.HttpServletUtils.getRequiredParameterValue;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.createViewUpdateTask;
|
||||
import static com.google.domain.registry.request.Action.Method.POST;
|
||||
import static com.google.domain.registry.util.FormattingLogger.getLoggerForCallerClass;
|
||||
|
||||
import com.google.api.client.extensions.appengine.http.UrlFetchTransport;
|
||||
import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential;
|
||||
import com.google.api.client.json.jackson2.JacksonFactory;
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.BigqueryScopes;
|
||||
import com.google.api.services.bigquery.model.Dataset;
|
||||
import com.google.api.services.bigquery.model.DatasetReference;
|
||||
import com.google.api.services.bigquery.model.Job;
|
||||
import com.google.api.services.bigquery.model.JobConfiguration;
|
||||
import com.google.api.services.bigquery.model.JobConfigurationLoad;
|
||||
import com.google.api.services.bigquery.model.JobReference;
|
||||
import com.google.api.services.bigquery.model.TableReference;
|
||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
|
|
@ -42,99 +34,82 @@ import com.google.common.base.Joiner;
|
|||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.net.MediaType;
|
||||
import com.google.domain.registry.bigquery.BigqueryFactory;
|
||||
import com.google.domain.registry.bigquery.BigqueryJobFailureException;
|
||||
import com.google.domain.registry.bigquery.BigqueryUtils.SourceFormat;
|
||||
import com.google.domain.registry.bigquery.BigqueryUtils.WriteDisposition;
|
||||
import com.google.domain.registry.config.RegistryEnvironment;
|
||||
import com.google.domain.registry.config.ConfigModule.Config;
|
||||
import com.google.domain.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
|
||||
import com.google.domain.registry.request.Action;
|
||||
import com.google.domain.registry.request.HttpException.BadRequestException;
|
||||
import com.google.domain.registry.request.HttpException.InternalServerErrorException;
|
||||
import com.google.domain.registry.request.Parameter;
|
||||
import com.google.domain.registry.util.Clock;
|
||||
import com.google.domain.registry.util.FormattingLogger;
|
||||
import com.google.domain.registry.util.NonFinalForTesting;
|
||||
import com.google.domain.registry.util.Retrier;
|
||||
import com.google.domain.registry.util.SystemClock;
|
||||
import com.google.domain.registry.util.SystemSleeper;
|
||||
import com.google.domain.registry.util.TaskEnqueuer;
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/** Load a datastore snapshot from Google Cloud Storage into BigQuery. */
|
||||
public class LoadSnapshotServlet extends HttpServlet {
|
||||
|
||||
private static final RegistryEnvironment ENVIRONMENT = RegistryEnvironment.get();
|
||||
/** Action to load a Datastore snapshot from Google Cloud Storage into BigQuery. */
|
||||
@Action(path = LoadSnapshotAction.PATH, method = POST)
|
||||
public class LoadSnapshotAction implements Runnable {
|
||||
|
||||
/** Parameter names for passing parameters into the servlet. */
|
||||
static final String SNAPSHOT_ID_PARAM = "id";
|
||||
static final String SNAPSHOT_FILE_PARAM = "file";
|
||||
static final String SNAPSHOT_KINDS_PARAM = "kinds";
|
||||
static final String LOAD_SNAPSHOT_ID_PARAM = "id";
|
||||
static final String LOAD_SNAPSHOT_FILE_PARAM = "file";
|
||||
static final String LOAD_SNAPSHOT_KINDS_PARAM = "kinds";
|
||||
|
||||
static final String SNAPSHOTS_DATASET = "snapshots";
|
||||
|
||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
||||
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
||||
static final String PATH = "/_dr/task/loadSnapshot"; // See web.xml.
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static Clock clock = new SystemClock();
|
||||
@Inject BigqueryFactory bigqueryFactory;
|
||||
@Inject BigqueryPollJobEnqueuer bigqueryPollEnqueuer;
|
||||
@Inject Clock clock;
|
||||
@Inject @Config("projectId") String projectId;
|
||||
@Inject @Parameter(LOAD_SNAPSHOT_FILE_PARAM) String snapshotFile;
|
||||
@Inject @Parameter(LOAD_SNAPSHOT_ID_PARAM) String snapshotId;
|
||||
@Inject @Parameter(LOAD_SNAPSHOT_KINDS_PARAM) String snapshotKinds;
|
||||
@Inject LoadSnapshotAction() {}
|
||||
|
||||
@NonFinalForTesting
|
||||
private static BigqueryFactory bigqueryFactory = new BigqueryFactory();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static BigqueryPollJobEnqueuer bigqueryPollEnqueuer = new BigqueryPollJobEnqueuer(
|
||||
new TaskEnqueuer(new Retrier(new SystemSleeper(), 5)));
|
||||
|
||||
/** Enqueue a task for starting a backup load. Not static for better testability. */
|
||||
public TaskHandle enqueueLoadTask(
|
||||
/** Enqueue a task for starting a backup load. */
|
||||
public static TaskHandle enqueueLoadSnapshotTask(
|
||||
String snapshotId, String gcsFile, ImmutableSet<String> kinds) {
|
||||
return QueueFactory.getQueue(QUEUE).add(
|
||||
return getQueue(QUEUE).add(
|
||||
TaskOptions.Builder.withUrl(PATH)
|
||||
.method(Method.POST)
|
||||
.param(SNAPSHOT_ID_PARAM, snapshotId)
|
||||
.param(SNAPSHOT_FILE_PARAM, gcsFile)
|
||||
.param(SNAPSHOT_KINDS_PARAM, Joiner.on(',').join(kinds)));
|
||||
.param(LOAD_SNAPSHOT_ID_PARAM, snapshotId)
|
||||
.param(LOAD_SNAPSHOT_FILE_PARAM, gcsFile)
|
||||
.param(LOAD_SNAPSHOT_KINDS_PARAM, Joiner.on(',').join(kinds)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
public void run() {
|
||||
try {
|
||||
String snapshotId = getRequiredParameterValue(req, SNAPSHOT_ID_PARAM);
|
||||
String gcsFilename = getRequiredParameterValue(req, SNAPSHOT_FILE_PARAM);
|
||||
ImmutableList<String> kinds = ImmutableList.copyOf(
|
||||
Splitter.on(',').splitToList(getRequiredParameterValue(req, SNAPSHOT_KINDS_PARAM)));
|
||||
|
||||
String message = loadSnapshot(snapshotId, gcsFilename, kinds);
|
||||
|
||||
rsp.setStatus(SC_OK);
|
||||
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
|
||||
rsp.getWriter().write("OK\n\n" + message);
|
||||
String message =
|
||||
loadSnapshot(snapshotId, snapshotFile, Splitter.on(',').split(snapshotKinds));
|
||||
logger.infofmt("Loaded snapshot successfully: %s", message);
|
||||
} catch (Throwable e) {
|
||||
logger.severe(e, e.toString());
|
||||
rsp.sendError(
|
||||
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
|
||||
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
|
||||
logger.severe(e, "Error loading snapshot");
|
||||
if (e instanceof IllegalArgumentException) {
|
||||
throw new BadRequestException("Error calling load snapshot: " + e.getMessage(), e);
|
||||
} else {
|
||||
throw new InternalServerErrorException(
|
||||
"Error loading snapshot: " + firstNonNull(e.getMessage(), e.toString()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String loadSnapshot(String snapshotId, String gcsFilename, ImmutableList<String> kinds)
|
||||
private String loadSnapshot(String snapshotId, String gcsFilename, Iterable<String> kinds)
|
||||
throws IOException {
|
||||
Bigquery bigquery = bigqueryFactory.create(
|
||||
getClass().getSimpleName(),
|
||||
new UrlFetchTransport(),
|
||||
new JacksonFactory(),
|
||||
new AppIdentityCredential(BigqueryScopes.all()));
|
||||
String projectId = ENVIRONMENT.config().getProjectId();
|
||||
Bigquery bigquery = bigqueryFactory.create(projectId, SNAPSHOTS_DATASET);
|
||||
DateTime now = clock.nowUtc();
|
||||
|
||||
ensureDataset(bigquery, projectId, ENVIRONMENT.config().getSnapshotsDataset());
|
||||
|
||||
String loadMessage =
|
||||
String.format("Loading datastore snapshot %s from %s...", snapshotId, gcsFilename);
|
||||
logger.info(loadMessage);
|
||||
|
|
@ -155,9 +130,8 @@ public class LoadSnapshotServlet extends HttpServlet {
|
|||
// well-known view in BigQuery to point at the newly loaded snapshot table for this kind.
|
||||
bigqueryPollEnqueuer.enqueuePollTask(
|
||||
jobRef,
|
||||
UpdateSnapshotViewAction.createViewUpdateTask(
|
||||
ENVIRONMENT.config().getSnapshotsDataset(), tableId, kindName),
|
||||
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
|
||||
createViewUpdateTask(SNAPSHOTS_DATASET, tableId, kindName),
|
||||
getQueue(UpdateSnapshotViewAction.QUEUE));
|
||||
|
||||
builder.append(String.format(" - %s:%s\n", projectId, jobId));
|
||||
logger.infofmt("Submitted load job %s:%s", projectId, jobId);
|
||||
|
|
@ -165,24 +139,6 @@ public class LoadSnapshotServlet extends HttpServlet {
|
|||
return builder.toString();
|
||||
}
|
||||
|
||||
private static void ensureDataset(Bigquery bigquery, String projectId, String datasetId)
|
||||
throws IOException {
|
||||
try {
|
||||
bigquery.datasets()
|
||||
.insert(projectId,
|
||||
new Dataset().setDatasetReference(
|
||||
new DatasetReference()
|
||||
.setProjectId(projectId)
|
||||
.setDatasetId(datasetId)))
|
||||
.execute();
|
||||
} catch (IOException e) {
|
||||
// Swallow errors about a duplicate dataset, and throw any other ones.
|
||||
if (!BigqueryJobFailureException.create(e).getReason().equals("duplicate")) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static String getBackupInfoFileForKind(String backupInfoFile, String kindName) {
|
||||
String extension = ".backup_info";
|
||||
checkArgument(backupInfoFile.endsWith(extension), "backup info file extension missing");
|
||||
|
|
@ -190,10 +146,10 @@ public class LoadSnapshotServlet extends HttpServlet {
|
|||
return Joiner.on('.').join(prefix, kindName, extension.substring(1));
|
||||
}
|
||||
|
||||
private static Job makeLoadJob(JobReference jobRef, String sourceUri, String tableId) {
|
||||
private Job makeLoadJob(JobReference jobRef, String sourceUri, String tableId) {
|
||||
TableReference tableReference = new TableReference()
|
||||
.setProjectId(jobRef.getProjectId())
|
||||
.setDatasetId(ENVIRONMENT.config().getSnapshotsDataset())
|
||||
.setDatasetId(SNAPSHOTS_DATASET)
|
||||
.setTableId(tableId);
|
||||
return new Job()
|
||||
.setJobReference(jobRef)
|
||||
|
|
@ -23,7 +23,7 @@ import com.google.api.services.bigquery.model.ViewDefinition;
|
|||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.domain.registry.bigquery.BigqueryFactory;
|
||||
import com.google.domain.registry.config.RegistryEnvironment;
|
||||
import com.google.domain.registry.config.ConfigModule.Config;
|
||||
import com.google.domain.registry.request.Action;
|
||||
import com.google.domain.registry.request.HttpException.InternalServerErrorException;
|
||||
import com.google.domain.registry.request.Parameter;
|
||||
|
|
@ -38,12 +38,12 @@ import javax.inject.Inject;
|
|||
@Action(path = UpdateSnapshotViewAction.PATH, method = POST)
|
||||
public class UpdateSnapshotViewAction implements Runnable {
|
||||
|
||||
private static final RegistryEnvironment ENVIRONMENT = RegistryEnvironment.get();
|
||||
|
||||
/** Headers for passing parameters into the servlet. */
|
||||
static final String SNAPSHOT_DATASET_ID_PARAM = "dataset";
|
||||
static final String SNAPSHOT_TABLE_ID_PARAM = "table";
|
||||
static final String SNAPSHOT_KIND_PARAM = "kind";
|
||||
static final String UPDATE_SNAPSHOT_DATASET_ID_PARAM = "dataset";
|
||||
static final String UPDATE_SNAPSHOT_TABLE_ID_PARAM = "table";
|
||||
static final String UPDATE_SNAPSHOT_KIND_PARAM = "kind";
|
||||
|
||||
static final String LATEST_SNAPSHOT_DATASET = "latest_snapshot";
|
||||
|
||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
||||
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
|
||||
|
|
@ -51,9 +51,10 @@ public class UpdateSnapshotViewAction implements Runnable {
|
|||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@Inject @Parameter(SNAPSHOT_DATASET_ID_PARAM) String datasetId;
|
||||
@Inject @Parameter(SNAPSHOT_TABLE_ID_PARAM) String tableId;
|
||||
@Inject @Parameter(SNAPSHOT_KIND_PARAM) String kindName;
|
||||
@Inject @Parameter(UPDATE_SNAPSHOT_DATASET_ID_PARAM) String datasetId;
|
||||
@Inject @Parameter(UPDATE_SNAPSHOT_TABLE_ID_PARAM) String tableId;
|
||||
@Inject @Parameter(UPDATE_SNAPSHOT_KIND_PARAM) String kindName;
|
||||
@Inject @Config("projectId") String projectId;
|
||||
@Inject BigqueryFactory bigqueryFactory;
|
||||
@Inject UpdateSnapshotViewAction() {}
|
||||
|
||||
|
|
@ -62,9 +63,9 @@ public class UpdateSnapshotViewAction implements Runnable {
|
|||
String datasetId, String tableId, String kindName) {
|
||||
return TaskOptions.Builder.withUrl(PATH)
|
||||
.method(Method.POST)
|
||||
.param(SNAPSHOT_DATASET_ID_PARAM, datasetId)
|
||||
.param(SNAPSHOT_TABLE_ID_PARAM, tableId)
|
||||
.param(SNAPSHOT_KIND_PARAM, kindName);
|
||||
.param(UPDATE_SNAPSHOT_DATASET_ID_PARAM, datasetId)
|
||||
.param(UPDATE_SNAPSHOT_TABLE_ID_PARAM, tableId)
|
||||
.param(UPDATE_SNAPSHOT_KIND_PARAM, kindName);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -79,14 +80,12 @@ public class UpdateSnapshotViewAction implements Runnable {
|
|||
|
||||
private void updateSnapshotView(String datasetId, String tableId, String kindName)
|
||||
throws IOException {
|
||||
String projectId = ENVIRONMENT.config().getProjectId();
|
||||
Bigquery bigquery =
|
||||
bigqueryFactory.create(projectId, ENVIRONMENT.config().getLatestSnapshotDataset());
|
||||
Bigquery bigquery = bigqueryFactory.create(projectId, LATEST_SNAPSHOT_DATASET);
|
||||
|
||||
updateTable(bigquery, new Table()
|
||||
.setTableReference(new TableReference()
|
||||
.setProjectId(projectId)
|
||||
.setDatasetId(ENVIRONMENT.config().getLatestSnapshotDataset())
|
||||
.setDatasetId(LATEST_SNAPSHOT_DATASET)
|
||||
.setTableId(kindName))
|
||||
.setView(new ViewDefinition().setQuery(
|
||||
SqlTemplate.create("SELECT * FROM [%DATASET%.%TABLE%]")
|
||||
|
|
@ -94,10 +93,12 @@ public class UpdateSnapshotViewAction implements Runnable {
|
|||
.put("TABLE", tableId)
|
||||
.build())));
|
||||
|
||||
String message = String.format(
|
||||
logger.infofmt(
|
||||
"Updated view %s:%s to point at snapshot table %s:%s.",
|
||||
ENVIRONMENT.config().getLatestSnapshotDataset(), kindName, datasetId, tableId);
|
||||
logger.info(message);
|
||||
LATEST_SNAPSHOT_DATASET,
|
||||
kindName,
|
||||
datasetId,
|
||||
tableId);
|
||||
}
|
||||
|
||||
private static void updateTable(Bigquery bigquery, Table table) throws IOException {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import com.google.domain.registry.export.BigqueryPollJobAction;
|
|||
import com.google.domain.registry.export.ExportDomainListsAction;
|
||||
import com.google.domain.registry.export.ExportRequestModule;
|
||||
import com.google.domain.registry.export.ExportReservedTermsAction;
|
||||
import com.google.domain.registry.export.LoadSnapshotAction;
|
||||
import com.google.domain.registry.export.SyncGroupMembersAction;
|
||||
import com.google.domain.registry.export.UpdateSnapshotViewAction;
|
||||
import com.google.domain.registry.export.sheet.SheetModule;
|
||||
|
|
@ -89,6 +90,7 @@ interface BackendRequestComponent {
|
|||
ExportCommitLogDiffAction exportCommitLogDiffAction();
|
||||
ExportDomainListsAction exportDomainListsAction();
|
||||
ExportReservedTermsAction exportReservedTermsAction();
|
||||
LoadSnapshotAction loadSnapshotAction();
|
||||
MetricsExportAction metricsExportAction();
|
||||
NordnUploadAction nordnUploadAction();
|
||||
NordnVerifyAction nordnVerifyAction();
|
||||
|
|
|
|||
|
|
@ -17,9 +17,9 @@ package com.google.domain.registry.request;
|
|||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Strings.nullToEmpty;
|
||||
import static com.google.common.net.HttpHeaders.LOCATION;
|
||||
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
|
||||
import static com.google.domain.registry.security.XsrfTokenManager.X_CSRF_TOKEN;
|
||||
import static com.google.domain.registry.security.XsrfTokenManager.validateToken;
|
||||
import static com.google.domain.registry.util.HttpServletUtils.sendOk;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_METHOD_NOT_ALLOWED;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_MOVED_TEMPORARILY;
|
||||
|
|
@ -149,7 +149,8 @@ public final class RequestHandler<C> {
|
|||
try {
|
||||
route.get().instantiator().apply(component).run();
|
||||
if (route.get().action().automaticallyPrintOk()) {
|
||||
sendOk(rsp);
|
||||
rsp.setContentType(PLAIN_TEXT_UTF_8.toString());
|
||||
rsp.getWriter().write("OK\n");
|
||||
}
|
||||
} catch (HttpException e) {
|
||||
e.send(rsp);
|
||||
|
|
|
|||
|
|
@ -1,48 +0,0 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.util;
|
||||
|
||||
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
|
||||
import static com.google.domain.registry.util.PreconditionsUtils.checkArgumentNotNull;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
/** Utility methods for working with {@link HttpServlet}-related classes. */
|
||||
public final class HttpServletUtils {
|
||||
|
||||
/**
|
||||
* Returns the value of the given request's first {@code name} parameter, or throws
|
||||
* {@code IllegalArgumentException} if the parameter is not present.
|
||||
*
|
||||
* @deprecated in favor of <code>RequestParameters.extractRequiredParameter</code>
|
||||
*/
|
||||
@Deprecated
|
||||
public static String getRequiredParameterValue(HttpServletRequest req, String name) {
|
||||
return checkArgumentNotNull(req.getParameter(name), "Missing required parameter: %s", name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the content type on an HttpServletResponse to UTF-8, returns 200 OK status, and sends
|
||||
* "OK".
|
||||
*/
|
||||
public static void sendOk(HttpServletResponse rsp) throws IOException {
|
||||
rsp.setContentType(PLAIN_TEXT_UTF_8.toString());
|
||||
rsp.getWriter().write("OK\n");
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue