zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-06-27 06:44:51 +02:00
Code Issues Projects Releases Packages Wiki Activity

mv com/google/domain/registry google/registry

Browse source 
This change renames directories in preparation for the great package
rename. The repository is now in a broken state because the code
itself hasn't been updated. However this should ensure that git
correctly preserves history for each file.
This commit is contained in:
Justine Tunney 2016-05-13 18:55:08 -04:00
parent a41677aea1
commit 5012893c1d
2396 changed files with 0 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

45
java/google/registry/export/BUILD Normal file
View file

@ -0,0 +1,45 @@
package(
default_visibility = ["//java/com/google/domain/registry:registry_project"],
)
java_library(
name = "export",
srcs = glob(["*.java"]),
deps = [
"//apiserving/discoverydata/bigquery:bigqueryv2",
"//apiserving/discoverydata/drive",
"//java/com/google/api/client/extensions/appengine/http",
"//java/com/google/api/client/googleapis/extensions/appengine/auth/oauth2",
"//java/com/google/api/client/googleapis/json",
"//java/com/google/api/client/http",
"//java/com/google/api/client/json",
"//java/com/google/api/client/json/jackson2",
"//java/com/google/common/annotations",
"//java/com/google/common/base",
"//java/com/google/common/collect",
"//java/com/google/common/html",
"//java/com/google/common/io",
"//java/com/google/common/net",
"//java/com/google/domain/registry/bigquery",
"//java/com/google/domain/registry/config",
"//java/com/google/domain/registry/gcs",
"//java/com/google/domain/registry/groups",
"//java/com/google/domain/registry/mapreduce",
"//java/com/google/domain/registry/mapreduce/inputs",
"//java/com/google/domain/registry/model",
"//java/com/google/domain/registry/request",
"//java/com/google/domain/registry/storage/drive",
"//java/com/google/domain/registry/util",
"//third_party/java/appengine:appengine-api",
"//third_party/java/appengine_gcs_client",
"//third_party/java/appengine_mapreduce2:appengine_mapreduce",
"//third_party/java/dagger",
"//third_party/java/joda_time",
"//third_party/java/json_simple",
"//third_party/java/jsr305_annotations",
"//third_party/java/jsr330_inject",
"//third_party/java/objectify:objectify-v4_1",
"//third_party/java/servlet/servlet_api",
],
)

173
java/google/registry/export/BigqueryPollJobAction.java Normal file
View file

@ -0,0 +1,173 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
import static com.google.domain.registry.bigquery.BigqueryUtils.toJobReferenceString;
import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.bigquery.model.Job;
import com.google.api.services.bigquery.model.JobReference;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Header;
import com.google.domain.registry.request.HttpException.BadRequestException;
import com.google.domain.registry.request.HttpException.NotModifiedException;
import com.google.domain.registry.request.Payload;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.util.TaskEnqueuer;
import dagger.Lazy;
import org.joda.time.Duration;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import javax.inject.Inject;
/**
* An action which polls the state of a bigquery job. If it is completed then it will log its
* completion state; otherwise it will return a failure code so that the task will be retried.
*/
@Action(
path = BigqueryPollJobAction.PATH,
method = {Action.Method.GET, Action.Method.POST},
automaticallyPrintOk = true)
public class BigqueryPollJobAction implements Runnable {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
static final String QUEUE = "export-bigquery-poll"; // See queue.xml
static final String PATH = "/_dr/task/pollBigqueryJob"; // See web.xml
static final String CHAINED_TASK_QUEUE_HEADER = "X-DomainRegistry-ChainedTaskQueue";
static final String PROJECT_ID_HEADER = "X-DomainRegistry-ProjectId";
static final String JOB_ID_HEADER = "X-DomainRegistry-JobId";
static final Duration POLL_COUNTDOWN = Duration.standardSeconds(20);
@Inject Bigquery bigquery;
@Inject TaskEnqueuer enqueuer;
@Inject @Header(CHAINED_TASK_QUEUE_HEADER) Lazy<String> chainedQueueName;
@Inject @Header(PROJECT_ID_HEADER) String projectId;
@Inject @Header(JOB_ID_HEADER) String jobId;
@Inject @Payload byte[] payload;
@Inject BigqueryPollJobAction() {}
@Override
public void run() {
checkJobOutcome(); // Throws a NotModifiedException if the job hasn't completed.
if (payload == null || payload.length == 0) {
return;
}
// If there is a payload, it's a chained task, so enqueue it.
TaskOptions task;
try {
task = (TaskOptions) new ObjectInputStream(new ByteArrayInputStream(payload)).readObject();
} catch (ClassNotFoundException | IOException e) {
logger.severe(e, e.toString());
throw new BadRequestException("Cannot deserialize task from payload", e);
}
String taskName = enqueuer.enqueue(getQueue(chainedQueueName.get()), task).getName();
logger.infofmt(
"Added chained task %s for %s to queue %s: %s",
taskName,
task.getUrl(),
chainedQueueName.get(),
task.toString());
}
/**
* Returns true if the provided job succeeded, false if it failed, and throws an exception if it
* is still pending.
*/
private boolean checkJobOutcome() {
Job job = null;
String jobRefString =
toJobReferenceString(new JobReference().setProjectId(projectId).setJobId(jobId));
try {
job = bigquery.jobs().get(projectId, jobId).execute();
} catch (IOException e) {
// We will throw a new exception because done==false, but first log this exception.
logger.warning(e, e.getMessage());
}
// If job is not yet done, then throw an exception so that we'll return a failing HTTP status
// code and the task will be retried.
if (job == null || !job.getStatus().getState().equals("DONE")) {
throw new NotModifiedException(jobRefString);
}
// Check if the job ended with an error.
if (job.getStatus().getErrorResult() != null) {
logger.severefmt("Bigquery job failed - %s - %s", jobRefString, job);
return false;
}
logger.infofmt("Bigquery job succeeded - %s", jobRefString);
return true;
}
/** Helper class to enqueue a bigquery poll job. */
public static class BigqueryPollJobEnqueuer {
private final TaskEnqueuer enqueuer;
@Inject
BigqueryPollJobEnqueuer(TaskEnqueuer enqueuer) {
this.enqueuer = enqueuer;
}
/** Enqueue a task to poll for the success or failure of the referenced BigQuery job. */
public TaskHandle enqueuePollTask(JobReference jobRef) {
return enqueuer.enqueue(getQueue(QUEUE), createCommonPollTask(jobRef).method(Method.GET));
}
/**
* Enqueue a task to poll for the success or failure of the referenced BigQuery job and to
* launch the provided task in the specified queue if the job succeeds.
*/
public TaskHandle enqueuePollTask(
JobReference jobRef, TaskOptions chainedTask, Queue chainedTaskQueue) throws IOException {
// Serialize the chainedTask into a byte array to put in the task payload.
ByteArrayOutputStream taskBytes = new ByteArrayOutputStream();
new ObjectOutputStream(taskBytes).writeObject(chainedTask);
return enqueuer.enqueue(
getQueue(QUEUE),
createCommonPollTask(jobRef)
.method(Method.POST)
.header(CHAINED_TASK_QUEUE_HEADER, chainedTaskQueue.getQueueName())
.payload(taskBytes.toByteArray()));
}
/**
* Enqueue a task to poll for the success or failure of the referenced BigQuery job and to
* launch the provided task in the specified queue if the job succeeds.
*/
private static TaskOptions createCommonPollTask(JobReference jobRef) {
// Omit host header so that task will be run on the current backend/module.
return withUrl(PATH)
.countdownMillis(POLL_COUNTDOWN.getMillis())
.header(PROJECT_ID_HEADER, jobRef.getProjectId())
.header(JOB_ID_HEADER, jobRef.getJobId());
}
}
}

179
java/google/registry/export/CheckSnapshotServlet.java Normal file
View file

@ -0,0 +1,179 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.collect.Sets.intersection;
import static com.google.common.html.HtmlEscapers.htmlEscaper;
import static com.google.domain.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
import static com.google.domain.registry.request.RequestParameters.extractRequiredParameter;
import static com.google.domain.registry.util.FormattingLogger.getLoggerForCallerClass;
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.net.MediaType;
import com.google.domain.registry.export.DatastoreBackupInfo.BackupStatus;
import com.google.domain.registry.request.HttpException.BadRequestException;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.util.NonFinalForTesting;
import org.joda.time.Duration;
import org.joda.time.PeriodType;
import org.joda.time.format.PeriodFormat;
import java.io.IOException;
import java.util.Set;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/** Check the status of a snapshot, and if complete, trigger loading it into BigQuery. */
public class CheckSnapshotServlet extends HttpServlet {
/** Parameter names for passing parameters into this servlet. */
static final String SNAPSHOT_NAME_PARAM = "name";
static final String SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad";
/** Servlet-specific details needed for enqueuing tasks against itself. */
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml.
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
/** The maximum amount of time we allow a backup to run before abandoning it. */
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
private static final FormattingLogger logger = getLoggerForCallerClass();
@NonFinalForTesting
private static DatastoreBackupService backupService = DatastoreBackupService.get();
@Override
public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
try {
rsp.setStatus(SC_OK);
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
rsp.getWriter().write("OK\n\n");
super.service(req, rsp);
} catch (Throwable e) {
logger.severe(e, e.toString());
rsp.sendError(
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
}
}
@Override
public void doGet(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
try {
String snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
rsp.getWriter().write(backupService.findByName(snapshotName).getInformation());
} catch (BadRequestException e) {
throw new IllegalArgumentException(e.getMessage());
}
}
@Override
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
String snapshotName;
String kindsToLoadParam;
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
try {
snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
kindsToLoadParam = extractRequiredParameter(req, SNAPSHOT_KINDS_TO_LOAD_PARAM);
} catch (BadRequestException e) {
throw new IllegalArgumentException(e.getMessage());
}
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
// Look up the backup by the provided name, stopping if we can't find it.
DatastoreBackupInfo backup;
try {
backup = backupService.findByName(snapshotName);
} catch (IllegalArgumentException e) {
String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage());
logger.severe(e, message);
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be retried,
// but we might abandon backups that start late and haven't yet written to datastore.
// We could fix that by replacing this with a two-phase polling strategy.
rsp.sendError(SC_BAD_REQUEST, htmlEscaper().escape(message));
return;
}
// Stop now if the backup is not complete.
if (!backup.getStatus().equals(BackupStatus.COMPLETE)) {
Duration runningTime = backup.getRunningTime();
if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
// Backup might still be running, so send a 304 to have the task retry.
rsp.sendError(SC_NOT_MODIFIED,
htmlEscaper().escape(String.format("Datastore backup %s still pending", snapshotName)));
} else {
// Declare the backup a lost cause, and send 202 Accepted so the task will not be retried.
String message = String.format("Datastore backup %s abandoned - not complete after %s",
snapshotName,
PeriodFormat.getDefault().print(
runningTime.toPeriod().normalizedStandard(
PeriodType.dayTime().withMillisRemoved())));
logger.severe(message);
rsp.sendError(SC_ACCEPTED, htmlEscaper().escape(message));
}
return;
}
// Get a compact string to identify this snapshot in BigQuery by trying to parse the unique
// suffix out of the snapshot name and falling back to the start time as a string.
String snapshotId = snapshotName.startsWith(ExportSnapshotServlet.SNAPSHOT_PREFIX)
? snapshotName.substring(ExportSnapshotServlet.SNAPSHOT_PREFIX.length())
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
if (!backup.getKinds().containsAll(kindsToLoad)) {
logger.warningfmt(
"Kinds to load included non-exported kinds: %s",
Sets.difference(kindsToLoad, backup.getKinds()));
}
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
ImmutableSet<String> exportedKindsToLoad =
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
String message = String.format("Datastore backup %s complete - ", snapshotName);
if (exportedKindsToLoad.isEmpty()) {
message += "no kinds to load into BigQuery";
} else {
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
message += "BigQuery load task enqueued";
}
logger.info(message);
rsp.getWriter().write(message);
}
/** Enqueue a poll task to monitor the named snapshot for completion. */
TaskHandle enqueuePollTask(String snapshotName, ImmutableSet<String> kindsToLoad) {
return QueueFactory.getQueue(QUEUE).add(
TaskOptions.Builder.withUrl(PATH)
.method(Method.POST)
.countdownMillis(POLL_COUNTDOWN.getMillis())
.param(SNAPSHOT_NAME_PARAM, snapshotName)
.param(SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
}
}

150
java/google/registry/export/DatastoreBackupInfo.java Normal file
View file

@ -0,0 +1,150 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.joda.time.DateTimeZone.UTC;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.Text;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.util.Clock;
import com.google.domain.registry.util.NonFinalForTesting;
import com.google.domain.registry.util.SystemClock;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import java.util.Date;
import java.util.List;
/** Container for information about a datastore backup. */
public class DatastoreBackupInfo {
@NonFinalForTesting
private static Clock clock = new SystemClock();
/** The possible status values for a datastore backup. */
public enum BackupStatus { PENDING, COMPLETE }
/** The name of the datastore backup. */
private final String backupName;
/** The entity kinds included in this datastore backup. */
private final ImmutableSet<String> kinds;
/** The start time of the datastore backup. */
private final DateTime startTime;
/** The completion time of the datastore backup, present if it has completed. */
private final Optional<DateTime> completeTime;
/**
* The GCS filename to which the backup's top-level .backup_info manifest file has been written,
* present if the backup has completed.
*/
private final Optional<String> gcsFilename;
/** DatastoreBackupInfo instances should only be obtained via DatastoreBackupService. */
DatastoreBackupInfo(Entity backupEntity) {
backupName = (String) checkNotNull(backupEntity.getProperty("name"), "name");
@SuppressWarnings("unchecked")
List<String> rawKinds = (List<String>) checkNotNull(backupEntity.getProperty("kinds"), "kinds");
Date rawStartTime = (Date) checkNotNull(backupEntity.getProperty("start_time"), "start_time");
Date rawCompleteTime = (Date) backupEntity.getProperty("complete_time");
Text rawGcsFilename = (Text) backupEntity.getProperty("gs_handle");
kinds = ImmutableSet.copyOf(rawKinds);
startTime = new DateTime(rawStartTime).withZone(UTC);
completeTime = Optional.fromNullable(
rawCompleteTime == null ? null : new DateTime(rawCompleteTime).withZone(UTC));
gcsFilename = Optional.fromNullable(
rawGcsFilename == null ? null : gcsPathToUri(rawGcsFilename.getValue()));
}
/** This constructor is only exposed for test purposes. */
@VisibleForTesting
DatastoreBackupInfo(
String backupName,
DateTime startTime,
Optional<DateTime> completeTime,
ImmutableSet<String> kinds,
Optional<String> gcsFilename) {
this.backupName = backupName;
this.startTime = startTime;
this.completeTime = completeTime;
this.kinds = kinds;
this.gcsFilename = gcsFilename;
}
/**
* Rewrite a GCS path as stored by Datastore Admin (with a "/gs/" prefix) to the more standard
* URI format that uses a "gs://" scheme prefix.
*/
private static String gcsPathToUri(String backupGcsPath) {
checkArgument(backupGcsPath.startsWith("/gs/"), "GCS path not in expected format");
return backupGcsPath.replaceFirst("/gs/", "gs://");
}
public String getName() {
return backupName;
}
public ImmutableSet<String> getKinds() {
return kinds;
}
public BackupStatus getStatus() {
return completeTime.isPresent() ? BackupStatus.COMPLETE : BackupStatus.PENDING;
}
public DateTime getStartTime() {
return startTime;
}
public Optional<DateTime> getCompleteTime() {
return completeTime;
}
/**
* Returns the length of time the backup ran for (if completed) or the length of time since the
* backup started (if it has not completed).
*/
public Duration getRunningTime() {
return new Duration(startTime, completeTime.or(clock.nowUtc()));
}
public Optional<String> getGcsFilename() {
return gcsFilename;
}
/** Returns a string version of key information about the backup. */
public String getInformation() {
return Joiner.on('\n')
.join(
"Backup name: " + backupName,
"Status: " + getStatus(),
"Started: " + startTime,
"Ended: " + completeTime.orNull(),
"Duration: " + getRunningTime().toPeriod().toString().substring(2).toLowerCase(),
"GCS: " + gcsFilename.orNull(),
"Kinds: " + kinds,
"");
}
}

122
java/google/registry/export/DatastoreBackupService.java Normal file
View file

@ -0,0 +1,122 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.common.base.Strings.nullToEmpty;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.modules.ModulesService;
import com.google.appengine.api.modules.ModulesServiceFactory;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.domain.registry.util.NonFinalForTesting;
import java.util.NoSuchElementException;
/** An object providing methods for starting and querying datastore backups. */
public class DatastoreBackupService {
/** The internal kind name used for entities storing information about datastore backups. */
static final String BACKUP_INFO_KIND = "_AE_Backup_Information";
/** The name of the app version used for hosting the Datastore Admin functionality. */
static final String DATASTORE_ADMIN_VERSION_NAME = "ah-builtin-python-bundle";
@NonFinalForTesting
private static ModulesService modulesService = ModulesServiceFactory.getModulesService();
/**
* Returns an instance of this service.
*
* <p>This method exists to allow for making the service a singleton object if desired at some
* future point; the choice is meaningless right now because the service maintains no state.
* That means its client-facing methods could in theory be static methods, but they are not
* because that makes it difficult to mock this service in clients.
*/
public static DatastoreBackupService get() {
return new DatastoreBackupService();
}
/**
* Generates the TaskOptions needed to trigger an AppEngine datastore backup job.
*
* @see "https://developers.google.com/appengine/articles/scheduled_backups"
*/
private static TaskOptions makeTaskOptions(
String queue, String name, String gcsBucket, ImmutableSet<String> kinds) {
String hostname = modulesService.getVersionHostname("default", DATASTORE_ADMIN_VERSION_NAME);
TaskOptions options = TaskOptions.Builder.withUrl("/_ah/datastore_admin/backup.create")
.header("Host", hostname)
.method(Method.GET)
.param("name", name + "_") // Add underscore since the name will be used as a prefix.
.param("filesystem", "gs")
.param("gs_bucket_name", gcsBucket)
.param("queue", queue);
for (String kind : kinds) {
options.param("kind", kind);
}
return options;
}
/**
* Launches a new datastore backup with the given name, GCS bucket, and set of kinds by
* submitting a task to the given task queue, and returns a handle to that task.
*/
public TaskHandle launchNewBackup(
String queue, String name, String gcsBucket, ImmutableSet<String> kinds) {
return getQueue(queue).add(makeTaskOptions(queue, name, gcsBucket, kinds));
}
/** Return an iterable of all datastore backups whose names have the given string prefix. */
public Iterable<DatastoreBackupInfo> findAllByNamePrefix(final String namePrefix) {
// Need the raw DatastoreService to access the internal _AE_Backup_Information entities.
// TODO(b/19081037): make an Objectify entity class for these raw datastore entities instead.
return FluentIterable
.from(getDatastoreService().prepare(new Query(BACKUP_INFO_KIND)).asIterable())
.filter(new Predicate<Entity>() {
@Override
public boolean apply(Entity entity) {
return nullToEmpty((String) entity.getProperty("name")).startsWith(namePrefix);
}})
.transform(new Function<Entity, DatastoreBackupInfo>() {
@Override
public DatastoreBackupInfo apply(Entity entity) {
return new DatastoreBackupInfo(entity);
}});
}
/**
* Return a single DatastoreBackup that uniquely matches this name prefix. Throws an IAE
* if no backups match or if more than one backup matches.
*/
public DatastoreBackupInfo findByName(final String namePrefix) {
try {
return Iterables.getOnlyElement(findAllByNamePrefix(namePrefix));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("More than one backup with name prefix " + namePrefix, e);
} catch (NoSuchElementException e) {
throw new IllegalArgumentException("No backup found with name prefix " + namePrefix, e);
}
}
}

61
java/google/registry/export/DriveModule.java Normal file
View file

@ -0,0 +1,61 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static dagger.Provides.Type.SET_VALUES;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.DriveScopes;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.request.OAuthScopes;
import dagger.Module;
import dagger.Provides;
import java.util.Set;
/**
* Dagger module for Google {@link Drive} service connection objects.
*
* @see com.google.domain.registry.config.ConfigModule
* @see com.google.domain.registry.request.Modules.UrlFetchTransportModule
* @see com.google.domain.registry.request.Modules.Jackson2Module
* @see com.google.domain.registry.request.Modules.AppIdentityCredentialModule
* @see com.google.domain.registry.request.Modules.UseAppIdentityCredentialForGoogleApisModule
*/
@Module
public final class DriveModule {
/** Provides OAuth2 scopes for the Drive service needed by Domain Registry. */
@Provides(type = SET_VALUES)
@OAuthScopes
static Set<String> provideDriveOAuthScopes() {
return DriveScopes.all();
}
@Provides
static Drive provideDrive(
HttpTransport transport,
JsonFactory jsonFactory,
HttpRequestInitializer httpRequestInitializer,
@Config("projectId") String projectId) {
return new Drive.Builder(transport, jsonFactory, httpRequestInitializer)
.setApplicationName(projectId)
.build();
}
}

96
java/google/registry/export/ExportConstants.java Normal file
View file

@ -0,0 +1,96 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.common.base.Predicates.not;
import static com.google.domain.registry.model.EntityClasses.CLASS_TO_KIND_FUNCTION;
import static com.google.domain.registry.util.TypeUtils.hasAnnotation;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Ordering;
import com.google.domain.registry.model.EntityClasses;
import com.google.domain.registry.model.ImmutableObject;
import com.google.domain.registry.model.annotations.NotBackedUp;
import com.google.domain.registry.model.annotations.VirtualEntity;
import com.google.domain.registry.model.billing.BillingEvent.Cancellation;
import com.google.domain.registry.model.billing.BillingEvent.Modification;
import com.google.domain.registry.model.billing.BillingEvent.OneTime;
import com.google.domain.registry.model.billing.BillingEvent.Recurring;
import com.google.domain.registry.model.billing.RegistrarCredit;
import com.google.domain.registry.model.billing.RegistrarCreditBalance;
import com.google.domain.registry.model.contact.ContactResource;
import com.google.domain.registry.model.domain.DomainBase;
import com.google.domain.registry.model.host.HostResource;
import com.google.domain.registry.model.index.DomainApplicationIndex;
import com.google.domain.registry.model.index.EppResourceIndex;
import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyContactIndex;
import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
import com.google.domain.registry.model.registrar.Registrar;
import com.google.domain.registry.model.registrar.RegistrarContact;
import com.google.domain.registry.model.registry.Registry;
import com.google.domain.registry.model.registry.label.PremiumList;
import com.google.domain.registry.model.registry.label.PremiumList.PremiumListEntry;
import com.google.domain.registry.model.reporting.HistoryEntry;
/** Constants related to export code. */
public final class ExportConstants {
/** Set of entity classes to export into BigQuery for reporting purposes. */
@VisibleForTesting
@SuppressWarnings("unchecked") // varargs
static final ImmutableSet<Class<? extends ImmutableObject>> REPORTING_ENTITY_CLASSES =
ImmutableSet.of(
Cancellation.class,
ContactResource.class,
DomainApplicationIndex.class,
DomainBase.class,
EppResourceIndex.class,
ForeignKeyContactIndex.class,
ForeignKeyDomainIndex.class,
ForeignKeyHostIndex.class,
HistoryEntry.class,
HostResource.class,
Modification.class,
OneTime.class,
PremiumList.class,
PremiumListEntry.class,
Recurring.class,
Registrar.class,
RegistrarContact.class,
RegistrarCredit.class,
RegistrarCreditBalance.class,
Registry.class);
/** Returns the names of kinds to include in datastore backups. */
public static ImmutableSet<String> getBackupKinds() {
// Back up all entity classes that aren't annotated with @VirtualEntity (never even persisted
// to datastore, so they can't be backed up) or @NotBackedUp (intentionally omitted).
return FluentIterable.from(EntityClasses.ALL_CLASSES)
.filter(not(hasAnnotation(VirtualEntity.class)))
.filter(not(hasAnnotation(NotBackedUp.class)))
.transform(CLASS_TO_KIND_FUNCTION)
.toSortedSet(Ordering.natural());
}
/** Returns the names of kinds to import into reporting tools (e.g. BigQuery). */
public static ImmutableSet<String> getReportingKinds() {
return FluentIterable.from(REPORTING_ENTITY_CLASSES)
.transform(CLASS_TO_KIND_FUNCTION)
.toSortedSet(Ordering.natural());
}
}

137
java/google/registry/export/ExportDomainListsAction.java Normal file
View file

@ -0,0 +1,137 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
import static com.google.domain.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
import static com.google.domain.registry.model.EppResourceUtils.isActive;
import static com.google.domain.registry.model.registry.Registries.getTldsOfType;
import static com.google.domain.registry.util.PipelineUtils.createJobPath;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.joda.time.DateTimeZone.UTC;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.appengine.tools.mapreduce.Reducer;
import com.google.appengine.tools.mapreduce.ReducerInput;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.gcs.GcsUtils;
import com.google.domain.registry.mapreduce.MapreduceAction;
import com.google.domain.registry.mapreduce.MapreduceRunner;
import com.google.domain.registry.model.domain.DomainResource;
import com.google.domain.registry.model.registry.Registry.TldType;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Response;
import com.google.domain.registry.util.FormattingLogger;
import org.joda.time.DateTime;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import javax.inject.Inject;
/**
* A mapreduce that exports the list of active domains on all real TLDs to Google Cloud Storage.
*
* Each TLD's active domain names are exported as a newline-delimited flat text file with the name
* TLD.txt into the domain-lists bucket. Note that this overwrites the files in place.
*/
@Action(path = "/_dr/task/exportDomainLists")
public class ExportDomainListsAction implements MapreduceAction {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
private static final int MAX_NUM_REDUCE_SHARDS = 100;
@Inject MapreduceRunner mrRunner;
@Inject Response response;
@Inject @Config("domainListsGcsBucket") String gcsBucket;
@Inject @Config("gcsBufferSize") int gcsBufferSize;
@Inject ExportDomainListsAction() {}
@Override
public void run() {
ImmutableSet<String> realTlds = getTldsOfType(TldType.REAL);
logger.infofmt("Exporting domain lists for tlds %s", realTlds);
response.sendJavaScriptRedirect(createJobPath(mrRunner
.setJobName("Export domain lists")
.setModuleName("backend")
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
.runMapreduce(
new ExportDomainListsMapper(DateTime.now(UTC), realTlds),
new ExportDomainListsReducer(gcsBucket, gcsBufferSize),
ImmutableList.of(createEntityInput(DomainResource.class)))));
}
static class ExportDomainListsMapper extends Mapper<DomainResource, String, String> {
private static final long serialVersionUID = -7312206212434039854L;
private final DateTime exportTime;
private final ImmutableSet<String> realTlds;
ExportDomainListsMapper(DateTime exportTime, ImmutableSet<String> realTlds) {
this.exportTime = exportTime;
this.realTlds = realTlds;
}
@Override
public void map(DomainResource domain) {
if (realTlds.contains(domain.getTld()) && isActive(domain, exportTime)) {
emit(domain.getTld(), domain.getFullyQualifiedDomainName());
getContext().incrementCounter(String.format("domains in tld %s", domain.getTld()));
}
}
}
static class ExportDomainListsReducer extends Reducer<String, String, Void> {
private static final long serialVersionUID = 7035260977259119087L;
private final String gcsBucket;
private final int gcsBufferSize;
public ExportDomainListsReducer(String gcsBucket, int gcsBufferSize) {
this.gcsBucket = gcsBucket;
this.gcsBufferSize = gcsBufferSize;
}
@Override
public void reduce(String tld, ReducerInput<String> fqdns) {
GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt");
GcsUtils cloudStorage =
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8);
PrintWriter writer = new PrintWriter(osWriter)) {
long count;
for (count = 0; fqdns.hasNext(); count++) {
writer.println(fqdns.next());
}
writer.flush();
getContext().incrementCounter("tld domain lists written out");
logger.infofmt("Wrote out %d domains for tld %s.", count, tld);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
}

94
java/google/registry/export/ExportRequestModule.java Normal file
View file

@ -0,0 +1,94 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.domain.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
import static com.google.domain.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
import static com.google.domain.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
import static com.google.domain.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
import static com.google.domain.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
import static com.google.domain.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
import static com.google.domain.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM;
import static com.google.domain.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM;
import static com.google.domain.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM;
import static com.google.domain.registry.request.RequestParameters.extractRequiredHeader;
import static com.google.domain.registry.request.RequestParameters.extractRequiredParameter;
import com.google.domain.registry.request.Header;
import com.google.domain.registry.request.Parameter;
import dagger.Module;
import dagger.Provides;
import javax.servlet.http.HttpServletRequest;
/** Dagger module for data export tasks. */
@Module
public final class ExportRequestModule {
@Provides
@Parameter(UPDATE_SNAPSHOT_DATASET_ID_PARAM)
static String provideUpdateSnapshotDatasetId(HttpServletRequest req) {
return extractRequiredParameter(req, UPDATE_SNAPSHOT_DATASET_ID_PARAM);
}
@Provides
@Parameter(UPDATE_SNAPSHOT_TABLE_ID_PARAM)
static String provideUpdateSnapshotTableId(HttpServletRequest req) {
return extractRequiredParameter(req, UPDATE_SNAPSHOT_TABLE_ID_PARAM);
}
@Provides
@Parameter(UPDATE_SNAPSHOT_KIND_PARAM)
static String provideUpdateSnapshotKind(HttpServletRequest req) {
return extractRequiredParameter(req, UPDATE_SNAPSHOT_KIND_PARAM);
}
@Provides
@Parameter(LOAD_SNAPSHOT_FILE_PARAM)
static String provideLoadSnapshotFile(HttpServletRequest req) {
return extractRequiredParameter(req, LOAD_SNAPSHOT_FILE_PARAM);
}
@Provides
@Parameter(LOAD_SNAPSHOT_ID_PARAM)
static String provideLoadSnapshotId(HttpServletRequest req) {
return extractRequiredParameter(req, LOAD_SNAPSHOT_ID_PARAM);
}
@Provides
@Parameter(LOAD_SNAPSHOT_KINDS_PARAM)
static String provideLoadSnapshotKinds(HttpServletRequest req) {
return extractRequiredParameter(req, LOAD_SNAPSHOT_KINDS_PARAM);
}
@Provides
@Header(CHAINED_TASK_QUEUE_HEADER)
static String provideChainedTaskQueue(HttpServletRequest req) {
return extractRequiredHeader(req, CHAINED_TASK_QUEUE_HEADER);
}
@Provides
@Header(JOB_ID_HEADER)
static String provideJobId(HttpServletRequest req) {
return extractRequiredHeader(req, JOB_ID_HEADER);
}
@Provides
@Header(PROJECT_ID_HEADER)
static String provideProjectId(HttpServletRequest req) {
return extractRequiredHeader(req, PROJECT_ID_HEADER);
}
}

85
java/google/registry/export/ExportReservedTermsAction.java Normal file
View file

@ -0,0 +1,85 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Strings.isNullOrEmpty;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static com.google.domain.registry.export.ExportUtils.exportReservedTerms;
import static com.google.domain.registry.request.Action.Method.POST;
import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.common.net.MediaType;
import com.google.domain.registry.model.registry.Registry;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.request.RequestParameters;
import com.google.domain.registry.request.Response;
import com.google.domain.registry.storage.drive.DriveConnection;
import com.google.domain.registry.util.FormattingLogger;
import javax.inject.Inject;
/** Action that exports the publicly viewable reserved terms list for a TLD to Google Drive. */
@Action(path = "/_dr/task/exportReservedTerms", method = POST)
public class ExportReservedTermsAction implements Runnable {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
static final MediaType EXPORT_MIME_TYPE = MediaType.PLAIN_TEXT_UTF_8;
static final String RESERVED_TERMS_FILENAME = "reserved_terms.txt";
@Inject DriveConnection driveConnection;
@Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@Inject Response response;
@Inject ExportReservedTermsAction() {}
/**
* Exports the reserved terms for the TLD specified via the "tld" param to a newline-delimited
* UTF-8-formatted CSV file (with one column) named "reserved_terms.txt" in the Google Drive
* folder with the id specified for that TLD.
*
* <p>This servlet prints the ID of the file in GoogleDrive that was created/updated.
*/
@Override
public void run() {
response.setContentType(PLAIN_TEXT_UTF_8);
try {
Registry registry = Registry.get(tld);
String resultMsg;
if (registry.getReservedLists().isEmpty() && isNullOrEmpty(registry.getDriveFolderId())) {
resultMsg = "No reserved lists configured";
logger.infofmt("No reserved terms to export for TLD %s", tld);
} else {
checkNotNull(registry.getDriveFolderId(), "No drive folder associated with this TLD");
resultMsg = driveConnection.createOrUpdateFile(
RESERVED_TERMS_FILENAME,
EXPORT_MIME_TYPE,
registry.getDriveFolderId(),
exportReservedTerms(registry).getBytes(UTF_8));
logger.infofmt("Exporting reserved terms succeeded for TLD %s, response was: %s",
tld, resultMsg);
}
response.setStatus(SC_OK);
response.setPayload(resultMsg);
} catch (Throwable e) {
response.setStatus(SC_INTERNAL_SERVER_ERROR);
response.setPayload(e.getMessage());
throw new RuntimeException(
String.format("Exception occurred while exporting reserved terms for TLD %s.", tld), e);
}
}
}

92
java/google/registry/export/ExportSnapshotServlet.java Normal file
View file

@ -0,0 +1,92 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.html.HtmlEscapers.htmlEscaper;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.common.net.MediaType;
import com.google.domain.registry.config.RegistryEnvironment;
import com.google.domain.registry.util.Clock;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.util.NonFinalForTesting;
import com.google.domain.registry.util.SystemClock;
import java.io.IOException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/**
* Trigger a backup-as-a-service job that writes a snapshot to Google Cloud Storage.
*
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
* the next upon successful completion:<ol>
* <li>The snapshot is exported to Google Cloud Storage (this servlet).
* <li>The {@link CheckSnapshotServlet} polls until the export is completed.
* <li>The {@link LoadSnapshotAction} imports the data from GCS to BigQuery.
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_snapshot.
* </ol>
*/
public class ExportSnapshotServlet extends HttpServlet {
private static final RegistryEnvironment ENVIRONMENT = RegistryEnvironment.get();
/** Queue to use for enqueuing the task that will actually launch the backup. */
static final String QUEUE = "export-snapshot"; // See queue.xml.
/** Prefix to use for naming all snapshots that are started by this servlet. */
static final String SNAPSHOT_PREFIX = "auto_snapshot_";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@NonFinalForTesting
private static Clock clock = new SystemClock();
@NonFinalForTesting
private static DatastoreBackupService backupService = DatastoreBackupService.get();
@NonFinalForTesting
private static CheckSnapshotServlet checkSnapshotServlet = new CheckSnapshotServlet();
@Override
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
try {
// Use a unique name for the snapshot so we can explicitly check its completion later.
String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss");
backupService.launchNewBackup(
QUEUE,
snapshotName,
ENVIRONMENT.config().getSnapshotsBucket(),
ExportConstants.getBackupKinds());
// Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery.
checkSnapshotServlet.enqueuePollTask(snapshotName, ExportConstants.getReportingKinds());
String message = "Datastore backup started with name: " + snapshotName;
logger.info(message);
rsp.setStatus(SC_OK);
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
rsp.getWriter().write("OK\n\n" + message);
} catch (Throwable e) {
logger.severe(e, e.toString());
rsp.sendError(
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
}
}
}

53
java/google/registry/export/ExportUtils.java Normal file
View file

@ -0,0 +1,53 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.domain.registry.model.registry.label.ReservationType.UNRESERVED;
import com.google.common.base.Joiner;
import com.google.domain.registry.config.RegistryEnvironment;
import com.google.domain.registry.model.registry.Registry;
import com.google.domain.registry.model.registry.label.ReservedList;
import com.google.domain.registry.model.registry.label.ReservedList.ReservedListEntry;
import com.googlecode.objectify.Key;
import java.util.Set;
import java.util.TreeSet;
/** Container class for exported-related static utility methods. */
public class ExportUtils {
private ExportUtils() {}
/** Returns the file contents of the auto-export reserved terms document for the given TLD. */
public static String exportReservedTerms(Registry registry) {
StringBuilder termsBuilder =
new StringBuilder(RegistryEnvironment.get().config().getReservedTermsExportDisclaimer());
Set<String> reservedTerms = new TreeSet<>();
for (Key<ReservedList> key : registry.getReservedLists()) {
ReservedList reservedList = ReservedList.load(key).get();
if (reservedList.getShouldPublish()) {
for (ReservedListEntry entry : reservedList.getReservedListEntries().values()) {
if (entry.getValue() != UNRESERVED) {
reservedTerms.add(entry.getLabel());
}
}
}
}
Joiner.on("\n").appendTo(termsBuilder, reservedTerms);
return termsBuilder.append("\n").toString();
}
}

163
java/google/registry/export/LoadSnapshotAction.java Normal file
View file

@ -0,0 +1,163 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.domain.registry.export.UpdateSnapshotViewAction.createViewUpdateTask;
import static com.google.domain.registry.request.Action.Method.POST;
import static com.google.domain.registry.util.FormattingLogger.getLoggerForCallerClass;
import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.bigquery.model.Job;
import com.google.api.services.bigquery.model.JobConfiguration;
import com.google.api.services.bigquery.model.JobConfigurationLoad;
import com.google.api.services.bigquery.model.JobReference;
import com.google.api.services.bigquery.model.TableReference;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.bigquery.BigqueryFactory;
import com.google.domain.registry.bigquery.BigqueryUtils.SourceFormat;
import com.google.domain.registry.bigquery.BigqueryUtils.WriteDisposition;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.HttpException.BadRequestException;
import com.google.domain.registry.request.HttpException.InternalServerErrorException;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.util.Clock;
import com.google.domain.registry.util.FormattingLogger;
import org.joda.time.DateTime;
import java.io.IOException;
import javax.inject.Inject;
/** Action to load a Datastore snapshot from Google Cloud Storage into BigQuery. */
@Action(path = LoadSnapshotAction.PATH, method = POST)
public class LoadSnapshotAction implements Runnable {
/** Parameter names for passing parameters into the servlet. */
static final String LOAD_SNAPSHOT_ID_PARAM = "id";
static final String LOAD_SNAPSHOT_FILE_PARAM = "file";
static final String LOAD_SNAPSHOT_KINDS_PARAM = "kinds";
static final String SNAPSHOTS_DATASET = "snapshots";
/** Servlet-specific details needed for enqueuing tasks against itself. */
static final String QUEUE = "export-snapshot"; // See queue.xml.
static final String PATH = "/_dr/task/loadSnapshot"; // See web.xml.
private static final FormattingLogger logger = getLoggerForCallerClass();
@Inject BigqueryFactory bigqueryFactory;
@Inject BigqueryPollJobEnqueuer bigqueryPollEnqueuer;
@Inject Clock clock;
@Inject @Config("projectId") String projectId;
@Inject @Parameter(LOAD_SNAPSHOT_FILE_PARAM) String snapshotFile;
@Inject @Parameter(LOAD_SNAPSHOT_ID_PARAM) String snapshotId;
@Inject @Parameter(LOAD_SNAPSHOT_KINDS_PARAM) String snapshotKinds;
@Inject LoadSnapshotAction() {}
/** Enqueue a task for starting a backup load. */
public static TaskHandle enqueueLoadSnapshotTask(
String snapshotId, String gcsFile, ImmutableSet<String> kinds) {
return getQueue(QUEUE).add(
TaskOptions.Builder.withUrl(PATH)
.method(Method.POST)
.param(LOAD_SNAPSHOT_ID_PARAM, snapshotId)
.param(LOAD_SNAPSHOT_FILE_PARAM, gcsFile)
.param(LOAD_SNAPSHOT_KINDS_PARAM, Joiner.on(',').join(kinds)));
}
@Override
public void run() {
try {
String message =
loadSnapshot(snapshotId, snapshotFile, Splitter.on(',').split(snapshotKinds));
logger.infofmt("Loaded snapshot successfully: %s", message);
} catch (Throwable e) {
logger.severe(e, "Error loading snapshot");
if (e instanceof IllegalArgumentException) {
throw new BadRequestException("Error calling load snapshot: " + e.getMessage(), e);
} else {
throw new InternalServerErrorException(
"Error loading snapshot: " + firstNonNull(e.getMessage(), e.toString()));
}
}
}
private String loadSnapshot(String snapshotId, String gcsFilename, Iterable<String> kinds)
throws IOException {
Bigquery bigquery = bigqueryFactory.create(projectId, SNAPSHOTS_DATASET);
DateTime now = clock.nowUtc();
String loadMessage =
String.format("Loading datastore snapshot %s from %s...", snapshotId, gcsFilename);
logger.info(loadMessage);
StringBuilder builder = new StringBuilder(loadMessage + "\n");
builder.append("Load jobs:\n");
for (String kindName : kinds) {
String jobId = String.format("load-snapshot-%s-%s-%d", snapshotId, kindName, now.getMillis());
JobReference jobRef = new JobReference().setProjectId(projectId).setJobId(jobId);
String sourceUri = getBackupInfoFileForKind(gcsFilename, kindName);
String tableId = String.format("%s_%s", snapshotId, kindName);
// Launch the load job.
Job job = makeLoadJob(jobRef, sourceUri, tableId);
bigquery.jobs().insert(projectId, job).execute();
// Enqueue a task to check on the load job's completion, and if it succeeds, to update a
// well-known view in BigQuery to point at the newly loaded snapshot table for this kind.
bigqueryPollEnqueuer.enqueuePollTask(
jobRef,
createViewUpdateTask(SNAPSHOTS_DATASET, tableId, kindName),
getQueue(UpdateSnapshotViewAction.QUEUE));
builder.append(String.format(" - %s:%s\n", projectId, jobId));
logger.infofmt("Submitted load job %s:%s", projectId, jobId);
}
return builder.toString();
}
private static String getBackupInfoFileForKind(String backupInfoFile, String kindName) {
String extension = ".backup_info";
checkArgument(backupInfoFile.endsWith(extension), "backup info file extension missing");
String prefix = backupInfoFile.substring(0, backupInfoFile.length() - extension.length());
return Joiner.on('.').join(prefix, kindName, extension.substring(1));
}
private Job makeLoadJob(JobReference jobRef, String sourceUri, String tableId) {
TableReference tableReference = new TableReference()
.setProjectId(jobRef.getProjectId())
.setDatasetId(SNAPSHOTS_DATASET)
.setTableId(tableId);
return new Job()
.setJobReference(jobRef)
.setConfiguration(new JobConfiguration()
.setLoad(new JobConfigurationLoad()
.setWriteDisposition(WriteDisposition.WRITE_EMPTY.toString())
.setSourceFormat(SourceFormat.DATASTORE_BACKUP.toString())
.setSourceUris(ImmutableList.of(sourceUri))
.setDestinationTable(tableReference)));
}
}

124
java/google/registry/export/PublishDetailReportAction.java Normal file
View file

@ -0,0 +1,124 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.domain.registry.util.PreconditionsUtils.checkArgumentNotNull;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.ByteStreams;
import com.google.common.net.MediaType;
import com.google.domain.registry.gcs.GcsUtils;
import com.google.domain.registry.model.registrar.Registrar;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.HttpException.BadRequestException;
import com.google.domain.registry.request.HttpException.InternalServerErrorException;
import com.google.domain.registry.request.JsonActionRunner;
import com.google.domain.registry.request.JsonActionRunner.JsonAction;
import com.google.domain.registry.storage.drive.DriveConnection;
import com.google.domain.registry.util.FormattingLogger;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.util.Map;
import javax.inject.Inject;
/** Publish a single registrar detail report from GCS to Drive. */
@Action(
path = PublishDetailReportAction.PATH,
method = Action.Method.POST,
xsrfProtection = true,
xsrfScope = "admin")
public final class PublishDetailReportAction implements Runnable, JsonAction {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
/** MIME type to use for deposited report files in Drive. */
private static final MediaType REPORT_MIME_TYPE = MediaType.CSV_UTF_8;
/** Endpoint to which JSON should be sent for this servlet. See {@code web.xml}. */
public static final String PATH = "/_dr/publishDetailReport";
/** Name of parameter indicating the registrar for which this report will be published. */
public static final String REGISTRAR_ID_PARAM = "registrar";
/** Name of parameter providing a name for the report file placed in Drive (the base name). */
public static final String DETAIL_REPORT_NAME_PARAM = "report";
/**
* Name of parameter giving the prefix of the GCS object name to use as the report contents.
* Concatenating this value with the value of the "report" parameter gives the full object name.
*/
public static final String GCS_FOLDER_PREFIX_PARAM = "gcsFolder";
/** Name of parameter giving the GCS bucket name for the file to use as the report contents. */
public static final String GCS_BUCKET_PARAM = "bucket";
@Inject DriveConnection driveConnection;
@Inject GcsUtils gcsUtils;
@Inject JsonActionRunner runner;
@Inject PublishDetailReportAction() {}
@Override
public void run() {
runner.run(this);
}
/** Copy a detail report from Cloud Storage to Drive. */
@Override
public Map<String, Object> handleJsonRequest(Map<String, ?> json) {
try {
logger.infofmt("Publishing detail report for parameters: %s", json);
String registrarId = getParam(json, REGISTRAR_ID_PARAM);
Registrar registrar = checkArgumentNotNull(Registrar.loadByClientId(registrarId),
"Registrar %s not found", registrarId);
String driveFolderId = checkArgumentNotNull(registrar.getDriveFolderId(),
"No drive folder associated with registrar " + registrarId);
String gcsBucketName = getParam(json, GCS_BUCKET_PARAM);
String gcsObjectName =
getParam(json, GCS_FOLDER_PREFIX_PARAM) + getParam(json, DETAIL_REPORT_NAME_PARAM);
try (InputStream input =
gcsUtils.openInputStream(new GcsFilename(gcsBucketName, gcsObjectName))) {
String driveId =
driveConnection.createFile(
getParam(json, DETAIL_REPORT_NAME_PARAM),
REPORT_MIME_TYPE,
driveFolderId,
ByteStreams.toByteArray(input));
logger.infofmt("Published detail report for %s to folder %s using GCS file gs://%s/%s.",
registrarId,
driveFolderId,
gcsBucketName,
gcsObjectName);
return ImmutableMap.<String, Object>of("driveId", driveId);
} catch (FileNotFoundException e) {
throw new IllegalArgumentException(e.getMessage(), e);
}
} catch (Throwable e) {
logger.severe(e, e.toString());
String message = firstNonNull(e.getMessage(), e.toString());
throw e instanceof IllegalArgumentException
? new BadRequestException(message) : new InternalServerErrorException(message);
}
}
private String getParam(Map<String, ?> json, String paramName) {
return (String) checkArgumentNotNull(
json.get(paramName),
"Missing required parameter: %s", paramName);
}
}

237
java/google/registry/export/SyncGroupMembersAction.java Normal file
View file

@ -0,0 +1,237 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import static com.google.domain.registry.request.Action.Method.POST;
import static com.google.domain.registry.util.CollectionUtils.nullToEmpty;
import static com.google.domain.registry.util.RegistrarUtils.normalizeClientId;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.groups.GroupsConnection;
import com.google.domain.registry.groups.GroupsConnection.Role;
import com.google.domain.registry.model.registrar.Registrar;
import com.google.domain.registry.model.registrar.RegistrarContact;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Response;
import com.google.domain.registry.util.Concurrent;
import com.google.domain.registry.util.FormattingLogger;
import com.googlecode.objectify.VoidWork;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import javax.annotation.Nullable;
import javax.inject.Inject;
/**
* Action that syncs changes to {@link RegistrarContact} entities with Google Groups.
*
* <p>This uses the <a href="https://developers.google.com/admin-sdk/directory/">Directory API</a>.
*/
@Action(path = "/_dr/task/syncGroupMembers", method = POST)
public final class SyncGroupMembersAction implements Runnable {
/**
* The number of threads to run simultaneously (one per registrar) while processing group syncs.
* This number is purposefully low because App Engine will complain about a large number of
* requests per second, so it's better to spread the work out (as we are only running this servlet
* once per hour anyway).
*/
private static final int NUM_WORK_THREADS = 2;
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
private enum Result {
OK(SC_OK, "Group memberships successfully updated."),
NOT_MODIFIED(SC_OK, "No registrar contacts have been updated since the last time servlet ran."),
FAILED(SC_INTERNAL_SERVER_ERROR, "Error occurred while updating registrar contacts.") {
@Override
protected void log(Throwable cause) {
logger.severefmt(cause, "%s", message);
}};
final int statusCode;
final String message;
private Result(int statusCode, String message) {
this.statusCode = statusCode;
this.message = message;
}
/** Log an error message. Results that use log levels other than info should override this. */
void log(@Nullable Throwable cause) {
logger.infofmt(cause, "%s", message);
}
}
@Inject GroupsConnection groupsConnection;
@Inject Response response;
@Inject @Config("publicDomainName") String publicDomainName;
@Inject SyncGroupMembersAction() {}
private void sendResponse(Result result, @Nullable List<Throwable> causes) {
for (Throwable cause : nullToEmpty(causes)) {
result.log(cause);
}
response.setStatus(result.statusCode);
response.setPayload(String.format("%s %s\n", result.name(), result.message));
}
/**
* Returns the Google Groups email address for the given registrar clientId and
* RegistrarContact.Type
*/
public static String getGroupEmailAddressForContactType(
String clientId,
RegistrarContact.Type type,
String publicDomainName) {
// Take the registrar's clientId, make it lowercase, and remove all characters that aren't
// alphanumeric, hyphens, or underscores.
return String.format(
"%s-%s-contacts@%s", normalizeClientId(clientId), type.getDisplayName(), publicDomainName);
}
/**
* Loads all Registrars, and for each one that is marked dirty, grabs the existing group
* memberships and updates them to reflect the current state of the RegistrarContacts.
*/
@Override
public void run() {
List<Registrar> dirtyRegistrars = Registrar
.loadAllActive()
.filter(new Predicate<Registrar>() {
@Override
public boolean apply(Registrar registrar) {
// Only grab registrars that require syncing and are of the correct type.
return registrar.getContactsRequireSyncing()
&& registrar.getType() == Registrar.Type.REAL;
}})
.toList();
if (dirtyRegistrars.isEmpty()) {
sendResponse(Result.NOT_MODIFIED, null);
return;
}
// Run multiple threads to communicate with Google Groups simultaneously.
ImmutableList<Optional<Throwable>> results = Concurrent.transform(
dirtyRegistrars,
NUM_WORK_THREADS,
new Function<Registrar, Optional<Throwable>>() {
@Override
public Optional<Throwable> apply(final Registrar registrar) {
try {
syncRegistrarContacts(registrar);
return Optional.<Throwable> absent();
} catch (Throwable e) {
logger.severe(e, e.getMessage());
return Optional.of(e);
}
}});
List<Throwable> errors = getErrorsAndUpdateFlagsForSuccesses(dirtyRegistrars, results);
// If there were no errors, return success; otherwise return a failed status and log the errors.
if (errors.isEmpty()) {
sendResponse(Result.OK, null);
} else {
sendResponse(Result.FAILED, errors);
}
}
/**
* Parses the results from Google Groups for each registrar, setting the dirty flag to false in
* Datastore for the calls that succeeded and accumulating the errors for the calls that failed.
*/
private List<Throwable> getErrorsAndUpdateFlagsForSuccesses(
List<Registrar> registrars,
List<Optional<Throwable>> results) {
final ImmutableList.Builder<Registrar> registrarsToSave = new ImmutableList.Builder<>();
List<Throwable> errors = new ArrayList<>();
for (int i = 0; i < results.size(); i++) {
Optional<Throwable> opt = results.get(i);
if (opt.isPresent()) {
errors.add(opt.get());
} else {
registrarsToSave.add(
registrars.get(i).asBuilder().setContactsRequireSyncing(false).build());
}
}
ofy().transactNew(new VoidWork() {
@Override
public void vrun() {
ofy().save().entities(registrarsToSave.build());
}});
return errors;
}
/** Syncs the contacts for an individual registrar to Google Groups. */
private void syncRegistrarContacts(Registrar registrar) {
String groupKey = "";
try {
Set<RegistrarContact> registrarContacts = registrar.getContacts();
long totalAdded = 0;
long totalRemoved = 0;
for (final RegistrarContact.Type type : RegistrarContact.Type.values()) {
groupKey = getGroupEmailAddressForContactType(
registrar.getClientIdentifier(), type, publicDomainName);
Set<String> currentMembers = groupsConnection.getMembersOfGroup(groupKey);
Set<String> desiredMembers = FluentIterable.from(registrarContacts)
.filter(new Predicate<RegistrarContact>() {
@Override
public boolean apply(RegistrarContact contact) {
return contact.getTypes().contains(type);
}})
.transform(new Function<RegistrarContact, String>() {
@Override
public String apply(RegistrarContact contact) {
return contact.getEmailAddress();
}})
.toSet();
for (String email : Sets.difference(desiredMembers, currentMembers)) {
groupsConnection.addMemberToGroup(groupKey, email, Role.MEMBER);
totalAdded++;
}
for (String email : Sets.difference(currentMembers, desiredMembers)) {
groupsConnection.removeMemberFromGroup(groupKey, email);
totalRemoved++;
}
}
logger.infofmt("Successfully synced contacts for registrar %s: added %d and removed %d",
registrar.getClientIdentifier(),
totalAdded,
totalRemoved);
} catch (IOException e) {
// Bail out of the current sync job if an error occurs. This is OK because (a) errors usually
// indicate that retrying won't succeed at all, or at least not immediately, and (b) the sync
// job will run within an hour anyway and effectively resume where it left off if this was a
// transient error.
String msg = String.format("Couldn't sync contacts for registrar %s to group %s",
registrar.getClientIdentifier(), groupKey);
throw new RuntimeException(msg, e);
}
}
}

116
java/google/registry/export/UpdateSnapshotViewAction.java Normal file
View file

@ -0,0 +1,116 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export;
import static com.google.domain.registry.request.Action.Method.POST;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.bigquery.model.Table;
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.ViewDefinition;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.domain.registry.bigquery.BigqueryFactory;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.HttpException.InternalServerErrorException;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.util.SqlTemplate;
import java.io.IOException;
import javax.inject.Inject;
/** Update a well-known view to point at a certain datastore snapshot table in BigQuery. */
@Action(path = UpdateSnapshotViewAction.PATH, method = POST)
public class UpdateSnapshotViewAction implements Runnable {
/** Headers for passing parameters into the servlet. */
static final String UPDATE_SNAPSHOT_DATASET_ID_PARAM = "dataset";
static final String UPDATE_SNAPSHOT_TABLE_ID_PARAM = "table";
static final String UPDATE_SNAPSHOT_KIND_PARAM = "kind";
static final String LATEST_SNAPSHOT_DATASET = "latest_snapshot";
/** Servlet-specific details needed for enqueuing tasks against itself. */
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
static final String PATH = "/_dr/task/updateSnapshotView"; // See web.xml.
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject @Parameter(UPDATE_SNAPSHOT_DATASET_ID_PARAM) String datasetId;
@Inject @Parameter(UPDATE_SNAPSHOT_TABLE_ID_PARAM) String tableId;
@Inject @Parameter(UPDATE_SNAPSHOT_KIND_PARAM) String kindName;
@Inject @Config("projectId") String projectId;
@Inject BigqueryFactory bigqueryFactory;
@Inject UpdateSnapshotViewAction() {}
/** Create a task for updating a snapshot view. */
public static TaskOptions createViewUpdateTask(
String datasetId, String tableId, String kindName) {
return TaskOptions.Builder.withUrl(PATH)
.method(Method.POST)
.param(UPDATE_SNAPSHOT_DATASET_ID_PARAM, datasetId)
.param(UPDATE_SNAPSHOT_TABLE_ID_PARAM, tableId)
.param(UPDATE_SNAPSHOT_KIND_PARAM, kindName);
}
@Override
public void run() {
try {
updateSnapshotView(datasetId, tableId, kindName);
} catch (Throwable e) {
logger.severefmt(e, "Could not update snapshot view for table %s", tableId);
throw new InternalServerErrorException("Error in update snapshot view action");
}
}
private void updateSnapshotView(String datasetId, String tableId, String kindName)
throws IOException {
Bigquery bigquery = bigqueryFactory.create(projectId, LATEST_SNAPSHOT_DATASET);
updateTable(bigquery, new Table()
.setTableReference(new TableReference()
.setProjectId(projectId)
.setDatasetId(LATEST_SNAPSHOT_DATASET)
.setTableId(kindName))
.setView(new ViewDefinition().setQuery(
SqlTemplate.create("SELECT * FROM [%DATASET%.%TABLE%]")
.put("DATASET", datasetId)
.put("TABLE", tableId)
.build())));
logger.infofmt(
"Updated view %s:%s to point at snapshot table %s:%s.",
LATEST_SNAPSHOT_DATASET,
kindName,
datasetId,
tableId);
}
private static void updateTable(Bigquery bigquery, Table table) throws IOException {
TableReference ref = table.getTableReference();
try {
bigquery.tables()
.update(ref.getProjectId(), ref.getDatasetId(), ref.getTableId(), table)
.execute();
} catch (GoogleJsonResponseException e) {
if (e.getDetails().getCode() == 404) {
bigquery.tables().insert(ref.getProjectId(), ref.getDatasetId(), table).execute();
}
}
}
}

16
java/google/registry/export/package-info.java Normal file
View file

@ -0,0 +1,16 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
@javax.annotation.ParametersAreNonnullByDefault
package com.google.domain.registry.export;

26
java/google/registry/export/sheet/BUILD Normal file
View file

@ -0,0 +1,26 @@
package(default_visibility = ["//java/com/google/domain/registry:registry_project"])
java_library(
name = "sheet",
srcs = glob(["*.java"]),
deps = [
"//java/com/google/api/client/googleapis/auth/oauth2",
"//java/com/google/common/base",
"//java/com/google/common/collect",
"//java/com/google/common/io",
"//java/com/google/common/net",
"//java/com/google/domain/registry/config",
"//java/com/google/domain/registry/model",
"//java/com/google/domain/registry/request",
"//java/com/google/domain/registry/util",
"//java/com/google/gdata",
"//java/com/google/gdata:spreadsheet",
"//third_party/java/appengine:appengine-api",
"//third_party/java/dagger",
"//third_party/java/joda_time",
"//third_party/java/jsr305_annotations",
"//third_party/java/jsr330_inject",
"//third_party/java/servlet/servlet_api",
],
)

36
java/google/registry/export/sheet/SheetModule.java Normal file
View file

@ -0,0 +1,36 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export.sheet;
import static com.google.common.base.Strings.emptyToNull;
import com.google.common.base.Optional;
import com.google.domain.registry.request.Parameter;
import dagger.Module;
import dagger.Provides;
import javax.servlet.http.HttpServletRequest;
/** Dagger module for the sheet package. */
@Module
public final class SheetModule {
@Provides
@Parameter("id")
static Optional<String> provideId(HttpServletRequest req) {
return Optional.fromNullable(emptyToNull(req.getParameter("id")));
}
}

106
java/google/registry/export/sheet/SheetSynchronizer.java Normal file
View file

@ -0,0 +1,106 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export.sheet;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import com.google.gdata.data.spreadsheet.CustomElementCollection;
import com.google.gdata.data.spreadsheet.ListEntry;
import com.google.gdata.data.spreadsheet.ListFeed;
import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
import com.google.gdata.data.spreadsheet.WorksheetEntry;
import com.google.gdata.util.ServiceException;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import javax.inject.Inject;
/** Generic data synchronization utility for Google Spreadsheets. */
class SheetSynchronizer {
private static final String SPREADSHEET_URL_PREFIX =
"https://spreadsheets.google.com/feeds/spreadsheets/";
@Inject SpreadsheetService spreadsheetService;
@Inject SheetSynchronizer() {}
/**
* Replace the contents of a Google Spreadsheet with {@code data}.
*
* <p>In order for this to work, you must create a spreadsheet with a header row, each containing
* the column name, without any spaces. All subsequent rows are considered data, so long as
* they're not blank. If you have a blank row in the middle of your data, you're going to have
* problems. You must also make sure that the spreadsheet has been shared with the API client
* credential email address.
*
* <p>The algorithm works by first assuming that the spreadsheet is sorted in the same way that
* {@code data} is sorted. It then iterates through the existing rows and comparing them to the
* items in {@code data}. Iteration continues until we either run out of rows, or items in
* {@code data}. If there's any rows remaining, they'll be deleted. If instead, items remain in
* data, they'll be inserted.
*
* @param spreadsheetId The ID of your spreadsheet. This can be obtained by opening the Google
* spreadsheet in your browser and copying the ID from the URL.
* @param data This should be a <i>sorted</i> list of rows containing the enterity of the
* spreadsheet. Each row is a map, where the key must be exactly the same as the column header
* cell in the spreadsheet, and value is an arbitrary object which will be converted to a
* string before storing it in the spreadsheet.
* @throws IOException error communicating with the GData service.
* @throws ServiceException if a system error occurred when retrieving the entry.
* @throws com.google.gdata.util.ParseException error parsing the returned entry.
* @throws com.google.gdata.util.ResourceNotFoundException if an entry URL is not valid.
* @throws com.google.gdata.util.ServiceForbiddenException if the GData service cannot get the
* entry resource due to access constraints.
* @see "https://developers.google.com/google-apps/spreadsheets/"
*/
void synchronize(String spreadsheetId, ImmutableList<ImmutableMap<String, String>> data)
throws IOException, ServiceException {
URL url = new URL(SPREADSHEET_URL_PREFIX + spreadsheetId);
SpreadsheetEntry spreadsheet = spreadsheetService.getEntry(url, SpreadsheetEntry.class);
WorksheetEntry worksheet = spreadsheet.getWorksheets().get(0);
worksheet.setRowCount(data.size());
worksheet = worksheet.update();
ListFeed listFeed = spreadsheetService.getFeed(worksheet.getListFeedUrl(), ListFeed.class);
List<ListEntry> entries = listFeed.getEntries();
int commonSize = Math.min(entries.size(), data.size());
for (int i = 0; i < commonSize; i++) {
ListEntry entry = entries.get(i);
CustomElementCollection elements = entry.getCustomElements();
boolean mutated = false;
for (ImmutableMap.Entry<String, String> cell : data.get(i).entrySet()) {
if (!cell.getValue().equals(elements.getValue(cell.getKey()))) {
mutated = true;
elements.setValueLocal(cell.getKey(), cell.getValue());
}
}
if (mutated) {
entry.update();
}
}
if (data.size() > entries.size()) {
for (int i = entries.size(); i < data.size(); i++) {
ListEntry entry = listFeed.createEntry();
CustomElementCollection elements = entry.getCustomElements();
for (ImmutableMap.Entry<String, String> cell : data.get(i).entrySet()) {
elements.setValueLocal(cell.getKey(), cell.getValue());
}
listFeed.insert(entry);
}
}
}
}

39
java/google/registry/export/sheet/SpreadsheetServiceModule.java Normal file
View file

@ -0,0 +1,39 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export.sheet;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.common.collect.ImmutableList;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import dagger.Module;
import dagger.Provides;
/** Dagger module for {@link SpreadsheetService}. */
@Module
public final class SpreadsheetServiceModule {
private static final String APPLICATION_NAME = "google-registry-v1";
private static final ImmutableList<String> SCOPES = ImmutableList.of(
"https://spreadsheets.google.com/feeds",
"https://docs.google.com/feeds");
@Provides
static SpreadsheetService provideSpreadsheetService(GoogleCredential credential) {
SpreadsheetService service = new SpreadsheetService(APPLICATION_NAME);
service.setOAuth2Credentials(credential.createScoped(SCOPES));
return service;
}
}

211
java/google/registry/export/sheet/SyncRegistrarsSheet.java Normal file
View file

@ -0,0 +1,211 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export.sheet;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.ABUSE;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.ADMIN;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.BILLING;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.LEGAL;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.MARKETING;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.TECH;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.WHOIS;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Ordering;
import com.google.domain.registry.model.registrar.Registrar;
import com.google.domain.registry.model.registrar.RegistrarAddress;
import com.google.domain.registry.model.registrar.RegistrarContact;
import com.google.domain.registry.util.Clock;
import com.google.domain.registry.util.DateTimeUtils;
import com.google.gdata.util.ServiceException;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import java.io.IOException;
import javax.annotation.Nullable;
import javax.inject.Inject;
/**
* Class for synchronizing all {@link Registrar} datastore objects to a Google Spreadsheet.
*
* @see SyncRegistrarsSheetAction
*/
class SyncRegistrarsSheet {
@Inject Clock clock;
@Inject SheetSynchronizer sheetSynchronizer;
@Inject SyncRegistrarsSheet() {}
/** Returns true if a {@link Registrar} entity was modified in past {@code duration}. */
boolean wasRegistrarsModifiedInLast(Duration duration) {
DateTime watermark = clock.nowUtc().minus(duration);
for (Registrar registrar : Registrar.loadAll()) {
if (DateTimeUtils.isAtOrAfter(registrar.getLastUpdateTime(), watermark)) {
return true;
}
}
return false;
}
/** Performs the synchronization operation. */
void run(String spreadsheetId) throws IOException, ServiceException {
sheetSynchronizer.synchronize(
spreadsheetId,
FluentIterable
.from(
new Ordering<Registrar>() {
@Override
public int compare(Registrar left, Registrar right) {
return left.getClientIdentifier().compareTo(right.getClientIdentifier());
}
}.immutableSortedCopy(Registrar.loadAll()))
.filter(
new Predicate<Registrar>() {
@Override
public boolean apply(Registrar registrar) {
return registrar.getType() == Registrar.Type.REAL
|| registrar.getType() == Registrar.Type.OTE;
}
})
.transform(
new Function<Registrar, ImmutableMap<String, String>>() {
@Override
public ImmutableMap<String, String> apply(Registrar registrar) {
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
ImmutableSortedSet<RegistrarContact> contacts = registrar.getContacts();
RegistrarAddress address =
firstNonNull(
registrar.getLocalizedAddress(),
firstNonNull(
registrar.getInternationalizedAddress(),
new RegistrarAddress.Builder()
.setStreet(ImmutableList.of("UNKNOWN"))
.setCity("UNKNOWN")
.setCountryCode("US")
.build()));
//
// （╯°□°）╯ WARNING WARNING WARNING
//
// Do not change these mappings simply because the Registrar model changed. Only
// change these mappings if the people who use the spreadsheet requested it be
// changed.
//
// These values are hard-coded because they correspond to actual spreadsheet
// columns. If you change this dictionary, then you'll need to manually add new
// columns to the registrar spreadsheets for all environments before deployment,
// and you'll need to remove deleted columns probably like a week after
// deployment.
//
builder.put("clientIdentifier", convert(registrar.getClientIdentifier()));
builder.put("registrarName", convert(registrar.getRegistrarName()));
builder.put("state", convert(registrar.getState()));
builder.put("ianaIdentifier", convert(registrar.getIanaIdentifier()));
builder.put("billingIdentifier", convert(registrar.getBillingIdentifier()));
builder.put("primaryContacts", convertContacts(contacts, byType(ADMIN)));
builder.put("techContacts", convertContacts(contacts, byType(TECH)));
builder.put("marketingContacts", convertContacts(contacts, byType(MARKETING)));
builder.put("abuseContacts", convertContacts(contacts, byType(ABUSE)));
builder.put("whoisInquiryContacts", convertContacts(contacts, byType(WHOIS)));
builder.put("legalContacts", convertContacts(contacts, byType(LEGAL)));
builder.put("billingContacts", convertContacts(contacts, byType(BILLING)));
builder.put(
"contactsMarkedAsWhoisAdmin",
convertContacts(
contacts,
new Predicate<RegistrarContact>() {
@Override
public boolean apply(RegistrarContact contact) {
return contact.getVisibleInWhoisAsAdmin();
}
}));
builder.put(
"contactsMarkedAsWhoisTech",
convertContacts(
contacts,
new Predicate<RegistrarContact>() {
@Override
public boolean apply(RegistrarContact contact) {
return contact.getVisibleInWhoisAsTech();
}
}));
builder.put("emailAddress", convert(registrar.getEmailAddress()));
builder.put("address.street", convert(address.getStreet()));
builder.put("address.city", convert(address.getCity()));
builder.put("address.state", convert(address.getState()));
builder.put("address.zip", convert(address.getZip()));
builder.put("address.countryCode", convert(address.getCountryCode()));
builder.put("phoneNumber", convert(registrar.getPhoneNumber()));
builder.put("faxNumber", convert(registrar.getFaxNumber()));
builder.put("creationTime", convert(registrar.getCreationTime()));
builder.put("lastUpdateTime", convert(registrar.getLastUpdateTime()));
builder.put("allowedTlds", convert(registrar.getAllowedTlds()));
builder.put("whoisServer", convert(registrar.getWhoisServer()));
builder.put("blockPremiumNames", convert(registrar.getBlockPremiumNames()));
builder.put("ipAddressWhitelist", convert(registrar.getIpAddressWhitelist()));
builder.put("url", convert(registrar.getUrl()));
builder.put("referralUrl", convert(registrar.getReferralUrl()));
builder.put("icannReferralEmail", convert(registrar.getIcannReferralEmail()));
return builder.build();
}
})
.toList());
}
private static String convertContacts(
Iterable<RegistrarContact> contacts, Predicate<RegistrarContact> filter) {
StringBuilder result = new StringBuilder();
boolean first = true;
for (RegistrarContact contact : contacts) {
if (!filter.apply(contact)) {
continue;
}
if (first) {
first = false;
} else {
result.append("\n");
}
result.append(contact.toStringMultilinePlainText());
}
return result.toString();
}
private static Predicate<RegistrarContact> byType(final RegistrarContact.Type type) {
return new Predicate<RegistrarContact>() {
@Override
public boolean apply(RegistrarContact contact) {
return contact.getTypes().contains(type);
}};
}
/** Converts a value to a string representation that can be stored in a spreadsheet cell. */
private static String convert(@Nullable Object value) {
if (value == null) {
return "";
} else if (value instanceof Iterable) {
return Joiner.on('\n').join((Iterable<?>) value);
} else {
return value.toString();
}
}
}

162
java/google/registry/export/sheet/SyncRegistrarsSheetAction.java Normal file
View file

@ -0,0 +1,162 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export.sheet;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static com.google.domain.registry.request.Action.Method.POST;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.appengine.api.modules.ModulesService;
import com.google.appengine.api.modules.ModulesServiceFactory;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Optional;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.model.server.Lock;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.request.Response;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.util.NonFinalForTesting;
import com.google.gdata.util.ServiceException;
import org.joda.time.Duration;
import java.io.IOException;
import java.util.concurrent.Callable;
import javax.annotation.Nullable;
import javax.inject.Inject;
/**
* Action for synchronizing the registrars spreadsheet.
*
* <p>You can specify the spreadsheet ID by passing the "id" parameter. If this parameter is not
* specified, then the spreadsheet ID will be obtained from the registry configuration.
*
* <p>Cron will run this action hourly. So in order to minimize Google Spreadsheets I/O, this action
* will iterate through all registrars and check if any entries were modified in the past hour. If
* no modifications were made, the action will exit without performing any syncing.
*
* <p><b>Note:</b> Setting the "id" parameter will disable the registrar update check.
*
* <p>Before using this service, you should make sure all the column headers listed in this source
* file are present. You also need to share the spreadsheet with the email address from the JSON
* credential file and give it edit permission.
*
* @see SyncRegistrarsSheet
*/
@Action(path = SyncRegistrarsSheetAction.PATH, method = POST)
public class SyncRegistrarsSheetAction implements Runnable {
private enum Result {
OK(SC_OK, "Sheet successfully updated."),
NOTMODIFIED(SC_OK, "Registrars table hasn't been modified in past hour."),
LOCKED(SC_NO_CONTENT, "Another task is currently writing to this sheet; dropping task."),
MISSINGNO(SC_BAD_REQUEST, "No sheet ID specified or configured; dropping task.") {
@Override
protected void log(Exception cause) {
logger.warningfmt(cause, "%s", message);
}},
FAILED(SC_INTERNAL_SERVER_ERROR, "Spreadsheet synchronization failed") {
@Override
protected void log(Exception cause) {
logger.severefmt(cause, "%s", message);
}};
private final int statusCode;
protected final String message;
private Result(int statusCode, String message) {
this.statusCode = statusCode;
this.message = message;
}
/** Log an error message. Results that use log levels other than info should override this. */
protected void log(@Nullable Exception cause) {
logger.infofmt(cause, "%s", message);
}
private void send(Response response, @Nullable Exception cause) {
log(cause);
response.setStatus(statusCode);
response.setContentType(PLAIN_TEXT_UTF_8);
response.setPayload(String.format("%s %s\n", name(), message));
}
}
public static final String PATH = "/_dr/task/syncRegistrarsSheet";
private static final String QUEUE = "sheet";
private static final String LOCK_NAME = "Synchronize registrars sheet";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@NonFinalForTesting
private static ModulesService modulesService = ModulesServiceFactory.getModulesService();
@Inject Response response;
@Inject SyncRegistrarsSheet syncRegistrarsSheet;
@Inject @Config("sheetLockTimeout") Duration timeout;
@Inject @Config("sheetRegistrarId") Optional<String> idConfig;
@Inject @Config("sheetRegistrarInterval") Duration interval;
@Inject @Parameter("id") Optional<String> idParam;
@Inject SyncRegistrarsSheetAction() {}
@Override
public void run() {
final Optional<String> sheetId = idParam.or(idConfig);
if (!sheetId.isPresent()) {
Result.MISSINGNO.send(response, null);
return;
}
if (!idParam.isPresent()) {
// TODO(b/19082368): Use a cursor.
if (!syncRegistrarsSheet.wasRegistrarsModifiedInLast(interval)) {
Result.NOTMODIFIED.send(response, null);
return;
}
}
String sheetLockName = String.format("%s: %s", LOCK_NAME, sheetId.get());
Callable<Void> runner = new Callable<Void>() {
@Nullable
@Override
public Void call() throws IOException {
try {
syncRegistrarsSheet.run(sheetId.get());
Result.OK.send(response, null);
} catch (IOException | ServiceException e) {
Result.FAILED.send(response, e);
}
return null;
}
};
if (!Lock.executeWithLocks(runner, getClass(), "", timeout, sheetLockName)) {
// If we fail to acquire the lock, it probably means lots of updates are happening at once, in
// which case it should be safe to not bother. The task queue definition should *not* specify
// max-concurrent-requests for this very reason.
Result.LOCKED.send(response, null);
}
}
/** Creates, enqueues, and returns a new backend task to sync registrar spreadsheets. */
public static TaskHandle enqueueBackendTask() {
String hostname = modulesService.getVersionHostname("backend", null);
return getQueue(QUEUE).add(withUrl(PATH).method(Method.GET).header("Host", hostname));
}
}

16
java/google/registry/export/sheet/package-info.java Normal file
View file

@ -0,0 +1,16 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
@javax.annotation.ParametersAreNonnullByDefault
package com.google.domain.registry.export.sheet;