diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index 55ec838ec..d91460daa 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -1362,15 +1362,6 @@ public final class RegistryConfig { return CONFIG_SETTINGS.get().appEngine.projectId; } - /** - * Returns the Google Cloud Storage bucket for storing backup snapshots. - * - * @see google.registry.export.ExportSnapshotAction - */ - public static String getSnapshotsBucket() { - return getProjectId() + "-snapshots"; - } - /** * Returns the Google Cloud Storage bucket for storing Datastore backups. * diff --git a/java/google/registry/env/common/backend/WEB-INF/web.xml b/java/google/registry/env/common/backend/WEB-INF/web.xml index b80efaa01..c9abe53b2 100644 --- a/java/google/registry/env/common/backend/WEB-INF/web.xml +++ b/java/google/registry/env/common/backend/WEB-INF/web.xml @@ -192,24 +192,6 @@ /_dr/task/uploadDatastoreBackup - - - backend-servlet - /_dr/task/exportSnapshot - - - - - backend-servlet - /_dr/task/checkSnapshot - - - - - backend-servlet - /_dr/task/loadSnapshot - - backend-servlet diff --git a/java/google/registry/export/BackupDatastoreAction.java b/java/google/registry/export/BackupDatastoreAction.java index 4c773bb25..1f03e3d99 100644 --- a/java/google/registry/export/BackupDatastoreAction.java +++ b/java/google/registry/export/BackupDatastoreAction.java @@ -28,9 +28,7 @@ import google.registry.request.auth.Auth; import javax.inject.Inject; /** - * Action to trigger a Datastore backup job that writes a snapshot to Google Cloud Storage. This - * class is introduced as an experimental feature, and will eventually replace {@link - * ExportSnapshotAction}. + * Action to trigger a Datastore backup job that writes a snapshot to Google Cloud Storage. * *

This is the first step of a four step workflow for exporting snapshots, with each step calling * the next upon successful completion: diff --git a/java/google/registry/export/CheckSnapshotAction.java b/java/google/registry/export/CheckSnapshotAction.java deleted file mode 100644 index 2a46678ec..000000000 --- a/java/google/registry/export/CheckSnapshotAction.java +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.common.collect.Sets.intersection; -import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask; -import static google.registry.request.Action.Method.GET; -import static google.registry.request.Action.Method.POST; - -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TaskHandle; -import com.google.appengine.api.taskqueue.TaskOptions; -import com.google.appengine.api.taskqueue.TaskOptions.Method; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import com.google.common.flogger.FluentLogger; -import google.registry.export.DatastoreBackupInfo.BackupStatus; -import google.registry.request.Action; -import google.registry.request.HttpException.BadRequestException; -import google.registry.request.HttpException.NoContentException; -import google.registry.request.HttpException.NotModifiedException; -import google.registry.request.Parameter; -import google.registry.request.RequestMethod; -import google.registry.request.Response; -import google.registry.request.auth.Auth; -import java.util.Set; -import javax.inject.Inject; -import org.joda.time.Duration; -import org.joda.time.PeriodType; -import org.joda.time.format.PeriodFormat; - -/** - * Action that checks the status of a snapshot, and if complete, trigger loading it into BigQuery. - */ -@Action( - service = Action.Service.BACKEND, - path = CheckSnapshotAction.PATH, - method = {POST, GET}, - automaticallyPrintOk = true, - auth = Auth.AUTH_INTERNAL_ONLY) -public class CheckSnapshotAction implements Runnable { - - /** Parameter names for passing parameters into this action. */ - static final String CHECK_SNAPSHOT_NAME_PARAM = "name"; - - static final String CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM = "kindsToLoad"; - - /** Action-specific details needed for enqueuing tasks against itself. */ - static final String QUEUE = "export-snapshot-poll"; // See queue.xml. - - static final String PATH = "/_dr/task/checkSnapshot"; // See web.xml. - static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2); - - /** The maximum amount of time we allow a backup to run before abandoning it. */ - static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20); - - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - - @Inject Response response; - @Inject @RequestMethod Action.Method requestMethod; - @Inject DatastoreBackupService backupService; - @Inject @Parameter(CHECK_SNAPSHOT_NAME_PARAM) String snapshotName; - @Inject @Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM) String kindsToLoadParam; - @Inject CheckSnapshotAction() {} - - @Override - public void run() { - if (requestMethod == POST) { - checkAndLoadSnapshotIfComplete(); - } else { - // This is a GET request. - response.setPayload(getBackup().getInformation()); - } - } - - private DatastoreBackupInfo getBackup() { - try { - return backupService.findByName(snapshotName); - } catch (IllegalArgumentException e) { - String message = String.format("Bad backup name %s: %s", snapshotName, e.getMessage()); - // TODO(b/19081569): Ideally this would return a 2XX error so the task would not be - // retried but we might abandon backups that start late and haven't yet written to - // Datastore. We could fix that by replacing this with a two-phase polling strategy. - throw new BadRequestException(message, e); - } - } - - private void checkAndLoadSnapshotIfComplete() { - Set kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam)); - DatastoreBackupInfo backup = getBackup(); - // Stop now if the backup is not complete. - if (!backup.getStatus().equals(BackupStatus.COMPLETE)) { - Duration runningTime = backup.getRunningTime(); - if (runningTime.isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) { - // Backup might still be running, so send a 304 to have the task retry. - throw new NotModifiedException( - String.format("Datastore backup %s still pending", snapshotName)); - } else { - // Declare the backup a lost cause, and send 204 No Content so the task will - // not be retried. - String message = - String.format( - "Datastore backup %s abandoned - not complete after %s", - snapshotName, - PeriodFormat.getDefault() - .print( - runningTime - .toPeriod() - .normalizedStandard(PeriodType.dayTime().withMillisRemoved()))); - throw new NoContentException(message); - } - } - // Get a compact string to identify this snapshot in BigQuery by trying to parse the unique - // suffix out of the snapshot name and falling back to the start time as a string. - String snapshotId = - snapshotName.startsWith(ExportSnapshotAction.SNAPSHOT_PREFIX) - ? snapshotName.substring(ExportSnapshotAction.SNAPSHOT_PREFIX.length()) - : backup.getStartTime().toString("YYYYMMdd_HHmmss"); - // Log a warning if kindsToLoad is not a subset of the exported snapshot kinds. - if (!backup.getKinds().containsAll(kindsToLoad)) { - logger.atWarning().log( - "Kinds to load included non-exported kinds: %s", - Sets.difference(kindsToLoad, backup.getKinds())); - } - // Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present). - ImmutableSet exportedKindsToLoad = - ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad)); - String message = String.format("Datastore backup %s complete - ", snapshotName); - if (exportedKindsToLoad.isEmpty()) { - message += "no kinds to load into BigQuery"; - } else { - enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad); - message += "BigQuery load task enqueued"; - } - logger.atInfo().log(message); - response.setPayload(message); - } - - /** Enqueue a poll task to monitor the named snapshot for completion. */ - static TaskHandle enqueuePollTask(String snapshotName, ImmutableSet kindsToLoad) { - return QueueFactory.getQueue(QUEUE) - .add( - TaskOptions.Builder.withUrl(PATH) - .method(Method.POST) - .countdownMillis(POLL_COUNTDOWN.getMillis()) - .param(CHECK_SNAPSHOT_NAME_PARAM, snapshotName) - .param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad))); - } -} diff --git a/java/google/registry/export/DatastoreBackupInfo.java b/java/google/registry/export/DatastoreBackupInfo.java deleted file mode 100644 index 96fff8c22..000000000 --- a/java/google/registry/export/DatastoreBackupInfo.java +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static org.joda.time.DateTimeZone.UTC; - -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.Text; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Ascii; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableSet; -import google.registry.util.Clock; -import google.registry.util.NonFinalForTesting; -import google.registry.util.SystemClock; -import java.util.Date; -import java.util.List; -import java.util.Optional; -import org.joda.time.DateTime; -import org.joda.time.Duration; - -/** Container for information about a Datastore backup. */ -public class DatastoreBackupInfo { - - @NonFinalForTesting - private static Clock clock = new SystemClock(); - - /** The possible status values for a Datastore backup. */ - public enum BackupStatus { PENDING, COMPLETE } - - /** The name of the Datastore backup. */ - private final String backupName; - - /** The entity kinds included in this Datastore backup. */ - private final ImmutableSet kinds; - - /** The start time of the Datastore backup. */ - private final DateTime startTime; - - /** The completion time of the Datastore backup, present if it has completed. */ - private final Optional completeTime; - - /** - * The GCS filename to which the backup's top-level .backup_info manifest file has been written, - * present if the backup has completed. - */ - private final Optional gcsFilename; - - /** DatastoreBackupInfo instances should only be obtained via DatastoreBackupService. */ - DatastoreBackupInfo(Entity backupEntity) { - backupName = (String) checkNotNull(backupEntity.getProperty("name"), "name"); - @SuppressWarnings("unchecked") - List rawKinds = (List) checkNotNull(backupEntity.getProperty("kinds"), "kinds"); - Date rawStartTime = (Date) checkNotNull(backupEntity.getProperty("start_time"), "start_time"); - Date rawCompleteTime = (Date) backupEntity.getProperty("complete_time"); - Text rawGcsFilename = (Text) backupEntity.getProperty("gs_handle"); - - kinds = ImmutableSet.copyOf(rawKinds); - startTime = new DateTime(rawStartTime).withZone(UTC); - completeTime = Optional.ofNullable( - rawCompleteTime == null ? null : new DateTime(rawCompleteTime).withZone(UTC)); - gcsFilename = Optional.ofNullable( - rawGcsFilename == null ? null : gcsPathToUri(rawGcsFilename.getValue())); - } - - /** This constructor is only exposed for test purposes. */ - @VisibleForTesting - DatastoreBackupInfo( - String backupName, - DateTime startTime, - Optional completeTime, - ImmutableSet kinds, - Optional gcsFilename) { - this.backupName = backupName; - this.startTime = startTime; - this.completeTime = completeTime; - this.kinds = kinds; - this.gcsFilename = gcsFilename; - } - - /** - * Rewrite a GCS path as stored by Datastore Admin (with a "/gs/" prefix) to the more standard - * URI format that uses a "gs://" scheme prefix. - */ - private static String gcsPathToUri(String backupGcsPath) { - checkArgument(backupGcsPath.startsWith("/gs/"), "GCS path not in expected format"); - return backupGcsPath.replaceFirst("/gs/", "gs://"); - } - - public String getName() { - return backupName; - } - - public ImmutableSet getKinds() { - return kinds; - } - - public BackupStatus getStatus() { - return completeTime.isPresent() ? BackupStatus.COMPLETE : BackupStatus.PENDING; - } - - public DateTime getStartTime() { - return startTime; - } - - public Optional getCompleteTime() { - return completeTime; - } - - /** - * Returns the length of time the backup ran for (if completed) or the length of time since the - * backup started (if it has not completed). - */ - public Duration getRunningTime() { - return new Duration(startTime, completeTime.orElse(clock.nowUtc())); - } - - public Optional getGcsFilename() { - return gcsFilename; - } - - /** Returns a string version of key information about the backup. */ - public String getInformation() { - return Joiner.on('\n') - .join( - "Backup name: " + backupName, - "Status: " + getStatus(), - "Started: " + startTime, - "Ended: " + completeTime.orElse(null), - "Duration: " + Ascii.toLowerCase(getRunningTime().toPeriod().toString().substring(2)), - "GCS: " + gcsFilename.orElse(null), - "Kinds: " + kinds, - ""); - } -} diff --git a/java/google/registry/export/DatastoreBackupService.java b/java/google/registry/export/DatastoreBackupService.java deleted file mode 100644 index 2a29f7d95..000000000 --- a/java/google/registry/export/DatastoreBackupService.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService; -import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; -import static com.google.common.base.Strings.nullToEmpty; -import static com.google.common.collect.ImmutableList.toImmutableList; - -import com.google.appengine.api.datastore.Query; -import com.google.appengine.api.taskqueue.TaskHandle; -import com.google.appengine.api.taskqueue.TaskOptions; -import com.google.appengine.api.taskqueue.TaskOptions.Method; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Streams; -import google.registry.util.AppEngineServiceUtils; -import java.util.NoSuchElementException; -import javax.inject.Inject; - -/** An object providing methods for starting and querying Datastore backups. */ -public class DatastoreBackupService { - - /** The internal kind name used for entities storing information about Datastore backups. */ - static final String BACKUP_INFO_KIND = "_AE_Backup_Information"; - - /** The name of the app version used for hosting the Datastore Admin functionality. */ - static final String DATASTORE_ADMIN_VERSION_NAME = "ah-builtin-python-bundle"; - - private final AppEngineServiceUtils appEngineServiceUtils; - - @Inject - public DatastoreBackupService(AppEngineServiceUtils appEngineServiceUtils) { - this.appEngineServiceUtils = appEngineServiceUtils; - } - - /** - * Generates the TaskOptions needed to trigger an AppEngine Datastore backup job. - * - * @see Scheduled Backups - */ - private TaskOptions makeTaskOptions( - String queue, String name, String gcsBucket, ImmutableSet kinds) { - String hostname = - appEngineServiceUtils.getVersionHostname("default", DATASTORE_ADMIN_VERSION_NAME); - TaskOptions options = TaskOptions.Builder.withUrl("/_ah/datastore_admin/backup.create") - .header("Host", hostname) - .method(Method.GET) - .param("name", name + "_") // Add underscore since the name will be used as a prefix. - .param("filesystem", "gs") - .param("gs_bucket_name", gcsBucket) - .param("queue", queue); - for (String kind : kinds) { - options.param("kind", kind); - } - return options; - } - - /** - * Launches a new Datastore backup with the given name, GCS bucket, and set of kinds by - * submitting a task to the given task queue, and returns a handle to that task. - */ - public TaskHandle launchNewBackup( - String queue, String name, String gcsBucket, ImmutableSet kinds) { - return getQueue(queue).add(makeTaskOptions(queue, name, gcsBucket, kinds)); - } - - /** Return an iterable of all Datastore backups whose names have the given string prefix. */ - public Iterable findAllByNamePrefix(final String namePrefix) { - // Need the raw DatastoreService to access the internal _AE_Backup_Information entities. - // TODO(b/19081037): make an Objectify entity class for these raw Datastore entities instead. - return Streams.stream(getDatastoreService().prepare(new Query(BACKUP_INFO_KIND)).asIterable()) - .filter(entity -> nullToEmpty((String) entity.getProperty("name")).startsWith(namePrefix)) - .map(DatastoreBackupInfo::new) - .collect(toImmutableList()); - } - - /** - * Return a single DatastoreBackup that uniquely matches this name prefix. Throws an IAE - * if no backups match or if more than one backup matches. - */ - public DatastoreBackupInfo findByName(final String namePrefix) { - try { - return Iterables.getOnlyElement(findAllByNamePrefix(namePrefix)); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("More than one backup with name prefix " + namePrefix, e); - } catch (NoSuchElementException e) { - throw new IllegalArgumentException("No backup found with name prefix " + namePrefix, e); - } - } -} diff --git a/java/google/registry/export/ExportRequestModule.java b/java/google/registry/export/ExportRequestModule.java index e1e7148cf..1c47cac4d 100644 --- a/java/google/registry/export/ExportRequestModule.java +++ b/java/google/registry/export/ExportRequestModule.java @@ -17,16 +17,15 @@ package google.registry.export; import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER; import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER; import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER; -import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM; -import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM; -import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM; -import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM; -import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM; +import static google.registry.export.CheckBackupAction.CHECK_BACKUP_KINDS_TO_LOAD_PARAM; +import static google.registry.export.CheckBackupAction.CHECK_BACKUP_NAME_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM; import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_FOLDER_PARAM; +import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_ID_PARAM; +import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_KINDS_PARAM; import static google.registry.request.RequestParameters.extractRequiredHeader; import static google.registry.request.RequestParameters.extractRequiredParameter; @@ -64,12 +63,6 @@ public final class ExportRequestModule { return extractRequiredParameter(req, UPDATE_SNAPSHOT_VIEWNAME_PARAM); } - @Provides - @Parameter(LOAD_SNAPSHOT_FILE_PARAM) - static String provideLoadSnapshotFile(HttpServletRequest req) { - return extractRequiredParameter(req, LOAD_SNAPSHOT_FILE_PARAM); - } - @Provides @Parameter(UPLOAD_BACKUP_FOLDER_PARAM) static String provideSnapshotUrlPrefix(HttpServletRequest req) { @@ -77,27 +70,27 @@ public final class ExportRequestModule { } @Provides - @Parameter(LOAD_SNAPSHOT_ID_PARAM) + @Parameter(UPLOAD_BACKUP_ID_PARAM) static String provideLoadSnapshotId(HttpServletRequest req) { - return extractRequiredParameter(req, LOAD_SNAPSHOT_ID_PARAM); + return extractRequiredParameter(req, UPLOAD_BACKUP_ID_PARAM); } @Provides - @Parameter(LOAD_SNAPSHOT_KINDS_PARAM) + @Parameter(UPLOAD_BACKUP_KINDS_PARAM) static String provideLoadSnapshotKinds(HttpServletRequest req) { - return extractRequiredParameter(req, LOAD_SNAPSHOT_KINDS_PARAM); + return extractRequiredParameter(req, UPLOAD_BACKUP_KINDS_PARAM); } @Provides - @Parameter(CHECK_SNAPSHOT_NAME_PARAM) + @Parameter(CHECK_BACKUP_NAME_PARAM) static String provideCheckSnapshotName(HttpServletRequest req) { - return extractRequiredParameter(req, CHECK_SNAPSHOT_NAME_PARAM); + return extractRequiredParameter(req, CHECK_BACKUP_NAME_PARAM); } @Provides - @Parameter(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM) + @Parameter(CHECK_BACKUP_KINDS_TO_LOAD_PARAM) static String provideCheckSnapshotKindsToLoad(HttpServletRequest req) { - return extractRequiredParameter(req, CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM); + return extractRequiredParameter(req, CHECK_BACKUP_KINDS_TO_LOAD_PARAM); } @Provides diff --git a/java/google/registry/export/ExportSnapshotAction.java b/java/google/registry/export/ExportSnapshotAction.java deleted file mode 100644 index c47b08fdc..000000000 --- a/java/google/registry/export/ExportSnapshotAction.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static google.registry.export.CheckSnapshotAction.enqueuePollTask; -import static google.registry.request.Action.Method.POST; - -import com.google.common.flogger.FluentLogger; -import google.registry.config.RegistryConfig; -import google.registry.request.Action; -import google.registry.request.Response; -import google.registry.request.auth.Auth; -import google.registry.util.Clock; -import javax.inject.Inject; - -/** - * Action to trigger a Datastore backup job that writes a snapshot to Google Cloud Storage. - * - *

This is the first step of a four step workflow for exporting snapshots, with each step calling - * the next upon successful completion: - * - *

    - *
  1. The snapshot is exported to Google Cloud Storage (this action). - *
  2. The {@link CheckSnapshotAction} polls until the export is completed. - *
  3. The {@link LoadSnapshotAction} imports the data from GCS to BigQuery. - *
  4. The {@link UpdateSnapshotViewAction} updates the view in latest_datastore_export. - *
- */ -@Action( - service = Action.Service.BACKEND, - path = ExportSnapshotAction.PATH, - method = POST, - automaticallyPrintOk = true, - auth = Auth.AUTH_INTERNAL_ONLY) -public class ExportSnapshotAction implements Runnable { - - /** Queue to use for enqueuing the task that will actually launch the backup. */ - static final String QUEUE = "export-snapshot"; // See queue.xml. - - static final String PATH = "/_dr/task/exportSnapshot"; // See web.xml. - - /** Prefix to use for naming all snapshots that are started by this servlet. */ - static final String SNAPSHOT_PREFIX = "auto_snapshot_"; - - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - - @Inject Clock clock; - @Inject DatastoreBackupService backupService; - @Inject Response response; - - @Inject - ExportSnapshotAction() {} - - @Override - public void run() { - // Use a unique name for the snapshot so we can explicitly check its completion later. - String snapshotName = SNAPSHOT_PREFIX + clock.nowUtc().toString("YYYYMMdd_HHmmss"); - backupService.launchNewBackup( - QUEUE, snapshotName, RegistryConfig.getSnapshotsBucket(), ExportConstants.getBackupKinds()); - // Enqueue a poll task to monitor the backup and load reporting-related kinds into bigquery. - enqueuePollTask(snapshotName, ExportConstants.getReportingKinds()); - String message = "Datastore backup started with name: " + snapshotName; - logger.atInfo().log(message); - response.setPayload(message); - } -} diff --git a/java/google/registry/export/LoadSnapshotAction.java b/java/google/registry/export/LoadSnapshotAction.java deleted file mode 100644 index b6b6a178f..000000000 --- a/java/google/registry/export/LoadSnapshotAction.java +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.google.common.base.Preconditions.checkArgument; -import static google.registry.export.UpdateSnapshotViewAction.createViewUpdateTask; -import static google.registry.request.Action.Method.POST; - -import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.JobConfiguration; -import com.google.api.services.bigquery.model.JobConfigurationLoad; -import com.google.api.services.bigquery.model.JobReference; -import com.google.api.services.bigquery.model.TableReference; -import com.google.appengine.api.taskqueue.TaskHandle; -import com.google.appengine.api.taskqueue.TaskOptions; -import com.google.appengine.api.taskqueue.TaskOptions.Method; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.flogger.FluentLogger; -import google.registry.bigquery.BigqueryUtils.SourceFormat; -import google.registry.bigquery.BigqueryUtils.WriteDisposition; -import google.registry.bigquery.CheckedBigquery; -import google.registry.config.RegistryConfig.Config; -import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer; -import google.registry.request.Action; -import google.registry.request.HttpException.BadRequestException; -import google.registry.request.HttpException.InternalServerErrorException; -import google.registry.request.Parameter; -import google.registry.request.auth.Auth; -import google.registry.util.Clock; -import java.io.IOException; -import javax.inject.Inject; -import org.joda.time.DateTime; - -/** Action to load a Datastore snapshot from Google Cloud Storage into BigQuery. */ -@Action( - service = Action.Service.BACKEND, - path = LoadSnapshotAction.PATH, - method = POST, - auth = Auth.AUTH_INTERNAL_ONLY) -public class LoadSnapshotAction implements Runnable { - - /** Parameter names for passing parameters into the servlet. */ - static final String LOAD_SNAPSHOT_ID_PARAM = "id"; - static final String LOAD_SNAPSHOT_FILE_PARAM = "file"; - static final String LOAD_SNAPSHOT_KINDS_PARAM = "kinds"; - - static final String SNAPSHOTS_DATASET = "snapshots"; - - static final String LATEST_SNAPSHOT_VIEW_NAME = "latest_datastore_export"; - - /** Servlet-specific details needed for enqueuing tasks against itself. */ - static final String QUEUE = "export-snapshot"; // See queue.xml. - static final String PATH = "/_dr/task/loadSnapshot"; // See web.xml. - - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - - @Inject CheckedBigquery checkedBigquery; - @Inject BigqueryPollJobEnqueuer bigqueryPollEnqueuer; - @Inject Clock clock; - @Inject @Config("projectId") String projectId; - @Inject @Parameter(LOAD_SNAPSHOT_FILE_PARAM) String snapshotFile; - @Inject @Parameter(LOAD_SNAPSHOT_ID_PARAM) String snapshotId; - @Inject @Parameter(LOAD_SNAPSHOT_KINDS_PARAM) String snapshotKinds; - @Inject LoadSnapshotAction() {} - - /** Enqueue a task for starting a backup load. */ - public static TaskHandle enqueueLoadSnapshotTask( - String snapshotId, String gcsFile, ImmutableSet kinds) { - return getQueue(QUEUE).add( - TaskOptions.Builder.withUrl(PATH) - .method(Method.POST) - .param(LOAD_SNAPSHOT_ID_PARAM, snapshotId) - .param(LOAD_SNAPSHOT_FILE_PARAM, gcsFile) - .param(LOAD_SNAPSHOT_KINDS_PARAM, Joiner.on(',').join(kinds))); - } - - @Override - public void run() { - try { - String message = - loadSnapshot(snapshotId, snapshotFile, Splitter.on(',').split(snapshotKinds)); - logger.atInfo().log("Loaded snapshot successfully: %s", message); - } catch (Throwable e) { - logger.atSevere().withCause(e).log("Error loading snapshot"); - if (e instanceof IllegalArgumentException) { - throw new BadRequestException("Error calling load snapshot: " + e.getMessage(), e); - } else { - throw new InternalServerErrorException( - "Error loading snapshot: " + firstNonNull(e.getMessage(), e.toString())); - } - } - } - - private String loadSnapshot(String snapshotId, String gcsFilename, Iterable kinds) - throws IOException { - Bigquery bigquery = checkedBigquery.ensureDataSetExists(projectId, SNAPSHOTS_DATASET); - DateTime now = clock.nowUtc(); - String loadMessage = - String.format("Loading Datastore snapshot %s from %s...", snapshotId, gcsFilename); - logger.atInfo().log(loadMessage); - StringBuilder builder = new StringBuilder(loadMessage + "\n"); - builder.append("Load jobs:\n"); - - for (String kindName : kinds) { - String jobId = String.format("load-snapshot-%s-%s-%d", snapshotId, kindName, now.getMillis()); - JobReference jobRef = new JobReference().setProjectId(projectId).setJobId(jobId); - String sourceUri = getBackupInfoFileForKind(gcsFilename, kindName); - String tableId = String.format("%s_%s", snapshotId, kindName); - - // Launch the load job. - Job job = makeLoadJob(jobRef, sourceUri, tableId); - bigquery.jobs().insert(projectId, job).execute(); - - // Enqueue a task to check on the load job's completion, and if it succeeds, to update a - // well-known view in BigQuery to point at the newly loaded snapshot table for this kind. - bigqueryPollEnqueuer.enqueuePollTask( - jobRef, - createViewUpdateTask(SNAPSHOTS_DATASET, tableId, kindName, LATEST_SNAPSHOT_VIEW_NAME), - getQueue(UpdateSnapshotViewAction.QUEUE)); - - builder.append(String.format(" - %s:%s\n", projectId, jobId)); - logger.atInfo().log("Submitted load job %s:%s", projectId, jobId); - } - return builder.toString(); - } - - private static String getBackupInfoFileForKind(String backupInfoFile, String kindName) { - String extension = ".backup_info"; - checkArgument(backupInfoFile.endsWith(extension), "backup info file extension missing"); - String prefix = backupInfoFile.substring(0, backupInfoFile.length() - extension.length()); - return Joiner.on('.').join(prefix, kindName, extension.substring(1)); - } - - private Job makeLoadJob(JobReference jobRef, String sourceUri, String tableId) { - TableReference tableReference = new TableReference() - .setProjectId(jobRef.getProjectId()) - .setDatasetId(SNAPSHOTS_DATASET) - .setTableId(tableId); - return new Job() - .setJobReference(jobRef) - .setConfiguration(new JobConfiguration() - .setLoad(new JobConfigurationLoad() - .setWriteDisposition(WriteDisposition.WRITE_EMPTY.toString()) - .setSourceFormat(SourceFormat.DATASTORE_BACKUP.toString()) - .setSourceUris(ImmutableList.of(sourceUri)) - .setDestinationTable(tableReference))); - } -} diff --git a/java/google/registry/export/UpdateSnapshotViewAction.java b/java/google/registry/export/UpdateSnapshotViewAction.java index 715991eb0..d3afca967 100644 --- a/java/google/registry/export/UpdateSnapshotViewAction.java +++ b/java/google/registry/export/UpdateSnapshotViewAction.java @@ -50,8 +50,7 @@ public class UpdateSnapshotViewAction implements Runnable { static final String UPDATE_SNAPSHOT_VIEWNAME_PARAM = "viewname"; /** Servlet-specific details needed for enqueuing tasks against itself. */ - // For now this queue is shared by the backup workflows started by both ExportSnapshotAction - // and BackupDatastoreAction. + // For now this queue is shared by the backup workflows started by BackupDatastoreAction. // TODO(weiminyu): update queue name (snapshot->backup) after ExportSnapshot flow is removed. static final String QUEUE = "export-snapshot-update-view"; // See queue.xml. diff --git a/java/google/registry/module/backend/BackendRequestComponent.java b/java/google/registry/module/backend/BackendRequestComponent.java index 47a875e8a..f54e402a6 100644 --- a/java/google/registry/module/backend/BackendRequestComponent.java +++ b/java/google/registry/module/backend/BackendRequestComponent.java @@ -42,13 +42,10 @@ import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule; import google.registry.export.BackupDatastoreAction; import google.registry.export.BigqueryPollJobAction; import google.registry.export.CheckBackupAction; -import google.registry.export.CheckSnapshotAction; import google.registry.export.ExportDomainListsAction; import google.registry.export.ExportPremiumTermsAction; import google.registry.export.ExportRequestModule; import google.registry.export.ExportReservedTermsAction; -import google.registry.export.ExportSnapshotAction; -import google.registry.export.LoadSnapshotAction; import google.registry.export.SyncGroupMembersAction; import google.registry.export.UpdateSnapshotViewAction; import google.registry.export.UploadDatastoreBackupAction; @@ -121,7 +118,6 @@ interface BackendRequestComponent { BigqueryPollJobAction bigqueryPollJobAction(); BrdaCopyAction brdaCopyAction(); CheckBackupAction checkBackupAction(); - CheckSnapshotAction checkSnapshotAction(); CommitLogCheckpointAction commitLogCheckpointAction(); CommitLogFanoutAction commitLogFanoutAction(); CopyDetailReportsAction copyDetailReportAction(); @@ -134,12 +130,10 @@ interface BackendRequestComponent { ExportDomainListsAction exportDomainListsAction(); ExportPremiumTermsAction exportPremiumTermsAction(); ExportReservedTermsAction exportReservedTermsAction(); - ExportSnapshotAction exportSnapshotAction(); GenerateInvoicesAction generateInvoicesAction(); GenerateSpec11ReportAction generateSpec11ReportAction(); IcannReportingStagingAction icannReportingStagingAction(); IcannReportingUploadAction icannReportingUploadAction(); - LoadSnapshotAction loadSnapshotAction(); NordnUploadAction nordnUploadAction(); NordnVerifyAction nordnVerifyAction(); PublishDnsUpdatesAction publishDnsUpdatesAction(); diff --git a/java/google/registry/tools/CheckSnapshotCommand.java b/java/google/registry/tools/CheckSnapshotCommand.java deleted file mode 100644 index 81b11065a..000000000 --- a/java/google/registry/tools/CheckSnapshotCommand.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.collect.Iterables; -import com.google.common.collect.Streams; -import google.registry.export.DatastoreBackupInfo; -import google.registry.export.DatastoreBackupService; -import javax.inject.Inject; - -/** - * Command to check the status of a Datastore backup, or "snapshot". - */ -@Parameters(separators = " =", commandDescription = "Check the status of a Datastore snapshot") -public class CheckSnapshotCommand implements CommandWithRemoteApi { - - @Parameter( - names = {"-s", "--snapshot"}, - description = "Unique prefix of the snapshot to check", - required = true) - private String snapshotName; - - @Inject DatastoreBackupService datastoreBackupService; - - @Override - public void run() { - Iterable backups = - datastoreBackupService.findAllByNamePrefix(snapshotName); - if (Iterables.isEmpty(backups)) { - System.err.println("No snapshot found with name: " + snapshotName); - return; - } - Streams.stream(backups).map(DatastoreBackupInfo::getInformation).forEach(System.out::println); - } -} diff --git a/java/google/registry/tools/RegistryTool.java b/java/google/registry/tools/RegistryTool.java index 6c5322c7e..460d159f6 100644 --- a/java/google/registry/tools/RegistryTool.java +++ b/java/google/registry/tools/RegistryTool.java @@ -32,7 +32,6 @@ public final class RegistryTool { .put("canonicalize_labels", CanonicalizeLabelsCommand.class) .put("check_domain", CheckDomainCommand.class) .put("check_domain_claims", CheckDomainClaimsCommand.class) - .put("check_snapshot", CheckSnapshotCommand.class) .put("convert_idn", ConvertIdnCommand.class) .put("count_domains", CountDomainsCommand.class) .put("create_anchor_tenant", CreateAnchorTenantCommand.class) diff --git a/java/google/registry/tools/RegistryToolComponent.java b/java/google/registry/tools/RegistryToolComponent.java index 08f0acad1..9b4678bfe 100644 --- a/java/google/registry/tools/RegistryToolComponent.java +++ b/java/google/registry/tools/RegistryToolComponent.java @@ -75,7 +75,6 @@ import javax.inject.Singleton; interface RegistryToolComponent { void inject(CheckDomainClaimsCommand command); void inject(CheckDomainCommand command); - void inject(CheckSnapshotCommand command); void inject(CountDomainsCommand command); void inject(CreateAnchorTenantCommand command); void inject(CreateCdnsTld command); diff --git a/javatests/google/registry/export/CheckSnapshotActionTest.java b/javatests/google/registry/export/CheckSnapshotActionTest.java deleted file mode 100644 index e61a2f5e5..000000000 --- a/javatests/google/registry/export/CheckSnapshotActionTest.java +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM; -import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM; -import static google.registry.testing.JUnitBackports.assertThrows; -import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued; -import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import google.registry.request.Action.Method; -import google.registry.request.HttpException.BadRequestException; -import google.registry.request.HttpException.NoContentException; -import google.registry.request.HttpException.NotModifiedException; -import google.registry.testing.AppEngineRule; -import google.registry.testing.FakeClock; -import google.registry.testing.FakeResponse; -import google.registry.testing.InjectRule; -import google.registry.testing.TaskQueueHelper.TaskMatcher; -import java.util.Optional; -import org.joda.time.DateTime; -import org.joda.time.Duration; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link CheckSnapshotAction}. */ -@RunWith(JUnit4.class) -public class CheckSnapshotActionTest { - - static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z"); - static final DateTime COMPLETE_TIME = START_TIME.plus(Duration.standardMinutes(30)); - - @Rule public final InjectRule inject = new InjectRule(); - @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build(); - private final DatastoreBackupService backupService = mock(DatastoreBackupService.class); - - private DatastoreBackupInfo backupInfo; - - private final FakeResponse response = new FakeResponse(); - private final FakeClock clock = new FakeClock(COMPLETE_TIME.plusMillis(1000)); - private final CheckSnapshotAction action = new CheckSnapshotAction(); - - @Before - public void before() { - inject.setStaticField(DatastoreBackupInfo.class, "clock", clock); - action.requestMethod = Method.POST; - action.snapshotName = "some_backup"; - action.kindsToLoadParam = "one,two"; - action.response = response; - action.backupService = backupService; - - backupInfo = - new DatastoreBackupInfo( - "some_backup", - START_TIME, - Optional.of(COMPLETE_TIME), - ImmutableSet.of("one", "two", "three"), - Optional.of("gs://somebucket/some_backup_20140801.backup_info")); - - when(backupService.findByName("some_backup")).thenReturn(backupInfo); - } - - private void setPendingBackup() { - backupInfo = - new DatastoreBackupInfo( - backupInfo.getName(), - backupInfo.getStartTime(), - Optional.empty(), - backupInfo.getKinds(), - backupInfo.getGcsFilename()); - - when(backupService.findByName("some_backup")).thenReturn(backupInfo); - } - - private static void assertLoadTaskEnqueued(String id, String file, String kinds) { - assertTasksEnqueued( - "export-snapshot", - new TaskMatcher() - .url("/_dr/task/loadSnapshot") - .method("POST") - .param("id", id) - .param("file", file) - .param("kinds", kinds)); - } - - @Test - public void testSuccess_enqueuePollTask() { - CheckSnapshotAction.enqueuePollTask( - "some_snapshot_name", ImmutableSet.of("one", "two", "three")); - assertTasksEnqueued( - CheckSnapshotAction.QUEUE, - new TaskMatcher() - .url(CheckSnapshotAction.PATH) - .param(CHECK_SNAPSHOT_NAME_PARAM, "some_snapshot_name") - .param(CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, "one,two,three") - .method("POST")); - } - - @Test - public void testPost_forPendingBackup_returnsNotModified() { - setPendingBackup(); - - NotModifiedException thrown = assertThrows(NotModifiedException.class, action::run); - assertThat(thrown).hasMessageThat().contains("Datastore backup some_backup still pending"); - } - - @Test - public void testPost_forStalePendingBackupBackup_returnsNoContent() { - setPendingBackup(); - - when(backupService.findByName("some_backup")).thenReturn(backupInfo); - - clock.setTo( - START_TIME - .plus(Duration.standardHours(20)) - .plus(Duration.standardMinutes(3)) - .plus(Duration.millis(1234))); - - NoContentException thrown = assertThrows(NoContentException.class, action::run); - assertThat(thrown) - .hasMessageThat() - .contains( - "Datastore backup some_backup abandoned - " - + "not complete after 20 hours, 3 minutes and 1 second"); - } - - @Test - public void testPost_forCompleteBackup_enqueuesLoadTask() { - - action.run(); - assertLoadTaskEnqueued( - "20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one,two"); - } - - @Test - public void testPost_forCompleteAutoBackup_enqueuesLoadTask_usingBackupName() { - action.snapshotName = "auto_snapshot_somestring"; - when(backupService.findByName("auto_snapshot_somestring")).thenReturn(backupInfo); - - action.run(); - assertLoadTaskEnqueued( - "somestring", "gs://somebucket/some_backup_20140801.backup_info", "one,two"); - } - - @Test - public void testPost_forCompleteBackup_withExtraKindsToLoad_enqueuesLoadTask() { - action.kindsToLoadParam = "one,foo"; - - action.run(); - assertLoadTaskEnqueued( - "20140801_010203", "gs://somebucket/some_backup_20140801.backup_info", "one"); - } - - @Test - public void testPost_forCompleteBackup_withEmptyKindsToLoad_skipsLoadTask() { - action.kindsToLoadParam = ""; - - action.run(); - assertNoTasksEnqueued("export-snapshot"); - } - - @Test - public void testPost_forBadBackup_returnsBadRequest() { - when(backupService.findByName("some_backup")) - .thenThrow(new IllegalArgumentException("No backup found")); - - BadRequestException thrown = assertThrows(BadRequestException.class, action::run); - assertThat(thrown).hasMessageThat().contains("Bad backup name some_backup: No backup found"); - } - - @Test - public void testGet_returnsInformation() { - action.requestMethod = Method.GET; - - action.run(); - assertThat(response.getPayload()) - .isEqualTo( - Joiner.on("\n") - .join( - ImmutableList.of( - "Backup name: some_backup", - "Status: COMPLETE", - "Started: 2014-08-01T01:02:03.000Z", - "Ended: 2014-08-01T01:32:03.000Z", - "Duration: 30m", - "GCS: gs://somebucket/some_backup_20140801.backup_info", - "Kinds: [one, two, three]", - ""))); - } - - @Test - public void testGet_forBadBackup_returnsError() { - action.requestMethod = Method.GET; - when(backupService.findByName("some_backup")) - .thenThrow(new IllegalArgumentException("No backup found")); - - BadRequestException thrown = assertThrows(BadRequestException.class, action::run); - assertThat(thrown).hasMessageThat().contains("Bad backup name some_backup: No backup found"); - } -} diff --git a/javatests/google/registry/export/DatastoreBackupInfoTest.java b/javatests/google/registry/export/DatastoreBackupInfoTest.java deleted file mode 100644 index 504d48340..000000000 --- a/javatests/google/registry/export/DatastoreBackupInfoTest.java +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService; -import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth8.assertThat; -import static google.registry.testing.JUnitBackports.assertThrows; - -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityNotFoundException; -import com.google.appengine.api.datastore.Text; -import com.google.common.collect.ImmutableList; -import google.registry.export.DatastoreBackupInfo.BackupStatus; -import google.registry.testing.AppEngineRule; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectRule; -import java.util.Date; -import org.joda.time.DateTime; -import org.joda.time.Duration; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link DatastoreBackupInfo}. */ -@RunWith(JUnit4.class) -public class DatastoreBackupInfoTest { - @Rule - public final InjectRule inject = new InjectRule(); - - @Rule - public final AppEngineRule appEngine = AppEngineRule.builder() - .withDatastore() - .build(); - - private FakeClock clock = new FakeClock(); - - private DateTime startTime = DateTime.parse("2014-08-01T01:02:03Z"); - - private Entity backupEntity; // Can't initialize until AppEngineRule has set up Datastore. - - @Before - public void before() { - inject.setStaticField(DatastoreBackupInfo.class, "clock", clock); - backupEntity = new Entity("_unused_"); - backupEntity.setProperty("name", "backup1"); - backupEntity.setProperty("kinds", ImmutableList.of("one", "two", "three")); - backupEntity.setProperty("start_time", new Date(startTime.getMillis())); - } - - /** Force a roundtrip to Datastore to ensure that we're simulating a freshly fetched entity. */ - private static Entity persistEntity(Entity entity) throws EntityNotFoundException { - return getDatastoreService().get(getDatastoreService().put(entity)); - } - - @Test - public void testSuccess_pendingBackup() throws Exception { - DatastoreBackupInfo backup = new DatastoreBackupInfo(persistEntity(backupEntity)); - - assertThat(backup.getName()).isEqualTo("backup1"); - assertThat(backup.getKinds()).containsExactly("one", "two", "three"); - assertThat(backup.getStartTime()).isEqualTo(startTime); - assertThat(backup.getCompleteTime()).isEmpty(); - assertThat(backup.getGcsFilename()).isEmpty(); - assertThat(backup.getStatus()).isEqualTo(BackupStatus.PENDING); - - clock.setTo(startTime.plusMinutes(1)); - assertThat(backup.getRunningTime()).isEqualTo(Duration.standardMinutes(1)); - clock.setTo(startTime.plusMinutes(1).plusSeconds(42)); - assertThat(backup.getRunningTime()).isEqualTo(Duration.standardSeconds(102)); - } - - @Test - public void testSuccess_completeBackup() throws Exception { - DateTime completeTime = startTime.plusMinutes(1).plusSeconds(42); - backupEntity.setProperty("complete_time", new Date(completeTime.getMillis())); - backupEntity.setProperty("gs_handle", new Text("/gs/somebucket/timestamp.backup_info")); - DatastoreBackupInfo backup = new DatastoreBackupInfo(persistEntity(backupEntity)); - - assertThat(backup.getName()).isEqualTo("backup1"); - assertThat(backup.getKinds()).containsExactly("one", "two", "three"); - assertThat(backup.getStartTime()).isEqualTo(startTime); - assertThat(backup.getCompleteTime().get()).isEqualTo(completeTime); - assertThat(backup.getGcsFilename()).hasValue("gs://somebucket/timestamp.backup_info"); - assertThat(backup.getStatus()).isEqualTo(BackupStatus.COMPLETE); - assertThat(backup.getRunningTime()).isEqualTo(Duration.standardSeconds(102)); - } - - @Test - public void testFailure_missingName() { - backupEntity.removeProperty("name"); - assertThrows( - NullPointerException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity))); - } - - @Test - public void testFailure_missingKinds() { - backupEntity.removeProperty("kinds"); - assertThrows( - NullPointerException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity))); - } - - @Test - public void testFailure_missingStartTime() { - backupEntity.removeProperty("start_time"); - assertThrows( - NullPointerException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity))); - } - - @Test - public void testFailure_badGcsFilenameFormat() { - backupEntity.setProperty("gs_handle", new Text("foo")); - assertThrows( - IllegalArgumentException.class, () -> new DatastoreBackupInfo(persistEntity(backupEntity))); - } -} diff --git a/javatests/google/registry/export/DatastoreBackupServiceTest.java b/javatests/google/registry/export/DatastoreBackupServiceTest.java deleted file mode 100644 index ba49863a2..000000000 --- a/javatests/google/registry/export/DatastoreBackupServiceTest.java +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService; -import static com.google.common.collect.Iterables.transform; -import static com.google.common.truth.Truth.assertThat; -import static google.registry.testing.JUnitBackports.assertThrows; -import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; -import static org.mockito.Mockito.when; - -import com.google.appengine.api.datastore.Entity; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import google.registry.testing.AppEngineRule; -import google.registry.testing.MockitoJUnitRule; -import google.registry.testing.TaskQueueHelper.TaskMatcher; -import google.registry.util.AppEngineServiceUtils; -import java.util.Date; -import org.joda.time.DateTime; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.Mock; - -/** Unit tests for {@link DatastoreBackupService}. */ -@RunWith(JUnit4.class) -public class DatastoreBackupServiceTest { - - @Rule - public final AppEngineRule appEngine = - AppEngineRule.builder().withDatastore().withTaskQueue().build(); - - @Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create(); - - @Mock private AppEngineServiceUtils appEngineServiceUtils; - - private static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z"); - - private DatastoreBackupService backupService; - - @Before - public void before() { - backupService = new DatastoreBackupService(appEngineServiceUtils); - when(appEngineServiceUtils.getVersionHostname("default", "ah-builtin-python-bundle")) - .thenReturn("ah-builtin-python-bundle.default.localhost"); - - persistBackupEntityWithName("backupA1"); - persistBackupEntityWithName("backupA2"); - persistBackupEntityWithName("backupA3"); - persistBackupEntityWithName("backupB1"); - persistBackupEntityWithName("backupB42"); - } - - private static void persistBackupEntityWithName(String name) { - Entity entity = new Entity(DatastoreBackupService.BACKUP_INFO_KIND); - entity.setProperty("name", name); - entity.setProperty("kinds", ImmutableList.of("one", "two", "three")); - entity.setProperty("start_time", new Date(START_TIME.getMillis())); - getDatastoreService().put(entity); - } - - @Test - public void testSuccess_launchBackup() { - backupService.launchNewBackup( - "export-snapshot", "backup1", "somebucket", ImmutableSet.of("foo", "bar")); - assertTasksEnqueued("export-snapshot", - new TaskMatcher() - .url("/_ah/datastore_admin/backup.create") - .header("Host", "ah-builtin-python-bundle.default.localhost") - .method("GET") - .param("name", "backup1_") - .param("filesystem", "gs") - .param("gs_bucket_name", "somebucket") - .param("queue", "export-snapshot") - .param("kind", "foo") - .param("kind", "bar")); - } - - @Test - public void testSuccess_findAllByNamePrefix() { - assertThat( - transform(backupService.findAllByNamePrefix("backupA"), DatastoreBackupInfo::getName)) - .containsExactly("backupA1", "backupA2", "backupA3"); - assertThat( - transform(backupService.findAllByNamePrefix("backupB"), DatastoreBackupInfo::getName)) - .containsExactly("backupB1", "backupB42"); - assertThat( - transform(backupService.findAllByNamePrefix("backupB4"), DatastoreBackupInfo::getName)) - .containsExactly("backupB42"); - assertThat(backupService.findAllByNamePrefix("backupX")).isEmpty(); - } - - @Test - public void testSuccess_findByName() { - assertThat(backupService.findByName("backupA1").getName()).isEqualTo("backupA1"); - assertThat(backupService.findByName("backupB4").getName()).isEqualTo("backupB42"); - } - - @Test - public void testFailure_findByName_multipleMatchingBackups() { - assertThrows(IllegalArgumentException.class, () -> backupService.findByName("backupA")); - } - - @Test - public void testFailure_findByName_noMatchingBackups() { - assertThrows(IllegalArgumentException.class, () -> backupService.findByName("backupX")); - } -} diff --git a/javatests/google/registry/export/ExportSnapshotActionTest.java b/javatests/google/registry/export/ExportSnapshotActionTest.java deleted file mode 100644 index a790d3a9f..000000000 --- a/javatests/google/registry/export/ExportSnapshotActionTest.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM; -import static google.registry.export.CheckSnapshotAction.CHECK_SNAPSHOT_NAME_PARAM; -import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -import com.google.common.base.Joiner; -import google.registry.testing.AppEngineRule; -import google.registry.testing.FakeClock; -import google.registry.testing.FakeResponse; -import google.registry.testing.TaskQueueHelper.TaskMatcher; -import org.joda.time.DateTime; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link ExportSnapshotAction}. */ -@RunWith(JUnit4.class) -public class ExportSnapshotActionTest { - - @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build(); - - private final DatastoreBackupService backupService = mock(DatastoreBackupService.class); - - private final FakeResponse response = new FakeResponse(); - private final FakeClock clock = new FakeClock(DateTime.parse("2014-08-01T01:02:03Z")); - private final ExportSnapshotAction action = new ExportSnapshotAction(); - - @Before - public void before() { - action.clock = clock; - action.backupService = backupService; - action.response = response; - } - - @Test - public void testPost_launchesBackup_andEnqueuesPollTask() { - action.run(); - verify(backupService) - .launchNewBackup( - ExportSnapshotAction.QUEUE, - "auto_snapshot_20140801_010203", - "registry-project-id-snapshots", - ExportConstants.getBackupKinds()); - assertTasksEnqueued( - CheckSnapshotAction.QUEUE, - new TaskMatcher() - .url(CheckSnapshotAction.PATH) - .param(CHECK_SNAPSHOT_NAME_PARAM, "auto_snapshot_20140801_010203") - .param( - CHECK_SNAPSHOT_KINDS_TO_LOAD_PARAM, - Joiner.on(",").join(ExportConstants.getReportingKinds())) - .method("POST")); - assertThat(response.getPayload()) - .isEqualTo("Datastore backup started with name: auto_snapshot_20140801_010203"); - } -} diff --git a/javatests/google/registry/export/LoadSnapshotActionTest.java b/javatests/google/registry/export/LoadSnapshotActionTest.java deleted file mode 100644 index 5e081df03..000000000 --- a/javatests/google/registry/export/LoadSnapshotActionTest.java +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.export; - -import static com.google.common.collect.Iterables.transform; -import static com.google.common.truth.Truth.assertThat; -import static google.registry.export.LoadSnapshotAction.LATEST_SNAPSHOT_VIEW_NAME; -import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM; -import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM; -import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM; -import static google.registry.export.LoadSnapshotAction.PATH; -import static google.registry.export.LoadSnapshotAction.QUEUE; -import static google.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask; -import static google.registry.testing.JUnitBackports.assertThrows; -import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; -import static org.joda.time.DateTimeZone.UTC; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.model.Dataset; -import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.JobConfigurationLoad; -import com.google.api.services.bigquery.model.JobReference; -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import google.registry.bigquery.CheckedBigquery; -import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer; -import google.registry.request.HttpException.BadRequestException; -import google.registry.request.HttpException.InternalServerErrorException; -import google.registry.testing.AppEngineRule; -import google.registry.testing.FakeClock; -import google.registry.testing.TaskQueueHelper.TaskMatcher; -import java.io.IOException; -import java.util.List; -import org.joda.time.DateTime; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; - -/** Unit tests for {@link LoadSnapshotAction}. */ -@RunWith(JUnit4.class) -public class LoadSnapshotActionTest { - - @Rule - public final AppEngineRule appEngine = AppEngineRule.builder() - .withTaskQueue() - .build(); - private final CheckedBigquery checkedBigquery = mock(CheckedBigquery.class); - private final Bigquery bigquery = mock(Bigquery.class); - private final Bigquery.Jobs bigqueryJobs = mock(Bigquery.Jobs.class); - private final Bigquery.Jobs.Insert bigqueryJobsInsert = mock(Bigquery.Jobs.Insert.class); - private final Bigquery.Datasets bigqueryDatasets = mock(Bigquery.Datasets.class); - private final Bigquery.Datasets.Insert bigqueryDatasetsInsert = - mock(Bigquery.Datasets.Insert.class); - private final BigqueryPollJobEnqueuer bigqueryPollEnqueuer = mock(BigqueryPollJobEnqueuer.class); - - private FakeClock clock = new FakeClock(new DateTime(1391096117045L, UTC)); - private LoadSnapshotAction action; - - @Before - public void before() throws Exception { - when(checkedBigquery.ensureDataSetExists("Project-Id", "snapshots")).thenReturn(bigquery); - when(bigquery.jobs()).thenReturn(bigqueryJobs); - when(bigqueryJobs.insert(eq("Project-Id"), any(Job.class))).thenReturn(bigqueryJobsInsert); - when(bigquery.datasets()).thenReturn(bigqueryDatasets); - when(bigqueryDatasets.insert(eq("Project-Id"), any(Dataset.class))) - .thenReturn(bigqueryDatasetsInsert); - action = new LoadSnapshotAction(); - action.checkedBigquery = checkedBigquery; - action.bigqueryPollEnqueuer = bigqueryPollEnqueuer; - action.clock = clock; - action.projectId = "Project-Id"; - action.snapshotFile = "gs://bucket/snapshot.backup_info"; - action.snapshotId = "id12345"; - action.snapshotKinds = "one,two,three"; - } - - @Test - public void testSuccess_enqueueLoadTask() { - enqueueLoadSnapshotTask( - "id12345", "gs://bucket/snapshot.backup_info", ImmutableSet.of("one", "two", "three")); - assertTasksEnqueued( - QUEUE, - new TaskMatcher() - .url(PATH) - .method("POST") - .param(LOAD_SNAPSHOT_ID_PARAM, "id12345") - .param(LOAD_SNAPSHOT_FILE_PARAM, "gs://bucket/snapshot.backup_info") - .param(LOAD_SNAPSHOT_KINDS_PARAM, "one,two,three")); - } - - @Test - public void testSuccess_doPost() throws Exception { - action.run(); - - // Verify that checkedBigquery was called in a way that would create the dataset if it didn't - // already exist. - verify(checkedBigquery).ensureDataSetExists("Project-Id", "snapshots"); - - // Capture the load jobs we inserted to do additional checking on them. - ArgumentCaptor jobArgument = ArgumentCaptor.forClass(Job.class); - verify(bigqueryJobs, times(3)).insert(eq("Project-Id"), jobArgument.capture()); - List jobs = jobArgument.getAllValues(); - assertThat(jobs).hasSize(3); - - // Check properties that should be common to all load jobs. - for (Job job : jobs) { - assertThat(job.getJobReference().getProjectId()).isEqualTo("Project-Id"); - JobConfigurationLoad config = job.getConfiguration().getLoad(); - assertThat(config.getSourceFormat()).isEqualTo("DATASTORE_BACKUP"); - assertThat(config.getDestinationTable().getProjectId()).isEqualTo("Project-Id"); - assertThat(config.getDestinationTable().getDatasetId()).isEqualTo("snapshots"); - } - - // Check the job IDs for each load job. - assertThat(transform(jobs, job -> job.getJobReference().getJobId())) - .containsExactly( - "load-snapshot-id12345-one-1391096117045", - "load-snapshot-id12345-two-1391096117045", - "load-snapshot-id12345-three-1391096117045"); - - // Check the source URI for each load job. - assertThat( - transform( - jobs, - job -> Iterables.getOnlyElement(job.getConfiguration().getLoad().getSourceUris()))) - .containsExactly( - "gs://bucket/snapshot.one.backup_info", - "gs://bucket/snapshot.two.backup_info", - "gs://bucket/snapshot.three.backup_info"); - - // Check the destination table ID for each load job. - assertThat( - transform( - jobs, job -> job.getConfiguration().getLoad().getDestinationTable().getTableId())) - .containsExactly("id12345_one", "id12345_two", "id12345_three"); - - // Check that we executed the inserted jobs. - verify(bigqueryJobsInsert, times(3)).execute(); - - // Check that the poll tasks for each load job were enqueued. - verify(bigqueryPollEnqueuer) - .enqueuePollTask( - new JobReference() - .setProjectId("Project-Id") - .setJobId("load-snapshot-id12345-one-1391096117045"), - UpdateSnapshotViewAction.createViewUpdateTask( - "snapshots", "id12345_one", "one", LATEST_SNAPSHOT_VIEW_NAME), - QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE)); - verify(bigqueryPollEnqueuer) - .enqueuePollTask( - new JobReference() - .setProjectId("Project-Id") - .setJobId("load-snapshot-id12345-two-1391096117045"), - UpdateSnapshotViewAction.createViewUpdateTask( - "snapshots", "id12345_two", "two", LATEST_SNAPSHOT_VIEW_NAME), - QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE)); - verify(bigqueryPollEnqueuer) - .enqueuePollTask( - new JobReference() - .setProjectId("Project-Id") - .setJobId("load-snapshot-id12345-three-1391096117045"), - UpdateSnapshotViewAction.createViewUpdateTask( - "snapshots", "id12345_three", "three", LATEST_SNAPSHOT_VIEW_NAME), - QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE)); - } - - @Test - public void testFailure_doPost_badGcsFilename() { - action.snapshotFile = "gs://bucket/snapshot"; - BadRequestException thrown = assertThrows(BadRequestException.class, action::run); - assertThat(thrown) - .hasMessageThat() - .contains("Error calling load snapshot: backup info file extension missing"); - } - - @Test - public void testFailure_doPost_bigqueryThrowsException() throws Exception { - when(bigqueryJobsInsert.execute()).thenThrow(new IOException("The Internet has gone missing")); - InternalServerErrorException thrown = - assertThrows(InternalServerErrorException.class, action::run); - assertThat(thrown) - .hasMessageThat() - .contains("Error loading snapshot: The Internet has gone missing"); - } -} diff --git a/javatests/google/registry/module/backend/testdata/backend_routing.txt b/javatests/google/registry/module/backend/testdata/backend_routing.txt index b2a9ecef1..c27f0b630 100644 --- a/javatests/google/registry/module/backend/testdata/backend_routing.txt +++ b/javatests/google/registry/module/backend/testdata/backend_routing.txt @@ -7,7 +7,6 @@ PATH CLASS METHOD /_dr/task/backupDatastore BackupDatastoreAction POST y INTERNAL APP IGNORED /_dr/task/brdaCopy BrdaCopyAction POST y INTERNAL APP IGNORED /_dr/task/checkDatastoreBackup CheckBackupAction POST,GET y INTERNAL APP IGNORED -/_dr/task/checkSnapshot CheckSnapshotAction POST,GET y INTERNAL APP IGNORED /_dr/task/copyDetailReports CopyDetailReportsAction POST n INTERNAL,API APP ADMIN /_dr/task/deleteContactsAndHosts DeleteContactsAndHostsAction GET n INTERNAL APP IGNORED /_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n INTERNAL APP IGNORED @@ -18,7 +17,6 @@ PATH CLASS METHOD /_dr/task/exportDomainLists ExportDomainListsAction POST n INTERNAL APP IGNORED /_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n INTERNAL APP IGNORED /_dr/task/exportReservedTerms ExportReservedTermsAction POST n INTERNAL APP IGNORED -/_dr/task/exportSnapshot ExportSnapshotAction POST y INTERNAL APP IGNORED /_dr/task/generateInvoices GenerateInvoicesAction POST n INTERNAL APP IGNORED /_dr/task/generateSpec11 GenerateSpec11ReportAction POST n INTERNAL APP IGNORED /_dr/task/icannReportingStaging IcannReportingStagingAction POST n INTERNAL APP IGNORED @@ -27,7 +25,6 @@ PATH CLASS METHOD /_dr/task/importRdeDomains RdeDomainImportAction GET n INTERNAL APP IGNORED /_dr/task/importRdeHosts RdeHostImportAction GET n INTERNAL APP IGNORED /_dr/task/linkRdeHosts RdeHostLinkAction GET n INTERNAL APP IGNORED -/_dr/task/loadSnapshot LoadSnapshotAction POST n INTERNAL APP IGNORED /_dr/task/nordnUpload NordnUploadAction POST y INTERNAL APP IGNORED /_dr/task/nordnVerify NordnVerifyAction POST y INTERNAL APP IGNORED /_dr/task/pollBigqueryJob BigqueryPollJobAction GET,POST y INTERNAL APP IGNORED