diff --git a/docs/app-engine-architecture.md b/docs/app-engine-architecture.md index 2d8b028bb..7ea9140e0 100644 --- a/docs/app-engine-architecture.md +++ b/docs/app-engine-architecture.md @@ -138,9 +138,6 @@ explicitly marked as otherwise. * `export-commits` -- Queue for tasks to export commit log checkpoints. Tasks are enqueued by `CommitLogCheckpointAction` (which is run every minute by cron) and executed by `ExportCommitLogDiffAction`. -* `export-reserved-terms` -- Cron queue for tasks to export the list of - reserved terms for each TLD. The tasks are executed by - `ExportReservedTermsAction`. * `export-snapshot` -- Cron and push queue for tasks to load a Datastore snapshot that was stored in Google Cloud Storage and export it to BigQuery. Tasks are enqueued by both cron and `CheckSnapshotServlet` and are executed @@ -179,6 +176,8 @@ explicitly marked as otherwise. * `rde-upload` -- Cron queue for tasks to upload already-generated RDE files from Cloud Storage to the escrow provider. Tasks are executed by `RdeUploadAction`. +* `retryable-cron-tasks` -- Catch-all cron queue for various cron tasks that + run infrequently, such as exporting reserved terms. * `sheet` -- Queue for tasks to sync registrar updates to a Google Sheets spreadsheet. Tasks are enqueued by `RegistrarServlet` when changes are made to registrar fields and are executed by `SyncRegistrarsSheetAction`. diff --git a/java/google/registry/env/alpha/default/WEB-INF/cron.xml b/java/google/registry/env/alpha/default/WEB-INF/cron.xml index 3c55d462a..4f4eb8d3c 100644 --- a/java/google/registry/env/alpha/default/WEB-INF/cron.xml +++ b/java/google/registry/env/alpha/default/WEB-INF/cron.xml @@ -96,7 +96,7 @@ - + Reserved terms export to Google Drive job for creating once-daily exports. diff --git a/java/google/registry/env/common/default/WEB-INF/queue.xml b/java/google/registry/env/common/default/WEB-INF/queue.xml index 34ef32cab..88fb6c1cc 100644 --- a/java/google/registry/env/common/default/WEB-INF/queue.xml +++ b/java/google/registry/env/common/default/WEB-INF/queue.xml @@ -1,12 +1,6 @@ - - default - 1/s - 5 - - dns-pull pull @@ -51,16 +45,6 @@ - - - export-reserved-terms - 1/s - 100 - - 3 - - - export-bigquery-poll @@ -213,12 +197,13 @@ retryable-cron-tasks 1/s - 100 3 + load0 500/s @@ -278,5 +263,4 @@ 500/s 500 - diff --git a/java/google/registry/env/crash/default/WEB-INF/cron.xml b/java/google/registry/env/crash/default/WEB-INF/cron.xml index c0e2ecec1..23887f73a 100644 --- a/java/google/registry/env/crash/default/WEB-INF/cron.xml +++ b/java/google/registry/env/crash/default/WEB-INF/cron.xml @@ -85,7 +85,7 @@ - + Reserved terms export to Google Drive job for creating once-daily exports. diff --git a/java/google/registry/env/production/default/WEB-INF/cron.xml b/java/google/registry/env/production/default/WEB-INF/cron.xml index a0a31ec0b..4b64bbc2f 100644 --- a/java/google/registry/env/production/default/WEB-INF/cron.xml +++ b/java/google/registry/env/production/default/WEB-INF/cron.xml @@ -202,7 +202,7 @@ - + Reserved terms export to Google Drive job for creating once-daily exports. diff --git a/java/google/registry/env/sandbox/default/WEB-INF/cron.xml b/java/google/registry/env/sandbox/default/WEB-INF/cron.xml index b33831513..f15851fb8 100644 --- a/java/google/registry/env/sandbox/default/WEB-INF/cron.xml +++ b/java/google/registry/env/sandbox/default/WEB-INF/cron.xml @@ -75,7 +75,7 @@ - + Reserved terms export to Google Drive job for creating once-daily exports. diff --git a/javatests/google/registry/export/BigqueryPollJobActionTest.java b/javatests/google/registry/export/BigqueryPollJobActionTest.java index d53128b8a..e928a1483 100644 --- a/javatests/google/registry/export/BigqueryPollJobActionTest.java +++ b/javatests/google/registry/export/BigqueryPollJobActionTest.java @@ -14,10 +14,12 @@ package google.registry.export; +import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assert_; import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.logging.Level.INFO; import static java.util.logging.Level.SEVERE; import static org.mockito.Mockito.when; @@ -27,7 +29,6 @@ import com.google.api.services.bigquery.model.ErrorProto; import com.google.api.services.bigquery.model.Job; import com.google.api.services.bigquery.model.JobReference; import com.google.api.services.bigquery.model.JobStatus; -import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.api.taskqueue.TaskOptions; import com.google.appengine.api.taskqueue.TaskOptions.Method; import com.google.appengine.api.taskqueue.dev.QueueStateInfo.TaskStateInfo; @@ -78,7 +79,7 @@ public class BigqueryPollJobActionTest { static final String PROJECT_ID = "project_id"; static final String JOB_ID = "job_id"; - static final String CHAINED_QUEUE_NAME = "default"; + static final String CHAINED_QUEUE_NAME = UpdateSnapshotViewAction.QUEUE; static final TaskEnqueuer ENQUEUER = new TaskEnqueuer(new Retrier(new FakeSleeper(new FakeClock()), 1)); @@ -127,7 +128,7 @@ public class BigqueryPollJobActionTest { new BigqueryPollJobEnqueuer(ENQUEUER).enqueuePollTask( new JobReference().setProjectId(PROJECT_ID).setJobId(JOB_ID), chainedTask, - QueueFactory.getQueue(CHAINED_QUEUE_NAME)); + getQueue(CHAINED_QUEUE_NAME)); assertTasksEnqueued(BigqueryPollJobAction.QUEUE, newPollJobTaskMatcher("POST")); TaskStateInfo taskInfo = getOnlyElement( TaskQueueHelper.getQueueInfo(BigqueryPollJobAction.QUEUE).getTaskInfo()); @@ -174,7 +175,7 @@ public class BigqueryPollJobActionTest { String.format("Bigquery job succeeded - %s:%s", PROJECT_ID, JOB_ID)); assertLogMessage( INFO, - "Added chained task my_task_name for /_dr/something to queue default"); + "Added chained task my_task_name for /_dr/something to queue " + CHAINED_QUEUE_NAME); assertTasksEnqueued(CHAINED_QUEUE_NAME, new TaskMatcher() .url("/_dr/something") .method("POST") @@ -213,7 +214,7 @@ public class BigqueryPollJobActionTest { public void testFailure_badChainedTaskPayload() throws Exception { when(bigqueryJobsGet.execute()).thenReturn( new Job().setStatus(new JobStatus().setState("DONE"))); - action.payload = "payload".getBytes(); + action.payload = "payload".getBytes(UTF_8); thrown.expect(BadRequestException.class, "Cannot deserialize task from payload"); action.run(); } diff --git a/javatests/google/registry/export/DatastoreBackupServiceTest.java b/javatests/google/registry/export/DatastoreBackupServiceTest.java index 2597b7c09..cf49c5ad9 100644 --- a/javatests/google/registry/export/DatastoreBackupServiceTest.java +++ b/javatests/google/registry/export/DatastoreBackupServiceTest.java @@ -51,6 +51,7 @@ public class DatastoreBackupServiceTest { @Rule public final AppEngineRule appEngine = AppEngineRule.builder() .withDatastore() + .withTaskQueue() .build(); @Mock @@ -84,8 +85,8 @@ public class DatastoreBackupServiceTest { @Test public void testSuccess_launchBackup() throws Exception { backupService.launchNewBackup( - "default", "backup1", "somebucket", ImmutableSet.of("foo", "bar")); - assertTasksEnqueued("default", + "export-snapshot", "backup1", "somebucket", ImmutableSet.of("foo", "bar")); + assertTasksEnqueued("export-snapshot", new TaskMatcher() .url("/_ah/datastore_admin/backup.create") .header("Host", "ah-builtin-python-bundle.default.localhost") @@ -93,7 +94,7 @@ public class DatastoreBackupServiceTest { .param("name", "backup1_") .param("filesystem", "gs") .param("gs_bucket_name", "somebucket") - .param("queue", "default") + .param("queue", "export-snapshot") .param("kind", "foo") .param("kind", "bar")); } diff --git a/javatests/google/registry/export/UpdateSnapshotViewActionTest.java b/javatests/google/registry/export/UpdateSnapshotViewActionTest.java index 2d0404d9c..06d83d11d 100644 --- a/javatests/google/registry/export/UpdateSnapshotViewActionTest.java +++ b/javatests/google/registry/export/UpdateSnapshotViewActionTest.java @@ -14,10 +14,13 @@ package google.registry.export; +import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; import static com.google.common.truth.Truth.assertThat; +import static google.registry.export.UpdateSnapshotViewAction.QUEUE; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM; +import static google.registry.export.UpdateSnapshotViewAction.createViewUpdateTask; import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; @@ -28,7 +31,6 @@ import static org.mockito.Mockito.when; import com.google.api.services.bigquery.Bigquery; import com.google.api.services.bigquery.model.Dataset; import com.google.api.services.bigquery.model.Table; -import com.google.appengine.api.taskqueue.QueueFactory; import google.registry.bigquery.BigqueryFactory; import google.registry.request.HttpException.InternalServerErrorException; import google.registry.testing.AppEngineRule; @@ -95,9 +97,8 @@ public class UpdateSnapshotViewActionTest { @Test public void testSuccess_createViewUpdateTask() throws Exception { - QueueFactory.getDefaultQueue().add( - UpdateSnapshotViewAction.createViewUpdateTask("some_dataset", "12345_fookind", "fookind")); - assertTasksEnqueued("default", + getQueue(QUEUE).add(createViewUpdateTask("some_dataset", "12345_fookind", "fookind")); + assertTasksEnqueued(QUEUE, new TaskMatcher() .url(UpdateSnapshotViewAction.PATH) .method("POST") diff --git a/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java b/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java index 46f89fcf5..468afa19f 100644 --- a/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java +++ b/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java @@ -158,10 +158,6 @@ public abstract class MapreduceTestCase extends ShardableTestCase { } } - protected void executeTasksUntilEmpty() throws Exception { - executeTasksUntilEmpty("default"); - } - protected void executeTasksUntilEmpty(String queueName) throws Exception { executeTasksUntilEmpty(queueName, null); }