diff --git a/docs/app-engine-architecture.md b/docs/app-engine-architecture.md index ce54e7a67..dc287733a 100644 --- a/docs/app-engine-architecture.md +++ b/docs/app-engine-architecture.md @@ -111,6 +111,11 @@ specification for the cron task. Here are the task queues in use by the system. All are push queues unless explicitly marked as otherwise. +* `async-delete-pull` and `async-host-rename-pull` -- Pull queues for tasks to + asynchronously delete contacts/hosts and to asynchronously refresh DNS + for renamed hosts, respectively. Tasks are enqueued during EPP + flows and then handled in batches by the regularly running cron tasks + `DeleteContactsAndHostsAction` and `RefreshDnsOnHostRenameAction`. * `bigquery-streaming-metrics` -- Queue for metrics that are asynchronously streamed to BigQuery in the `Metrics` class. Tasks are enqueued during EPP flows in `EppController`. This means that there is a lag of a few seconds to @@ -151,10 +156,6 @@ explicitly marked as otherwise. * `export-snapshot-update-view` -- Queue for tasks to update the BigQuery views to point to the most recently uploaded snapshot. Tasks are enqueued by `LoadSnapshotAction` and executed by `UpdateSnapshotViewAction`. -* `flows-async` -- Queue for asynchronous tasks that are enqueued during EPP - command flows. Currently all of these tasks correspond to invocations of any - of the following three MapReduces: `DnsRefreshForHostRenameAction`, - `DeleteHostResourceAction`, or `DeleteContactResourceAction`. * `group-members-sync` -- Cron queue for tasks to sync registrar contacts (not domain contacts!) to Google Groups. Tasks are executed by `SyncGroupMembersAction`. diff --git a/java/google/registry/env/common/default/WEB-INF/queue.xml b/java/google/registry/env/common/default/WEB-INF/queue.xml index d4286cfd4..34ef32cab 100644 --- a/java/google/registry/env/common/default/WEB-INF/queue.xml +++ b/java/google/registry/env/common/default/WEB-INF/queue.xml @@ -12,14 +12,6 @@ pull - - - - flows-async - 1/m - backend - - dns-publish 100/s diff --git a/java/google/registry/flows/async/AsyncFlowEnqueuer.java b/java/google/registry/flows/async/AsyncFlowEnqueuer.java index fb314572d..577d1d03e 100644 --- a/java/google/registry/flows/async/AsyncFlowEnqueuer.java +++ b/java/google/registry/flows/async/AsyncFlowEnqueuer.java @@ -17,7 +17,7 @@ package google.registry.flows.async; import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_IS_SUPERUSER; import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_REQUESTING_CLIENT_ID; import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_RESOURCE_KEY; -import static google.registry.flows.async.DnsRefreshForHostRenameAction.PARAM_HOST_KEY; +import static google.registry.flows.async.RefreshDnsOnHostRenameAction.PARAM_HOST_KEY; import static google.registry.flows.async.RefreshDnsOnHostRenameAction.QUEUE_ASYNC_HOST_RENAME; import com.google.appengine.api.taskqueue.Queue; diff --git a/java/google/registry/flows/async/AsyncFlowsModule.java b/java/google/registry/flows/async/AsyncFlowsModule.java index 3d402bd21..18e70a216 100644 --- a/java/google/registry/flows/async/AsyncFlowsModule.java +++ b/java/google/registry/flows/async/AsyncFlowsModule.java @@ -15,17 +15,13 @@ package google.registry.flows.async; import static google.registry.flows.async.DeleteContactsAndHostsAction.QUEUE_ASYNC_DELETE; -import static google.registry.flows.async.DnsRefreshForHostRenameAction.PARAM_HOST_KEY; import static google.registry.flows.async.RefreshDnsOnHostRenameAction.QUEUE_ASYNC_HOST_RENAME; -import static google.registry.request.RequestParameters.extractRequiredParameter; import com.google.appengine.api.taskqueue.Queue; import com.google.appengine.api.taskqueue.QueueFactory; import dagger.Module; import dagger.Provides; -import google.registry.request.Parameter; import javax.inject.Named; -import javax.servlet.http.HttpServletRequest; /** Dagger module for the async flows package. */ @Module @@ -42,11 +38,4 @@ public final class AsyncFlowsModule { static Queue provideAsyncHostRenamePullQueue() { return QueueFactory.getQueue(QUEUE_ASYNC_HOST_RENAME); } - - //TODO(b/26140521): Delete this method once non-batched DNS host refresh mapreduce is deleted. - @Provides - @Parameter(PARAM_HOST_KEY) - static String provideHostKey(HttpServletRequest req) { - return extractRequiredParameter(req, PARAM_HOST_KEY); - } } diff --git a/java/google/registry/flows/async/DnsRefreshForHostRenameAction.java b/java/google/registry/flows/async/DnsRefreshForHostRenameAction.java deleted file mode 100644 index 9c54c2b1b..000000000 --- a/java/google/registry/flows/async/DnsRefreshForHostRenameAction.java +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2016 The Domain Registry Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.flows.async; - -import static google.registry.model.EppResourceUtils.isActive; -import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.util.PipelineUtils.createJobPath; -import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; - -import com.google.appengine.tools.mapreduce.Mapper; -import com.google.common.collect.ImmutableList; -import com.googlecode.objectify.Key; -import google.registry.dns.DnsQueue; -import google.registry.mapreduce.MapreduceRunner; -import google.registry.mapreduce.inputs.EppResourceInputs; -import google.registry.model.domain.DomainResource; -import google.registry.model.host.HostResource; -import google.registry.request.Action; -import google.registry.request.HttpException.BadRequestException; -import google.registry.request.Parameter; -import google.registry.request.Response; -import google.registry.util.FormattingLogger; -import google.registry.util.NonFinalForTesting; -import javax.inject.Inject; -import org.joda.time.DateTime; - -/** - * Enqueues DNS refreshes for applicable domains following a host rename. - */ -//TODO(b/26140521): Delete this once non-batched DNS host refresh mapreduce is deleted. -@Action(path = "/_dr/task/dnsRefreshForHostRename") -public class DnsRefreshForHostRenameAction implements Runnable { - - /** The HTTP parameter name used to specify the websafe key of the host to rename. */ - public static final String PARAM_HOST_KEY = "hostKey"; - - private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); - - @NonFinalForTesting - static DnsQueue dnsQueue = DnsQueue.create(); - - @Inject @Parameter(PARAM_HOST_KEY) String hostKeyString; - @Inject MapreduceRunner mrRunner; - @Inject Response response; - @Inject DnsRefreshForHostRenameAction() {} - - @Override - public void run() { - Key resourceKey = null; - HostResource host; - try { - resourceKey = Key.create(hostKeyString); - host = checkArgumentNotNull(ofy().load().key(resourceKey).now()); - } catch (IllegalArgumentException e) { - throw new BadRequestException(resourceKey == null - ? "Could not parse key string: " + hostKeyString - : "Could not load resource for key: " + resourceKey); - } - response.sendJavaScriptRedirect(createJobPath(mrRunner - .setJobName("Enqueue DNS refreshes for domains following a host rename") - .setModuleName("backend") - .runMapOnly( - new DnsRefreshForHostRenameMapper(host), - ImmutableList.of(EppResourceInputs.createEntityInput(DomainResource.class))))); - } - - /** Map over domains and refresh the dns of those that referenced this host. */ - public static class DnsRefreshForHostRenameMapper extends Mapper { - - private static final long serialVersionUID = -4707015136971008447L; - - private final DateTime hostUpdateTime; - private final Key targetHostKey; - - DnsRefreshForHostRenameMapper(HostResource host) { - this.targetHostKey = Key.create(host); - this.hostUpdateTime = host.getUpdateAutoTimestamp().getTimestamp(); - } - - @Override - public final void map(DomainResource domain) { - if (isActive(domain, hostUpdateTime) - && domain.getNameservers().contains(targetHostKey)) { - try { - dnsQueue.addDomainRefreshTask(domain.getFullyQualifiedDomainName()); - logger.infofmt("Enqueued refresh for domain %s", domain.getFullyQualifiedDomainName()); - } catch (Throwable t) { - logger.severefmt(t, "Error while refreshing DNS for host rename %s", targetHostKey); - } - } - } - } -} diff --git a/java/google/registry/module/backend/BackendRequestComponent.java b/java/google/registry/module/backend/BackendRequestComponent.java index f035725fa..d73d68d21 100644 --- a/java/google/registry/module/backend/BackendRequestComponent.java +++ b/java/google/registry/module/backend/BackendRequestComponent.java @@ -43,7 +43,6 @@ import google.registry.export.sheet.SheetModule; import google.registry.export.sheet.SyncRegistrarsSheetAction; import google.registry.flows.async.AsyncFlowsModule; import google.registry.flows.async.DeleteContactsAndHostsAction; -import google.registry.flows.async.DnsRefreshForHostRenameAction; import google.registry.flows.async.RefreshDnsOnHostRenameAction; import google.registry.mapreduce.MapreduceModule; import google.registry.monitoring.whitebox.MetricsExportAction; @@ -93,7 +92,6 @@ interface BackendRequestComponent { DeleteContactsAndHostsAction deleteContactsAndHostsAction(); DeleteOldCommitLogsAction deleteOldCommitLogsAction(); DeleteProberDataAction deleteProberDataAction(); - DnsRefreshForHostRenameAction dnsRefreshForHostRenameAction(); ExpandRecurringBillingEventsAction expandRecurringBillingEventsAction(); ExportCommitLogDiffAction exportCommitLogDiffAction(); ExportDomainListsAction exportDomainListsAction(); diff --git a/javatests/google/registry/flows/async/DnsRefreshForHostRenameActionTest.java b/javatests/google/registry/flows/async/DnsRefreshForHostRenameActionTest.java deleted file mode 100644 index be03b203d..000000000 --- a/javatests/google/registry/flows/async/DnsRefreshForHostRenameActionTest.java +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2016 The Domain Registry Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.flows.async; - -import static google.registry.testing.DatastoreHelper.createTld; -import static google.registry.testing.DatastoreHelper.newDomainApplication; -import static google.registry.testing.DatastoreHelper.newDomainResource; -import static google.registry.testing.DatastoreHelper.newHostResource; -import static google.registry.testing.DatastoreHelper.persistActiveHost; -import static google.registry.testing.DatastoreHelper.persistResource; -import static google.registry.util.DateTimeUtils.START_OF_TIME; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyZeroInteractions; - -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.dns.DnsQueue; -import google.registry.model.host.HostResource; -import google.registry.request.HttpException.BadRequestException; -import google.registry.testing.ExceptionRule; -import google.registry.testing.FakeResponse; -import google.registry.testing.InjectRule; -import google.registry.testing.mapreduce.MapreduceTestCase; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link DnsRefreshForHostRenameAction}. */ -@RunWith(JUnit4.class) -public class DnsRefreshForHostRenameActionTest - extends MapreduceTestCase { - - @Rule - public final ExceptionRule thrown = new ExceptionRule(); - - @Rule - public InjectRule inject = new InjectRule(); - - private final DnsQueue dnsQueue = mock(DnsQueue.class); - - @Before - public void setup() throws Exception { - inject.setStaticField(DnsRefreshForHostRenameAction.class, "dnsQueue", dnsQueue); - } - - private void runMapreduce(String hostKeyString) throws Exception { - action = new DnsRefreshForHostRenameAction(); - action.hostKeyString = hostKeyString; - action.mrRunner = makeDefaultRunner(); - action.response = new FakeResponse(); - action.run(); - executeTasksUntilEmpty("mapreduce"); - } - - @Test - public void testSuccess_dnsUpdateEnqueued() throws Exception { - createTld("tld"); - Key renamedHostKey = Key.create(persistActiveHost("ns1.example.tld")); - Key otherHostKey = Key.create(persistActiveHost("ns2.example.tld")); - persistResource(newDomainApplication("notadomain.tld").asBuilder() - .setNameservers(ImmutableSet.of(renamedHostKey)) - .build()); - persistResource(newDomainResource("example.tld").asBuilder() - .setNameservers(ImmutableSet.of(renamedHostKey)) - .build()); - persistResource(newDomainResource("otherexample.tld").asBuilder() - .setNameservers(ImmutableSet.of(renamedHostKey)) - .build()); - persistResource(newDomainResource("untouched.tld").asBuilder() - .setNameservers(ImmutableSet.of(otherHostKey)) - .build()); - runMapreduce(renamedHostKey.getString()); - verify(dnsQueue).addDomainRefreshTask("example.tld"); - verify(dnsQueue).addDomainRefreshTask("otherexample.tld"); - verifyNoMoreInteractions(dnsQueue); - } - - @Test - public void testSuccess_noDnsTasksForDeletedDomain() throws Exception { - createTld("tld"); - Key renamedHostKey = Key.create(persistActiveHost("ns1.example.tld")); - persistResource(newDomainResource("example.tld").asBuilder() - .setNameservers(ImmutableSet.of(renamedHostKey)) - .setDeletionTime(START_OF_TIME) - .build()); - runMapreduce(renamedHostKey.getString()); - verifyZeroInteractions(dnsQueue); - } - - @Test - public void testFailure_badKeyPassed() throws Exception { - createTld("tld"); - thrown.expect(BadRequestException.class, "Could not parse key string: a bad key"); - runMapreduce("a bad key"); - } - - @Test - public void testFailure_hostDoesntExist() throws Exception { - createTld("tld"); - HostResource notPersistedHost = newHostResource("ns1.example.tld"); - thrown.expect( - BadRequestException.class, - "Could not load resource for key: Key(HostResource(\"2-ROID\"))"); - runMapreduce(Key.create(notPersistedHost).getString()); - } -}