Cut over to batched DNS refreshing on host renames

TESTED=I deployed it on alpha, renamed some hosts, and verified that
the [] ran as expected.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=134991941
This commit is contained in:
mcilwain 2016-10-03 09:29:19 -07:00 committed by Ben McIlwain
parent 3613eff204
commit d853e59c7f
7 changed files with 9 additions and 50 deletions

View file

@ -218,9 +218,4 @@ public final class ProductionRegistryConfigExample implements RegistryConfig {
public String getCheckApiServletRegistrarClientId() { public String getCheckApiServletRegistrarClientId() {
return "TheRegistrar"; return "TheRegistrar";
} }
@Override
public Duration getAsyncFlowFailureBackoff() {
return Duration.standardMinutes(10);
}
} }

View file

@ -211,14 +211,5 @@ public interface RegistryConfig {
*/ */
public String getCheckApiServletRegistrarClientId(); public String getCheckApiServletRegistrarClientId();
/**
* Returns the amount of time to back off following an async flow task failure.
*
* This should be ~orders of magnitude larger than the rate on the queue, in order to prevent
* the logs from filling up with unnecessarily failures.
*/
// TODO(b/26140521): Remove this configuration option along with non-batched async operations.
public Duration getAsyncFlowFailureBackoff();
// XXX: Please consider using ConfigModule instead of adding new methods to this file. // XXX: Please consider using ConfigModule instead of adding new methods to this file.
} }

View file

@ -166,9 +166,4 @@ public class TestRegistryConfig implements RegistryConfig {
public String getCheckApiServletRegistrarClientId() { public String getCheckApiServletRegistrarClientId() {
return "TheRegistrar"; return "TheRegistrar";
} }
@Override
public Duration getAsyncFlowFailureBackoff() {
return Duration.standardMinutes(10);
}
} }

View file

@ -14,22 +14,18 @@
package google.registry.flows.async; package google.registry.flows.async;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_IS_SUPERUSER; import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_IS_SUPERUSER;
import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_REQUESTING_CLIENT_ID; import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_REQUESTING_CLIENT_ID;
import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_RESOURCE_KEY; import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_RESOURCE_KEY;
import static google.registry.flows.async.DnsRefreshForHostRenameAction.PARAM_HOST_KEY; import static google.registry.flows.async.DnsRefreshForHostRenameAction.PARAM_HOST_KEY;
import static google.registry.flows.async.RefreshDnsOnHostRenameAction.QUEUE_ASYNC_HOST_RENAME; import static google.registry.flows.async.RefreshDnsOnHostRenameAction.QUEUE_ASYNC_HOST_RENAME;
import static google.registry.request.Actions.getPathForAction;
import com.google.appengine.api.taskqueue.Queue; import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.RetryOptions;
import com.google.appengine.api.taskqueue.TaskOptions; import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method; import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.appengine.api.taskqueue.TransientFailureException; import com.google.appengine.api.taskqueue.TransientFailureException;
import com.googlecode.objectify.Key; import com.googlecode.objectify.Key;
import google.registry.config.ConfigModule.Config; import google.registry.config.ConfigModule.Config;
import google.registry.config.RegistryEnvironment;
import google.registry.model.EppResource; import google.registry.model.EppResource;
import google.registry.model.host.HostResource; import google.registry.model.host.HostResource;
import google.registry.util.FormattingLogger; import google.registry.util.FormattingLogger;
@ -55,7 +51,7 @@ public final class AsyncFlowEnqueuer {
EppResource resourceToDelete, String requestingClientId, boolean isSuperuser) { EppResource resourceToDelete, String requestingClientId, boolean isSuperuser) {
Key<EppResource> resourceKey = Key.create(resourceToDelete); Key<EppResource> resourceKey = Key.create(resourceToDelete);
logger.infofmt( logger.infofmt(
"Enqueueing async deletion of %s on behalf of registrar %s.", "Enqueuing async deletion of %s on behalf of registrar %s.",
resourceKey, requestingClientId); resourceKey, requestingClientId);
TaskOptions task = TaskOptions task =
TaskOptions.Builder TaskOptions.Builder
@ -70,29 +66,12 @@ public final class AsyncFlowEnqueuer {
/** Enqueues a task to asynchronously refresh DNS for a renamed host. */ /** Enqueues a task to asynchronously refresh DNS for a renamed host. */
public void enqueueAsyncDnsRefresh(HostResource host) { public void enqueueAsyncDnsRefresh(HostResource host) {
Key<HostResource> hostKey = Key.create(host); Key<HostResource> hostKey = Key.create(host);
logger.infofmt("Enqueueing async DNS refresh for renamed host %s.", hostKey); logger.infofmt("Enqueuing async DNS refresh for renamed host %s.", hostKey);
addTaskToQueueWithRetry( addTaskToQueueWithRetry(
asyncDnsRefreshPullQueue, asyncDnsRefreshPullQueue,
TaskOptions.Builder.withMethod(Method.PULL).param(PARAM_HOST_KEY, hostKey.getString())); TaskOptions.Builder.withMethod(Method.PULL).param(PARAM_HOST_KEY, hostKey.getString()));
} }
/** Enqueues a task to asynchronously refresh DNS for a renamed host. */
//TODO(b/26140521): Delete this once non-batched DNS host refresh mapreduce is deleted.
@Deprecated
public void enqueueLegacyAsyncDnsRefresh(HostResource host) {
logger.infofmt("Enqueueing async DNS refresh for host %s", Key.create(host));
// Aggressively back off if the task fails, to minimize flooding the logs.
RetryOptions retryOptions =
RetryOptions.Builder.withMinBackoffSeconds(
RegistryEnvironment.get().config().getAsyncFlowFailureBackoff().getStandardSeconds());
final TaskOptions task =
TaskOptions.Builder.withUrl(getPathForAction(DnsRefreshForHostRenameAction.class))
.retryOptions(retryOptions)
.param(PARAM_HOST_KEY, Key.create(host).getString())
.method(Method.GET);
addTaskToQueueWithRetry(getQueue("flows-async"), task);
}
/** /**
* Adds a task to a queue with retrying, to avoid aborting the entire flow over a transient issue * Adds a task to a queue with retrying, to avoid aborting the entire flow over a transient issue
* enqueuing a task. * enqueuing a task.

View file

@ -230,7 +230,7 @@ public final class HostUpdateFlow extends LoggedInFlow implements TransactionalF
} }
// We must also enqueue updates for all domains that use this host as their nameserver so // We must also enqueue updates for all domains that use this host as their nameserver so
// that their NS records can be updated to point at the new name. // that their NS records can be updated to point at the new name.
asyncFlowEnqueuer.enqueueLegacyAsyncDnsRefresh(existingResource); asyncFlowEnqueuer.enqueueAsyncDnsRefresh(existingResource);
} }
} }

View file

@ -126,10 +126,10 @@ import org.joda.time.Duration;
* Duplicate jobs may exist {@code <=cursor}. So a transaction will not bother changing the cursor * Duplicate jobs may exist {@code <=cursor}. So a transaction will not bother changing the cursor
* if it's already been rolled forward. * if it's already been rolled forward.
* *
* <p>Enqueueing {@code RdeUploadAction} is also part of the cursor transaction. This is necessary * <p>Enqueuing {@code RdeUploadAction} is also part of the cursor transaction. This is necessary
* because the first thing the upload task does is check the staging cursor to verify it's been * because the first thing the upload task does is check the staging cursor to verify it's been
* completed, so we can't enqueue before we roll. We also can't enqueue after the roll, because then * completed, so we can't enqueue before we roll. We also can't enqueue after the roll, because then
* if enqueueing fails, the upload might never be enqueued. * if enqueuing fails, the upload might never be enqueued.
* *
* <h3>Determinism</h3> * <h3>Determinism</h3>
* *

View file

@ -15,8 +15,8 @@
package google.registry.flows.host; package google.registry.flows.host;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static google.registry.flows.async.RefreshDnsOnHostRenameAction.QUEUE_ASYNC_HOST_RENAME;
import static google.registry.model.EppResourceUtils.loadByForeignKey; import static google.registry.model.EppResourceUtils.loadByForeignKey;
import static google.registry.request.Actions.getPathForAction;
import static google.registry.testing.DatastoreHelper.assertNoBillingEvents; import static google.registry.testing.DatastoreHelper.assertNoBillingEvents;
import static google.registry.testing.DatastoreHelper.createTld; import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.testing.DatastoreHelper.getOnlyHistoryEntryOfType; import static google.registry.testing.DatastoreHelper.getOnlyHistoryEntryOfType;
@ -43,7 +43,6 @@ import google.registry.flows.EppRequestSource;
import google.registry.flows.ResourceFlowTestCase; import google.registry.flows.ResourceFlowTestCase;
import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException; import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
import google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException; import google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException;
import google.registry.flows.async.DnsRefreshForHostRenameAction;
import google.registry.flows.exceptions.ResourceHasClientUpdateProhibitedException; import google.registry.flows.exceptions.ResourceHasClientUpdateProhibitedException;
import google.registry.flows.exceptions.ResourceStatusProhibitsOperationException; import google.registry.flows.exceptions.ResourceStatusProhibitsOperationException;
import google.registry.flows.exceptions.StatusNotClientSettableException; import google.registry.flows.exceptions.StatusNotClientSettableException;
@ -160,9 +159,9 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
HostResource renamedHost = doSuccessfulTest(); HostResource renamedHost = doSuccessfulTest();
assertThat(renamedHost.getSuperordinateDomain()).isNull(); assertThat(renamedHost.getSuperordinateDomain()).isNull();
// Task enqueued to change the NS record of the referencing domain via mapreduce. // Task enqueued to change the NS record of the referencing domain via mapreduce.
assertTasksEnqueued("flows-async", new TaskMatcher() assertTasksEnqueued(
.url(getPathForAction(DnsRefreshForHostRenameAction.class)) QUEUE_ASYNC_HOST_RENAME,
.param(DnsRefreshForHostRenameAction.PARAM_HOST_KEY, Key.create(renamedHost).getString())); new TaskMatcher().payload("hostKey=" + Key.create(renamedHost).getString()));
} }
@Test @Test