diff --git a/java/com/google/domain/registry/config/RegistryConfig.java b/java/com/google/domain/registry/config/RegistryConfig.java index 93c25f00e..a302a49c7 100644 --- a/java/com/google/domain/registry/config/RegistryConfig.java +++ b/java/com/google/domain/registry/config/RegistryConfig.java @@ -256,4 +256,12 @@ public interface RegistryConfig { * committed, and could potentially miss the reference. */ public Duration getAsyncDeleteFlowMapreduceDelay(); + + /** + * Returns the amount of time to back off following an async delete flow task failure. + * + * This should be ~orders of magnitude larger than the rate on the queue, in order to prevent + * the logs from filling up with unnecessarily failures. + */ + public Duration getAsyncDeleteFlowFailureBackoff(); } diff --git a/java/com/google/domain/registry/config/TestRegistryConfig.java b/java/com/google/domain/registry/config/TestRegistryConfig.java index e958a7f6d..0551ddf7a 100644 --- a/java/com/google/domain/registry/config/TestRegistryConfig.java +++ b/java/com/google/domain/registry/config/TestRegistryConfig.java @@ -193,4 +193,9 @@ public class TestRegistryConfig implements RegistryConfig { public Duration getAsyncDeleteFlowMapreduceDelay() { return Duration.standardSeconds(90); } + + @Override + public Duration getAsyncDeleteFlowFailureBackoff() { + return Duration.standardMinutes(10); + } } diff --git a/java/com/google/domain/registry/flows/async/AsyncFlowUtils.java b/java/com/google/domain/registry/flows/async/AsyncFlowUtils.java index 83ce3684f..843a6ea20 100644 --- a/java/com/google/domain/registry/flows/async/AsyncFlowUtils.java +++ b/java/com/google/domain/registry/flows/async/AsyncFlowUtils.java @@ -18,11 +18,13 @@ import static com.google.domain.registry.request.Actions.getPathForAction; import com.google.appengine.api.taskqueue.Queue; import com.google.appengine.api.taskqueue.QueueFactory; +import com.google.appengine.api.taskqueue.RetryOptions; import com.google.appengine.api.taskqueue.TaskHandle; import com.google.appengine.api.taskqueue.TaskOptions; import com.google.appengine.api.taskqueue.TaskOptions.Method; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; +import com.google.domain.registry.config.RegistryEnvironment; import com.google.domain.registry.mapreduce.MapreduceAction; import com.google.domain.registry.util.FormattingLogger; @@ -48,8 +50,12 @@ public final class AsyncFlowUtils { Queue queue = QueueFactory.getQueue(ASYNC_FLOW_QUEUE_NAME); String path = getPathForAction(action); logger.infofmt("Enqueueing async mapreduce action with path %s and params %s", path, params); + // Aggressively back off if the task fails, to minimize flooding the logs. + RetryOptions retryOptions = RetryOptions.Builder.withMinBackoffSeconds( + RegistryEnvironment.get().config().getAsyncDeleteFlowFailureBackoff().getStandardSeconds()); TaskOptions options = TaskOptions.Builder .withUrl(path) + .retryOptions(retryOptions) .countdownMillis(executionDelay.getMillis()) .method(Method.GET); for (Entry entry : params.entrySet()) {