Enqueue async re-saves with additional future re-saves

Async tasks will now re-enqueue themselves after completion if there are
additional pending future actions. This allows all parts of domain delete flows
to be successfully re-saved as the parts happen, without going past the maximum
allowed 30 day task ETA limit. The first task runs at 30 days out when the
redemption grace period ends, and that task then enqueues another task to run 5
more days in the future, when the deletion is final and the pending delete
status gets removed.

No data migration plan is necessary because future resaves defaults to empty,
and indeed will always be empty on transfers. So previously enqueued tasks will
still be valid.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=202949677

java/google/registry/batch/BatchModule.java

@@ -15,15 +15,18 @@
 package google.registry.batch;
 
 import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_REQUESTED_TIME;
+import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESAVE_TIMES;
 import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESOURCE_KEY;
 import static google.registry.request.RequestParameters.extractOptionalBooleanParameter;
 import static google.registry.request.RequestParameters.extractOptionalIntParameter;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
+import static google.registry.request.RequestParameters.extractSetOfDatetimeParameters;
 
 import com.google.api.services.bigquery.model.TableFieldSchema;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import dagger.Module;
 import dagger.Provides;
@@ -89,4 +92,10 @@ public class BatchModule {
   static DateTime provideRequestedTime(HttpServletRequest req) {
     return extractRequiredDatetimeParameter(req, PARAM_REQUESTED_TIME);
   }
+
+  @Provides
+  @Parameter(PARAM_RESAVE_TIMES)
+  static ImmutableSet<DateTime> provideResaveTimes(HttpServletRequest req) {
+    return extractSetOfDatetimeParameters(req, PARAM_RESAVE_TIMES, null);
+  }
 }

java/google/registry/batch/ResaveEntityAction.java

@@ -15,11 +15,15 @@
 package google.registry.batch;
 
 import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_REQUESTED_TIME;
+import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESAVE_TIMES;
 import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESOURCE_KEY;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
+import google.registry.flows.async.AsyncFlowEnqueuer;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.request.Action;
@@ -27,7 +31,6 @@ import google.registry.request.Action.Method;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.util.Clock;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -43,18 +46,21 @@ public class ResaveEntityAction implements Runnable {
 
   private final Key<ImmutableObject> resourceKey;
   private final DateTime requestedTime;
-  private final Clock clock;
+  private final ImmutableSortedSet<DateTime> resaveTimes;
+  private final AsyncFlowEnqueuer asyncFlowEnqueuer;
   private final Response response;
 
   @Inject
   ResaveEntityAction(
       @Parameter(PARAM_RESOURCE_KEY) Key<ImmutableObject> resourceKey,
       @Parameter(PARAM_REQUESTED_TIME) DateTime requestedTime,
-      Clock clock,
+      @Parameter(PARAM_RESAVE_TIMES) ImmutableSet<DateTime> resaveTimes,
+      AsyncFlowEnqueuer asyncFlowEnqueuer,
       Response response) {
     this.resourceKey = resourceKey;
     this.requestedTime = requestedTime;
-    this.clock = clock;
+    this.resaveTimes = ImmutableSortedSet.copyOf(resaveTimes);
+    this.asyncFlowEnqueuer = asyncFlowEnqueuer;
     this.response = response;
   }
 
@@ -66,8 +72,11 @@ public class ResaveEntityAction implements Runnable {
       ImmutableObject entity = ofy().load().key(resourceKey).now();
       ofy().save().entity(
           (entity instanceof EppResource)
-              ? ((EppResource) entity).cloneProjectedAtTime(clock.nowUtc()) : entity
+              ? ((EppResource) entity).cloneProjectedAtTime(ofy().getTransactionTime()) : entity
       );
+      if (!resaveTimes.isEmpty()) {
+        asyncFlowEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
+      }
     });
     response.setPayload("Entity re-saved.");
   }

java/google/registry/flows/async/AsyncFlowEnqueuer.java

@@ -23,6 +23,8 @@ import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
@@ -47,6 +49,7 @@ public final class AsyncFlowEnqueuer {
   public static final String PARAM_IS_SUPERUSER = "isSuperuser";
   public static final String PARAM_HOST_KEY = "hostKey";
   public static final String PARAM_REQUESTED_TIME = "requestedTime";
+  public static final String PARAM_RESAVE_TIMES = "resaveTimes";
 
   /** The task queue names used by async flows. */
   public static final String QUEUE_ASYNC_ACTIONS = "async-actions";
@@ -85,12 +88,25 @@ public final class AsyncFlowEnqueuer {
   /** Enqueues a task to asynchronously re-save an entity at some point in the future. */
   public void enqueueAsyncResave(
       ImmutableObject entityToResave, DateTime now, DateTime whenToResave) {
-    checkArgument(isBeforeOrAt(now, whenToResave), "Can't enqueue a resave to run in the past");
+    enqueueAsyncResave(entityToResave, now, ImmutableSortedSet.of(whenToResave));
+  }
+
+  /**
+   * Enqueues a task to asynchronously re-save an entity at some point(s) in the future.
+   *
+   * <p>Multiple re-save times are chained one after the other, i.e. any given run will re-enqueue
+   * itself to run at the next time if there are remaining re-saves scheduled.
+   */
+  public void enqueueAsyncResave(
+      ImmutableObject entityToResave, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
+    DateTime firstResave = whenToResave.first();
+    checkArgument(isBeforeOrAt(now, firstResave), "Can't enqueue a resave to run in the past");
     Key<ImmutableObject> entityKey = Key.create(entityToResave);
-    Duration etaDuration = new Duration(now, whenToResave);
+    Duration etaDuration = new Duration(now, firstResave);
     if (etaDuration.isLongerThan(MAX_ASYNC_ETA)) {
-      logger.atInfo().log("Ignoring async re-save of %s; %s is past the ETA threshold of %s.",
-          entityKey, whenToResave, MAX_ASYNC_ETA);
+      logger.atInfo().log(
+          "Ignoring async re-save of %s; %s is past the ETA threshold of %s.",
+          entityKey, firstResave, MAX_ASYNC_ETA);
       return;
     }
     logger.atInfo().log("Enqueuing async re-save of %s to run at %s.", entityKey, whenToResave);
@@ -102,6 +118,9 @@ public final class AsyncFlowEnqueuer {
             .countdownMillis(etaDuration.getMillis())
             .param(PARAM_RESOURCE_KEY, entityKey.getString())
             .param(PARAM_REQUESTED_TIME, now.toString());
+    if (whenToResave.size() > 1) {
+      task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
+    }
     addTaskToQueueWithRetry(asyncActionsPushQueue, task);
   }
 

java/google/registry/flows/domain/DomainDeleteFlow.java

@@ -39,6 +39,7 @@ import static google.registry.util.CollectionUtils.union;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Sets;
 import com.googlecode.objectify.Key;
 import google.registry.dns.DnsQueue;
@@ -185,8 +186,8 @@ public final class DomainDeleteFlow implements TransactionalFlow {
       PollMessage.OneTime deletePollMessage =
           createDeletePollMessage(existingDomain, historyEntry, deletionTime);
       entitiesToSave.add(deletePollMessage);
-      asyncFlowEnqueuer.enqueueAsyncResave(existingDomain, now, deletionTime);
-      asyncFlowEnqueuer.enqueueAsyncResave(existingDomain, now, redemptionTime);
+      asyncFlowEnqueuer.enqueueAsyncResave(
+          existingDomain, now, ImmutableSortedSet.of(redemptionTime, deletionTime));
       builder.setDeletionTime(deletionTime)
           .setStatusValues(ImmutableSet.of(StatusValue.PENDING_DELETE))
           // Clear out all old grace periods and add REDEMPTION, which does not include a key to a

javatests/google/registry/batch/ResaveEntityActionTest.java

@@ -14,29 +14,56 @@
 
 package google.registry.batch;
 
+import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static com.google.common.truth.Truth.assertThat;
+import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_REQUESTED_TIME;
+import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESOURCE_KEY;
+import static google.registry.flows.async.AsyncFlowEnqueuer.PATH_RESAVE_ENTITY;
+import static google.registry.flows.async.AsyncFlowEnqueuer.QUEUE_ASYNC_ACTIONS;
+import static google.registry.flows.async.AsyncFlowEnqueuer.QUEUE_ASYNC_DELETE;
+import static google.registry.flows.async.AsyncFlowEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.testing.DatastoreHelper.createTld;
+import static google.registry.testing.DatastoreHelper.newDomainResource;
 import static google.registry.testing.DatastoreHelper.persistActiveContact;
 import static google.registry.testing.DatastoreHelper.persistDomainWithDependentResources;
 import static google.registry.testing.DatastoreHelper.persistDomainWithPendingTransfer;
-import static org.mockito.Mockito.mock;
+import static google.registry.testing.DatastoreHelper.persistResource;
+import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
+import static org.joda.time.Duration.standardDays;
+import static org.joda.time.Duration.standardSeconds;
+import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
+import com.google.appengine.api.modules.ModulesService;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
 import com.googlecode.objectify.Key;
+import google.registry.flows.async.AsyncFlowEnqueuer;
 import google.registry.model.ImmutableObject;
 import google.registry.model.domain.DomainResource;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.rgp.GracePeriodStatus;
+import google.registry.model.eppcommon.StatusValue;
+import google.registry.model.ofy.Ofy;
 import google.registry.request.Response;
 import google.registry.testing.AppEngineRule;
 import google.registry.testing.FakeClock;
+import google.registry.testing.FakeSleeper;
+import google.registry.testing.InjectRule;
+import google.registry.testing.MockitoJUnitRule;
 import google.registry.testing.ShardableTestCase;
-import google.registry.util.Clock;
+import google.registry.testing.TaskQueueHelper.TaskMatcher;
+import google.registry.util.Retrier;
 import org.joda.time.DateTime;
+import org.joda.time.Duration;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
+import org.mockito.Mock;
 
 /** Unit tests for {@link ResaveEntityAction}. */
 @RunWith(JUnit4.class)
@@ -46,16 +73,37 @@ public class ResaveEntityActionTest extends ShardableTestCase {
   public final AppEngineRule appEngine =
       AppEngineRule.builder().withDatastore().withTaskQueue().build();
 
-  private final Clock clock = new FakeClock(DateTime.parse("2016-02-11T10:00:00Z"));
-  private final Response response = mock(Response.class);
+  @Rule public final InjectRule inject = new InjectRule();
+  @Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create();
+
+  @Mock private ModulesService modulesService;
+  @Mock private Response response;
+  private final FakeClock clock = new FakeClock(DateTime.parse("2016-02-11T10:00:00Z"));
+  private AsyncFlowEnqueuer asyncFlowEnqueuer;
 
   @Before
   public void before() {
+    inject.setStaticField(Ofy.class, "clock", clock);
+    when(modulesService.getVersionHostname(any(String.class), any(String.class)))
+        .thenReturn("backend.hostname.fake");
+    asyncFlowEnqueuer =
+        new AsyncFlowEnqueuer(
+            getQueue(QUEUE_ASYNC_ACTIONS),
+            getQueue(QUEUE_ASYNC_DELETE),
+            getQueue(QUEUE_ASYNC_HOST_RENAME),
+            Duration.ZERO,
+            modulesService,
+            new Retrier(new FakeSleeper(clock), 1));
     createTld("tld");
   }
 
-  private void runAction(Key<ImmutableObject> resourceKey, DateTime requestedTime) {
-    ResaveEntityAction action = new ResaveEntityAction(resourceKey, requestedTime, clock, response);
+  private void runAction(
+      Key<ImmutableObject> resourceKey,
+      DateTime requestedTime,
+      ImmutableSortedSet<DateTime> resaveTimes) {
+    ResaveEntityAction action =
+        new ResaveEntityAction(
+            resourceKey, requestedTime, resaveTimes, asyncFlowEnqueuer, response);
     action.run();
   }
 
@@ -74,10 +122,48 @@ public class ResaveEntityActionTest extends ShardableTestCase {
             DateTime.parse("2016-02-11T10:00:00Z"),
             DateTime.parse("2017-01-02T10:11:00Z"),
             DateTime.parse("2016-02-06T10:00:00Z"));
+    clock.advanceOneMilli();
     assertThat(domain.getCurrentSponsorClientId()).isEqualTo("TheRegistrar");
-    runAction(Key.create(domain), DateTime.parse("2016-02-06T10:00:01Z"));
+    runAction(Key.create(domain), DateTime.parse("2016-02-06T10:00:01Z"), ImmutableSortedSet.of());
     DomainResource resavedDomain = ofy().load().entity(domain).now();
     assertThat(resavedDomain.getCurrentSponsorClientId()).isEqualTo("NewRegistrar");
     verify(response).setPayload("Entity re-saved.");
   }
+
+  @Test
+  public void test_domainPendingDeletion_isResavedAndReenqueued() {
+    DomainResource domain =
+        persistResource(
+            newDomainResource("domain.tld")
+                .asBuilder()
+                .setDeletionTime(clock.nowUtc().plusDays(35))
+                .setStatusValues(ImmutableSet.of(StatusValue.PENDING_DELETE))
+                .setGracePeriods(
+                    ImmutableSet.of(
+                        GracePeriod.createWithoutBillingEvent(
+                            GracePeriodStatus.REDEMPTION,
+                            clock.nowUtc().plusDays(30),
+                            "TheRegistrar")))
+                .build());
+    clock.advanceBy(standardDays(30));
+    DateTime requestedTime = clock.nowUtc();
+
+    assertThat(domain.getGracePeriods()).isNotEmpty();
+    runAction(Key.create(domain), requestedTime, ImmutableSortedSet.of(requestedTime.plusDays(5)));
+    DomainResource resavedDomain = ofy().load().entity(domain).now();
+    assertThat(resavedDomain.getGracePeriods()).isEmpty();
+
+    assertTasksEnqueued(
+        QUEUE_ASYNC_ACTIONS,
+        new TaskMatcher()
+            .url(PATH_RESAVE_ENTITY)
+            .method("POST")
+            .header("Host", "backend.hostname.fake")
+            .header("content-type", "application/x-www-form-urlencoded")
+            .param(PARAM_RESOURCE_KEY, Key.create(resavedDomain).getString())
+            .param(PARAM_REQUESTED_TIME, requestedTime.toString())
+            .etaDelta(
+                standardDays(5).minus(standardSeconds(30)),
+                standardDays(5).plus(standardSeconds(30))));
+  }
 }

javatests/google/registry/flows/async/AsyncFlowEnqueuerTest.java

@@ -16,6 +16,7 @@ package google.registry.flows.async;
 
 import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_REQUESTED_TIME;
+import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESAVE_TIMES;
 import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESOURCE_KEY;
 import static google.registry.flows.async.AsyncFlowEnqueuer.PATH_RESAVE_ENTITY;
 import static google.registry.flows.async.AsyncFlowEnqueuer.QUEUE_ASYNC_ACTIONS;
@@ -26,11 +27,13 @@ import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
 import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.testing.TestLogHandlerUtils.assertLogMessage;
 import static org.joda.time.Duration.standardDays;
+import static org.joda.time.Duration.standardHours;
 import static org.joda.time.Duration.standardSeconds;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.when;
 
 import com.google.appengine.api.modules.ModulesService;
+import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.LoggerConfig;
 import com.googlecode.objectify.Key;
 import google.registry.model.contact.ContactResource;
@@ -103,6 +106,31 @@ public class AsyncFlowEnqueuerTest extends ShardableTestCase {
                 standardDays(5).plus(standardSeconds(30))));
   }
 
+  @Test
+  public void test_enqueueAsyncResave_multipleResaves() {
+    ContactResource contact = persistActiveContact("jd23456");
+    DateTime now = clock.nowUtc();
+    asyncFlowEnqueuer.enqueueAsyncResave(
+        contact,
+        now,
+        ImmutableSortedSet.of(now.plusHours(24), now.plusHours(50), now.plusHours(75)));
+    assertTasksEnqueued(
+        QUEUE_ASYNC_ACTIONS,
+        new TaskMatcher()
+            .url(PATH_RESAVE_ENTITY)
+            .method("POST")
+            .header("Host", "backend.hostname.fake")
+            .header("content-type", "application/x-www-form-urlencoded")
+            .param(PARAM_RESOURCE_KEY, Key.create(contact).getString())
+            .param(PARAM_REQUESTED_TIME, now.toString())
+            .param(
+                PARAM_RESAVE_TIMES,
+                "2015-05-20T14:34:56.000Z,2015-05-21T15:34:56.000Z")
+            .etaDelta(
+                standardHours(24).minus(standardSeconds(30)),
+                standardHours(24).plus(standardSeconds(30))));
+  }
+
   @Test
   public void test_enqueueAsyncResave_ignoresTasksTooFarIntoFuture() throws Exception {
     ContactResource contact = persistActiveContact("jd23456");

javatests/google/registry/flows/domain/DomainDeleteFlowTest.java

@@ -17,6 +17,7 @@ package google.registry.flows.domain;
 import static com.google.common.collect.MoreCollectors.onlyElement;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_REQUESTED_TIME;
+import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESAVE_TIMES;
 import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESOURCE_KEY;
 import static google.registry.flows.async.AsyncFlowEnqueuer.PATH_RESAVE_ENTITY;
 import static google.registry.flows.async.AsyncFlowEnqueuer.QUEUE_ASYNC_ACTIONS;
@@ -269,22 +270,18 @@ public class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow,
     setUpSuccessfulTest();
     clock.advanceOneMilli();
     runFlowAssertResponse(loadFile("domain_delete_response_pending.xml"));
-    // This seems like it's too long.
+    Duration when = standardDays(3);
     assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
-        createExpectedResaveTask(standardDays(3)),
-        createExpectedResaveTask(standardDays(5)));
-  }
-
-  private TaskMatcher createExpectedResaveTask(Duration when) {
-    return new TaskMatcher()
+        new TaskMatcher()
             .url(PATH_RESAVE_ENTITY)
             .method("POST")
             .header("Host", "backend.hostname.fake")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, Key.create(domain).getString())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
-        .etaDelta(when.minus(standardSeconds(30)), when.plus(standardSeconds(30)));
+            .param(PARAM_RESAVE_TIMES, clock.nowUtc().plusDays(5).toString())
+            .etaDelta(when.minus(standardSeconds(30)), when.plus(standardSeconds(30))));
   }
 
   @Test