Add delete expired domains action (#836)

* Add delete expired domains action

This will be scheduled to run daily via App Engine cron in a subsequent PR.

* Add test

core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java

@@ -0,0 +1,193 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static google.registry.flows.FlowUtils.marshalWithLenientRetry;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;
+import static google.registry.util.ResourceUtils.readResourceUtf8;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.flows.EppController;
+import google.registry.flows.EppRequestSource;
+import google.registry.flows.PasswordOnlyTransportCredentials;
+import google.registry.flows.StatelessRequestSessionMetadata;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.eppcommon.ProtocolDefinition;
+import google.registry.model.eppoutput.EppOutput;
+import google.registry.request.Action;
+import google.registry.request.Action.Method;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.request.lock.LockHandler;
+import google.registry.util.Clock;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/**
+ * An action that deletes all non-renewing domains whose expiration dates have now passed.
+ *
+ * <p>The registry runs on an autorenew domain model, so domains don't ever expire naturally; they
+ * are only ever autorenewed. However, in some situations (such as URS) we don't want this to
+ * happen. Thus, the domains are tagged as non-renewing and are deleted by the next daily invocation
+ * of this action once they are past the date at which they were to expire.
+ *
+ * <p>Note that this action works by running a superuser EPP domain delete command, and as a side
+ * effect of when domains are deleted (just past their expiration date), they are invariably in the
+ * autorenew grace period when this happens.
+ */
+@Action(
+    service = Action.Service.BACKEND,
+    path = DeleteExpiredDomainsAction.PATH,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN,
+    method = Method.POST)
+public class DeleteExpiredDomainsAction implements Runnable {
+
+  public static final String PATH = "/_dr/task/deleteExpiredDomains";
+  private static final String LOCK_NAME = "Delete expired domains";
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private final EppController eppController;
+  private final String registryAdminClientId;
+  private final Clock clock;
+  private final LockHandler lockHandler;
+  private final Response response;
+  private final String deleteXmlTmpl;
+
+  @Inject
+  DeleteExpiredDomainsAction(
+      EppController eppController,
+      @Config("registryAdminClientId") String registryAdminClientId,
+      Clock clock,
+      LockHandler lockHandler,
+      Response response) {
+    this.eppController = eppController;
+    this.registryAdminClientId = registryAdminClientId;
+    this.clock = clock;
+    this.lockHandler = lockHandler;
+    this.response = response;
+    this.deleteXmlTmpl =
+        readResourceUtf8(DeleteExpiredDomainsAction.class, "delete_expired_domain.xml");
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(PLAIN_TEXT_UTF_8);
+
+    Callable<Void> runner =
+        () -> {
+          try {
+            runLocked();
+            response.setStatus(SC_OK);
+          } catch (Exception e) {
+            response.setStatus(SC_INTERNAL_SERVER_ERROR);
+            response.setPayload("Encountered error; see GCP logs for full details.");
+          }
+          return null;
+        };
+
+    if (!lockHandler.executeWithLocks(runner, null, Duration.standardHours(1), LOCK_NAME)) {
+      // Send a 200-series status code to prevent this conflicting action from retrying.
+      response.setStatus(SC_NO_CONTENT);
+      response.setPayload("Could not acquire lock; already running?");
+    }
+  }
+
+  private void runLocked() {
+    DateTime runTime = clock.nowUtc();
+    logger.atInfo().log(
+        "Deleting non-renewing domains with autorenew end times up through %s.", runTime);
+
+    // Note: This query is (and must be) non-transactional, and thus, is only eventually consistent.
+    ImmutableList<DomainBase> domainsToDelete =
+        ofy().load().type(DomainBase.class).filter("autorenewEndTime <=", runTime).list().stream()
+            // Datastore can't do two inequalities in one query, so the second happens in-memory.
+            .filter(d -> d.getDeletionTime().isEqual(END_OF_TIME))
+            .collect(toImmutableList());
+    if (domainsToDelete.isEmpty()) {
+      logger.atInfo().log("Found 0 domains to delete.");
+      response.setPayload("Found 0 domains to delete.");
+      return;
+    }
+
+    logger.atInfo().log(
+        "Found %d domains to delete: %s.",
+        domainsToDelete.size(),
+        String.join(
+            ", ",
+            domainsToDelete.stream().map(DomainBase::getDomainName).collect(toImmutableList())));
+    domainsToDelete.forEach(this::runDomainDeleteFlow);
+    logger.atInfo().log("Finished deleting domains.");
+    response.setPayload("Finished deleting domains.");
+  }
+
+  private void runDomainDeleteFlow(DomainBase domain) {
+    logger.atInfo().log("Attempting to delete domain %s", domain.getDomainName());
+    // Create a new transaction that the flow's execution will be enlisted in that loads the domain
+    // transactionally. This way we can ensure that nothing else has modified the domain in question
+    // in the intervening period since the query above found it.
+    Optional<EppOutput> eppOutput =
+        tm().transact(
+                () -> {
+                  DomainBase transDomain = tm().loadByKey(domain.createVKey());
+                  if (!domain.getAutorenewEndTime().isPresent()
+                      || domain.getAutorenewEndTime().get().isAfter(tm().getTransactionTime())) {
+                    logger.atSevere().log(
+                        "Failed to delete domain %s because of its autorenew end time: %s.",
+                        transDomain.getDomainName(), transDomain.getAutorenewEndTime());
+                    return Optional.empty();
+                  } else if (domain.getDeletionTime().isBefore(END_OF_TIME)) {
+                    logger.atSevere().log(
+                        "Failed to delete domain %s because it was already deleted on %s.",
+                        transDomain.getDomainName(), transDomain.getDeletionTime());
+                    return Optional.empty();
+                  }
+                  return Optional.of(
+                      eppController.handleEppCommand(
+                          new StatelessRequestSessionMetadata(
+                              registryAdminClientId,
+                              ProtocolDefinition.getVisibleServiceExtensionUris()),
+                          new PasswordOnlyTransportCredentials(),
+                          EppRequestSource.BACKEND,
+                          false,
+                          true,
+                          deleteXmlTmpl
+                              .replace("%DOMAIN%", transDomain.getDomainName())
+                              .getBytes(UTF_8)));
+                });
+
+    if (eppOutput.isPresent()) {
+      if (eppOutput.get().isSuccess()) {
+        logger.atInfo().log("Successfully deleted domain %s", domain.getDomainName());
+      } else {
+        logger.atWarning().log(
+            "Failed to delete domain %s; EPP response:\n\n%s",
+            domain.getDomainName(), new String(marshalWithLenientRetry(eppOutput.get()), UTF_8));
+      }
+    }
+  }
+}

core/src/main/java/google/registry/batch/delete_expired_domain.xml

@@ -0,0 +1,17 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <delete>
+      <domain:delete
+       xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>%DOMAIN%</domain:name>
+      </domain:delete>
+    </delete>
+    <extension>
+      <metadata:metadata xmlns:metadata="urn:google:params:xml:ns:metadata-1.0">
+        <metadata:reason>Non-renewing domain has reached expiration date.</metadata:reason>
+        <metadata:requestedByRegistrar>false</metadata:requestedByRegistrar>
+      </metadata:metadata>
+    </extension>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>

core/src/main/java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml

@@ -25,11 +25,6 @@
     <property name="tld" direction="asc"/>
     <property name="creationTime" direction="desc"/>
   </datastore-index>
-  <!-- For finding non-autorenewing domains to be deleted. -->
-  <datastore-index kind="DomainBase" ancestor="false" source="manual">
-    <property name="autorenewEndTime" direction="asc"/>
-    <property name="deletionTime" direction="asc"/>
-  </datastore-index>
   <!-- For finding host resources by registrar. -->
   <datastore-index kind="HostResource" ancestor="false" source="manual">
     <property name="currentSponsorClientId" direction="asc"/>

core/src/main/java/google/registry/flows/EppRequestSource.java

@@ -22,5 +22,6 @@ public enum EppRequestSource {
   TLS,
   TOOL,
   CHECK_API,
-  UNIT_TEST
+  UNIT_TEST,
+  BACKEND
 }

core/src/main/java/google/registry/flows/ExtensionManager.java

@@ -104,11 +104,14 @@ public final class ExtensionManager {
         clientId, flowClass.getSimpleName(), undeclaredUris);
   }
 
+  private static final ImmutableSet<EppRequestSource> ALLOWED_METADATA_EPP_REQUEST_SOURCES =
+      ImmutableSet.of(EppRequestSource.TOOL, EppRequestSource.BACKEND);
+
   private void checkForRestrictedExtensions(
       ImmutableSet<Class<? extends CommandExtension>> suppliedExtensions)
       throws OnlyToolCanPassMetadataException, UnauthorizedForSuperuserExtensionException {
     if (suppliedExtensions.contains(MetadataExtension.class)
-        && !eppRequestSource.equals(EppRequestSource.TOOL)) {
+        && !ALLOWED_METADATA_EPP_REQUEST_SOURCES.contains(eppRequestSource)) {
       throw new OnlyToolCanPassMetadataException();
     }
     // Can't use suppliedExtension.contains() here because the SuperuserExtension has child classes.

core/src/test/java/google/registry/batch/DeleteExpiredDomainsActionTest.java

@@ -0,0 +1,183 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.eppcommon.StatusValue.PENDING_DELETE;
+import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_CREATE;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.newDomainBase;
+import static google.registry.testing.DatabaseHelper.persistActiveDomain;
+import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;
+
+import com.google.common.collect.ImmutableSet;
+import google.registry.flows.DaggerEppTestComponent;
+import google.registry.flows.EppController;
+import google.registry.flows.EppTestComponent.FakesAndMocksModule;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.billing.BillingEvent.Flag;
+import google.registry.model.billing.BillingEvent.Reason;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.ofy.Ofy;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.monitoring.whitebox.EppMetric;
+import google.registry.testing.AppEngineExtension;
+import google.registry.testing.FakeClock;
+import google.registry.testing.FakeLockHandler;
+import google.registry.testing.FakeResponse;
+import google.registry.testing.InjectExtension;
+import java.util.Optional;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit tests for {@link DeleteExpiredDomainsAction}. */
+class DeleteExpiredDomainsActionTest {
+
+  @RegisterExtension
+  public final AppEngineExtension appEngine =
+      AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
+
+  @RegisterExtension public final InjectExtension inject = new InjectExtension();
+
+  private final FakeClock clock = new FakeClock(DateTime.parse("2016-06-13T20:21:22Z"));
+  private final FakeResponse response = new FakeResponse();
+  private DeleteExpiredDomainsAction action;
+
+  @BeforeEach
+  void beforeEach() {
+    inject.setStaticField(Ofy.class, "clock", clock);
+    createTld("tld");
+    EppController eppController =
+        DaggerEppTestComponent.builder()
+            .fakesAndMocksModule(
+                FakesAndMocksModule.create(clock, EppMetric.builderForRequest(clock)))
+            .build()
+            .startRequest()
+            .eppController();
+    action =
+        new DeleteExpiredDomainsAction(
+            eppController, "NewRegistrar", clock, new FakeLockHandler(true), response);
+  }
+
+  @Test
+  void test_deletesOnlyExpiredDomain() {
+    // A normal, active autorenewing domain that shouldn't be touched.
+    DomainBase activeDomain = persistActiveDomain("foo.tld");
+    clock.advanceOneMilli();
+
+    // A non-autorenewing domain that is already pending delete and shouldn't be touched.
+    DomainBase alreadyDeletedDomain =
+        persistResource(
+            newDomainBase("bar.tld")
+                .asBuilder()
+                .setAutorenewEndTime(Optional.of(clock.nowUtc().minusDays(10)))
+                .setDeletionTime(clock.nowUtc().plusDays(17))
+                .build());
+    clock.advanceOneMilli();
+
+    // A non-autorenewing domain that hasn't reached its expiration time and shouldn't be touched.
+    DomainBase notYetExpiredDomain =
+        persistResource(
+            newDomainBase("baz.tld")
+                .asBuilder()
+                .setAutorenewEndTime(Optional.of(clock.nowUtc().plusDays(15)))
+                .build());
+    clock.advanceOneMilli();
+
+    // A non-autorenewing domain that is past its expiration time and should be deleted.
+    // (This is the only one that needs a full set of subsidiary resources, for the delete flow to
+    // to operate on.)
+    DomainBase pendingExpirationDomain = persistNonAutorenewingDomain("fizz.tld");
+
+    assertThat(tm().loadByEntity(pendingExpirationDomain).getStatusValues())
+        .doesNotContain(PENDING_DELETE);
+    action.run();
+
+    DomainBase reloadedActiveDomain = tm().loadByEntity(activeDomain);
+    assertThat(reloadedActiveDomain).isEqualTo(activeDomain);
+    assertThat(reloadedActiveDomain.getStatusValues()).doesNotContain(PENDING_DELETE);
+    assertThat(tm().loadByEntity(alreadyDeletedDomain)).isEqualTo(alreadyDeletedDomain);
+    assertThat(tm().loadByEntity(notYetExpiredDomain)).isEqualTo(notYetExpiredDomain);
+    DomainBase reloadedExpiredDomain = tm().loadByEntity(pendingExpirationDomain);
+    assertThat(reloadedExpiredDomain.getStatusValues()).contains(PENDING_DELETE);
+    assertThat(reloadedExpiredDomain.getDeletionTime()).isEqualTo(clock.nowUtc().plusDays(35));
+  }
+
+  @Test
+  void test_deletesThreeDomainsInOneRun() {
+    DomainBase domain1 = persistNonAutorenewingDomain("ecck1.tld");
+    DomainBase domain2 = persistNonAutorenewingDomain("veee2.tld");
+    DomainBase domain3 = persistNonAutorenewingDomain("tarm3.tld");
+
+    action.run();
+
+    assertThat(tm().loadByEntity(domain1).getStatusValues()).contains(PENDING_DELETE);
+    assertThat(tm().loadByEntity(domain2).getStatusValues()).contains(PENDING_DELETE);
+    assertThat(tm().loadByEntity(domain3).getStatusValues()).contains(PENDING_DELETE);
+  }
+
+  private DomainBase persistNonAutorenewingDomain(String domainName) {
+    DomainBase pendingExpirationDomain = persistActiveDomain(domainName);
+    HistoryEntry createHistoryEntry =
+        persistResource(
+            new HistoryEntry.Builder()
+                .setType(DOMAIN_CREATE)
+                .setParent(pendingExpirationDomain)
+                .setModificationTime(clock.nowUtc().minusMonths(9))
+                .build());
+    BillingEvent.Recurring autorenewBillingEvent =
+        persistResource(createAutorenewBillingEvent(createHistoryEntry).build());
+    PollMessage.Autorenew autorenewPollMessage =
+        persistResource(createAutorenewPollMessage(createHistoryEntry).build());
+    pendingExpirationDomain =
+        persistResource(
+            pendingExpirationDomain
+                .asBuilder()
+                .setAutorenewEndTime(Optional.of(clock.nowUtc().minusDays(10)))
+                .setAutorenewBillingEvent(autorenewBillingEvent.createVKey())
+                .setAutorenewPollMessage(autorenewPollMessage.createVKey())
+                .build());
+    clock.advanceOneMilli();
+
+    return pendingExpirationDomain;
+  }
+
+  private BillingEvent.Recurring.Builder createAutorenewBillingEvent(
+      HistoryEntry createHistoryEntry) {
+    return new BillingEvent.Recurring.Builder()
+        .setReason(Reason.RENEW)
+        .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
+        .setTargetId("fizz.tld")
+        .setClientId("TheRegistrar")
+        .setEventTime(clock.nowUtc().plusYears(1))
+        .setRecurrenceEndTime(END_OF_TIME)
+        .setParent(createHistoryEntry);
+  }
+
+  private PollMessage.Autorenew.Builder createAutorenewPollMessage(
+      HistoryEntry createHistoryEntry) {
+    return new PollMessage.Autorenew.Builder()
+        .setTargetId("fizz.tld")
+        .setClientId("TheRegistrar")
+        .setEventTime(clock.nowUtc().plusYears(1))
+        .setAutorenewEndTime(END_OF_TIME)
+        .setParent(createHistoryEntry);
+  }
+}

core/src/test/java/google/registry/flows/EppTestComponent.java

@@ -45,12 +45,8 @@ import javax.inject.Singleton;
 
 /** Dagger component for running EPP tests. */
 @Singleton
-@Component(
-    modules = {
-        ConfigModule.class,
-        EppTestComponent.FakesAndMocksModule.class
-    })
-interface EppTestComponent {
+@Component(modules = {ConfigModule.class, EppTestComponent.FakesAndMocksModule.class})
+public interface EppTestComponent {
 
   RequestComponent startRequest();