diff --git a/core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java b/core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java new file mode 100644 index 000000000..b66f16dcb --- /dev/null +++ b/core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java @@ -0,0 +1,193 @@ +// Copyright 2021 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.batch; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; +import static google.registry.flows.FlowUtils.marshalWithLenientRetry; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.persistence.transaction.TransactionManagerFactory.tm; +import static google.registry.util.DateTimeUtils.END_OF_TIME; +import static google.registry.util.ResourceUtils.readResourceUtf8; +import static java.nio.charset.StandardCharsets.UTF_8; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; +import static javax.servlet.http.HttpServletResponse.SC_OK; + +import com.google.common.collect.ImmutableList; +import com.google.common.flogger.FluentLogger; +import google.registry.config.RegistryConfig.Config; +import google.registry.flows.EppController; +import google.registry.flows.EppRequestSource; +import google.registry.flows.PasswordOnlyTransportCredentials; +import google.registry.flows.StatelessRequestSessionMetadata; +import google.registry.model.domain.DomainBase; +import google.registry.model.eppcommon.ProtocolDefinition; +import google.registry.model.eppoutput.EppOutput; +import google.registry.request.Action; +import google.registry.request.Action.Method; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import google.registry.request.lock.LockHandler; +import google.registry.util.Clock; +import java.util.Optional; +import java.util.concurrent.Callable; +import javax.inject.Inject; +import org.joda.time.DateTime; +import org.joda.time.Duration; + +/** + * An action that deletes all non-renewing domains whose expiration dates have now passed. + * + *

The registry runs on an autorenew domain model, so domains don't ever expire naturally; they + * are only ever autorenewed. However, in some situations (such as URS) we don't want this to + * happen. Thus, the domains are tagged as non-renewing and are deleted by the next daily invocation + * of this action once they are past the date at which they were to expire. + * + *

Note that this action works by running a superuser EPP domain delete command, and as a side + * effect of when domains are deleted (just past their expiration date), they are invariably in the + * autorenew grace period when this happens. + */ +@Action( + service = Action.Service.BACKEND, + path = DeleteExpiredDomainsAction.PATH, + auth = Auth.AUTH_INTERNAL_OR_ADMIN, + method = Method.POST) +public class DeleteExpiredDomainsAction implements Runnable { + + public static final String PATH = "/_dr/task/deleteExpiredDomains"; + private static final String LOCK_NAME = "Delete expired domains"; + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private final EppController eppController; + private final String registryAdminClientId; + private final Clock clock; + private final LockHandler lockHandler; + private final Response response; + private final String deleteXmlTmpl; + + @Inject + DeleteExpiredDomainsAction( + EppController eppController, + @Config("registryAdminClientId") String registryAdminClientId, + Clock clock, + LockHandler lockHandler, + Response response) { + this.eppController = eppController; + this.registryAdminClientId = registryAdminClientId; + this.clock = clock; + this.lockHandler = lockHandler; + this.response = response; + this.deleteXmlTmpl = + readResourceUtf8(DeleteExpiredDomainsAction.class, "delete_expired_domain.xml"); + } + + @Override + public void run() { + response.setContentType(PLAIN_TEXT_UTF_8); + + Callable runner = + () -> { + try { + runLocked(); + response.setStatus(SC_OK); + } catch (Exception e) { + response.setStatus(SC_INTERNAL_SERVER_ERROR); + response.setPayload("Encountered error; see GCP logs for full details."); + } + return null; + }; + + if (!lockHandler.executeWithLocks(runner, null, Duration.standardHours(1), LOCK_NAME)) { + // Send a 200-series status code to prevent this conflicting action from retrying. + response.setStatus(SC_NO_CONTENT); + response.setPayload("Could not acquire lock; already running?"); + } + } + + private void runLocked() { + DateTime runTime = clock.nowUtc(); + logger.atInfo().log( + "Deleting non-renewing domains with autorenew end times up through %s.", runTime); + + // Note: This query is (and must be) non-transactional, and thus, is only eventually consistent. + ImmutableList domainsToDelete = + ofy().load().type(DomainBase.class).filter("autorenewEndTime <=", runTime).list().stream() + // Datastore can't do two inequalities in one query, so the second happens in-memory. + .filter(d -> d.getDeletionTime().isEqual(END_OF_TIME)) + .collect(toImmutableList()); + if (domainsToDelete.isEmpty()) { + logger.atInfo().log("Found 0 domains to delete."); + response.setPayload("Found 0 domains to delete."); + return; + } + + logger.atInfo().log( + "Found %d domains to delete: %s.", + domainsToDelete.size(), + String.join( + ", ", + domainsToDelete.stream().map(DomainBase::getDomainName).collect(toImmutableList()))); + domainsToDelete.forEach(this::runDomainDeleteFlow); + logger.atInfo().log("Finished deleting domains."); + response.setPayload("Finished deleting domains."); + } + + private void runDomainDeleteFlow(DomainBase domain) { + logger.atInfo().log("Attempting to delete domain %s", domain.getDomainName()); + // Create a new transaction that the flow's execution will be enlisted in that loads the domain + // transactionally. This way we can ensure that nothing else has modified the domain in question + // in the intervening period since the query above found it. + Optional eppOutput = + tm().transact( + () -> { + DomainBase transDomain = tm().loadByKey(domain.createVKey()); + if (!domain.getAutorenewEndTime().isPresent() + || domain.getAutorenewEndTime().get().isAfter(tm().getTransactionTime())) { + logger.atSevere().log( + "Failed to delete domain %s because of its autorenew end time: %s.", + transDomain.getDomainName(), transDomain.getAutorenewEndTime()); + return Optional.empty(); + } else if (domain.getDeletionTime().isBefore(END_OF_TIME)) { + logger.atSevere().log( + "Failed to delete domain %s because it was already deleted on %s.", + transDomain.getDomainName(), transDomain.getDeletionTime()); + return Optional.empty(); + } + return Optional.of( + eppController.handleEppCommand( + new StatelessRequestSessionMetadata( + registryAdminClientId, + ProtocolDefinition.getVisibleServiceExtensionUris()), + new PasswordOnlyTransportCredentials(), + EppRequestSource.BACKEND, + false, + true, + deleteXmlTmpl + .replace("%DOMAIN%", transDomain.getDomainName()) + .getBytes(UTF_8))); + }); + + if (eppOutput.isPresent()) { + if (eppOutput.get().isSuccess()) { + logger.atInfo().log("Successfully deleted domain %s", domain.getDomainName()); + } else { + logger.atWarning().log( + "Failed to delete domain %s; EPP response:\n\n%s", + domain.getDomainName(), new String(marshalWithLenientRetry(eppOutput.get()), UTF_8)); + } + } + } +} diff --git a/core/src/main/java/google/registry/batch/delete_expired_domain.xml b/core/src/main/java/google/registry/batch/delete_expired_domain.xml new file mode 100644 index 000000000..8714f04b6 --- /dev/null +++ b/core/src/main/java/google/registry/batch/delete_expired_domain.xml @@ -0,0 +1,17 @@ + + + + + %DOMAIN% + + + + + Non-renewing domain has reached expiration date. + false + + + ABC-12345 + + diff --git a/core/src/main/java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml b/core/src/main/java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml index d08709338..9a8e71364 100644 --- a/core/src/main/java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml +++ b/core/src/main/java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml @@ -25,11 +25,6 @@ - - - - - diff --git a/core/src/main/java/google/registry/flows/EppRequestSource.java b/core/src/main/java/google/registry/flows/EppRequestSource.java index dcaed2597..a4961cfcf 100644 --- a/core/src/main/java/google/registry/flows/EppRequestSource.java +++ b/core/src/main/java/google/registry/flows/EppRequestSource.java @@ -22,5 +22,6 @@ public enum EppRequestSource { TLS, TOOL, CHECK_API, - UNIT_TEST + UNIT_TEST, + BACKEND } diff --git a/core/src/main/java/google/registry/flows/ExtensionManager.java b/core/src/main/java/google/registry/flows/ExtensionManager.java index ffab72421..cee53b9a7 100644 --- a/core/src/main/java/google/registry/flows/ExtensionManager.java +++ b/core/src/main/java/google/registry/flows/ExtensionManager.java @@ -104,11 +104,14 @@ public final class ExtensionManager { clientId, flowClass.getSimpleName(), undeclaredUris); } + private static final ImmutableSet ALLOWED_METADATA_EPP_REQUEST_SOURCES = + ImmutableSet.of(EppRequestSource.TOOL, EppRequestSource.BACKEND); + private void checkForRestrictedExtensions( ImmutableSet> suppliedExtensions) throws OnlyToolCanPassMetadataException, UnauthorizedForSuperuserExtensionException { if (suppliedExtensions.contains(MetadataExtension.class) - && !eppRequestSource.equals(EppRequestSource.TOOL)) { + && !ALLOWED_METADATA_EPP_REQUEST_SOURCES.contains(eppRequestSource)) { throw new OnlyToolCanPassMetadataException(); } // Can't use suppliedExtension.contains() here because the SuperuserExtension has child classes. diff --git a/core/src/test/java/google/registry/batch/DeleteExpiredDomainsActionTest.java b/core/src/test/java/google/registry/batch/DeleteExpiredDomainsActionTest.java new file mode 100644 index 000000000..d260821ec --- /dev/null +++ b/core/src/test/java/google/registry/batch/DeleteExpiredDomainsActionTest.java @@ -0,0 +1,183 @@ +// Copyright 2021 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.batch; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.model.eppcommon.StatusValue.PENDING_DELETE; +import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_CREATE; +import static google.registry.persistence.transaction.TransactionManagerFactory.tm; +import static google.registry.testing.DatabaseHelper.createTld; +import static google.registry.testing.DatabaseHelper.newDomainBase; +import static google.registry.testing.DatabaseHelper.persistActiveDomain; +import static google.registry.testing.DatabaseHelper.persistResource; +import static google.registry.util.DateTimeUtils.END_OF_TIME; + +import com.google.common.collect.ImmutableSet; +import google.registry.flows.DaggerEppTestComponent; +import google.registry.flows.EppController; +import google.registry.flows.EppTestComponent.FakesAndMocksModule; +import google.registry.model.billing.BillingEvent; +import google.registry.model.billing.BillingEvent.Flag; +import google.registry.model.billing.BillingEvent.Reason; +import google.registry.model.domain.DomainBase; +import google.registry.model.ofy.Ofy; +import google.registry.model.poll.PollMessage; +import google.registry.model.reporting.HistoryEntry; +import google.registry.monitoring.whitebox.EppMetric; +import google.registry.testing.AppEngineExtension; +import google.registry.testing.FakeClock; +import google.registry.testing.FakeLockHandler; +import google.registry.testing.FakeResponse; +import google.registry.testing.InjectExtension; +import java.util.Optional; +import org.joda.time.DateTime; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** Unit tests for {@link DeleteExpiredDomainsAction}. */ +class DeleteExpiredDomainsActionTest { + + @RegisterExtension + public final AppEngineExtension appEngine = + AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build(); + + @RegisterExtension public final InjectExtension inject = new InjectExtension(); + + private final FakeClock clock = new FakeClock(DateTime.parse("2016-06-13T20:21:22Z")); + private final FakeResponse response = new FakeResponse(); + private DeleteExpiredDomainsAction action; + + @BeforeEach + void beforeEach() { + inject.setStaticField(Ofy.class, "clock", clock); + createTld("tld"); + EppController eppController = + DaggerEppTestComponent.builder() + .fakesAndMocksModule( + FakesAndMocksModule.create(clock, EppMetric.builderForRequest(clock))) + .build() + .startRequest() + .eppController(); + action = + new DeleteExpiredDomainsAction( + eppController, "NewRegistrar", clock, new FakeLockHandler(true), response); + } + + @Test + void test_deletesOnlyExpiredDomain() { + // A normal, active autorenewing domain that shouldn't be touched. + DomainBase activeDomain = persistActiveDomain("foo.tld"); + clock.advanceOneMilli(); + + // A non-autorenewing domain that is already pending delete and shouldn't be touched. + DomainBase alreadyDeletedDomain = + persistResource( + newDomainBase("bar.tld") + .asBuilder() + .setAutorenewEndTime(Optional.of(clock.nowUtc().minusDays(10))) + .setDeletionTime(clock.nowUtc().plusDays(17)) + .build()); + clock.advanceOneMilli(); + + // A non-autorenewing domain that hasn't reached its expiration time and shouldn't be touched. + DomainBase notYetExpiredDomain = + persistResource( + newDomainBase("baz.tld") + .asBuilder() + .setAutorenewEndTime(Optional.of(clock.nowUtc().plusDays(15))) + .build()); + clock.advanceOneMilli(); + + // A non-autorenewing domain that is past its expiration time and should be deleted. + // (This is the only one that needs a full set of subsidiary resources, for the delete flow to + // to operate on.) + DomainBase pendingExpirationDomain = persistNonAutorenewingDomain("fizz.tld"); + + assertThat(tm().loadByEntity(pendingExpirationDomain).getStatusValues()) + .doesNotContain(PENDING_DELETE); + action.run(); + + DomainBase reloadedActiveDomain = tm().loadByEntity(activeDomain); + assertThat(reloadedActiveDomain).isEqualTo(activeDomain); + assertThat(reloadedActiveDomain.getStatusValues()).doesNotContain(PENDING_DELETE); + assertThat(tm().loadByEntity(alreadyDeletedDomain)).isEqualTo(alreadyDeletedDomain); + assertThat(tm().loadByEntity(notYetExpiredDomain)).isEqualTo(notYetExpiredDomain); + DomainBase reloadedExpiredDomain = tm().loadByEntity(pendingExpirationDomain); + assertThat(reloadedExpiredDomain.getStatusValues()).contains(PENDING_DELETE); + assertThat(reloadedExpiredDomain.getDeletionTime()).isEqualTo(clock.nowUtc().plusDays(35)); + } + + @Test + void test_deletesThreeDomainsInOneRun() { + DomainBase domain1 = persistNonAutorenewingDomain("ecck1.tld"); + DomainBase domain2 = persistNonAutorenewingDomain("veee2.tld"); + DomainBase domain3 = persistNonAutorenewingDomain("tarm3.tld"); + + action.run(); + + assertThat(tm().loadByEntity(domain1).getStatusValues()).contains(PENDING_DELETE); + assertThat(tm().loadByEntity(domain2).getStatusValues()).contains(PENDING_DELETE); + assertThat(tm().loadByEntity(domain3).getStatusValues()).contains(PENDING_DELETE); + } + + private DomainBase persistNonAutorenewingDomain(String domainName) { + DomainBase pendingExpirationDomain = persistActiveDomain(domainName); + HistoryEntry createHistoryEntry = + persistResource( + new HistoryEntry.Builder() + .setType(DOMAIN_CREATE) + .setParent(pendingExpirationDomain) + .setModificationTime(clock.nowUtc().minusMonths(9)) + .build()); + BillingEvent.Recurring autorenewBillingEvent = + persistResource(createAutorenewBillingEvent(createHistoryEntry).build()); + PollMessage.Autorenew autorenewPollMessage = + persistResource(createAutorenewPollMessage(createHistoryEntry).build()); + pendingExpirationDomain = + persistResource( + pendingExpirationDomain + .asBuilder() + .setAutorenewEndTime(Optional.of(clock.nowUtc().minusDays(10))) + .setAutorenewBillingEvent(autorenewBillingEvent.createVKey()) + .setAutorenewPollMessage(autorenewPollMessage.createVKey()) + .build()); + clock.advanceOneMilli(); + + return pendingExpirationDomain; + } + + private BillingEvent.Recurring.Builder createAutorenewBillingEvent( + HistoryEntry createHistoryEntry) { + return new BillingEvent.Recurring.Builder() + .setReason(Reason.RENEW) + .setFlags(ImmutableSet.of(Flag.AUTO_RENEW)) + .setTargetId("fizz.tld") + .setClientId("TheRegistrar") + .setEventTime(clock.nowUtc().plusYears(1)) + .setRecurrenceEndTime(END_OF_TIME) + .setParent(createHistoryEntry); + } + + private PollMessage.Autorenew.Builder createAutorenewPollMessage( + HistoryEntry createHistoryEntry) { + return new PollMessage.Autorenew.Builder() + .setTargetId("fizz.tld") + .setClientId("TheRegistrar") + .setEventTime(clock.nowUtc().plusYears(1)) + .setAutorenewEndTime(END_OF_TIME) + .setParent(createHistoryEntry); + } +} diff --git a/core/src/test/java/google/registry/flows/EppTestComponent.java b/core/src/test/java/google/registry/flows/EppTestComponent.java index 80a278265..7aa00f616 100644 --- a/core/src/test/java/google/registry/flows/EppTestComponent.java +++ b/core/src/test/java/google/registry/flows/EppTestComponent.java @@ -45,12 +45,8 @@ import javax.inject.Singleton; /** Dagger component for running EPP tests. */ @Singleton -@Component( - modules = { - ConfigModule.class, - EppTestComponent.FakesAndMocksModule.class - }) -interface EppTestComponent { +@Component(modules = {ConfigModule.class, EppTestComponent.FakesAndMocksModule.class}) +public interface EppTestComponent { RequestComponent startRequest();