Add a scrap command to backfill registry locks (#478)

* Add a scrap command to backfill registry locks

* fix tests

* Change comments and messages

* Use URS time (best effort) if one exists

* Don't bother with root cause

core/src/main/java/google/registry/tools/CommandWithRemoteApi.java

@@ -20,4 +20,4 @@ package google.registry.tools;
  * <p>Just implementing this is sufficient to use the remote api; {@link RegistryTool} will install
  * it as needed.
  */
-interface CommandWithRemoteApi extends Command {}
+public interface CommandWithRemoteApi extends Command {}

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -15,6 +15,7 @@
 package google.registry.tools;
 
 import com.google.common.collect.ImmutableMap;
+import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
 import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
 import google.registry.tools.javascrap.RemoveIpAddressCommand;
 
@@ -30,6 +31,7 @@ public final class RegistryTool {
   public static final ImmutableMap<String, Class<? extends Command>> COMMAND_MAP =
       new ImmutableMap.Builder<String, Class<? extends Command>>()
           .put("ack_poll_messages", AckPollMessagesCommand.class)
+          .put("backfill_registry_locks", BackfillRegistryLocksCommand.class)
           .put("canonicalize_labels", CanonicalizeLabelsCommand.class)
           .put("check_domain", CheckDomainCommand.class)
           .put("check_domain_claims", CheckDomainClaimsCommand.class)

core/src/main/java/google/registry/tools/javascrap/BackfillRegistryLocksCommand.java

@@ -0,0 +1,152 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import com.googlecode.objectify.Key;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.registry.RegistryLockDao;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.schema.domain.RegistryLock;
+import google.registry.tools.CommandWithRemoteApi;
+import google.registry.tools.ConfirmingCommand;
+import google.registry.util.Clock;
+import google.registry.util.StringGenerator;
+import java.util.Comparator;
+import java.util.List;
+import javax.inject.Inject;
+import javax.inject.Named;
+import org.joda.time.DateTime;
+
+/**
+ * Scrap tool to backfill {@link RegistryLock}s for domains previously locked.
+ *
+ * <p>This will save new objects for all existing domains that are locked but don't have any
+ * corresponding lock objects already in the database.
+ */
+@Parameters(
+    separators = " =",
+    commandDescription =
+        "Backfills RegistryLock objects for specified domain resource IDs that are locked but don't"
+            + " already have a corresponding RegistryLock object.")
+public class BackfillRegistryLocksCommand extends ConfirmingCommand
+    implements CommandWithRemoteApi {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final int VERIFICATION_CODE_LENGTH = 32;
+
+  @Parameter(
+      names = {"--domain_roids"},
+      description = "Comma-separated list of domain roids to check")
+  protected List<String> roids;
+
+  // Inject here so that we can create the command automatically for tests
+  @Inject Clock clock;
+
+  @Inject
+  @Config("registryAdminClientId")
+  String registryAdminClientId;
+
+  @Inject
+  @Named("base58StringGenerator")
+  StringGenerator stringGenerator;
+
+  private DateTime now;
+  private ImmutableList<DomainBase> lockedDomains;
+
+  @Override
+  protected String prompt() {
+    checkArgument(
+        roids != null && !roids.isEmpty(), "Must provide non-empty domain_roids argument");
+    now = clock.nowUtc();
+    lockedDomains = getLockedDomainsWithoutLocks();
+    ImmutableList<String> lockedDomainNames =
+        lockedDomains.stream()
+            .map(DomainBase::getFullyQualifiedDomainName)
+            .collect(toImmutableList());
+    return String.format(
+        "Locked domains for which there does not exist a RegistryLock object: %s",
+        lockedDomainNames);
+  }
+
+  @Override
+  protected String execute() {
+    ImmutableSet.Builder<DomainBase> failedDomainsBuilder = new ImmutableSet.Builder<>();
+    for (DomainBase domainBase : lockedDomains) {
+      try {
+        RegistryLockDao.save(
+            new RegistryLock.Builder()
+                .isSuperuser(true)
+                .setRegistrarId(registryAdminClientId)
+                .setRepoId(domainBase.getRepoId())
+                .setDomainName(domainBase.getFullyQualifiedDomainName())
+                .setLockCompletionTimestamp(getLockCompletionTimestamp(domainBase, now))
+                .setVerificationCode(stringGenerator.createString(VERIFICATION_CODE_LENGTH))
+                .build());
+      } catch (Throwable t) {
+        logger.atSevere().withCause(t).log(
+            "Error when creating lock object for domain %s.",
+            domainBase.getFullyQualifiedDomainName());
+        failedDomainsBuilder.add(domainBase);
+      }
+    }
+    ImmutableSet<DomainBase> failedDomains = failedDomainsBuilder.build();
+    if (failedDomains.isEmpty()) {
+      return String.format(
+          "Successfully created lock objects for %d domains.", lockedDomains.size());
+    } else {
+      return String.format(
+          "Successfully created lock objects for %d domains. We failed to create locks "
+              + "for the following domains: %s",
+          lockedDomains.size() - failedDomains.size(), lockedDomains);
+    }
+  }
+
+  private DateTime getLockCompletionTimestamp(DomainBase domainBase, DateTime now) {
+    // Best-effort, if a domain was URS-locked we should use that time
+    // If we can't find that, return now.
+    return ofy().load().type(HistoryEntry.class).ancestor(domainBase).list().stream()
+        // sort by modification time descending so we get the most recent one if it was locked twice
+        .sorted(Comparator.comparing(HistoryEntry::getModificationTime).reversed())
+        .filter(entry -> entry.getReason().equals("Uniform Rapid Suspension"))
+        .findFirst()
+        .map(HistoryEntry::getModificationTime)
+        .orElse(now);
+  }
+
+  private ImmutableList<DomainBase> getLockedDomainsWithoutLocks() {
+    return ImmutableList.copyOf(
+        ofy().load()
+            .keys(
+                roids.stream()
+                    .map(roid -> Key.create(DomainBase.class, roid))
+                    .collect(toImmutableList()))
+            .values().stream()
+            .filter(d -> d.getDeletionTime().isAfter(now))
+            .filter(d -> d.getStatusValues().containsAll(REGISTRY_LOCK_STATUSES))
+            .filter(d -> !RegistryLockDao.getMostRecentByRepoId(d.getRepoId()).isPresent())
+            .collect(toImmutableList()));
+  }
+}

core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java

@@ -32,6 +32,7 @@ import google.registry.tools.LockDomainCommandTest;
 import google.registry.tools.UnlockDomainCommandTest;
 import google.registry.tools.UpdateRegistrarCommandTest;
 import google.registry.tools.UpdateReservedListCommandTest;
+import google.registry.tools.javascrap.BackfillRegistryLocksCommandTest;
 import google.registry.tools.server.CreatePremiumListActionTest;
 import google.registry.tools.server.UpdatePremiumListActionTest;
 import google.registry.ui.server.registrar.RegistryLockGetActionTest;
@@ -56,6 +57,7 @@ import org.junit.runners.Suite.SuiteClasses;
  */
 @RunWith(Suite.class)
 @SuiteClasses({
+  BackfillRegistryLocksCommandTest.class,
   ClaimsListDaoTest.class,
   CreatePremiumListActionTest.class,
   CreateRegistrarCommandTest.class,

core/src/test/java/google/registry/tools/javascrap/BackfillRegistryLocksCommandTest.java

@@ -0,0 +1,176 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.DatastoreHelper.createTld;
+import static google.registry.testing.DatastoreHelper.persistActiveDomain;
+import static google.registry.testing.DatastoreHelper.persistDeletedDomain;
+import static google.registry.testing.DatastoreHelper.persistNewRegistrar;
+import static google.registry.testing.DatastoreHelper.persistResource;
+import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
+import static org.junit.Assert.assertThrows;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.truth.Truth8;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registry.RegistryLockDao;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.transaction.JpaTestRules;
+import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageRule;
+import google.registry.schema.domain.RegistryLock;
+import google.registry.testing.DeterministicStringGenerator;
+import google.registry.testing.FakeClock;
+import google.registry.tools.CommandTestCase;
+import google.registry.util.StringGenerator.Alphabets;
+import java.util.Optional;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link BackfillRegistryLocksCommand}. */
+@RunWith(JUnit4.class)
+public class BackfillRegistryLocksCommandTest
+    extends CommandTestCase<BackfillRegistryLocksCommand> {
+
+  private final FakeClock fakeClock = new FakeClock();
+
+  @Rule
+  public final JpaIntegrationWithCoverageRule jpaRule =
+      new JpaTestRules.Builder().buildIntegrationWithCoverageRule();
+
+  @Before
+  public void before() {
+    persistNewRegistrar("adminreg", "Admin Registrar", Registrar.Type.REAL, 693L);
+    createTld("tld");
+    command.registryAdminClientId = "adminreg";
+    command.clock = fakeClock;
+    command.stringGenerator = new DeterministicStringGenerator(Alphabets.BASE_58);
+  }
+
+  @Test
+  public void testSimpleBackfill() throws Exception {
+    DomainBase domain = persistLockedDomain("example.tld");
+    Truth8.assertThat(RegistryLockDao.getMostRecentByRepoId(domain.getRepoId())).isEmpty();
+
+    runCommandForced("--domain_roids", domain.getRepoId());
+
+    Optional<RegistryLock> lockOptional = RegistryLockDao.getMostRecentByRepoId(domain.getRepoId());
+    Truth8.assertThat(lockOptional).isPresent();
+    Truth8.assertThat(lockOptional.get().getLockCompletionTimestamp()).isPresent();
+  }
+
+  @Test
+  public void testBackfill_onlyLockedDomains() throws Exception {
+    DomainBase neverLockedDomain = persistActiveDomain("neverlocked.tld");
+    DomainBase previouslyLockedDomain = persistLockedDomain("unlocked.tld");
+    persistResource(previouslyLockedDomain.asBuilder().setStatusValues(ImmutableSet.of()).build());
+    DomainBase lockedDomain = persistLockedDomain("locked.tld");
+
+    runCommandForced(
+        "--domain_roids",
+        String.format(
+            "%s,%s,%s",
+            neverLockedDomain.getRepoId(),
+            previouslyLockedDomain.getRepoId(),
+            lockedDomain.getRepoId()));
+
+    ImmutableList<RegistryLock> locks = RegistryLockDao.getLockedDomainsByRegistrarId("adminreg");
+    assertThat(locks).hasSize(1);
+    assertThat(Iterables.getOnlyElement(locks).getDomainName()).isEqualTo("locked.tld");
+  }
+
+  @Test
+  public void testBackfill_skipsDeletedDomains() throws Exception {
+    DomainBase domain = persistDeletedDomain("example.tld", fakeClock.nowUtc());
+    persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build());
+    fakeClock.advanceBy(Duration.standardSeconds(1));
+    runCommandForced("--domain_roids", domain.getRepoId());
+    Truth8.assertThat(RegistryLockDao.getMostRecentByRepoId(domain.getRepoId())).isEmpty();
+  }
+
+  @Test
+  public void testBackfill_skipsDomains_ifLockAlreadyExists() throws Exception {
+    DomainBase domain = persistLockedDomain("example.tld");
+
+    RegistryLock previousLock =
+        RegistryLockDao.save(
+            new RegistryLock.Builder()
+                .isSuperuser(true)
+                .setRegistrarId("adminreg")
+                .setRepoId(domain.getRepoId())
+                .setDomainName(domain.getFullyQualifiedDomainName())
+                .setLockCompletionTimestamp(fakeClock.nowUtc())
+                .setVerificationCode(command.stringGenerator.createString(32))
+                .build());
+
+    fakeClock.advanceBy(Duration.standardDays(1));
+    runCommandForced("--domain_roids", domain.getRepoId());
+
+    assertThat(
+            RegistryLockDao.getMostRecentByRepoId(domain.getRepoId())
+                .get()
+                .getLockCompletionTimestamp())
+        .isEqualTo(previousLock.getLockCompletionTimestamp());
+  }
+
+  @Test
+  public void testBackfill_usesUrsTime_ifExists() throws Exception {
+    DateTime ursTime = fakeClock.nowUtc();
+    DomainBase ursDomain = persistLockedDomain("urs.tld");
+    HistoryEntry historyEntry =
+        new HistoryEntry.Builder()
+            .setBySuperuser(true)
+            .setClientId("adminreg")
+            .setModificationTime(ursTime)
+            .setParent(ursDomain)
+            .setReason("Uniform Rapid Suspension")
+            .setType(HistoryEntry.Type.DOMAIN_UPDATE)
+            .setRequestedByRegistrar(false)
+            .build();
+    persistResource(historyEntry);
+    DomainBase nonUrsDomain = persistLockedDomain("nonurs.tld");
+
+    fakeClock.advanceBy(Duration.standardDays(10));
+    runCommandForced(
+        "--domain_roids", String.format("%s,%s", ursDomain.getRepoId(), nonUrsDomain.getRepoId()));
+
+    RegistryLock ursLock =
+        RegistryLockDao.getMostRecentVerifiedLockByRepoId(ursDomain.getRepoId()).get();
+    assertThat(ursLock.getLockCompletionTimestamp().get()).isEqualTo(ursTime);
+    RegistryLock nonUrsLock =
+        RegistryLockDao.getMostRecentVerifiedLockByRepoId(nonUrsDomain.getRepoId()).get();
+    assertThat(nonUrsLock.getLockCompletionTimestamp().get()).isEqualTo(fakeClock.nowUtc());
+  }
+
+  @Test
+  public void testFailure_mustProvideDomainRoids() {
+    assertThat(assertThrows(IllegalArgumentException.class, () -> runCommandForced()))
+        .hasMessageThat()
+        .isEqualTo("Must provide non-empty domain_roids argument");
+  }
+
+  private static DomainBase persistLockedDomain(String domainName) {
+    DomainBase domain = persistActiveDomain(domainName);
+    return persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build());
+  }
+}