Check if lock owner is finished on lock acquisition

Sometimes requests "die" suddenly, without going through catch/finally blocks.
If this happens, any lock they own will remain locked until it times out (which
can take hours in some cases).

This cl implicitly unlocks any lock if the owner of the lock isn't running
anymore.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=168880938

java/google/registry/request/RequestModule.java

@@ -28,7 +28,7 @@ import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.HttpException.UnsupportedMediaTypeException;
 import google.registry.request.auth.AuthResult;
 import google.registry.request.lock.LockHandler;
-import google.registry.request.lock.LockHandlerPassthrough;
+import google.registry.request.lock.LockHandlerImpl;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.RequestStatusCheckerImpl;
 import java.io.IOException;
@@ -127,7 +127,7 @@ public final class RequestModule {
   }
 
   @Provides
-  static LockHandler provideLockHandler(LockHandlerPassthrough lockHandler) {
+  static LockHandler provideLockHandler(LockHandlerImpl lockHandler) {
     return lockHandler;
   }
 

java/google/registry/request/lock/BUILD

@@ -9,8 +9,10 @@ java_library(
     srcs = glob(["*.java"]),
     deps = [
         "//java/google/registry/model",
+        "//java/google/registry/util",
         "@com_google_code_findbugs_jsr305",
         "@com_google_dagger",
+        "@com_google_guava",
         "@joda_time",
     ],
 )

java/google/registry/request/lock/LockHandlerImpl.java

@@ -0,0 +1,127 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.request.lock;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Throwables.throwIfUnchecked;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSortedSet;
+import google.registry.model.server.Lock;
+import google.registry.util.AppEngineTimeLimiter;
+import google.registry.util.FormattingLogger;
+import google.registry.util.RequestStatusChecker;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import org.joda.time.Duration;
+
+/** Implementation of {@link LockHandler} that uses the datastore lock. */
+public class LockHandlerImpl implements LockHandler {
+
+  private static final long serialVersionUID = 6551645164118637767L;
+
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+
+  /** Fudge factor to make sure we kill threads before a lock actually expires. */
+  private static final Duration LOCK_TIMEOUT_FUDGE = Duration.standardSeconds(5);
+
+  @Inject RequestStatusChecker requestStatusChecker;
+
+  @Inject public LockHandlerImpl() {}
+
+  /**
+   * Acquire one or more locks and execute a Void {@link Callable}.
+   *
+   * <p>Thread will be killed if it doesn't complete before the lease expires.
+   *
+   * <p>Note that locks are specific either to a given tld or to the entire system (in which case
+   * tld should be passed as null).
+   *
+   * @return whether all locks were acquired and the callable was run.
+   */
+  @Override
+  public boolean executeWithLocks(
+      final Callable<Void> callable,
+      @Nullable String tld,
+      Duration leaseLength,
+      String... lockNames) {
+    try {
+      return AppEngineTimeLimiter.create().callWithTimeout(
+          new LockingCallable(callable, Strings.emptyToNull(tld), leaseLength, lockNames),
+          leaseLength.minus(LOCK_TIMEOUT_FUDGE).getMillis(),
+          TimeUnit.MILLISECONDS,
+          true);
+    } catch (Exception e) {
+      throwIfUnchecked(e);
+      throw new RuntimeException(e);
+    }
+  }
+
+  /** Allows injection of mock Lock in tests. */
+  @VisibleForTesting
+  Optional<Lock> acquire(String lockName, @Nullable String tld, Duration leaseLength) {
+    return Lock.acquire(lockName, tld, leaseLength, requestStatusChecker);
+  }
+
+  /** A {@link Callable} that acquires and releases a lock around a delegate {@link Callable}. */
+  private class LockingCallable implements Callable<Boolean> {
+    final Callable<Void> delegate;
+    @Nullable final String tld;
+    final Duration leaseLength;
+    final Set<String> lockNames;
+
+    LockingCallable(
+        Callable<Void> delegate,
+        String tld,
+        Duration leaseLength,
+        String... lockNames) {
+      checkArgument(leaseLength.isLongerThan(LOCK_TIMEOUT_FUDGE));
+      this.delegate = delegate;
+      this.tld = tld;
+      this.leaseLength = leaseLength;
+      // Make sure we join locks in a fixed (lexicographical) order to avoid deadlock.
+      this.lockNames = ImmutableSortedSet.copyOf(lockNames);
+    }
+
+    @Override
+    public Boolean call() throws Exception {
+      Set<Lock> acquiredLocks = new HashSet<>();
+      try {
+        for (String lockName : lockNames) {
+          Optional<Lock> lock = acquire(lockName, tld, leaseLength);
+          if (!lock.isPresent()) {
+            logger.infofmt("Couldn't acquire lock named: %s for TLD: %s", lockName, tld);
+            return false;
+          }
+          logger.infofmt("Acquired lock: %s", lock);
+          acquiredLocks.add(lock.get());
+        }
+        delegate.call();
+        return true;
+      } finally {
+        for (Lock lock : acquiredLocks) {
+          lock.release();
+          logger.infofmt("Released lock: %s", lock);
+        }
+      }
+    }
+  }
+}

java/google/registry/request/lock/LockHandlerPassthrough.java

@@ -1,54 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.request.lock;
-
-import google.registry.model.server.Lock;
-import java.util.concurrent.Callable;
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import org.joda.time.Duration;
-
-/**
- * Implementation of {@link LockHandler} that uses Lock as is.
- *
- * This is a temporary implementation to help migrate from Lock to LockHandler. Once the migration
- * is complete - we will create a "proper" LockHandlerImpl class and remove this one.
- *
- * TODO(guyben):delete this class once LockHandlerImpl is done.
- */
-public class LockHandlerPassthrough implements LockHandler {
-
-  private static final long serialVersionUID = 6551645164118637767L;
-
-  @Inject public LockHandlerPassthrough() {}
-
-  /**
-   * Acquire one or more locks and execute a Void {@link Callable}.
-   *
-   * <p>Runs on a thread that will be killed if it doesn't complete before the lease expires.
-   *
-   * <p>This is a simple passthrough to {@link Lock#executeWithLocks}.
-   *
-   * @return true if all locks were acquired and the callable was run; false otherwise.
-   */
-  @Override
-  public boolean executeWithLocks(
-      final Callable<Void> callable,
-      @Nullable String tld,
-      Duration leaseLength,
-      String... lockNames) {
-    return Lock.executeWithLocks(callable, tld, leaseLength, lockNames);
-  }
-}