Add logging statement for # of tasks in DNS queue

This will make DNS issues easier to debug retroactively as we will be
able to determine, by looking at the logs, if the queue size was growing
unbounded.

Also adds some logging helpers to allow programmatically choosing the level
of logging.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=163123783
This commit is contained in:
mcilwain 2017-07-25 14:31:19 -07:00 committed by Ben McIlwain
parent 1a1fdfd531
commit 8869814e96
3 changed files with 24 additions and 4 deletions

View file

@ -31,14 +31,17 @@ import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.appengine.api.taskqueue.TransientFailureException;
import com.google.apphosting.api.DeadlineExceededException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.net.InternetDomainName;
import google.registry.dns.DnsConstants.TargetType;
import google.registry.model.registry.Registries;
import google.registry.util.FormattingLogger;
import google.registry.util.NonFinalForTesting;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import javax.inject.Inject;
import javax.inject.Named;
import org.joda.time.Duration;
@ -66,7 +69,9 @@ public class DnsQueue {
return new DnsQueue(getQueue(DNS_PULL_QUEUE_NAME));
}
long writeBatchSize = QueueConstants.maxLeaseCount();
@NonFinalForTesting
@VisibleForTesting
long leaseTasksBatchSize = QueueConstants.maxLeaseCount();
/**
* Enqueues the given task type with the given target name to the DNS queue.
@ -107,7 +112,13 @@ public class DnsQueue {
/** Returns handles for a batch of tasks, leased for the specified duration. */
public List<TaskHandle> leaseTasks(Duration leaseDuration) {
try {
return queue.leaseTasks(leaseDuration.getMillis(), MILLISECONDS, writeBatchSize);
int numTasks = queue.fetchStatistics().getNumTasks();
logger.logfmt(
(numTasks >= leaseTasksBatchSize) ? Level.WARNING : Level.INFO,
"There are %d tasks in the DNS queue '%s'.",
numTasks,
DNS_PULL_QUEUE_NAME);
return queue.leaseTasks(leaseDuration.getMillis(), MILLISECONDS, leaseTasksBatchSize);
} catch (TransientFailureException | DeadlineExceededException e) {
logger.severe(e, "Failed leasing tasks too fast");
return ImmutableList.of();

View file

@ -19,6 +19,7 @@ import com.google.common.collect.FluentIterable;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.Nullable;
/** Logging wrapper. */
public class FormattingLogger {
@ -34,7 +35,15 @@ public class FormattingLogger {
this.logger = Logger.getLogger(name);
}
private void log(Level level, Throwable cause, String msg) {
public void logfmt(Level level, Throwable cause, String fmt, Object... args) {
log(level, cause, String.format(fmt, args));
}
public void logfmt(Level level, String fmt, Object... args) {
log(level, null, String.format(fmt, args));
}
private void log(Level level, @Nullable Throwable cause, String msg) {
StackTraceElement callerFrame = FluentIterable
.from(new Exception().getStackTrace())
.firstMatch(new Predicate<StackTraceElement>() {

View file

@ -45,7 +45,7 @@ public class DnsQueueTest {
@Before
public void init() {
dnsQueue = DnsQueue.create();
dnsQueue.writeBatchSize = 10;
dnsQueue.leaseTasksBatchSize = 10;
}
@Test