Add the Distribution data type for instrumentation

This is one of a series of CLs adding a new metric type, EventMetric, which
is used for tracking numerical distributions.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=132103552

java/google/registry/monitoring/metrics/BUILD

@@ -18,6 +18,7 @@ java_library(
         "//java/com/google/common/collect",
         "//java/com/google/common/html",
         "//java/com/google/common/math",
+        "//java/com/google/common/primitives",
         "//java/com/google/common/util/concurrent",
         "//third_party/java/appengine:appengine-api",
         "//third_party/java/auto:auto_value",

java/google/registry/monitoring/metrics/CustomFitter.java

@@ -0,0 +1,53 @@
+// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.monitoring.metrics;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.monitoring.metrics.MetricsUtils.checkDouble;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Ordering;
+
+/**
+ * Models a {@link DistributionFitter} with arbitrary sized intervals.
+ *
+ * <p>If only only one boundary is provided, then the fitter will consist of an overflow and
+ * underflow interval separated by that boundary.
+ */
+@AutoValue
+public abstract class CustomFitter implements DistributionFitter {
+
+  /**
+   * Create a new {@link CustomFitter} with the given interval boundaries.
+   *
+   * @param boundaries is a sorted list of interval boundaries
+   * @throws IllegalArgumentException if {@code boundaries} is empty or not sorted in ascending
+   *     order, or if a value in the set is infinite, {@code NaN}, or {@code -0.0}.
+   */
+  public static CustomFitter create(ImmutableSet<Double> boundaries) {
+    checkArgument(boundaries.size() > 0, "boundaries must not be empty");
+    checkArgument(Ordering.natural().isOrdered(boundaries), "boundaries must be sorted");
+    for (Double d : boundaries) {
+      checkDouble(d);
+    }
+
+    return new AutoValue_CustomFitter(ImmutableSortedSet.copyOf(boundaries));
+  }
+
+  @Override
+  public abstract ImmutableSortedSet<Double> boundaries();
+}

java/google/registry/monitoring/metrics/Distribution.java

@@ -0,0 +1,46 @@
+// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.monitoring.metrics;
+
+import com.google.common.collect.ImmutableRangeMap;
+
+/**
+ * Models a distribution of double-precision floating point sample data, and provides summary
+ * statistics of the distribution. This class also models the probability density function (PDF) of
+ * the distribution with a histogram.
+ *
+ * <p>The summary statistics provided are the mean and sumOfSquaredDeviation of the distribution.
+ *
+ * <p>The histogram fitting function is provided via a {@link DistributionFitter} implementation.
+ *
+ * @see DistributionFitter
+ */
+public interface Distribution {
+
+  /** Returns the mean of this distribution. */
+  double mean();
+
+  /** Returns the sum of squared deviations from the mean of this distribution. */
+  double sumOfSquaredDeviation();
+
+  /** Returns the count of samples in this distribution. */
+  long count();
+
+  /** Returns a histogram of the distribution's values. */
+  ImmutableRangeMap<Double, Long> intervalCounts();
+
+  /** Returns the {@link DistributionFitter} of this distribution. */
+  DistributionFitter distributionFitter();
+}

java/google/registry/monitoring/metrics/DistributionFitter.java

@@ -0,0 +1,33 @@
+// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.monitoring.metrics;
+
+import com.google.common.collect.ImmutableSortedSet;
+
+/**
+ * A companion interface to {@link Distribution} which fits samples to a histogram in order to
+ * estimate the probability density function (PDF) of the {@link Distribution}.
+ *
+ * <p>The fitter models the histogram with a set of finite boundaries. The closed-open interval
+ * [a,b) between two consecutive boundaries represents the domain of permissible values in that
+ * interval. The values less than the first boundary are in the underflow interval of (-inf, a) and
+ * values greater or equal to the last boundary in the array are in the overflow interval of [a,
+ * inf).
+ */
+public interface DistributionFitter {
+
+  /** Returns a sorted set of the boundaries modeled by this {@link DistributionFitter}. */
+  ImmutableSortedSet<Double> boundaries();
+}

java/google/registry/monitoring/metrics/ExponentialFitter.java

@@ -0,0 +1,68 @@
+// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.monitoring.metrics;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.monitoring.metrics.MetricsUtils.checkDouble;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableSortedSet;
+
+/**
+ * Models a {@link DistributionFitter} with intervals of exponentially increasing size.
+ *
+ * <p>The interval boundaries are defined by {@code scale * Math.pow(base, i)} for {@code i} in
+ * {@code [0, numFiniteIntervals]}.
+ *
+ * <p>For example, an {@link ExponentialFitter} with {@code numFiniteIntervals=3, base=4.0,
+ * scale=1.5} represents a histogram with intervals {@code (-inf, 1.5), [1.5, 6), [6, 24), [24, 96),
+ * [96, +inf)}.
+ */
+@AutoValue
+public abstract class ExponentialFitter implements DistributionFitter {
+
+  /**
+   * Create a new {@link ExponentialFitter}.
+   *
+   * @param numFiniteIntervals the number of intervals, excluding the underflow and overflow
+   *     intervals
+   * @param base the base of the exponent
+   * @param scale a multiplicative factor for the exponential function
+   * @throws IllegalArgumentException if {@code numFiniteIntervals <= 0}, {@code width <= 0} or
+   *     {@code base <= 1}
+   */
+  public static ExponentialFitter create(int numFiniteIntervals, double base, double scale) {
+    checkArgument(numFiniteIntervals > 0, "numFiniteIntervals must be greater than 0");
+    checkArgument(scale != 0, "scale must not be 0");
+    checkArgument(base > 1, "base must be greater than 1");
+    checkDouble(base);
+    checkDouble(scale);
+
+    ImmutableSortedSet.Builder<Double> boundaries = ImmutableSortedSet.naturalOrder();
+
+    for (int i = 0; i < numFiniteIntervals + 1; i++) {
+      boundaries.add(scale * Math.pow(base, i));
+    }
+
+    return new AutoValue_ExponentialFitter(base, scale, boundaries.build());
+  }
+
+  public abstract double base();
+
+  public abstract double scale();
+
+  @Override
+  public abstract ImmutableSortedSet<Double> boundaries();
+}

java/google/registry/monitoring/metrics/ImmutableDistribution.java

@@ -0,0 +1,54 @@
+// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.monitoring.metrics;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableRangeMap;
+import javax.annotation.concurrent.ThreadSafe;
+
+/**
+ * An immutable {@link Distribution}. Instances of this class can used to create {@link MetricPoint}
+ * instances, and should be used when exporting values to a {@link MetricWriter}.
+ *
+ * @see MutableDistribution
+ */
+@ThreadSafe
+@AutoValue
+public abstract class ImmutableDistribution implements Distribution {
+
+  public static ImmutableDistribution copyOf(Distribution distribution) {
+    return new AutoValue_ImmutableDistribution(
+        distribution.mean(),
+        distribution.sumOfSquaredDeviation(),
+        distribution.count(),
+        distribution.intervalCounts(),
+        distribution.distributionFitter());
+  }
+
+  @Override
+  public abstract double mean();
+
+  @Override
+  public abstract double sumOfSquaredDeviation();
+
+  @Override
+  public abstract long count();
+
+  @Override
+  public abstract ImmutableRangeMap<Double, Long> intervalCounts();
+
+  @Override
+  public abstract DistributionFitter distributionFitter();
+}

java/google/registry/monitoring/metrics/LinearFitter.java

@@ -0,0 +1,64 @@
+// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.monitoring.metrics;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.monitoring.metrics.MetricsUtils.checkDouble;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableSortedSet;
+
+/**
+ * Models a {@link DistributionFitter} with equally sized intervals.
+ *
+ * <p>The interval boundaries are defined by {@code width * i + offset} for {@code i} in {@code [0,
+ * numFiniteIntervals}.
+ *
+ * <p>For example, a {@link LinearFitter} with {@code numFiniteIntervals=2, width=10, offset=5}
+ * represents a histogram with intervals {@code (-inf, 5), [5, 15), [15, 25), [25, +inf)}.
+ */
+@AutoValue
+public abstract class LinearFitter implements DistributionFitter {
+
+  /**
+   * Create a new {@link LinearFitter}.
+   *
+   * @param numFiniteIntervals the number of intervals, excluding the underflow and overflow
+   *     intervals
+   * @param width the width of each interval
+   * @param offset the start value of the first interval
+   * @throws IllegalArgumentException if {@code numFiniteIntervals <= 0} or {@code width <= 0}
+   */
+  public static LinearFitter create(int numFiniteIntervals, double width, double offset) {
+    checkArgument(numFiniteIntervals > 0, "numFiniteIntervals must be greater than 0");
+    checkArgument(width > 0, "width must be greater than 0");
+    checkDouble(offset);
+
+    ImmutableSortedSet.Builder<Double> boundaries = ImmutableSortedSet.naturalOrder();
+
+    for (int i = 0; i < numFiniteIntervals + 1; i++) {
+      boundaries.add(width * i + offset);
+    }
+
+    return new AutoValue_LinearFitter(width, offset, boundaries.build());
+  }
+
+  public abstract double width();
+
+  public abstract double offset();
+
+  @Override
+  public abstract ImmutableSortedSet<Double> boundaries();
+}

java/google/registry/monitoring/metrics/MetricsUtils.java

@@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList;
 /** Static helper methods for the Metrics library. */
 final class MetricsUtils {
 
+  private static final Double NEGATIVE_ZERO = -0.0;
   private static final String LABEL_SIZE_ERROR =
       "The count of labelValues must be equal to the underlying Metric's count of labels.";
 
@@ -45,4 +46,11 @@ final class MetricsUtils {
   static void checkLabelValuesLength(Metric<?> metric, ImmutableList<String> labelValues) {
     checkArgument(labelValues.size() == metric.getMetricSchema().labels().size(), LABEL_SIZE_ERROR);
   }
+
+  /** Check that the given double is not infinite, {@code NaN}, or {@code -0.0}. */
+  static void checkDouble(double value) {
+    checkArgument(
+        !Double.isInfinite(value) && !Double.isNaN(value) && !NEGATIVE_ZERO.equals(value),
+        "value must be finite, not NaN, and not -0.0");
+  }
 }

java/google/registry/monitoring/metrics/MutableDistribution.java

@@ -0,0 +1,111 @@
+// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.monitoring.metrics;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static google.registry.monitoring.metrics.MetricsUtils.checkDouble;
+
+import com.google.common.collect.ImmutableRangeMap;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Range;
+import com.google.common.collect.TreeRangeMap;
+import com.google.common.primitives.Doubles;
+import java.util.Map;
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * A mutable {@link Distribution}. Instances of this class <b>should not</b> be used to construct
+ * {@link MetricPoint} instances as {@link MetricPoint} instances are supposed to represent
+ * immutable values.
+ *
+ * @see ImmutableDistribution
+ */
+@NotThreadSafe
+public final class MutableDistribution implements Distribution {
+
+  private final TreeRangeMap<Double, Long> intervalCounts;
+  private final DistributionFitter distributionFitter;
+  private double sumOfSquaredDeviation = 0.0;
+  private double mean = 0.0;
+  private int count = 0;
+
+  /** Constructs an empty Distribution with the specified {@link DistributionFitter}. */
+  public MutableDistribution(DistributionFitter distributionFitter) {
+    this.distributionFitter = checkNotNull(distributionFitter);
+    ImmutableSortedSet<Double> boundaries = distributionFitter.boundaries();
+
+    checkArgument(boundaries.size() > 0);
+    checkArgument(Ordering.natural().isOrdered(boundaries));
+
+    this.intervalCounts = TreeRangeMap.create();
+
+    double[] boundariesArray = Doubles.toArray(distributionFitter.boundaries());
+
+    // Add underflow and overflow intervals
+    this.intervalCounts.put(Range.lessThan(boundariesArray[0]), 0L);
+    this.intervalCounts.put(Range.atLeast(boundariesArray[boundariesArray.length - 1]), 0L);
+
+    // Add finite intervals
+    for (int i = 1; i < boundariesArray.length; i++) {
+      this.intervalCounts.put(Range.closedOpen(boundariesArray[i - 1], boundariesArray[i]), 0L);
+    }
+  }
+
+  public void add(double value) {
+    add(value, 1L);
+  }
+
+  public void add(double value, long numSamples) {
+    checkArgument(numSamples > 0, "numSamples must be greater than 0");
+    checkDouble(value);
+
+    Map.Entry<Range<Double>, Long> entry = intervalCounts.getEntry(value);
+    intervalCounts.put(entry.getKey(), entry.getValue() + numSamples);
+    this.count += numSamples;
+
+    // Update mean and sumOfSquaredDeviation using Welford's method
+    // See Knuth, "The Art of Computer Programming", Vol. 2, page 232, 3rd edition
+    double delta = value - mean;
+    mean += delta * numSamples / count;
+    sumOfSquaredDeviation += delta * (value - mean) * numSamples;
+  }
+
+  @Override
+  public double mean() {
+    return mean;
+  }
+
+  @Override
+  public double sumOfSquaredDeviation() {
+    return sumOfSquaredDeviation;
+  }
+
+  @Override
+  public long count() {
+    return count;
+  }
+
+  @Override
+  public ImmutableRangeMap<Double, Long> intervalCounts() {
+    return ImmutableRangeMap.copyOf(intervalCounts);
+  }
+
+  @Override
+  public DistributionFitter distributionFitter() {
+    return distributionFitter;
+  }
+}