From 64abebec82d977629c8d55a570b77f38aeb6df03 Mon Sep 17 00:00:00 2001 From: shikhman Date: Mon, 15 Aug 2016 09:14:31 -0700 Subject: [PATCH] Add StackDriver implementation, in monitoring/metrics package ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=130287319 --- .../monitoring/metrics/AbstractMetric.java | 60 ++++ java/google/registry/monitoring/metrics/BUILD | 29 ++ .../registry/monitoring/metrics/Counter.java | 97 ++++++ .../metrics/IncrementableMetric.java | 35 +++ .../monitoring/metrics/LabelDescriptor.java | 56 ++++ .../registry/monitoring/metrics/Metric.java | 43 +++ .../monitoring/metrics/MetricExporter.java | 52 ++++ .../monitoring/metrics/MetricMetrics.java | 84 ++++++ .../monitoring/metrics/MetricPoint.java | 54 ++++ .../monitoring/metrics/MetricRegistry.java | 113 +++++++ .../metrics/MetricRegistryImpl.java | 118 ++++++++ .../monitoring/metrics/MetricReporter.java | 114 +++++++ .../monitoring/metrics/MetricSchema.java | 71 +++++ .../monitoring/metrics/MetricWriter.java | 36 +++ .../monitoring/metrics/SettableMetric.java | 25 ++ .../monitoring/metrics/StackdriverWriter.java | 283 ++++++++++++++++++ .../monitoring/metrics/StoredMetric.java | 90 ++++++ .../monitoring/metrics/VirtualMetric.java | 86 ++++++ .../monitoring/metrics/package-info.java | 16 + java/google/registry/repositories.bzl | 6 + third_party/java/error_prone/BUILD | 8 + 21 files changed, 1476 insertions(+) create mode 100644 java/google/registry/monitoring/metrics/AbstractMetric.java create mode 100644 java/google/registry/monitoring/metrics/BUILD create mode 100644 java/google/registry/monitoring/metrics/Counter.java create mode 100644 java/google/registry/monitoring/metrics/IncrementableMetric.java create mode 100644 java/google/registry/monitoring/metrics/LabelDescriptor.java create mode 100644 java/google/registry/monitoring/metrics/Metric.java create mode 100644 java/google/registry/monitoring/metrics/MetricExporter.java create mode 100644 java/google/registry/monitoring/metrics/MetricMetrics.java create mode 100644 java/google/registry/monitoring/metrics/MetricPoint.java create mode 100644 java/google/registry/monitoring/metrics/MetricRegistry.java create mode 100644 java/google/registry/monitoring/metrics/MetricRegistryImpl.java create mode 100644 java/google/registry/monitoring/metrics/MetricReporter.java create mode 100644 java/google/registry/monitoring/metrics/MetricSchema.java create mode 100644 java/google/registry/monitoring/metrics/MetricWriter.java create mode 100644 java/google/registry/monitoring/metrics/SettableMetric.java create mode 100644 java/google/registry/monitoring/metrics/StackdriverWriter.java create mode 100644 java/google/registry/monitoring/metrics/StoredMetric.java create mode 100644 java/google/registry/monitoring/metrics/VirtualMetric.java create mode 100644 java/google/registry/monitoring/metrics/package-info.java create mode 100644 third_party/java/error_prone/BUILD diff --git a/java/google/registry/monitoring/metrics/AbstractMetric.java b/java/google/registry/monitoring/metrics/AbstractMetric.java new file mode 100644 index 000000000..58b33d7dd --- /dev/null +++ b/java/google/registry/monitoring/metrics/AbstractMetric.java @@ -0,0 +1,60 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableSet; +import google.registry.monitoring.metrics.MetricSchema.Kind; + +//TODO(shikhman): implement HistogramMetrics. +abstract class AbstractMetric implements Metric { + + private final Class valueClass; + private final MetricSchema metricSchema; + + AbstractMetric( + String name, + String description, + String valueDisplayName, + Kind kind, + ImmutableSet labels, + Class valueClass) { + this.metricSchema = MetricSchema.create(name, description, valueDisplayName, kind, labels); + this.valueClass = valueClass; + } + + /** Returns the schema of this metric. */ + @Override + public final MetricSchema getMetricSchema() { + return metricSchema; + } + + /** + * Returns the type for the value of this metric, which would otherwise be erased at compile-time. + * This is useful for implementors of {@link MetricWriter}. + */ + @Override + public final Class getValueClass() { + return valueClass; + } + + @Override + public final String toString() { + return MoreObjects.toStringHelper(this) + .add("valueClass", valueClass) + .add("schema", metricSchema) + .toString(); + } +} diff --git a/java/google/registry/monitoring/metrics/BUILD b/java/google/registry/monitoring/metrics/BUILD new file mode 100644 index 000000000..494fba08d --- /dev/null +++ b/java/google/registry/monitoring/metrics/BUILD @@ -0,0 +1,29 @@ +package( + default_visibility = ["//java/google/registry:registry_project"], +) + +licenses(["notice"]) # Apache 2.0 + + +java_library( + name = "metrics", + srcs = glob(["*.java"]), + deps = [ + "//google/monitoring:monitoring_java_lib", + "//java/com/google/api/client/json", + "//java/com/google/common/annotations", + "//java/com/google/common/base", + "//java/com/google/common/collect", + "//java/com/google/common/html", + "//java/com/google/common/math", + "//java/com/google/common/util/concurrent", + "//third_party/java/appengine:appengine-api", + "//third_party/java/auto:auto_value", + "//third_party/java/dagger", + "//third_party/java/error_prone:annotations", + "//third_party/java/joda_time", + "//third_party/java/jsr305_annotations", + "//third_party/java/jsr330_inject", + "//third_party/java/re2j", + ], +) diff --git a/java/google/registry/monitoring/metrics/Counter.java b/java/google/registry/monitoring/metrics/Counter.java new file mode 100644 index 000000000..51b413a1f --- /dev/null +++ b/java/google/registry/monitoring/metrics/Counter.java @@ -0,0 +1,97 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.AtomicLongMap; +import google.registry.monitoring.metrics.MetricSchema.Kind; +import java.util.Map.Entry; +import javax.annotation.concurrent.ThreadSafe; +import org.joda.time.Instant; + +/** + * A metric which stores Long values. It is stateful and can be changed in increments. + * + *

This metric is generally suitable for counters, such as requests served or errors generated. + */ +@ThreadSafe +public final class Counter extends AbstractMetric + implements SettableMetric, IncrementableMetric { + + private static final String LABEL_COUNT_ERROR = + "The count of labelValues must be equal to the underlying " + + "MetricDescriptor's count of labels."; + + private final AtomicLongMap> values = AtomicLongMap.create(); + + Counter( + String name, + String description, + String valueDisplayName, + ImmutableSet labels) { + super(name, description, valueDisplayName, Kind.CUMULATIVE, labels, Long.class); + } + + @VisibleForTesting + void incrementBy(Number offset, ImmutableList labelValues) { + values.addAndGet(labelValues, offset.longValue()); + } + + @Override + public final void incrementBy(long offset, String... labelValues) { + checkArgument(labelValues.length == this.getMetricSchema().labels().size(), LABEL_COUNT_ERROR); + + incrementBy(offset, ImmutableList.copyOf(labelValues)); + } + + /** + * Returns a snapshot of the metric's values. The timestamp of each {@link MetricPoint} will be + * the last modification time for that tuple of label values. + */ + @Override + public final ImmutableList> getTimestampedValues() { + return getTimestampedValues(Instant.now()); + } + + @Override + public final int getCardinality() { + return values.size(); + } + + @VisibleForTesting + final ImmutableList> getTimestampedValues(Instant timestamp) { + ImmutableList.Builder> timestampedValues = new ImmutableList.Builder<>(); + for (Entry, Long> entry : values.asMap().entrySet()) { + timestampedValues.add(MetricPoint.create(this, entry.getKey(), timestamp, entry.getValue())); + } + return timestampedValues.build(); + } + + @VisibleForTesting + final void set(Long value, ImmutableList labelValues) { + this.values.put(labelValues, value); + } + + @Override + public final void set(Long value, String... labelValues) { + checkArgument(labelValues.length == this.getMetricSchema().labels().size(), LABEL_COUNT_ERROR); + + set(value, ImmutableList.copyOf(labelValues)); + } +} diff --git a/java/google/registry/monitoring/metrics/IncrementableMetric.java b/java/google/registry/monitoring/metrics/IncrementableMetric.java new file mode 100644 index 000000000..5f017b188 --- /dev/null +++ b/java/google/registry/monitoring/metrics/IncrementableMetric.java @@ -0,0 +1,35 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +/** + * A {@link Metric} which can be incremented. + * + *

This is a view into a {@link Counter} to provide compile-time checking to disallow re-setting + * the metric, which is useful for metrics which should be monotonic. + */ +public interface IncrementableMetric extends Metric { + + /** + * Increments a metric for a given set of label values. + * + *

If the metric is undefined for given label values, it will first be set to zero. + * + *

The metric's timestamp will be updated to the current time for the given label values. + * + *

The count of {@code labelValues} must be equal to the underlying metric's count of labels. + */ + void incrementBy(long offset, String... labelValues); +} diff --git a/java/google/registry/monitoring/metrics/LabelDescriptor.java b/java/google/registry/monitoring/metrics/LabelDescriptor.java new file mode 100644 index 000000000..85862cfd5 --- /dev/null +++ b/java/google/registry/monitoring/metrics/LabelDescriptor.java @@ -0,0 +1,56 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.value.AutoValue; +import com.google.re2j.Pattern; + +/** + * Definition of a metric label. + * + *

If a metric is created with labels, corresponding label values must be provided when setting + * values on the metric. + */ +@AutoValue +public abstract class LabelDescriptor { + + private static final Pattern ALLOWED_LABEL_PATTERN = Pattern.compile("\\w+"); + + LabelDescriptor() {} + + /** + * Returns a new {@link LabelDescriptor}. + * + * @param name identifier for label + * @param description human-readable description of label + * @throws IllegalArgumentException if {@code name} isn't {@code \w+} or {@code description} is + * blank + */ + public static LabelDescriptor create(String name, String description) { + checkArgument( + ALLOWED_LABEL_PATTERN.matches(name), + "Label must match the regex %s", + ALLOWED_LABEL_PATTERN); + checkArgument(!description.isEmpty(), "Description must not be empty"); + + return new AutoValue_LabelDescriptor(name, description); + } + + public abstract String name(); + + public abstract String description(); +} diff --git a/java/google/registry/monitoring/metrics/Metric.java b/java/google/registry/monitoring/metrics/Metric.java new file mode 100644 index 000000000..796db663d --- /dev/null +++ b/java/google/registry/monitoring/metrics/Metric.java @@ -0,0 +1,43 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import com.google.common.collect.ImmutableList; + +/** + * A Metric which stores timestamped values. + * + *

This is a read-only view. + */ +public interface Metric { + + /** + * Returns the list of the latest {@link MetricPoint} instances for every label-value combination + * tracked for this metric. + */ + ImmutableList> getTimestampedValues(); + + /** Returns the count of values being stored with this metric. */ + int getCardinality(); + + /** Returns the schema of this metric. */ + MetricSchema getMetricSchema(); + + /** + * Returns the type for the value of this metric, which would otherwise be erased at compile-time. + * This is useful for implementors of {@link MetricWriter}. + */ + Class getValueClass(); +} diff --git a/java/google/registry/monitoring/metrics/MetricExporter.java b/java/google/registry/monitoring/metrics/MetricExporter.java new file mode 100644 index 000000000..7a73c29f0 --- /dev/null +++ b/java/google/registry/monitoring/metrics/MetricExporter.java @@ -0,0 +1,52 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.AbstractExecutionThreadService; +import java.util.concurrent.BlockingQueue; + +/** + * Background service to asynchronously push bundles of {@link MetricPoint} instances to a {@link + * MetricWriter}. + */ +class MetricExporter extends AbstractExecutionThreadService { + + private final BlockingQueue>>> writeQueue; + private final MetricWriter writer; + + MetricExporter( + BlockingQueue>>> writeQueue, MetricWriter writer) { + this.writeQueue = writeQueue; + this.writer = writer; + } + + @Override + protected void run() throws Exception { + while (isRunning()) { + Optional>> batch = writeQueue.take(); + if (batch.isPresent()) { + for (MetricPoint point : batch.get()) { + writer.write(point); + } + writer.flush(); + } else { + // An absent optional indicates that the Reporter wants this service to shut down. + return; + } + } + } +} diff --git a/java/google/registry/monitoring/metrics/MetricMetrics.java b/java/google/registry/monitoring/metrics/MetricMetrics.java new file mode 100644 index 000000000..e4ac1293f --- /dev/null +++ b/java/google/registry/monitoring/metrics/MetricMetrics.java @@ -0,0 +1,84 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.HashMap; + +/** Static store of metrics internal to this client library. */ +final class MetricMetrics { + + /** A counter representing the total number of push intervals since the start of the process. */ + static final IncrementableMetric pushIntervals = + MetricRegistryImpl.getDefault() + .newIncrementableMetric( + "/metrics/push_intervals", + "Count of push intervals.", + "Push Intervals", + ImmutableSet.of()); + private static final ImmutableSet LABELS = + ImmutableSet.of( + LabelDescriptor.create("kind", "Metric Kind"), + LabelDescriptor.create("valueType", "Metric Value Type")); + + /** + * A counter representing the total number of points pushed. Has {@link MetricSchema.Kind} and + * Metric value classes as LABELS. + */ + static final IncrementableMetric pushedPoints = + MetricRegistryImpl.getDefault() + .newIncrementableMetric( + "/metrics/points_pushed", + "Count of points pushed to Monitoring API.", + "Points Pushed", + LABELS); + + /** A gauge representing a snapshot of the number of active timeseries being reported. */ + private static final Metric timeseriesCount = + MetricRegistryImpl.getDefault() + .newGauge( + "/metrics/timeseries_count", + "Count of Timeseries being pushed to Monitoring API", + "Timeseries Count", + LABELS, + new Supplier, Long>>() { + @Override + public ImmutableMap, Long> get() { + HashMap, Long> timeseriesCount = new HashMap<>(); + + for (Metric metric : MetricRegistryImpl.getDefault().getRegisteredMetrics()) { + ImmutableList key = + ImmutableList.of( + metric.getMetricSchema().kind().toString(), + metric.getValueClass().toString()); + + int cardinality = metric.getCardinality(); + if (!timeseriesCount.containsKey(key)) { + timeseriesCount.put(key, (long) cardinality); + } else { + timeseriesCount.put(key, timeseriesCount.get(key) + cardinality); + } + } + + return ImmutableMap.copyOf(timeseriesCount); + } + }, + Long.class); + + private MetricMetrics() {} +} diff --git a/java/google/registry/monitoring/metrics/MetricPoint.java b/java/google/registry/monitoring/metrics/MetricPoint.java new file mode 100644 index 000000000..9eff0f766 --- /dev/null +++ b/java/google/registry/monitoring/metrics/MetricPoint.java @@ -0,0 +1,54 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import org.joda.time.Instant; + +/** + * Value type class to store a point-in-time snapshot of a {@link Metric} value for a given label + * value tuple. + */ +@AutoValue +public abstract class MetricPoint { + + private static final String LABEL_COUNT_ERROR = + "The count of labelsValues must be equal to the underlying " + + "MetricDescriptor's count of labels."; + + MetricPoint() {} + + /** + * Returns a new {@link MetricPoint}. Callers should insure that the count of {@code labelValues} + * matches the count of labels for the given metric. + */ + static MetricPoint create( + Metric metric, ImmutableList labelValues, Instant timestamp, V value) { + checkArgument( + labelValues.size() == metric.getMetricSchema().labels().size(), LABEL_COUNT_ERROR); + return new AutoValue_MetricPoint<>(metric, labelValues, timestamp, value); + } + + public abstract Metric metric(); + + public abstract ImmutableList labelValues(); + + public abstract Instant timestamp(); + + public abstract V value(); +} diff --git a/java/google/registry/monitoring/metrics/MetricRegistry.java b/java/google/registry/monitoring/metrics/MetricRegistry.java new file mode 100644 index 000000000..9494c259b --- /dev/null +++ b/java/google/registry/monitoring/metrics/MetricRegistry.java @@ -0,0 +1,113 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** An interface to create and keep track of metrics. */ +public interface MetricRegistry { + + /** + * Returns a new Gauge metric. + * + *

The metric's values are computed at sample time via the supplied callback function. The + * metric will be registered at the time of creation and collected for subsequent write intervals. + * + *

Since the metric's values are computed by a pre-defined callback function, this method only + * returns a read-only {@link Metric} view. + * + * @param name name of the metric. Should be in the form of '/foo/bar'. + * @param description human readable description of the metric. Must not be empty. + * @param valueDisplayName human readable description of the metric's value type. Must not be + * empty. + * @param labels list of the metric's labels. The labels (if there are any) must be of type + * STRING. + * @param metricCallback {@link Supplier} to compute the on-demand values of the metric. The + * function should be lightweight to compute and must be thread-safe. The map keys, which are + * lists of strings, must match in quantity and order with the provided labels. + * @param valueClass type hint to allow for compile-time encoding. Must match . + * @param value type of the metric. Must be one of {@link Boolean}, {@link Double}, {@link + * Long}, or {@link String}. + * @throws IllegalStateException if a metric of the same name is already registered. + */ + Metric newGauge( + String name, + String description, + String valueDisplayName, + ImmutableSet labels, + Supplier, V>> metricCallback, + Class valueClass); + + /** + * Returns a new {@link SettableMetric}. + * + *

The metric's value is stateful and can be set to different values over time. + * + *

The metric is thread-safe. + * + *

The metric will be registered at the time of creation and collected for subsequent write + * intervals. + * + * @param name name of the metric. Should be in the form of '/foo/bar'. + * @param description human readable description of the metric. + * @param valueDisplayName human readable description of the metric's value type. + * @param labels list of the metric's labels. The labels (if there are any) must be of type + * STRING. + * @param valueClass type hint to allow for compile-time encoding. Must match . + * @param value type of the metric. Must be one of {@link Boolean}, {@link Double}, {@link + * Long}, or {@link String}. + * @throws IllegalStateException if a metric of the same name is already registered. + */ + SettableMetric newSettableMetric( + String name, + String description, + String valueDisplayName, + ImmutableSet labels, + Class valueClass); + + /** + * Returns a new {@link IncrementableMetric}. + * + *

The metric's values are {@link Long}, and can be incremented, and decremented. The metric is + * thread-safe. The metric will be registered at the time of creation and collected for subsequent + * write intervals. + * + *

This metric type is generally intended for monotonically increasing values, although the + * metric can in fact be incremented by negative numbers. + * + * @param name name of the metric. Should be in the form of '/foo/bar'. + * @param description human readable description of the metric. + * @param valueDisplayName human readable description of the metric's value type. + * @param labels list of the metric's labels. The labels (if there are any) must be of type + * STRING. + * @throws IllegalStateException if a metric of the same name is already registered. + */ + IncrementableMetric newIncrementableMetric( + String name, + String description, + String valueDisplayName, + ImmutableSet labels); + + /** + * Fetches a snapshot of the currently registered metrics + * + *

Users who wish to manually sample and write metrics without relying on the scheduled {@link + * MetricReporter} can use this method to gather the list of metrics to report. + */ + ImmutableList> getRegisteredMetrics(); +} diff --git a/java/google/registry/monitoring/metrics/MetricRegistryImpl.java b/java/google/registry/monitoring/metrics/MetricRegistryImpl.java new file mode 100644 index 000000000..c47543558 --- /dev/null +++ b/java/google/registry/monitoring/metrics/MetricRegistryImpl.java @@ -0,0 +1,118 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Logger; +import javax.annotation.concurrent.ThreadSafe; + +/** A singleton registry of {@link Metric}s. */ +@ThreadSafe +public final class MetricRegistryImpl implements MetricRegistry { + + private static final Logger logger = Logger.getLogger(MetricRegistryImpl.class.getName()); + private static final MetricRegistryImpl INSTANCE = new MetricRegistryImpl(); + + /** The canonical registry for metrics. The map key is the metric name. */ + private final ConcurrentHashMap> registeredMetrics = new ConcurrentHashMap<>(); + + private MetricRegistryImpl() {} + + public static MetricRegistryImpl getDefault() { + return INSTANCE; + } + + @Override + @CanIgnoreReturnValue + public Metric newGauge( + String name, + String description, + String valueDisplayName, + ImmutableSet labels, + Supplier, V>> metricCallback, + Class valueClass) { + VirtualMetric metric = + new VirtualMetric<>( + name, description, valueDisplayName, labels, metricCallback, valueClass); + registerMetric(name, metric); + logger.info("Registered new callback metric: " + name); + + return metric; + } + + @Override + public SettableMetric newSettableMetric( + String name, + String description, + String valueDisplayName, + ImmutableSet labels, + Class valueClass) { + StoredMetric metric = + new StoredMetric<>(name, description, valueDisplayName, labels, valueClass); + registerMetric(name, metric); + logger.info("Registered new stored metric: " + name); + + return metric; + } + + @Override + public IncrementableMetric newIncrementableMetric( + String name, + String description, + String valueDisplayName, + ImmutableSet labels) { + + Counter metric = new Counter(name, description, valueDisplayName, labels); + registerMetric(name, metric); + logger.info("Registered new counter: " + name); + + return metric; + } + + @Override + public ImmutableList> getRegisteredMetrics() { + return ImmutableList.copyOf(registeredMetrics.values()); + } + + /** + * Unregisters a metric. + * + *

This is a no-op if the metric is not already registered. + * + *

{@link MetricWriter} implementations should not send unregistered metrics on subsequent + * write intervals. + */ + @VisibleForTesting + void unregisterMetric(String name) { + registeredMetrics.remove(name); + logger.info("Unregistered metric: " + name); + } + + /** Registers a metric. */ + @VisibleForTesting + void registerMetric(String name, Metric metric) { + Metric previousMetric = registeredMetrics.putIfAbsent(name, metric); + + checkState(previousMetric == null, "Duplicate metric of same name: %s", name); + } +} diff --git a/java/google/registry/monitoring/metrics/MetricReporter.java b/java/google/registry/monitoring/metrics/MetricReporter.java new file mode 100644 index 000000000..bff2a5f3e --- /dev/null +++ b/java/google/registry/monitoring/metrics/MetricReporter.java @@ -0,0 +1,114 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.AbstractScheduledService; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * Engine to write metrics to a {@link MetricWriter} on a regular periodic basis. + * + *

In the Producer/Consumer pattern, this class is the Producer and {@link MetricExporter} is the + * consumer. + */ +public class MetricReporter extends AbstractScheduledService { + + private static final Logger logger = Logger.getLogger(MetricReporter.class.getName()); + + private final long writeInterval; + private final MetricRegistry metricRegistry; + private final BlockingQueue>>> writeQueue; + private final MetricExporter metricExporter; + + /** + * Returns a new MetricReporter. + * + * @param metricWriter {@link MetricWriter} implementation to write metrics to. + * @param writeInterval time period between metric writes, in seconds. + */ + public MetricReporter(MetricWriter metricWriter, long writeInterval) { + this( + metricWriter, + writeInterval, + MetricRegistryImpl.getDefault(), + new ArrayBlockingQueue>>>(1000)); + } + + @VisibleForTesting + MetricReporter( + MetricWriter metricWriter, + long writeInterval, + MetricRegistry metricRegistry, + BlockingQueue>>> writeQueue) { + checkArgument(writeInterval > 0, "writeInterval must be greater than zero"); + + this.writeInterval = writeInterval; + this.metricRegistry = metricRegistry; + this.writeQueue = writeQueue; + this.metricExporter = new MetricExporter(writeQueue, metricWriter); + } + + @Override + protected void runOneIteration() { + logger.info("Running background metric push"); + ImmutableList.Builder> points = new ImmutableList.Builder<>(); + + /* + TODO(shikhman): Right now timestamps are recorded for each datapoint, which may use more storage + on the backend than if one timestamp were recorded for a batch. This should be configurable. + */ + for (Metric metric : metricRegistry.getRegisteredMetrics()) { + points.addAll(metric.getTimestampedValues()); + logger.fine(String.format("Enqueued metric %s", metric)); + MetricMetrics.pushedPoints.incrementBy( + 1, metric.getMetricSchema().kind().name(), metric.getValueClass().toString()); + } + + if (!writeQueue.offer(Optional.of(points.build()))) { + logger.warning("writeQueue full, dropped a reporting interval of points"); + } + + MetricMetrics.pushIntervals.incrementBy(1); + } + + @Override + protected void shutDown() { + // Make sure to run one iteration on shutdown so that short-lived programs still report at + // least once. + runOneIteration(); + + writeQueue.offer(Optional.>>absent()); + metricExporter.stopAsync().awaitTerminated(); + } + + @Override + protected void startUp() { + metricExporter.startAsync().awaitRunning(); + } + + @Override + protected Scheduler scheduler() { + // Start writing after waiting for one writeInterval. + return Scheduler.newFixedDelaySchedule(writeInterval, writeInterval, TimeUnit.SECONDS); + } +} diff --git a/java/google/registry/monitoring/metrics/MetricSchema.java b/java/google/registry/monitoring/metrics/MetricSchema.java new file mode 100644 index 000000000..d3a889056 --- /dev/null +++ b/java/google/registry/monitoring/metrics/MetricSchema.java @@ -0,0 +1,71 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableSet; + +/** The description of a metric's schema. */ +@AutoValue +public abstract class MetricSchema { + + MetricSchema() {} + + /** + * Returns an instance of {@link MetricSchema}. + * + * @param name must have a URL-like hierarchical name, for example "/cpu/utilization". + * @param description a human readable description of the metric. Must not be blank. + * @param valueDisplayName a human readable description of the metric's value. Must not be blank. + * @param kind the kind of metric. + * @param labels an ordered set of mandatory metric labels. For example, a metric may track error + * code as a label. If labels are set, corresponding label values must be provided when values + * are set. The set of labels may be empty. + */ + static MetricSchema create( + String name, + String description, + String valueDisplayName, + Kind kind, + ImmutableSet labels) { + checkArgument(!name.isEmpty(), "Name must not be blank"); + checkArgument(!description.isEmpty(), "Description must not be blank"); + checkArgument(!valueDisplayName.isEmpty(), "Value Display Name must not be empty"); + // TODO: strengthen metric name validation. + + return new AutoValue_MetricSchema(name, description, valueDisplayName, kind, labels); + } + + public abstract String name(); + + public abstract String description(); + + public abstract String valueDisplayName(); + + public abstract Kind kind(); + + public abstract ImmutableSet labels(); + + /** + * The kind of metric. CUMULATIVE metrics have values relative to an initial value, and GAUGE + * metrics have values which are standalone. + */ + public enum Kind { + CUMULATIVE, + GAUGE, + } +} diff --git a/java/google/registry/monitoring/metrics/MetricWriter.java b/java/google/registry/monitoring/metrics/MetricWriter.java new file mode 100644 index 000000000..2fcdedaa0 --- /dev/null +++ b/java/google/registry/monitoring/metrics/MetricWriter.java @@ -0,0 +1,36 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import java.io.Flushable; +import java.io.IOException; + +/** An interface for exporting Metrics. */ +public interface MetricWriter extends Flushable { + + /** + * Writes a {@link MetricPoint} to the writer's destination. + * + *

The write may be asynchronous. + * + * @throws IOException if the provided metric cannot be represented by the writer or if the metric + * cannot be flushed. + */ + void write(MetricPoint metricPoint) throws IOException; + + /** Forces the writer to synchronously write all buffered metric values. */ + @Override + void flush() throws IOException; +} diff --git a/java/google/registry/monitoring/metrics/SettableMetric.java b/java/google/registry/monitoring/metrics/SettableMetric.java new file mode 100644 index 000000000..77edf662c --- /dev/null +++ b/java/google/registry/monitoring/metrics/SettableMetric.java @@ -0,0 +1,25 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +/** A {@link Metric} which can be set to different values over time. */ +public interface SettableMetric extends Metric { + + /** + * Sets the metric's value for a given set of label values. The count of labelValues must equal to + * the underlying metric's count of labels. + */ + void set(V value, String... labelValues); +} diff --git a/java/google/registry/monitoring/metrics/StackdriverWriter.java b/java/google/registry/monitoring/metrics/StackdriverWriter.java new file mode 100644 index 000000000..5f0c7a86c --- /dev/null +++ b/java/google/registry/monitoring/metrics/StackdriverWriter.java @@ -0,0 +1,283 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; + +import com.google.api.services.monitoring.v3.Monitoring; +import com.google.api.services.monitoring.v3.model.CreateTimeSeriesRequest; +import com.google.api.services.monitoring.v3.model.LabelDescriptor; +import com.google.api.services.monitoring.v3.model.Metric; +import com.google.api.services.monitoring.v3.model.MetricDescriptor; +import com.google.api.services.monitoring.v3.model.MonitoredResource; +import com.google.api.services.monitoring.v3.model.Point; +import com.google.api.services.monitoring.v3.model.TimeInterval; +import com.google.api.services.monitoring.v3.model.TimeSeries; +import com.google.api.services.monitoring.v3.model.TypedValue; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.RateLimiter; +import google.registry.monitoring.metrics.MetricSchema.Kind; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Queue; +import java.util.logging.Logger; +import javax.annotation.concurrent.NotThreadSafe; +import javax.inject.Inject; +import javax.inject.Named; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + +/** + * Metrics writer for Google Cloud Monitoring V3 + * + *

This class communicates with the API via HTTP. In order to increase throughput and minimize + * CPU, it buffers points to be written until it has {@code maxPointsPerRequest} points buffered or + * until {@link #flush()} is called. + */ +// TODO(shikhman): add retry logic +@NotThreadSafe +public class StackdriverWriter implements MetricWriter { + + /** + * A counter representing the total number of points pushed. Has {@link MetricSchema.Kind} and + * metric value types as labels. + */ + private static final IncrementableMetric pushedPoints = + MetricRegistryImpl.getDefault() + .newIncrementableMetric( + "/metrics/stackdriver/points_pushed", + "Count of points pushed to Stackdriver Monitoring API.", + "Points Pushed", + ImmutableSet.of( + google.registry.monitoring.metrics.LabelDescriptor.create("kind", "Metric Kind"), + google.registry.monitoring.metrics.LabelDescriptor.create( + "valueType", "Metric Value Type"))); + private static final String METRIC_DOMAIN = "custom.googleapis.com"; + private static final String LABEL_VALUE_TYPE = "STRING"; + private static final DateTimeFormatter DATETIME_FORMATTER = ISODateTimeFormat.dateTime(); + private static final Logger logger = Logger.getLogger(StackdriverWriter.class.getName()); + // A map of native type to the equivalent Stackdriver metric type. + private static final ImmutableMap, String> ENCODED_METRIC_TYPES = + new ImmutableMap.Builder, String>() + .put(Long.class, "INT64") + .put(Double.class, "DOUBLE") + .put(Boolean.class, "BOOL") + .put(String.class, "STRING") + .build(); + // A map of native kind to the equivalent Stackdriver metric kind. + private static final ImmutableMap ENCODED_METRIC_KINDS = + new ImmutableMap.Builder() + .put(Kind.GAUGE.name(), "GAUGE") + .put(Kind.CUMULATIVE.name(), "CUMULATIVE") + .build(); + private static final String FLUSH_OVERFLOW_ERROR = "Cannot flush more than 200 points at a time"; + private static final String METRIC_KIND_ERROR = + "Unrecognized metric kind, must be one of " + + Joiner.on(",").join(ENCODED_METRIC_KINDS.keySet()); + private static final String METRIC_TYPE_ERROR = + "Unrecognized metric type, must be one of " + + Joiner.on(" ").join(ENCODED_METRIC_TYPES.keySet()); + private static final String METRIC_LABEL_COUNT_ERROR = + "Metric label value count does not match its MetricDescriptor's label count"; + + private final MonitoredResource monitoredResource; + private final Queue timeSeriesBuffer; + /** + * A local cache of MetricDescriptors. A metric's metadata (name, kind, type, label definitions) + * must be registered before points for the metric can be pushed. + */ + private final HashMap, MetricDescriptor> + registeredDescriptors = new HashMap<>(); + private final String project; + private final Monitoring monitoringClient; + private final int maxPointsPerRequest; + private final RateLimiter rateLimiter; + + /** + * Constructs a StackdriverWriter. + * + *

The monitoringClient must have read and write permissions to the Cloud Monitoring API v3 on + * the provided project. + */ + @Inject + public StackdriverWriter( + Monitoring monitoringClient, + String project, + MonitoredResource monitoredResource, + @Named("stackdriverMaxQps") int maxQps, + @Named("stackdriverMaxPointsPerRequest") int maxPointsPerRequest) { + this.monitoringClient = checkNotNull(monitoringClient); + this.project = "projects/" + checkNotNull(project); + this.monitoredResource = monitoredResource; + this.maxPointsPerRequest = maxPointsPerRequest; + this.timeSeriesBuffer = new ArrayDeque<>(maxPointsPerRequest); + this.rateLimiter = RateLimiter.create(maxQps); + } + + @VisibleForTesting + static ImmutableList createLabelDescriptors( + ImmutableSet labelDescriptors) { + List stackDriverLabelDescriptors = new ArrayList<>(labelDescriptors.size()); + + for (google.registry.monitoring.metrics.LabelDescriptor labelDescriptor : labelDescriptors) { + stackDriverLabelDescriptors.add( + new LabelDescriptor() + .setKey(labelDescriptor.name()) + .setDescription(labelDescriptor.description()) + .setValueType(LABEL_VALUE_TYPE)); + } + + return ImmutableList.copyOf(stackDriverLabelDescriptors); + } + + @VisibleForTesting + static MetricDescriptor createMetricDescriptor( + google.registry.monitoring.metrics.Metric metric) { + return new MetricDescriptor() + .setType(METRIC_DOMAIN + "/" + metric.getMetricSchema().name()) + .setDescription(metric.getMetricSchema().description()) + .setDisplayName(metric.getMetricSchema().valueDisplayName()) + .setValueType(ENCODED_METRIC_TYPES.get(metric.getValueClass())) + .setLabels(createLabelDescriptors(metric.getMetricSchema().labels())) + .setMetricKind(ENCODED_METRIC_KINDS.get(metric.getMetricSchema().kind().name())); + } + + /** Encodes and writes a metric point to Stackdriver. The point may be buffered. */ + @Override + public void write(google.registry.monitoring.metrics.MetricPoint point) + throws IOException { + checkNotNull(point); + google.registry.monitoring.metrics.Metric metric = point.metric(); + try { + checkArgument( + ENCODED_METRIC_KINDS.containsKey(metric.getMetricSchema().kind().name()), + METRIC_KIND_ERROR); + checkArgument(ENCODED_METRIC_TYPES.containsKey(metric.getValueClass()), METRIC_TYPE_ERROR); + } catch (IllegalArgumentException e) { + throw new IOException(e.getMessage()); + } + + MetricDescriptor descriptor = registerMetric(metric); + + if (point.labelValues().size() != point.metric().getMetricSchema().labels().size()) { + throw new IOException(METRIC_LABEL_COUNT_ERROR); + } + + V value = point.value(); + TypedValue encodedValue = new TypedValue(); + Class valueClass = metric.getValueClass(); + + if (valueClass == Long.class) { + encodedValue.setInt64Value((Long) value); + } else if (valueClass == Double.class) { + encodedValue.setDoubleValue((Double) value); + } else if (valueClass == Boolean.class) { + encodedValue.setBoolValue((Boolean) value); + } else if (valueClass == String.class) { + encodedValue.setStringValue((String) value); + } else { + // This is unreachable because the precondition checks will catch all NotSerializable + // exceptions. + throw new IllegalArgumentException("Invalid metric valueClass: " + metric.getValueClass()); + } + + Point encodedPoint = + new Point() + .setInterval(new TimeInterval().setEndTime(DATETIME_FORMATTER.print(point.timestamp()))) + .setValue(encodedValue); + + List encodedLabels = descriptor.getLabels(); + ImmutableMap.Builder labelValues = new ImmutableMap.Builder<>(); + int i = 0; + for (LabelDescriptor labelDescriptor : encodedLabels) { + labelValues.put(labelDescriptor.getKey(), point.labelValues().get(i++)); + } + + Metric encodedMetric = + new Metric().setType(descriptor.getType()).setLabels(labelValues.build()); + + timeSeriesBuffer.add( + new TimeSeries() + .setMetric(encodedMetric) + .setPoints(ImmutableList.of(encodedPoint)) + .setResource(monitoredResource) + // these two attributes are ignored by the API, we set them here to use elsewhere + // for internal metrics. + .setMetricKind(descriptor.getMetricKind()) + .setValueType(descriptor.getValueType())); + + logger.fine(String.format("Enqueued metric %s for writing", descriptor.getType())); + if (timeSeriesBuffer.size() == maxPointsPerRequest) { + flush(); + } + } + + /** Flushes all buffered metric points to Stackdriver. This call is blocking. */ + @Override + public void flush() throws IOException { + checkState(timeSeriesBuffer.size() <= 200, FLUSH_OVERFLOW_ERROR); + + ImmutableList timeSeriesList = ImmutableList.copyOf(timeSeriesBuffer); + timeSeriesBuffer.clear(); + + CreateTimeSeriesRequest request = new CreateTimeSeriesRequest().setTimeSeries(timeSeriesList); + + rateLimiter.acquire(); + monitoringClient.projects().timeSeries().create(project, request).execute(); + + for (TimeSeries timeSeries : timeSeriesList) { + pushedPoints.incrementBy(1, timeSeries.getMetricKind(), timeSeries.getValueType()); + } + logger.info(String.format("Flushed %d metrics to Stackdriver", timeSeriesList.size())); + } + + /** + * Registers a metric's {@link MetricDescriptor} with the Monitoring API. + * + * @param metric the metric to be registered. + */ + @VisibleForTesting + MetricDescriptor registerMetric(final google.registry.monitoring.metrics.Metric metric) { + if (registeredDescriptors.containsKey(metric)) { + logger.info( + String.format("Fetched existing metric descriptor %s", metric.getMetricSchema().name())); + return registeredDescriptors.get(metric); + } + + MetricDescriptor descriptor = createMetricDescriptor(metric); + + try { + rateLimiter.acquire(); + descriptor = + monitoringClient.projects().metricDescriptors().create(project, descriptor).execute(); + } catch (IOException e) { + throw new RuntimeException("Error creating a MetricDescriptor"); + } + + logger.info(String.format("Registered new metric descriptor %s", descriptor.getType())); + registeredDescriptors.put(metric, descriptor); + + return descriptor; + } +} diff --git a/java/google/registry/monitoring/metrics/StoredMetric.java b/java/google/registry/monitoring/metrics/StoredMetric.java new file mode 100644 index 000000000..a307aedf9 --- /dev/null +++ b/java/google/registry/monitoring/metrics/StoredMetric.java @@ -0,0 +1,90 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableSet; +import google.registry.monitoring.metrics.MetricSchema.Kind; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.concurrent.ThreadSafe; +import org.joda.time.Instant; + +/** + * A metric which is stateful. + * + *

The values are stored and set over time. This metric is generally suitable for state + * indicators, such as indicating that a server is in a RUNNING state or in a STOPPED state. + * + *

See {@link Counter} for a subclass which is suitable for incremental values. + */ +@ThreadSafe +public class StoredMetric extends AbstractMetric implements SettableMetric { + + private static final String LABEL_COUNT_ERROR = + "The count of labelValues must be equal to the underlying " + + "MetricDescriptor's count of labels."; + + private final ConcurrentHashMap, V> values = new ConcurrentHashMap<>(); + + StoredMetric( + String name, + String description, + String valueDisplayName, + ImmutableSet labels, + Class valueClass) { + super(name, description, valueDisplayName, Kind.GAUGE, labels, valueClass); + } + + @VisibleForTesting + final void set(V value, ImmutableList labelValues) { + this.values.put(labelValues, value); + } + + @Override + public final void set(V value, String... labelValues) { + checkArgument(labelValues.length == this.getMetricSchema().labels().size(), LABEL_COUNT_ERROR); + + set(value, ImmutableList.copyOf(labelValues)); + } + + /** + * Returns a snapshot of the metric's values. The timestamp of each MetricPoint will be the last + * modification time for that tuple of label values. + */ + @Override + public final ImmutableList> getTimestampedValues() { + return getTimestampedValues(Instant.now()); + } + + @Override + public final int getCardinality() { + return values.size(); + } + + @VisibleForTesting + final ImmutableList> getTimestampedValues(Instant timestamp) { + ImmutableList.Builder> timestampedValues = new Builder<>(); + for (Entry, V> entry : values.entrySet()) { + timestampedValues.add(MetricPoint.create(this, entry.getKey(), timestamp, entry.getValue())); + } + + return timestampedValues.build(); + } +} diff --git a/java/google/registry/monitoring/metrics/VirtualMetric.java b/java/google/registry/monitoring/metrics/VirtualMetric.java new file mode 100644 index 000000000..56270c758 --- /dev/null +++ b/java/google/registry/monitoring/metrics/VirtualMetric.java @@ -0,0 +1,86 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.monitoring.metrics; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import google.registry.monitoring.metrics.MetricSchema.Kind; +import java.util.Map.Entry; +import javax.annotation.concurrent.ThreadSafe; +import org.joda.time.Instant; + +/** + * A metric whose value is computed at sample-time. + * + *

This pattern works well for gauge-like metrics, such as CPU usage, memory usage, and file + * descriptor counts. + */ +@ThreadSafe +public final class VirtualMetric extends AbstractMetric { + + private final Supplier, V>> valuesSupplier; + + /** + * Local cache of the count of values so that we don't have to evaluate the callback function to + * get the metric's cardinality. + */ + private volatile int cardinality; + + VirtualMetric( + String name, + String description, + String valueDisplayName, + ImmutableSet labels, + Supplier, V>> valuesSupplier, + Class valueClass) { + super(name, description, valueDisplayName, Kind.GAUGE, labels, valueClass); + + this.valuesSupplier = valuesSupplier; + } + + /** + * Returns a snapshot of the metric's values. This will evaluate the stored callback function. The + * timestamp for each MetricPoint will be the current time. + */ + @Override + public ImmutableList> getTimestampedValues() { + return getTimestampedValues(Instant.now()); + } + + /** + * Returns the cached value of the cardinality of this metric. The cardinality is computed when + * the metric is evaluated. If the metric has never been evaluated, the cardinality is zero. + */ + @Override + public int getCardinality() { + return cardinality; + } + + @VisibleForTesting + ImmutableList> getTimestampedValues(Instant timestamp) { + ImmutableMap, V> values = valuesSupplier.get(); + + ImmutableList.Builder> metricPoints = ImmutableList.builder(); + for (Entry, V> entry : values.entrySet()) { + metricPoints.add(MetricPoint.create(this, entry.getKey(), timestamp, entry.getValue())); + } + + cardinality = values.size(); + return metricPoints.build(); + } +} diff --git a/java/google/registry/monitoring/metrics/package-info.java b/java/google/registry/monitoring/metrics/package-info.java new file mode 100644 index 000000000..25f075913 --- /dev/null +++ b/java/google/registry/monitoring/metrics/package-info.java @@ -0,0 +1,16 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +@javax.annotation.ParametersAreNonnullByDefault +package google.registry.monitoring.metrics; diff --git a/java/google/registry/repositories.bzl b/java/google/registry/repositories.bzl index f7bb17ad0..b63295b04 100644 --- a/java/google/registry/repositories.bzl +++ b/java/google/registry/repositories.bzl @@ -160,6 +160,12 @@ def domain_registry_repositories(): sha1 = "647e19b28c106a63a14401c0f5956289792adf2f", ) + native.maven_jar( + name = "error_prone_annotations", + artifact = "com.google.errorprone:error_prone_annotations:2.0.11", + sha1 = "3624d81fca4e93c67f43bafc222b06e1b1e3b260", + ) + native.maven_jar( name = "fastutil", artifact = "it.unimi.dsi:fastutil:6.4.3", diff --git a/third_party/java/error_prone/BUILD b/third_party/java/error_prone/BUILD new file mode 100644 index 000000000..a41567d64 --- /dev/null +++ b/third_party/java/error_prone/BUILD @@ -0,0 +1,8 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +java_library( + name = "annotations", + exports = ["@error_prone_annotations//jar"], +)