Use GKE-specific metrics in the proxy

Associate the custom metrics with the correct monitored resource type. The labels of the monitored resource are either obtained from environment variables for the container, configured in the GKE deployment file, or queried from GCE metadate server. Using the correct monitored resource can help performance and reduced out-of-order metric writes.

Also changed the metrics display name to be more descriptive.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=189184411

java/google/registry/proxy/MetricsModule.java

@@ -26,16 +26,16 @@ import com.google.monitoring.metrics.stackdriver.StackdriverWriter;
 import dagger.Component;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.proxy.ProxyConfig.Environment;
+import google.registry.proxy.metric.MetricParameters;
+import google.registry.util.FormattingLogger;
 import javax.inject.Singleton;
 
 /** Module that provides necessary bindings to instantiate a {@link MetricReporter} */
 @Module
 public class MetricsModule {
 
-  // TODO (b/64765479): change to GKE cluster and config in YAML file.
-  private static final String MONITORED_RESOURCE_TYPE = "gce_instance";
-  private static final String GCE_INSTANCE_ZONE = "us-east4-c";
-  private static final String GCE_INSTANCE_ID = "5401454098973297721";
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
 
   @Singleton
   @Provides
@@ -48,15 +48,12 @@ public class MetricsModule {
 
   @Singleton
   @Provides
-  static MetricWriter provideMetricWriter(Monitoring monitoringClient, ProxyConfig config) {
-    // The MonitoredResource for GAE apps is not writable (and missing fields anyway) so we just
-    // use the gce_instance resource type instead.
+  static MetricWriter provideMetricWriter(
+      Monitoring monitoringClient, MonitoredResource monitoredResource, ProxyConfig config) {
     return new StackdriverWriter(
         monitoringClient,
         config.projectId,
-        new MonitoredResource()
-            .setType(MONITORED_RESOURCE_TYPE)
-            .setLabels(ImmutableMap.of("zone", GCE_INSTANCE_ZONE, "instance_id", GCE_INSTANCE_ID)),
+        monitoredResource,
         config.metrics.stackdriverMaxQps,
         config.metrics.stackdriverMaxPointsPerRequest);
   }
@@ -70,6 +67,32 @@ public class MetricsModule {
         new ThreadFactoryBuilder().setDaemon(true).build());
   }
 
+  /**
+   * Provides a {@link MonitoredResource} appropriate for environment tha proxy runs in.
+   *
+   * <p>When running locally, the type of the monitored resource is set to {@code global}, otherwise
+   * it is {@code gke_container}.
+   *
+   * @see <a
+   *     href="https://cloud.google.com/monitoring/custom-metrics/creating-metrics#which-resource">
+   *     Choosing a monitored resource type</a>
+   */
+  @Singleton
+  @Provides
+  static MonitoredResource provideMonitoredResource(
+      Environment env, ProxyConfig config, MetricParameters metricParameters) {
+    MonitoredResource monitoredResource = new MonitoredResource();
+    if (env == Environment.LOCAL) {
+      monitoredResource
+          .setType("global")
+          .setLabels(ImmutableMap.of("project_id", config.projectId));
+    } else {
+      monitoredResource.setType("gke_container").setLabels(metricParameters.makeLabelsMap());
+    }
+    logger.infofmt("Monitored resource: %s", monitoredResource);
+    return monitoredResource;
+  }
+
   @Singleton
   @Component(modules = {MetricsModule.class, ProxyModule.class})
   interface MetricsComponent {

java/google/registry/proxy/kubernetes/proxy-deployment.yaml

@@ -46,3 +46,13 @@ spec:
         env:
         - name: GOOGLE_APPLICATION_CREDENTIALS
           value: /var/secrets/google/service-account.json
+        - name: POD_ID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE_ID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: CONTAINER_NAME
+          value: proxy

java/google/registry/proxy/metric/BackendMetrics.java

@@ -72,7 +72,7 @@ public class BackendMetrics {
           .newEventMetric(
               "/proxy/backend/request_bytes",
               "Size of the backend requests sent.",
-              "Bytes",
+              "Request Bytes",
               LABELS,
               DEFAULT_SIZE_FITTER);
 
@@ -81,7 +81,7 @@ public class BackendMetrics {
           .newEventMetric(
               "/proxy/backend/response_bytes",
               "Size of the backend responses received.",
-              "Bytes",
+              "Response Bytes",
               LABELS,
               DEFAULT_SIZE_FITTER);
 
@@ -90,7 +90,7 @@ public class BackendMetrics {
           .newEventMetric(
               "/proxy/backend/latency_ms",
               "Round-trip time between a request sent and its corresponding response received.",
-              "Milliseconds",
+              "Latency Milliseconds",
               LABELS,
               DEFAULT_LATENCY_FITTER);
 

java/google/registry/proxy/metric/FrontendMetrics.java

@@ -62,7 +62,7 @@ public class FrontendMetrics {
           .newGauge(
               "/proxy/frontend/active_connections",
               "Number of active connections from clients to the proxy.",
-              "Connections",
+              "Active Connections",
               LABELS,
               () ->
                   activeConnections
@@ -78,7 +78,7 @@ public class FrontendMetrics {
           .newIncrementableMetric(
               "/proxy/frontend/total_connections",
               "Total number connections ever made from clients to the proxy.",
-              "Connections",
+              "Total Connections",
               LABELS);
 
   static final IncrementableMetric quotaRejectionsCounter =
@@ -86,7 +86,7 @@ public class FrontendMetrics {
           .newIncrementableMetric(
               "/proxy/frontend/quota_rejections",
               "Total number rejected quota request made by proxy for each connection.",
-              "Rejections",
+              "Quota Rejections",
               LABELS);
 
   @Inject

java/google/registry/proxy/metric/MetricParameters.java

@@ -0,0 +1,144 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.proxy.metric;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.api.services.monitoring.v3.model.MonitoredResource;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.CharStreams;
+import google.registry.util.FormattingLogger;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Map;
+import java.util.function.Function;
+import javax.inject.Inject;
+
+/**
+ * Utility class to obtain labels for monitored resource of type {@code gke_container}.
+ *
+ * <p>Custom metrics collected by the proxy need to be associated with a {@link MonitoredResource}.
+ * When running on GKE, the type is {@code gke_container}. The labels for this type are used to
+ * group related metrics together, and to avoid out-of-order metrics writes. This class provides a
+ * map of the labels where the values are either read from environment variables (pod and container
+ * related labels) or queried from GCE metadata server (cluster and instance related labels).
+ *
+ * @see <a
+ *     href="https://cloud.google.com/monitoring/custom-metrics/creating-metrics#which-resource">
+ *     Creating Custom Metrics - Choosing a monitored resource type</a>
+ * @see <a href="https://cloud.google.com/monitoring/api/resources#tag_gke_container">Monitored
+ *     Resource Types - gke_container</a>
+ * @see <a href="https://cloud.google.com/compute/docs/storing-retrieving-metadata#querying">Storing
+ *     and Retrieving Instance Metadata - Getting metadata</a>
+ * @see <a
+ *     href="https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/">
+ *     Expose Pod Information to Containers Through Environment Variables </a>
+ */
+public class MetricParameters {
+
+  // Environment variable names, defined in the GKE deployment pod spec.
+  static final String NAMESPACE_ID_ENV = "NAMESPACE_ID";
+  static final String POD_ID_ENV = "POD_ID";
+  static final String CONTAINER_NAME_ENV = "CONTAINER_NAME";
+
+  // GCE metadata server URLs to retrieve instance related information.
+  private static final String GCE_METADATA_URL_BASE = "http://metadata.google.internal/";
+  static final String PROJECT_ID_PATH = "computeMetadata/v1/project/project-id";
+  static final String CLUSTER_NAME_PATH = "computeMetadata/v1/instance/attributes/cluster-name";
+  static final String INSTANCE_ID_PATH = "computeMetadata/v1/instance/id";
+  static final String ZONE_PATH = "computeMetadata/v1/instance/zone";
+
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+
+  private final Map<String, String> envVarMap;
+  private final Function<String, HttpURLConnection> connectionFactory;
+
+  MetricParameters(
+      Map<String, String> envVarMap, Function<String, HttpURLConnection> connectionFactory) {
+    this.envVarMap = envVarMap;
+    this.connectionFactory = connectionFactory;
+  }
+
+  @Inject
+  MetricParameters() {
+    this(ImmutableMap.copyOf(System.getenv()), MetricParameters::gceConnectionFactory);
+  }
+
+  private static final HttpURLConnection gceConnectionFactory(String path) {
+    String url = GCE_METADATA_URL_BASE + path;
+    try {
+      HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
+      connection.setRequestMethod("GET");
+      // The metadata server requires this header to be set when querying from a GCE instance.
+      connection.setRequestProperty("Metadata-Flavor", "Google");
+      connection.setDoOutput(true);
+      return connection;
+    } catch (IOException e) {
+      logger.warningfmt(e, "Incorrect GCE metadata server URL: %s", url);
+      throw new RuntimeException(e);
+    }
+  };
+
+  private String readEnvVar(String envVar) {
+    return envVarMap.getOrDefault(envVar, "");
+  }
+
+  private String readGceMetadata(String path) {
+    String value = "";
+    HttpURLConnection connection = connectionFactory.apply(path);
+    try {
+      connection.connect();
+      int responseCode = connection.getResponseCode();
+      if (responseCode < 200 || responseCode > 299) {
+        logger.warningfmt(
+            "Got an error response: %d\n%s",
+            responseCode,
+            CharStreams.toString(new InputStreamReader(connection.getErrorStream(), UTF_8)));
+      } else {
+        value = CharStreams.toString(new InputStreamReader(connection.getInputStream(), UTF_8));
+      }
+    } catch (IOException e) {
+      logger.warningfmt(e, "Cannot obtain GCE metadata from path %s", path);
+    }
+    return value;
+  }
+
+  public ImmutableMap<String, String> makeLabelsMap() {
+    // The zone metadata is in the form of "projects/<PROJECT_NUMERICAL_ID>/zones/<ZONE_NAME>".
+    // We only need the last part after the slash.
+    String fullZone = readGceMetadata(ZONE_PATH);
+    String zone;
+    String[] fullZoneArray = fullZone.split("/", -1);
+    if (fullZoneArray.length < 4) {
+      logger.warningfmt("Zone %s is valid.", fullZone);
+      // This will make the metric report throw, but it happens in a different thread and will not
+      // kill the whole application.
+      zone = "";
+    } else {
+      zone = fullZoneArray[3];
+    }
+    return new ImmutableMap.Builder<String, String>()
+        .put("project_id", readGceMetadata(PROJECT_ID_PATH))
+        .put("cluster_name", readGceMetadata(CLUSTER_NAME_PATH))
+        .put("namespace_id", readEnvVar(NAMESPACE_ID_ENV))
+        .put("instance_id", readGceMetadata(INSTANCE_ID_PATH))
+        .put("pod_id", readEnvVar(POD_ID_ENV))
+        .put("container_name", readEnvVar(CONTAINER_NAME_ENV))
+        .put("zone", zone)
+        .build();
+  }
+}

javatests/google/registry/proxy/metric/MetricParametersTest.java

@@ -0,0 +1,137 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.proxy.metric;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.proxy.metric.MetricParameters.CLUSTER_NAME_PATH;
+import static google.registry.proxy.metric.MetricParameters.CONTAINER_NAME_ENV;
+import static google.registry.proxy.metric.MetricParameters.INSTANCE_ID_PATH;
+import static google.registry.proxy.metric.MetricParameters.NAMESPACE_ID_ENV;
+import static google.registry.proxy.metric.MetricParameters.POD_ID_ENV;
+import static google.registry.proxy.metric.MetricParameters.PROJECT_ID_PATH;
+import static google.registry.proxy.metric.MetricParameters.ZONE_PATH;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableMap;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.util.HashMap;
+import java.util.Map.Entry;
+import java.util.function.Function;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link MetricParameters}. */
+@RunWith(JUnit4.class)
+public class MetricParametersTest {
+
+  private static final HashMap<String, String> RESULTS = new HashMap<>();
+
+  private final HttpURLConnection projectIdConnection = mock(HttpURLConnection.class);
+  private final HttpURLConnection clusterNameConnection = mock(HttpURLConnection.class);
+  private final HttpURLConnection instanceIdConnection = mock(HttpURLConnection.class);
+  private final HttpURLConnection zoneConnection = mock(HttpURLConnection.class);
+  private final ImmutableMap<String, HttpURLConnection> mockConnections =
+      ImmutableMap.of(
+          PROJECT_ID_PATH,
+          projectIdConnection,
+          CLUSTER_NAME_PATH,
+          clusterNameConnection,
+          INSTANCE_ID_PATH,
+          instanceIdConnection,
+          ZONE_PATH,
+          zoneConnection);
+  private final HashMap<String, String> fakeEnvVarMap = new HashMap<>();
+  private final Function<String, HttpURLConnection> fakeConnectionFactory =
+      path -> mockConnections.get(path);
+
+  private final MetricParameters metricParameters =
+      new MetricParameters(fakeEnvVarMap, fakeConnectionFactory);
+
+  private static InputStream makeInputStreamFromString(String input) {
+    return new ByteArrayInputStream(input.getBytes(UTF_8));
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    fakeEnvVarMap.put(NAMESPACE_ID_ENV, "some-namespace");
+    fakeEnvVarMap.put(POD_ID_ENV, "some-pod");
+    fakeEnvVarMap.put(CONTAINER_NAME_ENV, "some-container");
+    when(projectIdConnection.getInputStream())
+        .thenReturn(makeInputStreamFromString("some-project"));
+    when(clusterNameConnection.getInputStream())
+        .thenReturn(makeInputStreamFromString("some-cluster"));
+    when(instanceIdConnection.getInputStream())
+        .thenReturn(makeInputStreamFromString("some-instance"));
+    when(zoneConnection.getInputStream())
+        .thenReturn(makeInputStreamFromString("projects/some-project/zones/some-zone"));
+    for (Entry<String, HttpURLConnection> entry : mockConnections.entrySet()) {
+      when(entry.getValue().getResponseCode()).thenReturn(200);
+    }
+    RESULTS.put("project_id", "some-project");
+    RESULTS.put("cluster_name", "some-cluster");
+    RESULTS.put("namespace_id", "some-namespace");
+    RESULTS.put("instance_id", "some-instance");
+    RESULTS.put("pod_id", "some-pod");
+    RESULTS.put("container_name", "some-container");
+    RESULTS.put("zone", "some-zone");
+  }
+
+  @Test
+  public void testSuccess() {
+    assertThat(metricParameters.makeLabelsMap()).isEqualTo(ImmutableMap.copyOf(RESULTS));
+  }
+
+  @Test
+  public void testSuccess_missingEnvVar() {
+    fakeEnvVarMap.remove(POD_ID_ENV);
+    RESULTS.put("pod_id", "");
+    assertThat(metricParameters.makeLabelsMap()).isEqualTo(ImmutableMap.copyOf(RESULTS));
+  }
+
+  @Test
+  public void testSuccess_malformedZone() throws Exception {
+    when(zoneConnection.getInputStream()).thenReturn(makeInputStreamFromString("some-zone"));
+    RESULTS.put("zone", "");
+    assertThat(metricParameters.makeLabelsMap()).isEqualTo(ImmutableMap.copyOf(RESULTS));
+  }
+
+  @Test
+  public void testSuccess_errorResponseCode() throws Exception {
+    when(projectIdConnection.getResponseCode()).thenReturn(404);
+    when(projectIdConnection.getErrorStream())
+        .thenReturn(makeInputStreamFromString("some error message"));
+    RESULTS.put("project_id", "");
+    assertThat(metricParameters.makeLabelsMap()).isEqualTo(ImmutableMap.copyOf(RESULTS));
+  }
+
+  @Test
+  public void testSuccess_connectionError() throws Exception {
+    InputStream fakeInputStream = mock(InputStream.class);
+    when(projectIdConnection.getInputStream()).thenReturn(fakeInputStream);
+    when(fakeInputStream.read(any(byte[].class), anyInt(), anyInt()))
+        .thenThrow(new IOException("some exception"));
+    RESULTS.put("project_id", "");
+    assertThat(metricParameters.makeLabelsMap()).isEqualTo(ImmutableMap.copyOf(RESULTS));
+  }
+}