diff --git a/java/google/registry/proxy/MetricsModule.java b/java/google/registry/proxy/MetricsModule.java index c3f96ad82..6086f6e4f 100644 --- a/java/google/registry/proxy/MetricsModule.java +++ b/java/google/registry/proxy/MetricsModule.java @@ -26,16 +26,16 @@ import com.google.monitoring.metrics.stackdriver.StackdriverWriter; import dagger.Component; import dagger.Module; import dagger.Provides; +import google.registry.proxy.ProxyConfig.Environment; +import google.registry.proxy.metric.MetricParameters; +import google.registry.util.FormattingLogger; import javax.inject.Singleton; /** Module that provides necessary bindings to instantiate a {@link MetricReporter} */ @Module public class MetricsModule { - // TODO (b/64765479): change to GKE cluster and config in YAML file. - private static final String MONITORED_RESOURCE_TYPE = "gce_instance"; - private static final String GCE_INSTANCE_ZONE = "us-east4-c"; - private static final String GCE_INSTANCE_ID = "5401454098973297721"; + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); @Singleton @Provides @@ -48,15 +48,12 @@ public class MetricsModule { @Singleton @Provides - static MetricWriter provideMetricWriter(Monitoring monitoringClient, ProxyConfig config) { - // The MonitoredResource for GAE apps is not writable (and missing fields anyway) so we just - // use the gce_instance resource type instead. + static MetricWriter provideMetricWriter( + Monitoring monitoringClient, MonitoredResource monitoredResource, ProxyConfig config) { return new StackdriverWriter( monitoringClient, config.projectId, - new MonitoredResource() - .setType(MONITORED_RESOURCE_TYPE) - .setLabels(ImmutableMap.of("zone", GCE_INSTANCE_ZONE, "instance_id", GCE_INSTANCE_ID)), + monitoredResource, config.metrics.stackdriverMaxQps, config.metrics.stackdriverMaxPointsPerRequest); } @@ -70,6 +67,32 @@ public class MetricsModule { new ThreadFactoryBuilder().setDaemon(true).build()); } + /** + * Provides a {@link MonitoredResource} appropriate for environment tha proxy runs in. + * + *

When running locally, the type of the monitored resource is set to {@code global}, otherwise + * it is {@code gke_container}. + * + * @see + * Choosing a monitored resource type + */ + @Singleton + @Provides + static MonitoredResource provideMonitoredResource( + Environment env, ProxyConfig config, MetricParameters metricParameters) { + MonitoredResource monitoredResource = new MonitoredResource(); + if (env == Environment.LOCAL) { + monitoredResource + .setType("global") + .setLabels(ImmutableMap.of("project_id", config.projectId)); + } else { + monitoredResource.setType("gke_container").setLabels(metricParameters.makeLabelsMap()); + } + logger.infofmt("Monitored resource: %s", monitoredResource); + return monitoredResource; + } + @Singleton @Component(modules = {MetricsModule.class, ProxyModule.class}) interface MetricsComponent { diff --git a/java/google/registry/proxy/kubernetes/proxy-deployment.yaml b/java/google/registry/proxy/kubernetes/proxy-deployment.yaml index 314bad9c4..880ec6cdc 100644 --- a/java/google/registry/proxy/kubernetes/proxy-deployment.yaml +++ b/java/google/registry/proxy/kubernetes/proxy-deployment.yaml @@ -46,3 +46,13 @@ spec: env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /var/secrets/google/service-account.json + - name: POD_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_ID + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONTAINER_NAME + value: proxy diff --git a/java/google/registry/proxy/metric/BackendMetrics.java b/java/google/registry/proxy/metric/BackendMetrics.java index 624d8c14d..c6fa4310d 100644 --- a/java/google/registry/proxy/metric/BackendMetrics.java +++ b/java/google/registry/proxy/metric/BackendMetrics.java @@ -72,7 +72,7 @@ public class BackendMetrics { .newEventMetric( "/proxy/backend/request_bytes", "Size of the backend requests sent.", - "Bytes", + "Request Bytes", LABELS, DEFAULT_SIZE_FITTER); @@ -81,7 +81,7 @@ public class BackendMetrics { .newEventMetric( "/proxy/backend/response_bytes", "Size of the backend responses received.", - "Bytes", + "Response Bytes", LABELS, DEFAULT_SIZE_FITTER); @@ -90,7 +90,7 @@ public class BackendMetrics { .newEventMetric( "/proxy/backend/latency_ms", "Round-trip time between a request sent and its corresponding response received.", - "Milliseconds", + "Latency Milliseconds", LABELS, DEFAULT_LATENCY_FITTER); diff --git a/java/google/registry/proxy/metric/FrontendMetrics.java b/java/google/registry/proxy/metric/FrontendMetrics.java index 6e15f3564..feb414e83 100644 --- a/java/google/registry/proxy/metric/FrontendMetrics.java +++ b/java/google/registry/proxy/metric/FrontendMetrics.java @@ -62,7 +62,7 @@ public class FrontendMetrics { .newGauge( "/proxy/frontend/active_connections", "Number of active connections from clients to the proxy.", - "Connections", + "Active Connections", LABELS, () -> activeConnections @@ -78,7 +78,7 @@ public class FrontendMetrics { .newIncrementableMetric( "/proxy/frontend/total_connections", "Total number connections ever made from clients to the proxy.", - "Connections", + "Total Connections", LABELS); static final IncrementableMetric quotaRejectionsCounter = @@ -86,7 +86,7 @@ public class FrontendMetrics { .newIncrementableMetric( "/proxy/frontend/quota_rejections", "Total number rejected quota request made by proxy for each connection.", - "Rejections", + "Quota Rejections", LABELS); @Inject diff --git a/java/google/registry/proxy/metric/MetricParameters.java b/java/google/registry/proxy/metric/MetricParameters.java new file mode 100644 index 000000000..800e6ec6a --- /dev/null +++ b/java/google/registry/proxy/metric/MetricParameters.java @@ -0,0 +1,144 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.proxy.metric; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.api.services.monitoring.v3.model.MonitoredResource; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.CharStreams; +import google.registry.util.FormattingLogger; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Map; +import java.util.function.Function; +import javax.inject.Inject; + +/** + * Utility class to obtain labels for monitored resource of type {@code gke_container}. + * + *

Custom metrics collected by the proxy need to be associated with a {@link MonitoredResource}. + * When running on GKE, the type is {@code gke_container}. The labels for this type are used to + * group related metrics together, and to avoid out-of-order metrics writes. This class provides a + * map of the labels where the values are either read from environment variables (pod and container + * related labels) or queried from GCE metadata server (cluster and instance related labels). + * + * @see + * Creating Custom Metrics - Choosing a monitored resource type + * @see Monitored + * Resource Types - gke_container + * @see Storing + * and Retrieving Instance Metadata - Getting metadata + * @see + * Expose Pod Information to Containers Through Environment Variables + */ +public class MetricParameters { + + // Environment variable names, defined in the GKE deployment pod spec. + static final String NAMESPACE_ID_ENV = "NAMESPACE_ID"; + static final String POD_ID_ENV = "POD_ID"; + static final String CONTAINER_NAME_ENV = "CONTAINER_NAME"; + + // GCE metadata server URLs to retrieve instance related information. + private static final String GCE_METADATA_URL_BASE = "http://metadata.google.internal/"; + static final String PROJECT_ID_PATH = "computeMetadata/v1/project/project-id"; + static final String CLUSTER_NAME_PATH = "computeMetadata/v1/instance/attributes/cluster-name"; + static final String INSTANCE_ID_PATH = "computeMetadata/v1/instance/id"; + static final String ZONE_PATH = "computeMetadata/v1/instance/zone"; + + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); + + private final Map envVarMap; + private final Function connectionFactory; + + MetricParameters( + Map envVarMap, Function connectionFactory) { + this.envVarMap = envVarMap; + this.connectionFactory = connectionFactory; + } + + @Inject + MetricParameters() { + this(ImmutableMap.copyOf(System.getenv()), MetricParameters::gceConnectionFactory); + } + + private static final HttpURLConnection gceConnectionFactory(String path) { + String url = GCE_METADATA_URL_BASE + path; + try { + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setRequestMethod("GET"); + // The metadata server requires this header to be set when querying from a GCE instance. + connection.setRequestProperty("Metadata-Flavor", "Google"); + connection.setDoOutput(true); + return connection; + } catch (IOException e) { + logger.warningfmt(e, "Incorrect GCE metadata server URL: %s", url); + throw new RuntimeException(e); + } + }; + + private String readEnvVar(String envVar) { + return envVarMap.getOrDefault(envVar, ""); + } + + private String readGceMetadata(String path) { + String value = ""; + HttpURLConnection connection = connectionFactory.apply(path); + try { + connection.connect(); + int responseCode = connection.getResponseCode(); + if (responseCode < 200 || responseCode > 299) { + logger.warningfmt( + "Got an error response: %d\n%s", + responseCode, + CharStreams.toString(new InputStreamReader(connection.getErrorStream(), UTF_8))); + } else { + value = CharStreams.toString(new InputStreamReader(connection.getInputStream(), UTF_8)); + } + } catch (IOException e) { + logger.warningfmt(e, "Cannot obtain GCE metadata from path %s", path); + } + return value; + } + + public ImmutableMap makeLabelsMap() { + // The zone metadata is in the form of "projects//zones/". + // We only need the last part after the slash. + String fullZone = readGceMetadata(ZONE_PATH); + String zone; + String[] fullZoneArray = fullZone.split("/", -1); + if (fullZoneArray.length < 4) { + logger.warningfmt("Zone %s is valid.", fullZone); + // This will make the metric report throw, but it happens in a different thread and will not + // kill the whole application. + zone = ""; + } else { + zone = fullZoneArray[3]; + } + return new ImmutableMap.Builder() + .put("project_id", readGceMetadata(PROJECT_ID_PATH)) + .put("cluster_name", readGceMetadata(CLUSTER_NAME_PATH)) + .put("namespace_id", readEnvVar(NAMESPACE_ID_ENV)) + .put("instance_id", readGceMetadata(INSTANCE_ID_PATH)) + .put("pod_id", readEnvVar(POD_ID_ENV)) + .put("container_name", readEnvVar(CONTAINER_NAME_ENV)) + .put("zone", zone) + .build(); + } +} diff --git a/javatests/google/registry/proxy/metric/MetricParametersTest.java b/javatests/google/registry/proxy/metric/MetricParametersTest.java new file mode 100644 index 000000000..7dad2aadf --- /dev/null +++ b/javatests/google/registry/proxy/metric/MetricParametersTest.java @@ -0,0 +1,137 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.proxy.metric; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.proxy.metric.MetricParameters.CLUSTER_NAME_PATH; +import static google.registry.proxy.metric.MetricParameters.CONTAINER_NAME_ENV; +import static google.registry.proxy.metric.MetricParameters.INSTANCE_ID_PATH; +import static google.registry.proxy.metric.MetricParameters.NAMESPACE_ID_ENV; +import static google.registry.proxy.metric.MetricParameters.POD_ID_ENV; +import static google.registry.proxy.metric.MetricParameters.PROJECT_ID_PATH; +import static google.registry.proxy.metric.MetricParameters.ZONE_PATH; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.util.HashMap; +import java.util.Map.Entry; +import java.util.function.Function; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link MetricParameters}. */ +@RunWith(JUnit4.class) +public class MetricParametersTest { + + private static final HashMap RESULTS = new HashMap<>(); + + private final HttpURLConnection projectIdConnection = mock(HttpURLConnection.class); + private final HttpURLConnection clusterNameConnection = mock(HttpURLConnection.class); + private final HttpURLConnection instanceIdConnection = mock(HttpURLConnection.class); + private final HttpURLConnection zoneConnection = mock(HttpURLConnection.class); + private final ImmutableMap mockConnections = + ImmutableMap.of( + PROJECT_ID_PATH, + projectIdConnection, + CLUSTER_NAME_PATH, + clusterNameConnection, + INSTANCE_ID_PATH, + instanceIdConnection, + ZONE_PATH, + zoneConnection); + private final HashMap fakeEnvVarMap = new HashMap<>(); + private final Function fakeConnectionFactory = + path -> mockConnections.get(path); + + private final MetricParameters metricParameters = + new MetricParameters(fakeEnvVarMap, fakeConnectionFactory); + + private static InputStream makeInputStreamFromString(String input) { + return new ByteArrayInputStream(input.getBytes(UTF_8)); + } + + @Before + public void setUp() throws Exception { + fakeEnvVarMap.put(NAMESPACE_ID_ENV, "some-namespace"); + fakeEnvVarMap.put(POD_ID_ENV, "some-pod"); + fakeEnvVarMap.put(CONTAINER_NAME_ENV, "some-container"); + when(projectIdConnection.getInputStream()) + .thenReturn(makeInputStreamFromString("some-project")); + when(clusterNameConnection.getInputStream()) + .thenReturn(makeInputStreamFromString("some-cluster")); + when(instanceIdConnection.getInputStream()) + .thenReturn(makeInputStreamFromString("some-instance")); + when(zoneConnection.getInputStream()) + .thenReturn(makeInputStreamFromString("projects/some-project/zones/some-zone")); + for (Entry entry : mockConnections.entrySet()) { + when(entry.getValue().getResponseCode()).thenReturn(200); + } + RESULTS.put("project_id", "some-project"); + RESULTS.put("cluster_name", "some-cluster"); + RESULTS.put("namespace_id", "some-namespace"); + RESULTS.put("instance_id", "some-instance"); + RESULTS.put("pod_id", "some-pod"); + RESULTS.put("container_name", "some-container"); + RESULTS.put("zone", "some-zone"); + } + + @Test + public void testSuccess() { + assertThat(metricParameters.makeLabelsMap()).isEqualTo(ImmutableMap.copyOf(RESULTS)); + } + + @Test + public void testSuccess_missingEnvVar() { + fakeEnvVarMap.remove(POD_ID_ENV); + RESULTS.put("pod_id", ""); + assertThat(metricParameters.makeLabelsMap()).isEqualTo(ImmutableMap.copyOf(RESULTS)); + } + + @Test + public void testSuccess_malformedZone() throws Exception { + when(zoneConnection.getInputStream()).thenReturn(makeInputStreamFromString("some-zone")); + RESULTS.put("zone", ""); + assertThat(metricParameters.makeLabelsMap()).isEqualTo(ImmutableMap.copyOf(RESULTS)); + } + + @Test + public void testSuccess_errorResponseCode() throws Exception { + when(projectIdConnection.getResponseCode()).thenReturn(404); + when(projectIdConnection.getErrorStream()) + .thenReturn(makeInputStreamFromString("some error message")); + RESULTS.put("project_id", ""); + assertThat(metricParameters.makeLabelsMap()).isEqualTo(ImmutableMap.copyOf(RESULTS)); + } + + @Test + public void testSuccess_connectionError() throws Exception { + InputStream fakeInputStream = mock(InputStream.class); + when(projectIdConnection.getInputStream()).thenReturn(fakeInputStream); + when(fakeInputStream.read(any(byte[].class), anyInt(), anyInt())) + .thenThrow(new IOException("some exception")); + RESULTS.put("project_id", ""); + assertThat(metricParameters.makeLabelsMap()).isEqualTo(ImmutableMap.copyOf(RESULTS)); + } +}