mirror of
https://github.com/google/nomulus.git
synced 2025-05-28 13:51:15 +02:00
Stop exporting EPP flow metrics to BigQuery
These are simply too costly in their current form now that we are handling double-digit QPS, so at a minimum we'd want to refactor these for batched exports using a background thread (like how Stackdriver metrics work). However, upon further review, that work isn't worth doing if this BigQuery table isn't actually being used for anything, and it seems that we aren't using it anymore given that ICANN transaction reporting no longer requires it. So the simplest thing to do is simply to get rid of this entirely, and just use a combination of Stackdriver metrics and App Engine logs. The eppMetrics BigQuery table is ~1.2 billion rows and takes up 223 GB, so that's not an insignificant GCP billings saving if we can delete it. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=215905466
This commit is contained in:
mcilwain
Ben McIlwain
parent
7b9d562043
commit
218c4517eb
24 changed files with 17 additions and 763 deletions
|
|
@ -1,36 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.monitoring.whitebox;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
/**
|
||||
* A metric which can be encoded into a BigQuery row.
|
||||
*
|
||||
* @see BigQueryMetricsEnqueuer
|
||||
*/
|
||||
public interface BigQueryMetric {
|
||||
|
||||
/** Get the BigQuery table name for this metric. */
|
||||
String getTableId();
|
||||
|
||||
/** Get the schema description for the BigQuery table. */
|
||||
ImmutableList<TableFieldSchema> getSchemaFields();
|
||||
|
||||
/** Get a map of the row values for this metric instance. */
|
||||
ImmutableMap<String, String> getBigQueryRowEncoding();
|
||||
}
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.monitoring.whitebox;
|
||||
|
||||
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
||||
|
||||
import com.google.appengine.api.taskqueue.Queue;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TransientFailureException;
|
||||
import com.google.common.base.Supplier;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.util.AppEngineServiceUtils;
|
||||
import java.util.Map.Entry;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
|
||||
/**
|
||||
* A collector of metric information. Enqueues collected metrics to a task queue to be written to
|
||||
* BigQuery asynchronously.
|
||||
*
|
||||
* @see MetricsExportAction
|
||||
*/
|
||||
public class BigQueryMetricsEnqueuer {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
public static final String QUEUE_BIGQUERY_STREAMING_METRICS = "bigquery-streaming-metrics";
|
||||
|
||||
@Inject AppEngineServiceUtils appEngineServiceUtils;
|
||||
@Inject @Named("insertIdGenerator") Supplier<String> idGenerator;
|
||||
@Inject @Named(QUEUE_BIGQUERY_STREAMING_METRICS) Queue queue;
|
||||
|
||||
@Inject BigQueryMetricsEnqueuer() {}
|
||||
|
||||
public void export(BigQueryMetric metric) {
|
||||
try {
|
||||
String hostname = appEngineServiceUtils.getCurrentVersionHostname("backend");
|
||||
TaskOptions opts =
|
||||
withUrl(MetricsExportAction.PATH)
|
||||
.header("Host", hostname)
|
||||
.param("insertId", idGenerator.get());
|
||||
for (Entry<String, String> entry : metric.getBigQueryRowEncoding().entrySet()) {
|
||||
opts.param(entry.getKey(), entry.getValue());
|
||||
}
|
||||
opts.param("tableId", metric.getTableId());
|
||||
queue.add(opts);
|
||||
} catch (TransientFailureException e) {
|
||||
// Log and swallow. We may drop some metrics here but this should be rare.
|
||||
logger.atInfo().withCause(e).log(
|
||||
"Transient error occurred while recording metric; metric dropped.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -15,44 +15,19 @@
|
|||
package google.registry.monitoring.whitebox;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static google.registry.bigquery.BigqueryUtils.toBigqueryTimestamp;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Iterables;
|
||||
import google.registry.bigquery.BigqueryUtils.FieldType;
|
||||
import google.registry.model.eppoutput.Result.Code;
|
||||
import google.registry.model.registry.Registries;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.Optional;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
* A value class for recording attributes of an EPP metric.
|
||||
*
|
||||
* @see BigQueryMetricsEnqueuer
|
||||
*/
|
||||
/** A value class for recording attributes of an EPP metric. */
|
||||
@AutoValue
|
||||
public abstract class EppMetric implements BigQueryMetric {
|
||||
|
||||
static final String TABLE_ID = "eppMetrics";
|
||||
static final ImmutableList<TableFieldSchema> SCHEMA_FIELDS =
|
||||
ImmutableList.of(
|
||||
new TableFieldSchema().setName("requestId").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("startTime").setType(FieldType.TIMESTAMP.name()),
|
||||
new TableFieldSchema().setName("endTime").setType(FieldType.TIMESTAMP.name()),
|
||||
new TableFieldSchema().setName("commandName").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("clientId").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("tld").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("privilegeLevel").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("eppTarget").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("eppStatus").setType(FieldType.INTEGER.name()),
|
||||
new TableFieldSchema().setName("attempts").setType(FieldType.INTEGER.name()));
|
||||
|
||||
public abstract String getRequestId();
|
||||
public abstract class EppMetric {
|
||||
|
||||
public abstract DateTime getStartTimestamp();
|
||||
|
||||
|
|
@ -64,55 +39,8 @@ public abstract class EppMetric implements BigQueryMetric {
|
|||
|
||||
public abstract Optional<String> getTld();
|
||||
|
||||
public abstract Optional<String> getPrivilegeLevel();
|
||||
|
||||
public abstract Optional<String> getEppTarget();
|
||||
|
||||
public abstract Optional<Code> getStatus();
|
||||
|
||||
public abstract Integer getAttempts();
|
||||
|
||||
@Override
|
||||
public String getTableId() {
|
||||
return TABLE_ID;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ImmutableList<TableFieldSchema> getSchemaFields() {
|
||||
return SCHEMA_FIELDS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ImmutableMap<String, String> getBigQueryRowEncoding() {
|
||||
// Create map builder, start with required values
|
||||
ImmutableMap.Builder<String, String> map =
|
||||
new ImmutableMap.Builder<String, String>()
|
||||
.put("requestId", getRequestId())
|
||||
.put("startTime", toBigqueryTimestamp(getStartTimestamp()))
|
||||
.put("endTime", toBigqueryTimestamp(getEndTimestamp()))
|
||||
.put("attempts", getAttempts().toString());
|
||||
// Populate optional values, if present
|
||||
addOptional("commandName", getCommandName(), map);
|
||||
addOptional("clientId", getClientId(), map);
|
||||
addOptional("tld", getTld(), map);
|
||||
addOptional("privilegeLevel", getPrivilegeLevel(), map);
|
||||
addOptional("eppTarget", getEppTarget(), map);
|
||||
if (getStatus().isPresent()) {
|
||||
map.put("eppStatus", Integer.toString(getStatus().get().code));
|
||||
}
|
||||
|
||||
return map.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to populate an {@link com.google.common.collect.ImmutableMap.Builder} with an
|
||||
* {@link Optional} value if the value is {@link Optional#isPresent()}.
|
||||
*/
|
||||
private static <T> void addOptional(
|
||||
String key, Optional<T> value, ImmutableMap.Builder<String, String> map) {
|
||||
value.ifPresent(t -> map.put(key, t.toString()));
|
||||
}
|
||||
|
||||
/** Create an {@link EppMetric.Builder}. */
|
||||
public static Builder builder() {
|
||||
return new AutoValue_EppMetric.Builder();
|
||||
|
|
@ -124,9 +52,8 @@ public abstract class EppMetric implements BigQueryMetric {
|
|||
*
|
||||
* <p>The start timestamp is recorded now, and the end timestamp at {@code build()}.
|
||||
*/
|
||||
public static Builder builderForRequest(String requestId, Clock clock) {
|
||||
public static Builder builderForRequest(Clock clock) {
|
||||
return builder()
|
||||
.setRequestId(requestId)
|
||||
.setStartTimestamp(clock.nowUtc())
|
||||
.setClock(clock);
|
||||
}
|
||||
|
|
@ -135,14 +62,9 @@ public abstract class EppMetric implements BigQueryMetric {
|
|||
@AutoValue.Builder
|
||||
public abstract static class Builder {
|
||||
|
||||
/** Builder-only counter of the number of attempts, to support {@link #incrementAttempts()}. */
|
||||
private int attempts = 0;
|
||||
|
||||
/** Builder-only clock to support automatic recording of endTimestamp on {@link #build()}. */
|
||||
private Clock clock = null;
|
||||
|
||||
abstract Builder setRequestId(String requestId);
|
||||
|
||||
abstract Builder setStartTimestamp(DateTime startTimestamp);
|
||||
|
||||
abstract Builder setEndTimestamp(DateTime endTimestamp);
|
||||
|
|
@ -191,19 +113,8 @@ public abstract class EppMetric implements BigQueryMetric {
|
|||
return this;
|
||||
}
|
||||
|
||||
public abstract Builder setPrivilegeLevel(String privilegeLevel);
|
||||
|
||||
public abstract Builder setEppTarget(String eppTarget);
|
||||
|
||||
public abstract Builder setStatus(Code code);
|
||||
|
||||
abstract Builder setAttempts(Integer attempts);
|
||||
|
||||
public Builder incrementAttempts() {
|
||||
attempts++;
|
||||
return this;
|
||||
}
|
||||
|
||||
Builder setClock(Clock clock) {
|
||||
this.clock = clock;
|
||||
return this;
|
||||
|
|
@ -216,7 +127,6 @@ public abstract class EppMetric implements BigQueryMetric {
|
|||
* current timestamp of the clock; otherwise end timestamp must have been previously set.
|
||||
*/
|
||||
public EppMetric build() {
|
||||
setAttempts(attempts);
|
||||
if (clock != null) {
|
||||
setEndTimestamp(clock.nowUtc());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,104 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.monitoring.whitebox;
|
||||
|
||||
import static com.google.common.base.Predicates.in;
|
||||
import static com.google.common.base.Predicates.not;
|
||||
import static com.google.common.collect.Multimaps.filterKeys;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static java.util.stream.Collectors.joining;
|
||||
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.model.TableDataInsertAllRequest;
|
||||
import com.google.api.services.bigquery.model.TableDataInsertAllResponse;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableListMultimap;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.bigquery.CheckedBigquery;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.ParameterMap;
|
||||
import google.registry.request.auth.Auth;
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/** Action for exporting metrics to BigQuery. */
|
||||
@Action(
|
||||
path = MetricsExportAction.PATH,
|
||||
method = POST,
|
||||
auth = Auth.AUTH_INTERNAL_ONLY
|
||||
)
|
||||
public class MetricsExportAction implements Runnable {
|
||||
|
||||
public static final String PATH = "/_dr/task/metrics";
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final String DATASET_ID = "metrics";
|
||||
private static final ImmutableSet<String> SPECIAL_PARAMS = ImmutableSet.of("tableId", "insertId");
|
||||
|
||||
@Inject @Parameter("tableId") String tableId;
|
||||
@Inject @Parameter("insertId") String insertId;
|
||||
@Inject @Config("projectId") String projectId;
|
||||
|
||||
@Inject CheckedBigquery checkedBigquery;
|
||||
@Inject @ParameterMap ImmutableListMultimap<String, String> parameters;
|
||||
@Inject MetricsExportAction() {}
|
||||
|
||||
/** Exports metrics to BigQuery. */
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
Bigquery bigquery =
|
||||
checkedBigquery.ensureDataSetAndTableExist(projectId, DATASET_ID, tableId);
|
||||
// Filter out the special parameters that the Action is called with. Everything that's left
|
||||
// is returned in a Map that is suitable to pass to Bigquery as row data.
|
||||
Map<String, Object> jsonRows =
|
||||
ImmutableMap.copyOf(
|
||||
filterKeys(parameters, not(in(SPECIAL_PARAMS))).entries());
|
||||
TableDataInsertAllResponse response = bigquery.tabledata()
|
||||
.insertAll(
|
||||
projectId,
|
||||
DATASET_ID,
|
||||
tableId,
|
||||
new TableDataInsertAllRequest()
|
||||
.setRows(
|
||||
ImmutableList.of(new TableDataInsertAllRequest.Rows()
|
||||
.setInsertId(insertId)
|
||||
.setJson(jsonRows))))
|
||||
.execute();
|
||||
|
||||
if (response.getInsertErrors() != null && !response.getInsertErrors().isEmpty()) {
|
||||
throw new RuntimeException(
|
||||
response
|
||||
.getInsertErrors()
|
||||
.stream()
|
||||
.map(
|
||||
error -> {
|
||||
try {
|
||||
return error.toPrettyString();
|
||||
} catch (IOException e) {
|
||||
return error.toString();
|
||||
}
|
||||
})
|
||||
.collect(joining("\n")));
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.atWarning().withCause(e).log("Unknown error while exporting metrics to BigQuery.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -14,24 +14,9 @@
|
|||
|
||||
package google.registry.monitoring.whitebox;
|
||||
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static google.registry.monitoring.whitebox.BigQueryMetricsEnqueuer.QUEUE_BIGQUERY_STREAMING_METRICS;
|
||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.appengine.api.taskqueue.Queue;
|
||||
import com.google.common.base.Supplier;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import dagger.multibindings.IntoMap;
|
||||
import dagger.multibindings.StringKey;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.RequestLogId;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.UUID;
|
||||
import javax.inject.Named;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
/**
|
||||
* Dagger module for injecting common settings for Whitebox tasks.
|
||||
|
|
@ -39,46 +24,14 @@ import javax.servlet.http.HttpServletRequest;
|
|||
@Module
|
||||
public class WhiteboxModule {
|
||||
|
||||
@Provides
|
||||
@IntoMap
|
||||
@StringKey(EppMetric.TABLE_ID)
|
||||
static ImmutableList<TableFieldSchema> provideEppMetricsSchema() {
|
||||
return EppMetric.SCHEMA_FIELDS;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter("tableId")
|
||||
static String provideTableId(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, "tableId");
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter("insertId")
|
||||
static String provideInsertId(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, "insertId");
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Named("insertIdGenerator")
|
||||
static Supplier<String> provideInsertIdGenerator() {
|
||||
return () -> UUID.randomUUID().toString();
|
||||
}
|
||||
|
||||
/** Provides an EppMetric builder with the request ID and startTimestamp already initialized. */
|
||||
@Provides
|
||||
static EppMetric.Builder provideEppMetricBuilder(
|
||||
@RequestLogId String requestLogId, Clock clock) {
|
||||
return EppMetric.builderForRequest(requestLogId, clock);
|
||||
static EppMetric.Builder provideEppMetricBuilder(Clock clock) {
|
||||
return EppMetric.builderForRequest(clock);
|
||||
}
|
||||
|
||||
@Provides
|
||||
static CheckApiMetric.Builder provideCheckApiMetricBuilder(Clock clock) {
|
||||
return CheckApiMetric.builder(clock);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Named(QUEUE_BIGQUERY_STREAMING_METRICS)
|
||||
static Queue provideBigQueryStreamingMetricsQueue() {
|
||||
return getQueue(QUEUE_BIGQUERY_STREAMING_METRICS);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue