mirror of
https://github.com/google/nomulus.git
synced 2025-05-13 16:07:15 +02:00
Cleanup minor things in whitebox metrics code
Specifically: - remove @RequestScope from EppMetric since it's only for components - fix to call the better overload of toBigqueryTimestamp - use the same UUID provider for BigQueryMetricsEnqueuer that already exists for the VerifyEntityIntegrityStreamer - minor cleanup in VerifyEntityIntegrityStreamer (inject projectId vs whole env) ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=132721794
This commit is contained in:
parent
2537e95de5
commit
9dffd64dc7
6 changed files with 25 additions and 38 deletions
|
@ -20,11 +20,11 @@ import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
||||||
import com.google.appengine.api.modules.ModulesService;
|
import com.google.appengine.api.modules.ModulesService;
|
||||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||||
import com.google.appengine.api.taskqueue.TransientFailureException;
|
import com.google.appengine.api.taskqueue.TransientFailureException;
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.base.Supplier;
|
||||||
import google.registry.util.FormattingLogger;
|
import google.registry.util.FormattingLogger;
|
||||||
import java.util.Map.Entry;
|
import java.util.Map.Entry;
|
||||||
import java.util.UUID;
|
|
||||||
import javax.inject.Inject;
|
import javax.inject.Inject;
|
||||||
|
import javax.inject.Named;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A collector of metric information. Enqueues collected metrics to a task queue to be written to
|
* A collector of metric information. Enqueues collected metrics to a task queue to be written to
|
||||||
|
@ -39,18 +39,17 @@ public class BigQueryMetricsEnqueuer {
|
||||||
public static final String QUEUE = "bigquery-streaming-metrics";
|
public static final String QUEUE = "bigquery-streaming-metrics";
|
||||||
|
|
||||||
@Inject ModulesService modulesService;
|
@Inject ModulesService modulesService;
|
||||||
|
@Inject @Named("insertIdGenerator") Supplier<String> idGenerator;
|
||||||
|
|
||||||
@Inject
|
@Inject BigQueryMetricsEnqueuer() {}
|
||||||
BigQueryMetricsEnqueuer() {}
|
|
||||||
|
|
||||||
@VisibleForTesting
|
public void export(BigQueryMetric metric) {
|
||||||
void export(BigQueryMetric metric, String insertId) {
|
|
||||||
try {
|
try {
|
||||||
String hostname = modulesService.getVersionHostname("backend", null);
|
String hostname = modulesService.getVersionHostname("backend", null);
|
||||||
TaskOptions opts =
|
TaskOptions opts =
|
||||||
withUrl(MetricsExportAction.PATH)
|
withUrl(MetricsExportAction.PATH)
|
||||||
.header("Host", hostname)
|
.header("Host", hostname)
|
||||||
.param("insertId", insertId);
|
.param("insertId", idGenerator.get());
|
||||||
for (Entry<String, String> entry : metric.getBigQueryRowEncoding().entrySet()) {
|
for (Entry<String, String> entry : metric.getBigQueryRowEncoding().entrySet()) {
|
||||||
opts.param(entry.getKey(), entry.getValue());
|
opts.param(entry.getKey(), entry.getValue());
|
||||||
}
|
}
|
||||||
|
@ -61,9 +60,4 @@ public class BigQueryMetricsEnqueuer {
|
||||||
logger.info(e, e.getMessage());
|
logger.info(e, e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Enqueue a metric to be exported to BigQuery. */
|
|
||||||
public void export(BigQueryMetric metric) {
|
|
||||||
export(metric, UUID.randomUUID().toString());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,9 +23,7 @@ import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import google.registry.bigquery.BigqueryUtils.FieldType;
|
import google.registry.bigquery.BigqueryUtils.FieldType;
|
||||||
import google.registry.model.eppoutput.Result.Code;
|
import google.registry.model.eppoutput.Result.Code;
|
||||||
import google.registry.request.RequestScope;
|
|
||||||
import google.registry.util.Clock;
|
import google.registry.util.Clock;
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
import org.joda.time.DateTime;
|
import org.joda.time.DateTime;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -34,7 +32,6 @@ import org.joda.time.DateTime;
|
||||||
* @see BigQueryMetricsEnqueuer
|
* @see BigQueryMetricsEnqueuer
|
||||||
*/
|
*/
|
||||||
@AutoValue
|
@AutoValue
|
||||||
@RequestScope
|
|
||||||
public abstract class EppMetric implements BigQueryMetric {
|
public abstract class EppMetric implements BigQueryMetric {
|
||||||
|
|
||||||
static final String TABLE_ID = "eppMetrics";
|
static final String TABLE_ID = "eppMetrics";
|
||||||
|
@ -84,12 +81,8 @@ public abstract class EppMetric implements BigQueryMetric {
|
||||||
ImmutableMap.Builder<String, String> map =
|
ImmutableMap.Builder<String, String> map =
|
||||||
ImmutableMap.<String, String>builder()
|
ImmutableMap.<String, String>builder()
|
||||||
.put("requestId", getRequestId())
|
.put("requestId", getRequestId())
|
||||||
.put(
|
.put("startTime", toBigqueryTimestamp(getStartTimestamp()))
|
||||||
"startTime",
|
.put("endTime", toBigqueryTimestamp(getEndTimestamp()))
|
||||||
toBigqueryTimestamp(getStartTimestamp().getMillis(), TimeUnit.MILLISECONDS))
|
|
||||||
.put(
|
|
||||||
"endTime",
|
|
||||||
toBigqueryTimestamp(getEndTimestamp().getMillis(), TimeUnit.MILLISECONDS))
|
|
||||||
.put("attempts", getAttempts().toString());
|
.put("attempts", getAttempts().toString());
|
||||||
// Populate optional values, if present
|
// Populate optional values, if present
|
||||||
addOptional("commandName", getCommandName(), map);
|
addOptional("commandName", getCommandName(), map);
|
||||||
|
|
|
@ -37,13 +37,14 @@ import com.google.common.collect.FluentIterable;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import google.registry.bigquery.BigqueryFactory;
|
import google.registry.bigquery.BigqueryFactory;
|
||||||
import google.registry.config.RegistryEnvironment;
|
import google.registry.config.ConfigModule.Config;
|
||||||
import google.registry.util.Retrier;
|
import google.registry.util.Retrier;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.Callable;
|
import java.util.concurrent.Callable;
|
||||||
import javax.annotation.Nullable;
|
import javax.annotation.Nullable;
|
||||||
|
import javax.inject.Named;
|
||||||
import org.joda.time.DateTime;
|
import org.joda.time.DateTime;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -52,22 +53,21 @@ import org.joda.time.DateTime;
|
||||||
@AutoFactory(allowSubclasses = true)
|
@AutoFactory(allowSubclasses = true)
|
||||||
public class VerifyEntityIntegrityStreamer {
|
public class VerifyEntityIntegrityStreamer {
|
||||||
|
|
||||||
|
private final String projectId;
|
||||||
|
private final BigqueryFactory bigqueryFactory;
|
||||||
|
private final Supplier<String> idGenerator;
|
||||||
|
private final Retrier retrier;
|
||||||
private final DateTime scanTime;
|
private final DateTime scanTime;
|
||||||
private Bigquery bigquery;
|
private Bigquery bigquery;
|
||||||
|
|
||||||
BigqueryFactory bigqueryFactory;
|
|
||||||
RegistryEnvironment environment;
|
|
||||||
Retrier retrier;
|
|
||||||
Supplier<String> idGenerator;
|
|
||||||
|
|
||||||
public VerifyEntityIntegrityStreamer(
|
public VerifyEntityIntegrityStreamer(
|
||||||
|
@Provided @Config("projectId") String projectId,
|
||||||
@Provided BigqueryFactory bigqueryFactory,
|
@Provided BigqueryFactory bigqueryFactory,
|
||||||
@Provided RegistryEnvironment environment,
|
|
||||||
@Provided Retrier retrier,
|
@Provided Retrier retrier,
|
||||||
@Provided Supplier<String> idGenerator,
|
@Provided @Named("insertIdGenerator") Supplier<String> idGenerator,
|
||||||
DateTime scanTime) {
|
DateTime scanTime) {
|
||||||
|
this.projectId = projectId;
|
||||||
this.bigqueryFactory = bigqueryFactory;
|
this.bigqueryFactory = bigqueryFactory;
|
||||||
this.environment = environment;
|
|
||||||
this.retrier = retrier;
|
this.retrier = retrier;
|
||||||
this.idGenerator = idGenerator;
|
this.idGenerator = idGenerator;
|
||||||
this.scanTime = scanTime;
|
this.scanTime = scanTime;
|
||||||
|
@ -78,9 +78,7 @@ public class VerifyEntityIntegrityStreamer {
|
||||||
// BigQuery.
|
// BigQuery.
|
||||||
private Bigquery getBigquery() throws IOException {
|
private Bigquery getBigquery() throws IOException {
|
||||||
if (bigquery == null) {
|
if (bigquery == null) {
|
||||||
bigquery =
|
bigquery = bigqueryFactory.create(projectId, DATASET, TABLE_ID);
|
||||||
bigqueryFactory.create(
|
|
||||||
environment.config().getProjectId(), DATASET, TABLE_ID);
|
|
||||||
}
|
}
|
||||||
return bigquery;
|
return bigquery;
|
||||||
}
|
}
|
||||||
|
@ -179,7 +177,7 @@ public class VerifyEntityIntegrityStreamer {
|
||||||
getBigquery()
|
getBigquery()
|
||||||
.tabledata()
|
.tabledata()
|
||||||
.insertAll(
|
.insertAll(
|
||||||
environment.config().getProjectId(),
|
projectId,
|
||||||
DATASET,
|
DATASET,
|
||||||
TABLE_ID,
|
TABLE_ID,
|
||||||
new TableDataInsertAllRequest().setRows(rows));
|
new TableDataInsertAllRequest().setRows(rows));
|
||||||
|
|
|
@ -65,7 +65,8 @@ public class WhiteboxModule {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
static Supplier<String> provideIdGenerator() {
|
@Named("insertIdGenerator")
|
||||||
|
static Supplier<String> provideInsertIdGenerator() {
|
||||||
return new Supplier<String>() {
|
return new Supplier<String>() {
|
||||||
@Override
|
@Override
|
||||||
public String get() {
|
public String get() {
|
||||||
|
|
|
@ -21,6 +21,7 @@ import static org.mockito.Mockito.when;
|
||||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||||
import com.google.appengine.api.modules.ModulesService;
|
import com.google.appengine.api.modules.ModulesService;
|
||||||
import com.google.auto.value.AutoValue;
|
import com.google.auto.value.AutoValue;
|
||||||
|
import com.google.common.base.Suppliers;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import google.registry.testing.AppEngineRule;
|
import google.registry.testing.AppEngineRule;
|
||||||
|
@ -56,6 +57,7 @@ public class BigQueryMetricsEnqueuerTest {
|
||||||
@Before
|
@Before
|
||||||
public void setUp() {
|
public void setUp() {
|
||||||
enqueuer = new BigQueryMetricsEnqueuer();
|
enqueuer = new BigQueryMetricsEnqueuer();
|
||||||
|
enqueuer.idGenerator = Suppliers.ofInstance("laffo");
|
||||||
enqueuer.modulesService = modulesService;
|
enqueuer.modulesService = modulesService;
|
||||||
when(modulesService.getVersionHostname(Matchers.anyString(), Matchers.anyString()))
|
when(modulesService.getVersionHostname(Matchers.anyString(), Matchers.anyString()))
|
||||||
.thenReturn("1.backend.test.localhost");
|
.thenReturn("1.backend.test.localhost");
|
||||||
|
@ -67,7 +69,7 @@ public class BigQueryMetricsEnqueuerTest {
|
||||||
TestMetric.create(
|
TestMetric.create(
|
||||||
DateTime.parse("1984-12-18TZ"), DateTime.parse("1984-12-18TZ").plusMillis(1));
|
DateTime.parse("1984-12-18TZ"), DateTime.parse("1984-12-18TZ").plusMillis(1));
|
||||||
|
|
||||||
enqueuer.export(metric, "laffo");
|
enqueuer.export(metric);
|
||||||
|
|
||||||
assertTasksEnqueued("bigquery-streaming-metrics",
|
assertTasksEnqueued("bigquery-streaming-metrics",
|
||||||
new TaskMatcher()
|
new TaskMatcher()
|
||||||
|
|
|
@ -43,7 +43,6 @@ import com.google.common.collect.ImmutableMap;
|
||||||
import com.google.common.collect.ImmutableSet;
|
import com.google.common.collect.ImmutableSet;
|
||||||
import com.googlecode.objectify.Key;
|
import com.googlecode.objectify.Key;
|
||||||
import google.registry.bigquery.BigqueryFactory;
|
import google.registry.bigquery.BigqueryFactory;
|
||||||
import google.registry.config.RegistryEnvironment;
|
|
||||||
import google.registry.mapreduce.MapreduceRunner;
|
import google.registry.mapreduce.MapreduceRunner;
|
||||||
import google.registry.model.contact.ContactResource;
|
import google.registry.model.contact.ContactResource;
|
||||||
import google.registry.model.domain.DomainResource;
|
import google.registry.model.domain.DomainResource;
|
||||||
|
@ -108,8 +107,8 @@ public class VerifyEntityIntegrityActionTest
|
||||||
inject.setStaticField(VerifyEntityIntegrityAction.class, "component", component);
|
inject.setStaticField(VerifyEntityIntegrityAction.class, "component", component);
|
||||||
integrity =
|
integrity =
|
||||||
new VerifyEntityIntegrityStreamer(
|
new VerifyEntityIntegrityStreamer(
|
||||||
|
"project-id",
|
||||||
bigqueryFactory,
|
bigqueryFactory,
|
||||||
RegistryEnvironment.UNITTEST,
|
|
||||||
new Retrier(new FakeSleeper(new FakeClock()), 1),
|
new Retrier(new FakeSleeper(new FakeClock()), 1),
|
||||||
Suppliers.ofInstance("rowid"),
|
Suppliers.ofInstance("rowid"),
|
||||||
now);
|
now);
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue