Cleanup minor things in whitebox metrics code

Specifically:
- remove @RequestScope from EppMetric since it's only for components
- fix to call the better overload of toBigqueryTimestamp
- use the same UUID provider for BigQueryMetricsEnqueuer that already exists for
  the VerifyEntityIntegrityStreamer
- minor cleanup in VerifyEntityIntegrityStreamer (inject projectId vs whole env)

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=132721794
This commit is contained in:
nickfelt 2016-09-09 15:11:09 -07:00 committed by Ben McIlwain
parent 2537e95de5
commit 9dffd64dc7
6 changed files with 25 additions and 38 deletions

View file

@ -20,11 +20,11 @@ import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
import com.google.appengine.api.modules.ModulesService; import com.google.appengine.api.modules.ModulesService;
import com.google.appengine.api.taskqueue.TaskOptions; import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TransientFailureException; import com.google.appengine.api.taskqueue.TransientFailureException;
import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Supplier;
import google.registry.util.FormattingLogger; import google.registry.util.FormattingLogger;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.UUID;
import javax.inject.Inject; import javax.inject.Inject;
import javax.inject.Named;
/** /**
* A collector of metric information. Enqueues collected metrics to a task queue to be written to * A collector of metric information. Enqueues collected metrics to a task queue to be written to
@ -39,18 +39,17 @@ public class BigQueryMetricsEnqueuer {
public static final String QUEUE = "bigquery-streaming-metrics"; public static final String QUEUE = "bigquery-streaming-metrics";
@Inject ModulesService modulesService; @Inject ModulesService modulesService;
@Inject @Named("insertIdGenerator") Supplier<String> idGenerator;
@Inject @Inject BigQueryMetricsEnqueuer() {}
BigQueryMetricsEnqueuer() {}
@VisibleForTesting public void export(BigQueryMetric metric) {
void export(BigQueryMetric metric, String insertId) {
try { try {
String hostname = modulesService.getVersionHostname("backend", null); String hostname = modulesService.getVersionHostname("backend", null);
TaskOptions opts = TaskOptions opts =
withUrl(MetricsExportAction.PATH) withUrl(MetricsExportAction.PATH)
.header("Host", hostname) .header("Host", hostname)
.param("insertId", insertId); .param("insertId", idGenerator.get());
for (Entry<String, String> entry : metric.getBigQueryRowEncoding().entrySet()) { for (Entry<String, String> entry : metric.getBigQueryRowEncoding().entrySet()) {
opts.param(entry.getKey(), entry.getValue()); opts.param(entry.getKey(), entry.getValue());
} }
@ -61,9 +60,4 @@ public class BigQueryMetricsEnqueuer {
logger.info(e, e.getMessage()); logger.info(e, e.getMessage());
} }
} }
/** Enqueue a metric to be exported to BigQuery. */
public void export(BigQueryMetric metric) {
export(metric, UUID.randomUUID().toString());
}
} }

View file

@ -23,9 +23,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import google.registry.bigquery.BigqueryUtils.FieldType; import google.registry.bigquery.BigqueryUtils.FieldType;
import google.registry.model.eppoutput.Result.Code; import google.registry.model.eppoutput.Result.Code;
import google.registry.request.RequestScope;
import google.registry.util.Clock; import google.registry.util.Clock;
import java.util.concurrent.TimeUnit;
import org.joda.time.DateTime; import org.joda.time.DateTime;
/** /**
@ -34,7 +32,6 @@ import org.joda.time.DateTime;
* @see BigQueryMetricsEnqueuer * @see BigQueryMetricsEnqueuer
*/ */
@AutoValue @AutoValue
@RequestScope
public abstract class EppMetric implements BigQueryMetric { public abstract class EppMetric implements BigQueryMetric {
static final String TABLE_ID = "eppMetrics"; static final String TABLE_ID = "eppMetrics";
@ -84,12 +81,8 @@ public abstract class EppMetric implements BigQueryMetric {
ImmutableMap.Builder<String, String> map = ImmutableMap.Builder<String, String> map =
ImmutableMap.<String, String>builder() ImmutableMap.<String, String>builder()
.put("requestId", getRequestId()) .put("requestId", getRequestId())
.put( .put("startTime", toBigqueryTimestamp(getStartTimestamp()))
"startTime", .put("endTime", toBigqueryTimestamp(getEndTimestamp()))
toBigqueryTimestamp(getStartTimestamp().getMillis(), TimeUnit.MILLISECONDS))
.put(
"endTime",
toBigqueryTimestamp(getEndTimestamp().getMillis(), TimeUnit.MILLISECONDS))
.put("attempts", getAttempts().toString()); .put("attempts", getAttempts().toString());
// Populate optional values, if present // Populate optional values, if present
addOptional("commandName", getCommandName(), map); addOptional("commandName", getCommandName(), map);

View file

@ -37,13 +37,14 @@ import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import google.registry.bigquery.BigqueryFactory; import google.registry.bigquery.BigqueryFactory;
import google.registry.config.RegistryEnvironment; import google.registry.config.ConfigModule.Config;
import google.registry.util.Retrier; import google.registry.util.Retrier;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import javax.inject.Named;
import org.joda.time.DateTime; import org.joda.time.DateTime;
/** /**
@ -52,22 +53,21 @@ import org.joda.time.DateTime;
@AutoFactory(allowSubclasses = true) @AutoFactory(allowSubclasses = true)
public class VerifyEntityIntegrityStreamer { public class VerifyEntityIntegrityStreamer {
private final String projectId;
private final BigqueryFactory bigqueryFactory;
private final Supplier<String> idGenerator;
private final Retrier retrier;
private final DateTime scanTime; private final DateTime scanTime;
private Bigquery bigquery; private Bigquery bigquery;
BigqueryFactory bigqueryFactory;
RegistryEnvironment environment;
Retrier retrier;
Supplier<String> idGenerator;
public VerifyEntityIntegrityStreamer( public VerifyEntityIntegrityStreamer(
@Provided @Config("projectId") String projectId,
@Provided BigqueryFactory bigqueryFactory, @Provided BigqueryFactory bigqueryFactory,
@Provided RegistryEnvironment environment,
@Provided Retrier retrier, @Provided Retrier retrier,
@Provided Supplier<String> idGenerator, @Provided @Named("insertIdGenerator") Supplier<String> idGenerator,
DateTime scanTime) { DateTime scanTime) {
this.projectId = projectId;
this.bigqueryFactory = bigqueryFactory; this.bigqueryFactory = bigqueryFactory;
this.environment = environment;
this.retrier = retrier; this.retrier = retrier;
this.idGenerator = idGenerator; this.idGenerator = idGenerator;
this.scanTime = scanTime; this.scanTime = scanTime;
@ -78,9 +78,7 @@ public class VerifyEntityIntegrityStreamer {
// BigQuery. // BigQuery.
private Bigquery getBigquery() throws IOException { private Bigquery getBigquery() throws IOException {
if (bigquery == null) { if (bigquery == null) {
bigquery = bigquery = bigqueryFactory.create(projectId, DATASET, TABLE_ID);
bigqueryFactory.create(
environment.config().getProjectId(), DATASET, TABLE_ID);
} }
return bigquery; return bigquery;
} }
@ -179,7 +177,7 @@ public class VerifyEntityIntegrityStreamer {
getBigquery() getBigquery()
.tabledata() .tabledata()
.insertAll( .insertAll(
environment.config().getProjectId(), projectId,
DATASET, DATASET,
TABLE_ID, TABLE_ID,
new TableDataInsertAllRequest().setRows(rows)); new TableDataInsertAllRequest().setRows(rows));

View file

@ -65,7 +65,8 @@ public class WhiteboxModule {
} }
@Provides @Provides
static Supplier<String> provideIdGenerator() { @Named("insertIdGenerator")
static Supplier<String> provideInsertIdGenerator() {
return new Supplier<String>() { return new Supplier<String>() {
@Override @Override
public String get() { public String get() {

View file

@ -21,6 +21,7 @@ import static org.mockito.Mockito.when;
import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.appengine.api.modules.ModulesService; import com.google.appengine.api.modules.ModulesService;
import com.google.auto.value.AutoValue; import com.google.auto.value.AutoValue;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import google.registry.testing.AppEngineRule; import google.registry.testing.AppEngineRule;
@ -56,6 +57,7 @@ public class BigQueryMetricsEnqueuerTest {
@Before @Before
public void setUp() { public void setUp() {
enqueuer = new BigQueryMetricsEnqueuer(); enqueuer = new BigQueryMetricsEnqueuer();
enqueuer.idGenerator = Suppliers.ofInstance("laffo");
enqueuer.modulesService = modulesService; enqueuer.modulesService = modulesService;
when(modulesService.getVersionHostname(Matchers.anyString(), Matchers.anyString())) when(modulesService.getVersionHostname(Matchers.anyString(), Matchers.anyString()))
.thenReturn("1.backend.test.localhost"); .thenReturn("1.backend.test.localhost");
@ -67,7 +69,7 @@ public class BigQueryMetricsEnqueuerTest {
TestMetric.create( TestMetric.create(
DateTime.parse("1984-12-18TZ"), DateTime.parse("1984-12-18TZ").plusMillis(1)); DateTime.parse("1984-12-18TZ"), DateTime.parse("1984-12-18TZ").plusMillis(1));
enqueuer.export(metric, "laffo"); enqueuer.export(metric);
assertTasksEnqueued("bigquery-streaming-metrics", assertTasksEnqueued("bigquery-streaming-metrics",
new TaskMatcher() new TaskMatcher()

View file

@ -43,7 +43,6 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import com.googlecode.objectify.Key; import com.googlecode.objectify.Key;
import google.registry.bigquery.BigqueryFactory; import google.registry.bigquery.BigqueryFactory;
import google.registry.config.RegistryEnvironment;
import google.registry.mapreduce.MapreduceRunner; import google.registry.mapreduce.MapreduceRunner;
import google.registry.model.contact.ContactResource; import google.registry.model.contact.ContactResource;
import google.registry.model.domain.DomainResource; import google.registry.model.domain.DomainResource;
@ -108,8 +107,8 @@ public class VerifyEntityIntegrityActionTest
inject.setStaticField(VerifyEntityIntegrityAction.class, "component", component); inject.setStaticField(VerifyEntityIntegrityAction.class, "component", component);
integrity = integrity =
new VerifyEntityIntegrityStreamer( new VerifyEntityIntegrityStreamer(
"project-id",
bigqueryFactory, bigqueryFactory,
RegistryEnvironment.UNITTEST,
new Retrier(new FakeSleeper(new FakeClock()), 1), new Retrier(new FakeSleeper(new FakeClock()), 1),
Suppliers.ofInstance("rowid"), Suppliers.ofInstance("rowid"),
now); now);