Make metrics export run on backend module

This fixes a previous issue where metrics generated on the
tools/backend modules weren't being successfully exported to BigQuery.
-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=119047757
This commit is contained in:
mcilwain 2016-04-05 07:35:05 -07:00 committed by Justine Tunney
parent 17c4ad83c8
commit e6d3f42e60
7 changed files with 34 additions and 15 deletions

View file

@ -13,6 +13,16 @@
<load-on-startup>1</load-on-startup> <load-on-startup>1</load-on-startup>
</servlet> </servlet>
<!-- Whitebox Metrics servlet. -->
<servlet>
<servlet-name>metrics</servlet-name>
<servlet-class>com.google.domain.registry.monitoring.whitebox.MetricsTaskServlet</servlet-class>
</servlet>
<servlet-mapping>
<servlet-name>metrics</servlet-name>
<url-pattern>/_dr/task/metrics</url-pattern>
</servlet-mapping>
<!-- RDE --> <!-- RDE -->
<!-- <!--

View file

@ -103,16 +103,6 @@
<url-pattern>/check</url-pattern> <url-pattern>/check</url-pattern>
</servlet-mapping> </servlet-mapping>
<!-- Whitebox Metrics servlet. -->
<servlet>
<servlet-name>metrics</servlet-name>
<servlet-class>com.google.domain.registry.monitoring.whitebox.MetricsTaskServlet</servlet-class>
</servlet>
<servlet-mapping>
<servlet-name>metrics</servlet-name>
<url-pattern>/_dr/task/metrics</url-pattern>
</servlet-mapping>
<!-- Security config --> <!-- Security config -->
<security-constraint> <security-constraint>
<web-resource-collection> <web-resource-collection>

View file

@ -18,6 +18,8 @@ import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl; import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
import static com.google.domain.registry.bigquery.BigqueryUtils.toBigqueryTimestamp; import static com.google.domain.registry.bigquery.BigqueryUtils.toBigqueryTimestamp;
import com.google.appengine.api.modules.ModulesService;
import com.google.appengine.api.modules.ModulesServiceFactory;
import com.google.appengine.api.taskqueue.TaskOptions; import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TransientFailureException; import com.google.appengine.api.taskqueue.TransientFailureException;
import com.google.common.base.Supplier; import com.google.common.base.Supplier;
@ -33,12 +35,15 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
/** A collector of metric information. */ /** A collector of metric information. */
abstract class Metrics { public abstract class Metrics {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
public static final String QUEUE = "bigquery-streaming-metrics"; public static final String QUEUE = "bigquery-streaming-metrics";
@NonFinalForTesting
private static ModulesService modulesService = ModulesServiceFactory.getModulesService();
@NonFinalForTesting @NonFinalForTesting
private static Clock clock = new SystemClock(); private static Clock clock = new SystemClock();
@ -60,7 +65,9 @@ abstract class Metrics {
public void export() { public void export() {
try { try {
TaskOptions opts = withUrl("/_dr/task/metrics") String hostname = modulesService.getVersionHostname("backend", null);
TaskOptions opts = withUrl(MetricsTaskServlet.PATH)
.header("Host", hostname)
.param("insertId", idGenerator.get()) .param("insertId", idGenerator.get())
.param("startTime", toBigqueryTimestamp(startTimeMillis, TimeUnit.MILLISECONDS)) .param("startTime", toBigqueryTimestamp(startTimeMillis, TimeUnit.MILLISECONDS))
.param("endTime", toBigqueryTimestamp(clock.nowUtc().getMillis(), TimeUnit.MILLISECONDS)); .param("endTime", toBigqueryTimestamp(clock.nowUtc().getMillis(), TimeUnit.MILLISECONDS));

View file

@ -51,6 +51,8 @@ import javax.servlet.http.HttpServletResponse;
/** Servlet for exporting metrics to BigQuery. */ /** Servlet for exporting metrics to BigQuery. */
public class MetricsTaskServlet extends HttpServlet { public class MetricsTaskServlet extends HttpServlet {
public static final String PATH = "/_dr/task/metrics";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
private static final String DATASET_ID = "metrics"; private static final String DATASET_ID = "metrics";
private static final String PROJECT_ID = RegistryEnvironment.get().config().getProjectId(); private static final String PROJECT_ID = RegistryEnvironment.get().config().getProjectId();
@ -81,7 +83,7 @@ public class MetricsTaskServlet extends HttpServlet {
ImmutableMap.Builder<String, Object> b = new ImmutableMap.Builder<>(); ImmutableMap.Builder<String, Object> b = new ImmutableMap.Builder<>();
@SuppressWarnings({"cast", "unchecked"}) // Return type is always a Set<String>. @SuppressWarnings({"cast", "unchecked"}) // Return type is always a Set<String>.
Set<String> parameterKeys = (Set<String>) req.getParameterMap().keySet(); Set<String> parameterKeys = req.getParameterMap().keySet();
for (String key : Sets.difference(parameterKeys, filter)) { for (String key : Sets.difference(parameterKeys, filter)) {
b.put(key, req.getParameter(key)); b.put(key, req.getParameter(key));

View file

@ -31,6 +31,7 @@ java_library(
"//java/com/google/domain/registry/flows", "//java/com/google/domain/registry/flows",
"//java/com/google/domain/registry/mapreduce", "//java/com/google/domain/registry/mapreduce",
"//java/com/google/domain/registry/model", "//java/com/google/domain/registry/model",
"//java/com/google/domain/registry/monitoring/whitebox",
"//java/com/google/domain/registry/request", "//java/com/google/domain/registry/request",
"//java/com/google/domain/registry/security", "//java/com/google/domain/registry/security",
"//java/com/google/domain/registry/security:servlets", "//java/com/google/domain/registry/security:servlets",

View file

@ -28,12 +28,14 @@ import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import com.google.appengine.api.modules.ModulesService;
import com.google.common.base.Optional; import com.google.common.base.Optional;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.model.ofy.Ofy; import com.google.domain.registry.model.ofy.Ofy;
import com.google.domain.registry.model.registrar.Registrar; import com.google.domain.registry.model.registrar.Registrar;
import com.google.domain.registry.model.tmch.ClaimsListShard.ClaimsListSingleton; import com.google.domain.registry.model.tmch.ClaimsListShard.ClaimsListSingleton;
import com.google.domain.registry.monitoring.whitebox.Metrics;
import com.google.domain.registry.security.XsrfProtectedServlet; import com.google.domain.registry.security.XsrfProtectedServlet;
import com.google.domain.registry.testing.FakeClock; import com.google.domain.registry.testing.FakeClock;
import com.google.domain.registry.testing.FakeServletInputStream; import com.google.domain.registry.testing.FakeServletInputStream;
@ -74,6 +76,9 @@ public abstract class EppServletTestCase<S extends HttpServlet> {
@Mock @Mock
HttpServletResponse rsp; HttpServletResponse rsp;
@Mock
ModulesService modulesService;
HttpSession session; HttpSession session;
FakeClock clock = new FakeClock(); FakeClock clock = new FakeClock();
@ -96,6 +101,8 @@ public abstract class EppServletTestCase<S extends HttpServlet> {
public final void init() throws Exception { public final void init() throws Exception {
inject.setStaticField(Ofy.class, "clock", clock); // For transactional flows. inject.setStaticField(Ofy.class, "clock", clock); // For transactional flows.
inject.setStaticField(FlowRunner.class, "clock", clock); // For non-transactional flows. inject.setStaticField(FlowRunner.class, "clock", clock); // For non-transactional flows.
inject.setStaticField(Metrics.class, "modulesService", modulesService);
when(modulesService.getVersionHostname("backend", null)).thenReturn("backend.hostname");
// Create RegistryData for all TLDs used in these tests. // Create RegistryData for all TLDs used in these tests.
// We want to create all of these even for tests that don't use them to make sure that // We want to create all of these even for tests that don't use them to make sure that

View file

@ -27,10 +27,10 @@ import org.junit.Before;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.junit.runners.JUnit4; import org.mockito.runners.MockitoJUnitRunner;
/** Unit tests for {@link Metrics}. */ /** Unit tests for {@link Metrics}. */
@RunWith(JUnit4.class) @RunWith(MockitoJUnitRunner.class)
public class MetricsTest { public class MetricsTest {
@Rule @Rule
@ -39,6 +39,7 @@ public class MetricsTest {
@Rule @Rule
public final AppEngineRule appEngine = AppEngineRule.builder() public final AppEngineRule appEngine = AppEngineRule.builder()
.withDatastore() .withDatastore()
.withLocalModules()
.withTaskQueue() .withTaskQueue()
.build(); .build();
@ -60,6 +61,7 @@ public class MetricsTest {
assertTasksEnqueued("bigquery-streaming-metrics", assertTasksEnqueued("bigquery-streaming-metrics",
new TaskMatcher() new TaskMatcher()
.url("/_dr/task/metrics") .url("/_dr/task/metrics")
.header("Host", "1.backend.test.localhost")
.param("tableId", "test") .param("tableId", "test")
.param("startTime", "472176000.000000") .param("startTime", "472176000.000000")
.param("endTime", "472176000.001000") .param("endTime", "472176000.001000")