mirror of
https://github.com/google/nomulus.git
synced 2025-05-28 15:11:26 +02:00
Make metrics export run on backend module
This fixes a previous issue where metrics generated on the tools/backend modules weren't being successfully exported to BigQuery. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=119047757
This commit is contained in:
parent
17c4ad83c8
commit
e6d3f42e60
7 changed files with 34 additions and 15 deletions
|
@ -13,6 +13,16 @@
|
||||||
<load-on-startup>1</load-on-startup>
|
<load-on-startup>1</load-on-startup>
|
||||||
</servlet>
|
</servlet>
|
||||||
|
|
||||||
|
<!-- Whitebox Metrics servlet. -->
|
||||||
|
<servlet>
|
||||||
|
<servlet-name>metrics</servlet-name>
|
||||||
|
<servlet-class>com.google.domain.registry.monitoring.whitebox.MetricsTaskServlet</servlet-class>
|
||||||
|
</servlet>
|
||||||
|
<servlet-mapping>
|
||||||
|
<servlet-name>metrics</servlet-name>
|
||||||
|
<url-pattern>/_dr/task/metrics</url-pattern>
|
||||||
|
</servlet-mapping>
|
||||||
|
|
||||||
<!-- RDE -->
|
<!-- RDE -->
|
||||||
|
|
||||||
<!--
|
<!--
|
||||||
|
|
|
@ -103,16 +103,6 @@
|
||||||
<url-pattern>/check</url-pattern>
|
<url-pattern>/check</url-pattern>
|
||||||
</servlet-mapping>
|
</servlet-mapping>
|
||||||
|
|
||||||
<!-- Whitebox Metrics servlet. -->
|
|
||||||
<servlet>
|
|
||||||
<servlet-name>metrics</servlet-name>
|
|
||||||
<servlet-class>com.google.domain.registry.monitoring.whitebox.MetricsTaskServlet</servlet-class>
|
|
||||||
</servlet>
|
|
||||||
<servlet-mapping>
|
|
||||||
<servlet-name>metrics</servlet-name>
|
|
||||||
<url-pattern>/_dr/task/metrics</url-pattern>
|
|
||||||
</servlet-mapping>
|
|
||||||
|
|
||||||
<!-- Security config -->
|
<!-- Security config -->
|
||||||
<security-constraint>
|
<security-constraint>
|
||||||
<web-resource-collection>
|
<web-resource-collection>
|
||||||
|
|
|
@ -18,6 +18,8 @@ import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||||
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
||||||
import static com.google.domain.registry.bigquery.BigqueryUtils.toBigqueryTimestamp;
|
import static com.google.domain.registry.bigquery.BigqueryUtils.toBigqueryTimestamp;
|
||||||
|
|
||||||
|
import com.google.appengine.api.modules.ModulesService;
|
||||||
|
import com.google.appengine.api.modules.ModulesServiceFactory;
|
||||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||||
import com.google.appengine.api.taskqueue.TransientFailureException;
|
import com.google.appengine.api.taskqueue.TransientFailureException;
|
||||||
import com.google.common.base.Supplier;
|
import com.google.common.base.Supplier;
|
||||||
|
@ -33,12 +35,15 @@ import java.util.UUID;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
/** A collector of metric information. */
|
/** A collector of metric information. */
|
||||||
abstract class Metrics {
|
public abstract class Metrics {
|
||||||
|
|
||||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||||
|
|
||||||
public static final String QUEUE = "bigquery-streaming-metrics";
|
public static final String QUEUE = "bigquery-streaming-metrics";
|
||||||
|
|
||||||
|
@NonFinalForTesting
|
||||||
|
private static ModulesService modulesService = ModulesServiceFactory.getModulesService();
|
||||||
|
|
||||||
@NonFinalForTesting
|
@NonFinalForTesting
|
||||||
private static Clock clock = new SystemClock();
|
private static Clock clock = new SystemClock();
|
||||||
|
|
||||||
|
@ -60,7 +65,9 @@ abstract class Metrics {
|
||||||
|
|
||||||
public void export() {
|
public void export() {
|
||||||
try {
|
try {
|
||||||
TaskOptions opts = withUrl("/_dr/task/metrics")
|
String hostname = modulesService.getVersionHostname("backend", null);
|
||||||
|
TaskOptions opts = withUrl(MetricsTaskServlet.PATH)
|
||||||
|
.header("Host", hostname)
|
||||||
.param("insertId", idGenerator.get())
|
.param("insertId", idGenerator.get())
|
||||||
.param("startTime", toBigqueryTimestamp(startTimeMillis, TimeUnit.MILLISECONDS))
|
.param("startTime", toBigqueryTimestamp(startTimeMillis, TimeUnit.MILLISECONDS))
|
||||||
.param("endTime", toBigqueryTimestamp(clock.nowUtc().getMillis(), TimeUnit.MILLISECONDS));
|
.param("endTime", toBigqueryTimestamp(clock.nowUtc().getMillis(), TimeUnit.MILLISECONDS));
|
||||||
|
|
|
@ -51,6 +51,8 @@ import javax.servlet.http.HttpServletResponse;
|
||||||
/** Servlet for exporting metrics to BigQuery. */
|
/** Servlet for exporting metrics to BigQuery. */
|
||||||
public class MetricsTaskServlet extends HttpServlet {
|
public class MetricsTaskServlet extends HttpServlet {
|
||||||
|
|
||||||
|
public static final String PATH = "/_dr/task/metrics";
|
||||||
|
|
||||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||||
private static final String DATASET_ID = "metrics";
|
private static final String DATASET_ID = "metrics";
|
||||||
private static final String PROJECT_ID = RegistryEnvironment.get().config().getProjectId();
|
private static final String PROJECT_ID = RegistryEnvironment.get().config().getProjectId();
|
||||||
|
@ -81,7 +83,7 @@ public class MetricsTaskServlet extends HttpServlet {
|
||||||
ImmutableMap.Builder<String, Object> b = new ImmutableMap.Builder<>();
|
ImmutableMap.Builder<String, Object> b = new ImmutableMap.Builder<>();
|
||||||
|
|
||||||
@SuppressWarnings({"cast", "unchecked"}) // Return type is always a Set<String>.
|
@SuppressWarnings({"cast", "unchecked"}) // Return type is always a Set<String>.
|
||||||
Set<String> parameterKeys = (Set<String>) req.getParameterMap().keySet();
|
Set<String> parameterKeys = req.getParameterMap().keySet();
|
||||||
|
|
||||||
for (String key : Sets.difference(parameterKeys, filter)) {
|
for (String key : Sets.difference(parameterKeys, filter)) {
|
||||||
b.put(key, req.getParameter(key));
|
b.put(key, req.getParameter(key));
|
||||||
|
|
|
@ -31,6 +31,7 @@ java_library(
|
||||||
"//java/com/google/domain/registry/flows",
|
"//java/com/google/domain/registry/flows",
|
||||||
"//java/com/google/domain/registry/mapreduce",
|
"//java/com/google/domain/registry/mapreduce",
|
||||||
"//java/com/google/domain/registry/model",
|
"//java/com/google/domain/registry/model",
|
||||||
|
"//java/com/google/domain/registry/monitoring/whitebox",
|
||||||
"//java/com/google/domain/registry/request",
|
"//java/com/google/domain/registry/request",
|
||||||
"//java/com/google/domain/registry/security",
|
"//java/com/google/domain/registry/security",
|
||||||
"//java/com/google/domain/registry/security:servlets",
|
"//java/com/google/domain/registry/security:servlets",
|
||||||
|
|
|
@ -28,12 +28,14 @@ import static org.mockito.Mockito.reset;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
import com.google.appengine.api.modules.ModulesService;
|
||||||
import com.google.common.base.Optional;
|
import com.google.common.base.Optional;
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import com.google.common.collect.ImmutableSet;
|
import com.google.common.collect.ImmutableSet;
|
||||||
import com.google.domain.registry.model.ofy.Ofy;
|
import com.google.domain.registry.model.ofy.Ofy;
|
||||||
import com.google.domain.registry.model.registrar.Registrar;
|
import com.google.domain.registry.model.registrar.Registrar;
|
||||||
import com.google.domain.registry.model.tmch.ClaimsListShard.ClaimsListSingleton;
|
import com.google.domain.registry.model.tmch.ClaimsListShard.ClaimsListSingleton;
|
||||||
|
import com.google.domain.registry.monitoring.whitebox.Metrics;
|
||||||
import com.google.domain.registry.security.XsrfProtectedServlet;
|
import com.google.domain.registry.security.XsrfProtectedServlet;
|
||||||
import com.google.domain.registry.testing.FakeClock;
|
import com.google.domain.registry.testing.FakeClock;
|
||||||
import com.google.domain.registry.testing.FakeServletInputStream;
|
import com.google.domain.registry.testing.FakeServletInputStream;
|
||||||
|
@ -74,6 +76,9 @@ public abstract class EppServletTestCase<S extends HttpServlet> {
|
||||||
@Mock
|
@Mock
|
||||||
HttpServletResponse rsp;
|
HttpServletResponse rsp;
|
||||||
|
|
||||||
|
@Mock
|
||||||
|
ModulesService modulesService;
|
||||||
|
|
||||||
HttpSession session;
|
HttpSession session;
|
||||||
|
|
||||||
FakeClock clock = new FakeClock();
|
FakeClock clock = new FakeClock();
|
||||||
|
@ -96,6 +101,8 @@ public abstract class EppServletTestCase<S extends HttpServlet> {
|
||||||
public final void init() throws Exception {
|
public final void init() throws Exception {
|
||||||
inject.setStaticField(Ofy.class, "clock", clock); // For transactional flows.
|
inject.setStaticField(Ofy.class, "clock", clock); // For transactional flows.
|
||||||
inject.setStaticField(FlowRunner.class, "clock", clock); // For non-transactional flows.
|
inject.setStaticField(FlowRunner.class, "clock", clock); // For non-transactional flows.
|
||||||
|
inject.setStaticField(Metrics.class, "modulesService", modulesService);
|
||||||
|
when(modulesService.getVersionHostname("backend", null)).thenReturn("backend.hostname");
|
||||||
|
|
||||||
// Create RegistryData for all TLDs used in these tests.
|
// Create RegistryData for all TLDs used in these tests.
|
||||||
// We want to create all of these even for tests that don't use them to make sure that
|
// We want to create all of these even for tests that don't use them to make sure that
|
||||||
|
|
|
@ -27,10 +27,10 @@ import org.junit.Before;
|
||||||
import org.junit.Rule;
|
import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.runner.RunWith;
|
import org.junit.runner.RunWith;
|
||||||
import org.junit.runners.JUnit4;
|
import org.mockito.runners.MockitoJUnitRunner;
|
||||||
|
|
||||||
/** Unit tests for {@link Metrics}. */
|
/** Unit tests for {@link Metrics}. */
|
||||||
@RunWith(JUnit4.class)
|
@RunWith(MockitoJUnitRunner.class)
|
||||||
public class MetricsTest {
|
public class MetricsTest {
|
||||||
|
|
||||||
@Rule
|
@Rule
|
||||||
|
@ -39,6 +39,7 @@ public class MetricsTest {
|
||||||
@Rule
|
@Rule
|
||||||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
public final AppEngineRule appEngine = AppEngineRule.builder()
|
||||||
.withDatastore()
|
.withDatastore()
|
||||||
|
.withLocalModules()
|
||||||
.withTaskQueue()
|
.withTaskQueue()
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
@ -60,6 +61,7 @@ public class MetricsTest {
|
||||||
assertTasksEnqueued("bigquery-streaming-metrics",
|
assertTasksEnqueued("bigquery-streaming-metrics",
|
||||||
new TaskMatcher()
|
new TaskMatcher()
|
||||||
.url("/_dr/task/metrics")
|
.url("/_dr/task/metrics")
|
||||||
|
.header("Host", "1.backend.test.localhost")
|
||||||
.param("tableId", "test")
|
.param("tableId", "test")
|
||||||
.param("startTime", "472176000.000000")
|
.param("startTime", "472176000.000000")
|
||||||
.param("endTime", "472176000.001000")
|
.param("endTime", "472176000.001000")
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue