mirror of
https://github.com/google/nomulus.git
synced 2025-08-05 09:21:49 +02:00
Stop exporting EPP flow metrics to BigQuery
These are simply too costly in their current form now that we are handling double-digit QPS, so at a minimum we'd want to refactor these for batched exports using a background thread (like how Stackdriver metrics work). However, upon further review, that work isn't worth doing if this BigQuery table isn't actually being used for anything, and it seems that we aren't using it anymore given that ICANN transaction reporting no longer requires it. So the simplest thing to do is simply to get rid of this entirely, and just use a combination of Stackdriver metrics and App Engine logs. The eppMetrics BigQuery table is ~1.2 billion rows and takes up 223 GB, so that's not an insignificant GCP billings saving if we can delete it. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=215905466
This commit is contained in:
parent
7b9d562043
commit
218c4517eb
24 changed files with 17 additions and 763 deletions
|
@ -1,111 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.monitoring.whitebox;
|
||||
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static google.registry.bigquery.BigqueryUtils.toBigqueryTimestamp;
|
||||
import static google.registry.monitoring.whitebox.BigQueryMetricsEnqueuer.QUEUE_BIGQUERY_STREAMING_METRICS;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.common.base.Suppliers;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import google.registry.testing.AppEngineRule;
|
||||
import google.registry.testing.MockitoJUnitRule;
|
||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||
import google.registry.util.AppEngineServiceUtils;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
import org.mockito.Mock;
|
||||
|
||||
/** Unit tests for {@link BigQueryMetricsEnqueuer}. */
|
||||
@RunWith(JUnit4.class)
|
||||
public class BigQueryMetricsEnqueuerTest {
|
||||
|
||||
@Rule
|
||||
public final AppEngineRule appEngine =
|
||||
AppEngineRule.builder().withDatastore().withLocalModules().withTaskQueue().build();
|
||||
|
||||
@Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create();
|
||||
|
||||
@Mock private AppEngineServiceUtils appEngineServiceUtils;
|
||||
|
||||
private BigQueryMetricsEnqueuer enqueuer;
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
enqueuer = new BigQueryMetricsEnqueuer();
|
||||
enqueuer.idGenerator = Suppliers.ofInstance("laffo");
|
||||
enqueuer.appEngineServiceUtils = appEngineServiceUtils;
|
||||
enqueuer.queue = getQueue(QUEUE_BIGQUERY_STREAMING_METRICS);
|
||||
when(appEngineServiceUtils.getCurrentVersionHostname("backend"))
|
||||
.thenReturn("backend.test.localhost");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExport() {
|
||||
TestMetric metric =
|
||||
TestMetric.create(
|
||||
DateTime.parse("1984-12-18TZ"), DateTime.parse("1984-12-18TZ").plusMillis(1));
|
||||
|
||||
enqueuer.export(metric);
|
||||
|
||||
assertTasksEnqueued("bigquery-streaming-metrics",
|
||||
new TaskMatcher()
|
||||
.url("/_dr/task/metrics")
|
||||
.header("Host", "backend.test.localhost")
|
||||
.param("tableId", "test")
|
||||
.param("startTime", "472176000.000000")
|
||||
.param("endTime", "472176000.001000")
|
||||
.param("insertId", "laffo"));
|
||||
}
|
||||
|
||||
/** A stub implementation of {@link BigQueryMetric}. */
|
||||
@AutoValue
|
||||
abstract static class TestMetric implements BigQueryMetric {
|
||||
|
||||
static TestMetric create(DateTime startTimestamp, DateTime endTimestamp) {
|
||||
return new AutoValue_BigQueryMetricsEnqueuerTest_TestMetric(startTimestamp, endTimestamp);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTableId() {
|
||||
return "test";
|
||||
}
|
||||
|
||||
@Override
|
||||
public ImmutableList<TableFieldSchema> getSchemaFields() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ImmutableMap<String, String> getBigQueryRowEncoding() {
|
||||
return ImmutableMap.of(
|
||||
"startTime", toBigqueryTimestamp(getStartTimestamp()),
|
||||
"endTime", toBigqueryTimestamp(getEndTimestamp()));
|
||||
}
|
||||
|
||||
abstract DateTime getStartTimestamp();
|
||||
|
||||
abstract DateTime getEndTimestamp();
|
||||
}
|
||||
}
|
|
@ -14,18 +14,13 @@
|
|||
|
||||
package google.registry.monitoring.whitebox;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth8.assertThat;
|
||||
import static google.registry.testing.DatastoreHelper.createTld;
|
||||
import static google.registry.testing.DatastoreHelper.createTlds;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.model.eppoutput.Result.Code;
|
||||
import google.registry.testing.AppEngineRule;
|
||||
import google.registry.testing.FakeClock;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
@ -40,7 +35,7 @@ public class EppMetricTest {
|
|||
@Test
|
||||
public void test_invalidTld_isRecordedAsInvalid() {
|
||||
EppMetric metric =
|
||||
EppMetric.builderForRequest("request-id-1", new FakeClock())
|
||||
EppMetric.builderForRequest(new FakeClock())
|
||||
.setTlds(ImmutableSet.of("notarealtld"))
|
||||
.build();
|
||||
assertThat(metric.getTld()).hasValue("_invalid");
|
||||
|
@ -50,9 +45,7 @@ public class EppMetricTest {
|
|||
public void test_validTld_isRecorded() {
|
||||
createTld("example");
|
||||
EppMetric metric =
|
||||
EppMetric.builderForRequest("request-id-1", new FakeClock())
|
||||
.setTlds(ImmutableSet.of("example"))
|
||||
.build();
|
||||
EppMetric.builderForRequest(new FakeClock()).setTlds(ImmutableSet.of("example")).build();
|
||||
assertThat(metric.getTld()).hasValue("example");
|
||||
}
|
||||
|
||||
|
@ -60,7 +53,7 @@ public class EppMetricTest {
|
|||
public void test_multipleTlds_areRecordedAsVarious() {
|
||||
createTlds("foo", "bar");
|
||||
EppMetric metric =
|
||||
EppMetric.builderForRequest("request-id-1", new FakeClock())
|
||||
EppMetric.builderForRequest(new FakeClock())
|
||||
.setTlds(ImmutableSet.of("foo", "bar", "baz"))
|
||||
.build();
|
||||
assertThat(metric.getTld()).hasValue("_various");
|
||||
|
@ -69,64 +62,7 @@ public class EppMetricTest {
|
|||
@Test
|
||||
public void test_zeroTlds_areRecordedAsAbsent() {
|
||||
EppMetric metric =
|
||||
EppMetric.builderForRequest("request-id-1", new FakeClock())
|
||||
.setTlds(ImmutableSet.of())
|
||||
.build();
|
||||
EppMetric.builderForRequest(new FakeClock()).setTlds(ImmutableSet.of()).build();
|
||||
assertThat(metric.getTld()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetBigQueryRowEncoding_encodesCorrectly() {
|
||||
EppMetric metric =
|
||||
EppMetric.builder()
|
||||
.setRequestId("request-id-1")
|
||||
.setStartTimestamp(new DateTime(1337))
|
||||
.setEndTimestamp(new DateTime(1338))
|
||||
.setCommandName("command")
|
||||
.setClientId("client")
|
||||
.setTld("example")
|
||||
.setPrivilegeLevel("level")
|
||||
.setEppTarget("target")
|
||||
.setStatus(Code.COMMAND_USE_ERROR)
|
||||
.incrementAttempts()
|
||||
.build();
|
||||
|
||||
assertThat(metric.getBigQueryRowEncoding())
|
||||
.containsExactlyEntriesIn(
|
||||
new ImmutableMap.Builder<String, String>()
|
||||
.put("requestId", "request-id-1")
|
||||
.put("startTime", "1.337000")
|
||||
.put("endTime", "1.338000")
|
||||
.put("commandName", "command")
|
||||
.put("clientId", "client")
|
||||
.put("tld", "example")
|
||||
.put("privilegeLevel", "level")
|
||||
.put("eppTarget", "target")
|
||||
.put("eppStatus", "2002")
|
||||
.put("attempts", "1")
|
||||
.build());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetBigQueryRowEncoding_hasAllSchemaFields() {
|
||||
EppMetric metric =
|
||||
EppMetric.builder()
|
||||
.setRequestId("request-id-1")
|
||||
.setStartTimestamp(new DateTime(1337))
|
||||
.setEndTimestamp(new DateTime(1338))
|
||||
.setCommandName("command")
|
||||
.setClientId("client")
|
||||
.setTld("example")
|
||||
.setPrivilegeLevel("level")
|
||||
.setEppTarget("target")
|
||||
.setStatus(Code.COMMAND_USE_ERROR)
|
||||
.incrementAttempts()
|
||||
.build();
|
||||
ImmutableSet.Builder<String> schemaFieldNames = new ImmutableSet.Builder<>();
|
||||
for (TableFieldSchema schemaField : metric.getSchemaFields()) {
|
||||
schemaFieldNames.add(schemaField.getName());
|
||||
}
|
||||
|
||||
assertThat(metric.getBigQueryRowEncoding().keySet()).isEqualTo(schemaFieldNames.build());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,108 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.monitoring.whitebox;
|
||||
|
||||
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||
import static org.mockito.Matchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.Bigquery.Tabledata;
|
||||
import com.google.api.services.bigquery.Bigquery.Tabledata.InsertAll;
|
||||
import com.google.api.services.bigquery.model.TableDataInsertAllRequest;
|
||||
import com.google.api.services.bigquery.model.TableDataInsertAllResponse;
|
||||
import com.google.api.services.bigquery.model.TableDataInsertAllResponse.InsertErrors;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableListMultimap;
|
||||
import google.registry.bigquery.CheckedBigquery;
|
||||
import google.registry.testing.AppEngineRule;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
import org.mockito.Matchers;
|
||||
|
||||
/** Unit tests for {@link MetricsExportAction}. */
|
||||
@RunWith(JUnit4.class)
|
||||
public class MetricsExportActionTest {
|
||||
|
||||
@Rule
|
||||
public final AppEngineRule appEngine =
|
||||
AppEngineRule.builder().withDatastore().withTaskQueue().build();
|
||||
|
||||
private final CheckedBigquery checkedBigquery = mock(CheckedBigquery.class);
|
||||
private final Bigquery bigquery = mock(Bigquery.class);
|
||||
private final Tabledata tabledata = mock(Tabledata.class);
|
||||
private final InsertAll insertAll = mock(InsertAll.class);
|
||||
|
||||
private TableDataInsertAllResponse response = new TableDataInsertAllResponse();
|
||||
private long currentTimeMillis = 1000000000000L;
|
||||
|
||||
private ImmutableListMultimap<String, String> parameters =
|
||||
new ImmutableListMultimap.Builder<String, String>()
|
||||
.put("startTime", String.valueOf(MILLISECONDS.toSeconds(currentTimeMillis - 100)))
|
||||
.put("endTime", String.valueOf(MILLISECONDS.toSeconds(currentTimeMillis)))
|
||||
.put("jobname", "test job")
|
||||
.put("status", "success")
|
||||
.put("tld", "test")
|
||||
.build();
|
||||
|
||||
MetricsExportAction action;
|
||||
|
||||
@Before
|
||||
public void setup() throws Exception {
|
||||
when(checkedBigquery.ensureDataSetAndTableExist(anyString(), anyString(), anyString()))
|
||||
.thenReturn(bigquery);
|
||||
|
||||
when(bigquery.tabledata()).thenReturn(tabledata);
|
||||
when(tabledata.insertAll(
|
||||
anyString(),
|
||||
anyString(),
|
||||
anyString(),
|
||||
Matchers.any(TableDataInsertAllRequest.class))).thenReturn(insertAll);
|
||||
action = new MetricsExportAction();
|
||||
action.checkedBigquery = checkedBigquery;
|
||||
action.insertId = "insert id";
|
||||
action.parameters = parameters;
|
||||
action.projectId = "project id";
|
||||
action.tableId = "eppMetrics";
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSuccess_nullErrors() throws Exception {
|
||||
when(insertAll.execute()).thenReturn(response);
|
||||
response.setInsertErrors(null);
|
||||
action.run();
|
||||
verify(insertAll).execute();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSuccess_emptyErrors() throws Exception {
|
||||
when(insertAll.execute()).thenReturn(response);
|
||||
response.setInsertErrors(ImmutableList.of());
|
||||
action.run();
|
||||
verify(insertAll).execute();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFailure_errors() throws Exception {
|
||||
when(insertAll.execute()).thenReturn(response);
|
||||
response.setInsertErrors(ImmutableList.of(new InsertErrors()));
|
||||
action.run();
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue