mirror of
https://github.com/google/nomulus.git
synced 2025-08-05 01:11:50 +02:00
Consolidate BigQuery handling into one place
I'm writing a follow-up CL that will send integrity checking data to BigQuery, and that is made a lot easier by centralizing the BigQuery connection logic. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=119375766
This commit is contained in:
mcilwain
Justine Tunney
parent
c880a042a7
commit
755fce9e52
12 changed files with 329 additions and 347 deletions
|
|
@ -11,12 +11,16 @@ java_library(
|
|||
resources = glob(["testdata/*"]),
|
||||
deps = [
|
||||
"//apiserving/discoverydata/bigquery:bigqueryv2",
|
||||
"//java/com/google/api/client/http",
|
||||
"//java/com/google/api/client/json",
|
||||
"//java/com/google/common/collect",
|
||||
"//java/com/google/domain/registry/bigquery",
|
||||
"//java/com/google/domain/registry/util",
|
||||
"//javatests/com/google/domain/registry/testing",
|
||||
"//third_party/java/joda_time",
|
||||
"//third_party/java/jsr305_annotations",
|
||||
"//third_party/java/junit",
|
||||
"//third_party/java/mockito",
|
||||
"//third_party/java/truth",
|
||||
],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,129 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.bigquery;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.domain.registry.bigquery.BigqueryUtils.FieldType.STRING;
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Matchers.anyString;
|
||||
import static org.mockito.Matchers.eq;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.client.http.HttpRequestInitializer;
|
||||
import com.google.api.client.http.HttpTransport;
|
||||
import com.google.api.client.json.JsonFactory;
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.model.Dataset;
|
||||
import com.google.api.services.bigquery.model.Table;
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.api.services.bigquery.model.TableReference;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.domain.registry.testing.InjectRule;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
/** Unit tests for {@link BigqueryFactory}. */
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class BigqueryFactoryTest {
|
||||
|
||||
@Rule
|
||||
public final InjectRule inject = new InjectRule();
|
||||
|
||||
@Mock
|
||||
private BigqueryFactory.Subfactory subfactory;
|
||||
|
||||
@Mock
|
||||
private Bigquery bigquery;
|
||||
|
||||
@Mock
|
||||
private Bigquery.Datasets bigqueryDatasets;
|
||||
|
||||
@Mock
|
||||
private Bigquery.Datasets.Insert bigqueryDatasetsInsert;
|
||||
|
||||
@Mock
|
||||
private Bigquery.Tables bigqueryTables;
|
||||
|
||||
@Mock
|
||||
private Bigquery.Tables.Insert bigqueryTablesInsert;
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
when(subfactory.create(
|
||||
anyString(),
|
||||
any(HttpTransport.class),
|
||||
any(JsonFactory.class),
|
||||
any(HttpRequestInitializer.class)))
|
||||
.thenReturn(bigquery);
|
||||
when(bigquery.datasets()).thenReturn(bigqueryDatasets);
|
||||
when(bigqueryDatasets.insert(eq("Project-Id"), any(Dataset.class)))
|
||||
.thenReturn(bigqueryDatasetsInsert);
|
||||
when(bigquery.tables()).thenReturn(bigqueryTables);
|
||||
when(bigqueryTables.insert(eq("Project-Id"), any(String.class), any(Table.class)))
|
||||
.thenReturn(bigqueryTablesInsert);
|
||||
BigquerySchemas.knownTableSchemas =
|
||||
ImmutableMap.of(
|
||||
"Table-Id",
|
||||
ImmutableList.of(new TableFieldSchema().setName("column1").setType(STRING.name())));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSuccess_datastoreCreation() throws Exception {
|
||||
BigqueryFactory factory = new BigqueryFactory();
|
||||
factory.subfactory = subfactory;
|
||||
factory.create("Project-Id", "Dataset-Id");
|
||||
|
||||
ArgumentCaptor<Dataset> datasetArg = ArgumentCaptor.forClass(Dataset.class);
|
||||
verify(bigqueryDatasets).insert(eq("Project-Id"), datasetArg.capture());
|
||||
assertThat(datasetArg.getValue().getDatasetReference().getProjectId())
|
||||
.isEqualTo("Project-Id");
|
||||
assertThat(datasetArg.getValue().getDatasetReference().getDatasetId())
|
||||
.isEqualTo("Dataset-Id");
|
||||
verify(bigqueryDatasetsInsert).execute();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSuccess_datastoreAndTableCreation() throws Exception {
|
||||
BigqueryFactory factory = new BigqueryFactory();
|
||||
factory.subfactory = subfactory;
|
||||
factory.create("Project-Id", "Dataset-Id", "Table-Id");
|
||||
|
||||
ArgumentCaptor<Dataset> datasetArg = ArgumentCaptor.forClass(Dataset.class);
|
||||
verify(bigqueryDatasets).insert(eq("Project-Id"), datasetArg.capture());
|
||||
assertThat(datasetArg.getValue().getDatasetReference().getProjectId())
|
||||
.isEqualTo("Project-Id");
|
||||
assertThat(datasetArg.getValue().getDatasetReference().getDatasetId())
|
||||
.isEqualTo("Dataset-Id");
|
||||
verify(bigqueryDatasetsInsert).execute();
|
||||
|
||||
ArgumentCaptor<Table> tableArg = ArgumentCaptor.forClass(Table.class);
|
||||
verify(bigqueryTables).insert(eq("Project-Id"), eq("Dataset-Id"), tableArg.capture());
|
||||
TableReference ref = tableArg.getValue().getTableReference();
|
||||
assertThat(ref.getProjectId()).isEqualTo("Project-Id");
|
||||
assertThat(ref.getDatasetId()).isEqualTo("Dataset-Id");
|
||||
assertThat(ref.getTableId()).isEqualTo("Table-Id");
|
||||
assertThat(tableArg.getValue().getSchema().getFields())
|
||||
.containsExactly(new TableFieldSchema().setName("column1").setType(STRING.name()));
|
||||
verify(bigqueryTablesInsert).execute();
|
||||
}
|
||||
}
|
||||
|
|
@ -25,9 +25,6 @@ import static org.mockito.Mockito.mock;
|
|||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.client.http.HttpRequestInitializer;
|
||||
import com.google.api.client.http.HttpTransport;
|
||||
import com.google.api.client.json.JsonFactory;
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.model.Dataset;
|
||||
import com.google.api.services.bigquery.model.Table;
|
||||
|
|
@ -107,17 +104,11 @@ public class UpdateSnapshotViewServletTest {
|
|||
@Before
|
||||
public void before() throws Exception {
|
||||
inject.setStaticField(UpdateSnapshotViewServlet.class, "bigqueryFactory", bigqueryFactory);
|
||||
when(bigqueryFactory.create(anyString(), anyString())).thenReturn(bigquery);
|
||||
|
||||
when(req.getMethod()).thenReturn("POST");
|
||||
when(rsp.getWriter()).thenReturn(new PrintWriter(httpOutput));
|
||||
|
||||
when(bigqueryFactory.create(
|
||||
anyString(),
|
||||
any(HttpTransport.class),
|
||||
any(JsonFactory.class),
|
||||
any(HttpRequestInitializer.class)))
|
||||
.thenReturn(bigquery);
|
||||
|
||||
when(bigquery.datasets()).thenReturn(bigqueryDatasets);
|
||||
when(bigqueryDatasets.insert(eq("Project-Id"), any(Dataset.class)))
|
||||
.thenReturn(bigqueryDatasetsInsert);
|
||||
|
|
@ -154,15 +145,6 @@ public class UpdateSnapshotViewServletTest {
|
|||
|
||||
servlet.service(req, rsp);
|
||||
|
||||
// Check that we attempted to create the latest_snapshot dataset.
|
||||
ArgumentCaptor<Dataset> datasetArgument = ArgumentCaptor.forClass(Dataset.class);
|
||||
verify(bigqueryDatasets).insert(eq("Project-Id"), datasetArgument.capture());
|
||||
assertThat(datasetArgument.getValue().getDatasetReference().getProjectId())
|
||||
.isEqualTo("Project-Id");
|
||||
assertThat(datasetArgument.getValue().getDatasetReference().getDatasetId())
|
||||
.isEqualTo("testdataset");
|
||||
verify(bigqueryDatasetsInsert).execute();
|
||||
|
||||
// Check that we updated the view.
|
||||
ArgumentCaptor<Table> tableArg = ArgumentCaptor.forClass(Table.class);
|
||||
verify(bigqueryTables).update(
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
package com.google.domain.registry.monitoring.whitebox;
|
||||
|
||||
import static org.mockito.Matchers.anyString;
|
||||
import static org.mockito.Mockito.doNothing;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
|
|
@ -28,12 +27,9 @@ import com.google.api.services.bigquery.Bigquery.Tabledata.InsertAll;
|
|||
import com.google.api.services.bigquery.model.TableDataInsertAllRequest;
|
||||
import com.google.api.services.bigquery.model.TableDataInsertAllResponse;
|
||||
import com.google.api.services.bigquery.model.TableDataInsertAllResponse.InsertErrors;
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.api.services.bigquery.model.TableReference;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.domain.registry.bigquery.BigqueryFactory;
|
||||
import com.google.domain.registry.bigquery.BigqueryHelper;
|
||||
import com.google.domain.registry.testing.AppEngineRule;
|
||||
import com.google.domain.registry.testing.InjectRule;
|
||||
|
||||
|
|
@ -73,9 +69,6 @@ public class MetricsTaskServletTest {
|
|||
@Mock
|
||||
BigqueryFactory bigqueryFactory;
|
||||
|
||||
@Mock
|
||||
BigqueryHelper bigqueryHelper;
|
||||
|
||||
@Mock
|
||||
Bigquery bigquery;
|
||||
|
||||
|
|
@ -119,6 +112,8 @@ public class MetricsTaskServletTest {
|
|||
when(rsp.getWriter()).thenReturn(new PrintWriter(httpOutput));
|
||||
|
||||
inject.setStaticField(MetricsTaskServlet.class, "bigqueryFactory", bigqueryFactory);
|
||||
when(bigqueryFactory.create(anyString(), anyString(), anyString()))
|
||||
.thenReturn(bigquery);
|
||||
when(bigqueryFactory.create(
|
||||
anyString(),
|
||||
Matchers.any(HttpTransport.class),
|
||||
|
|
@ -126,14 +121,6 @@ public class MetricsTaskServletTest {
|
|||
Matchers.any(HttpRequestInitializer.class)))
|
||||
.thenReturn(bigquery);
|
||||
|
||||
inject.setStaticField(MetricsTaskServlet.class, "bigqueryHelper", bigqueryHelper);
|
||||
doNothing().when(bigqueryHelper).ensureDataset(Matchers.any(Bigquery.class), anyString(),
|
||||
anyString());
|
||||
doNothing().when(bigqueryHelper).ensureTable(
|
||||
Matchers.any(Bigquery.class),
|
||||
Matchers.any(TableReference.class),
|
||||
Matchers.<ImmutableList<TableFieldSchema>>any());
|
||||
|
||||
when(bigquery.tabledata()).thenReturn(tabledata);
|
||||
when(tabledata.insertAll(
|
||||
anyString(),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue