mirror of
https://github.com/google/nomulus.git
synced 2025-05-17 01:47:14 +02:00
Use Multimap bindings to construct BigQuery schema map
This refactoring allows the schemas to live in the same package they are used, rather than all having to be inside the bigquery package. This is a follow-up to [] ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=120551011
This commit is contained in:
mcilwain
Justine Tunney
parent
d65bf2a714
commit
e14faaa7e6
9 changed files with 143 additions and 125 deletions
|
|
@ -16,7 +16,8 @@ package com.google.domain.registry.bigquery;
|
|||
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.domain.registry.bigquery.BigquerySchemas.knownTableSchemas;
|
||||
import static com.google.common.collect.Sets.newConcurrentHashSet;
|
||||
import static com.google.domain.registry.util.FormattingLogger.getLoggerForCallerClass;
|
||||
|
||||
import com.google.api.client.extensions.appengine.http.UrlFetchTransport;
|
||||
import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential;
|
||||
|
|
@ -32,13 +33,12 @@ import com.google.api.services.bigquery.model.Table;
|
|||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.api.services.bigquery.model.TableReference;
|
||||
import com.google.api.services.bigquery.model.TableSchema;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.domain.registry.util.FormattingLogger;
|
||||
import com.google.domain.registry.util.NonFinalForTesting;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
|
@ -46,24 +46,21 @@ import javax.inject.Inject;
|
|||
/** Factory for creating {@link Bigquery} connections. */
|
||||
public class BigqueryFactory {
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
|
||||
// Cross-request caches to avoid unnecessary RPCs.
|
||||
@NonFinalForTesting
|
||||
private Set<String> knownTables = Sets.newConcurrentHashSet();
|
||||
|
||||
@NonFinalForTesting
|
||||
private Set<String> datasets = Sets.newConcurrentHashSet();
|
||||
|
||||
@NonFinalForTesting
|
||||
@VisibleForTesting
|
||||
Subfactory subfactory = new Subfactory();
|
||||
private static Set<String> knownExistingDatasets = newConcurrentHashSet();
|
||||
private static Set<String> knownExistingTables = newConcurrentHashSet();
|
||||
|
||||
@Inject Map<String, ImmutableList<TableFieldSchema>> bigquerySchemas;
|
||||
@Inject Subfactory subfactory;
|
||||
@Inject BigqueryFactory() {}
|
||||
|
||||
/** This class is broken out solely so that it can be mocked inside of tests. */
|
||||
static class Subfactory {
|
||||
|
||||
@Inject Subfactory() {}
|
||||
|
||||
public Bigquery create(
|
||||
String applicationName,
|
||||
HttpTransport transport,
|
||||
|
|
@ -96,9 +93,9 @@ public class BigqueryFactory {
|
|||
new AppIdentityCredential(BigqueryScopes.all()));
|
||||
|
||||
// Note: it's safe for multiple threads to call this as the dataset will only be created once.
|
||||
if (!datasets.contains(datasetId)) {
|
||||
if (!knownExistingDatasets.contains(datasetId)) {
|
||||
ensureDataset(bigquery, projectId, datasetId);
|
||||
datasets.add(datasetId);
|
||||
knownExistingDatasets.add(datasetId);
|
||||
}
|
||||
|
||||
return bigquery;
|
||||
|
|
@ -111,17 +108,17 @@ public class BigqueryFactory {
|
|||
public Bigquery create(String projectId, String datasetId, String tableId)
|
||||
throws IOException {
|
||||
Bigquery bigquery = create(projectId, datasetId);
|
||||
checkArgument(knownTableSchemas.containsKey(tableId), "Unknown table ID: %s", tableId);
|
||||
checkArgument(bigquerySchemas.containsKey(tableId), "Unknown table ID: %s", tableId);
|
||||
|
||||
if (!knownTables.contains(tableId)) {
|
||||
if (!knownExistingTables.contains(tableId)) {
|
||||
ensureTable(
|
||||
bigquery,
|
||||
new TableReference()
|
||||
.setDatasetId(datasetId)
|
||||
.setProjectId(projectId)
|
||||
.setTableId(tableId),
|
||||
knownTableSchemas.get(tableId));
|
||||
knownTables.add(tableId);
|
||||
bigquerySchemas.get(tableId));
|
||||
knownExistingTables.add(tableId);
|
||||
}
|
||||
|
||||
return bigquery;
|
||||
|
|
@ -151,8 +148,7 @@ public class BigqueryFactory {
|
|||
}
|
||||
|
||||
/** Ensures the table exists in Bigquery. */
|
||||
private void ensureTable(
|
||||
Bigquery bigquery, TableReference table, ImmutableList<TableFieldSchema> schema)
|
||||
private void ensureTable(Bigquery bigquery, TableReference table, List<TableFieldSchema> schema)
|
||||
throws IOException {
|
||||
try {
|
||||
bigquery.tables().insert(table.getProjectId(), table.getDatasetId(), new Table()
|
||||
|
|
|
|||
|
|
@ -21,12 +21,16 @@ import com.google.api.client.http.HttpTransport;
|
|||
import com.google.api.client.json.JsonFactory;
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.BigqueryScopes;
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.domain.registry.config.ConfigModule.Config;
|
||||
import com.google.domain.registry.request.OAuthScopes;
|
||||
|
||||
import dagger.Module;
|
||||
import dagger.Multibindings;
|
||||
import dagger.Provides;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
|
|
@ -41,6 +45,13 @@ import java.util.Set;
|
|||
@Module
|
||||
public final class BigqueryModule {
|
||||
|
||||
@Multibindings
|
||||
interface BigQueryMultibindings {
|
||||
|
||||
/** Provides a map of BigQuery table names to field names. */
|
||||
Map<String, ImmutableList<TableFieldSchema>> bigquerySchemas();
|
||||
}
|
||||
|
||||
/** Provides OAuth2 scopes for the Bigquery service needed by Domain Registry. */
|
||||
@Provides(type = SET_VALUES)
|
||||
@OAuthScopes
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.bigquery;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.domain.registry.bigquery.BigqueryUtils.FieldType;
|
||||
import com.google.domain.registry.util.NonFinalForTesting;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/** Schemas for BigQuery tables. */
|
||||
public final class BigquerySchemas {
|
||||
|
||||
public static final String EPPMETRICS_TABLE_ID = "eppMetrics";
|
||||
public static final String ENTITY_INTEGRITY_ALERTS_TABLE_ID = "alerts";
|
||||
public static final String ENTITY_INTEGRITY_ALERTS_FIELD_SCANTIME = "scanTime";
|
||||
public static final String ENTITY_INTEGRITY_ALERTS_FIELD_SOURCE = "source";
|
||||
public static final String ENTITY_INTEGRITY_ALERTS_FIELD_TARGET = "target";
|
||||
public static final String ENTITY_INTEGRITY_ALERTS_FIELD_MESSAGE = "message";
|
||||
|
||||
static final ImmutableList<TableFieldSchema> EPPMETRICS_SCHEMA_FIELDS =
|
||||
ImmutableList.<TableFieldSchema>of(
|
||||
new TableFieldSchema().setName("requestId").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("startTime").setType(FieldType.TIMESTAMP.name()),
|
||||
new TableFieldSchema().setName("endTime").setType(FieldType.TIMESTAMP.name()),
|
||||
new TableFieldSchema().setName("commandName").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("clientId").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("privilegeLevel").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("eppTarget").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("eppStatus").setType(FieldType.INTEGER.name()),
|
||||
new TableFieldSchema().setName("attempts").setType(FieldType.INTEGER.name()));
|
||||
|
||||
static final ImmutableList<TableFieldSchema> ENTITY_INTEGRITY_ALERTS_SCHEMA_FIELDS =
|
||||
ImmutableList.<TableFieldSchema>of(
|
||||
new TableFieldSchema()
|
||||
.setName(ENTITY_INTEGRITY_ALERTS_FIELD_SCANTIME)
|
||||
.setType(FieldType.TIMESTAMP.name()),
|
||||
new TableFieldSchema()
|
||||
.setName(ENTITY_INTEGRITY_ALERTS_FIELD_SOURCE)
|
||||
.setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema()
|
||||
.setName(ENTITY_INTEGRITY_ALERTS_FIELD_TARGET)
|
||||
.setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema()
|
||||
.setName(ENTITY_INTEGRITY_ALERTS_FIELD_MESSAGE)
|
||||
.setType(FieldType.STRING.name()));
|
||||
|
||||
@NonFinalForTesting
|
||||
static Map<String, ImmutableList<TableFieldSchema>> knownTableSchemas =
|
||||
new ImmutableMap.Builder<String, ImmutableList<TableFieldSchema>>()
|
||||
.put(EPPMETRICS_TABLE_ID, EPPMETRICS_SCHEMA_FIELDS)
|
||||
.put(ENTITY_INTEGRITY_ALERTS_TABLE_ID, ENTITY_INTEGRITY_ALERTS_SCHEMA_FIELDS)
|
||||
.build();
|
||||
|
||||
private BigquerySchemas() {}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.domain.registry.monitoring.whitebox;
|
||||
|
||||
import static com.google.domain.registry.bigquery.BigqueryUtils.FieldType.STRING;
|
||||
import static com.google.domain.registry.bigquery.BigqueryUtils.FieldType.TIMESTAMP;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
/** The Bigquery schema for the entity integrity alerts table. */
|
||||
final class EntityIntegrityAlertsSchema {
|
||||
|
||||
static final String DATASET = "entity_integrity";
|
||||
static final String TABLE_ID = "alerts";
|
||||
static final String FIELD_SCANTIME = "scanTime";
|
||||
static final String FIELD_SOURCE = "source";
|
||||
static final String FIELD_TARGET = "target";
|
||||
static final String FIELD_MESSAGE = "message";
|
||||
|
||||
static final ImmutableList<TableFieldSchema> ENTITY_INTEGRITY_ALERTS_SCHEMA_FIELDS =
|
||||
ImmutableList.of(
|
||||
new TableFieldSchema().setName(FIELD_SCANTIME).setType(TIMESTAMP.name()),
|
||||
new TableFieldSchema().setName(FIELD_SOURCE).setType(STRING.name()),
|
||||
new TableFieldSchema().setName(FIELD_TARGET).setType(STRING.name()),
|
||||
new TableFieldSchema().setName(FIELD_MESSAGE).setType(STRING.name()));
|
||||
|
||||
private EntityIntegrityAlertsSchema() {}
|
||||
}
|
||||
|
||||
|
|
@ -14,14 +14,30 @@
|
|||
|
||||
package com.google.domain.registry.monitoring.whitebox;
|
||||
|
||||
import com.google.domain.registry.bigquery.BigquerySchemas;
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.domain.registry.bigquery.BigqueryUtils.FieldType;
|
||||
import com.google.domain.registry.model.eppoutput.Result.Code;
|
||||
|
||||
/** The EPP Metrics collector. See {@link Metrics}. */
|
||||
public class EppMetrics extends Metrics {
|
||||
|
||||
static final String EPPMETRICS_TABLE_ID = "eppMetrics";
|
||||
|
||||
static final ImmutableList<TableFieldSchema> EPPMETRICS_SCHEMA_FIELDS =
|
||||
ImmutableList.of(
|
||||
new TableFieldSchema().setName("requestId").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("startTime").setType(FieldType.TIMESTAMP.name()),
|
||||
new TableFieldSchema().setName("endTime").setType(FieldType.TIMESTAMP.name()),
|
||||
new TableFieldSchema().setName("commandName").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("clientId").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("privilegeLevel").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("eppTarget").setType(FieldType.STRING.name()),
|
||||
new TableFieldSchema().setName("eppStatus").setType(FieldType.INTEGER.name()),
|
||||
new TableFieldSchema().setName("attempts").setType(FieldType.INTEGER.name()));
|
||||
|
||||
public EppMetrics() {
|
||||
setTableId(BigquerySchemas.EPPMETRICS_TABLE_ID);
|
||||
setTableId(EPPMETRICS_TABLE_ID);
|
||||
fields.put("attempts", 0);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15,11 +15,12 @@
|
|||
package com.google.domain.registry.monitoring.whitebox;
|
||||
|
||||
import static com.google.api.client.util.Data.NULL_STRING;
|
||||
import static com.google.domain.registry.bigquery.BigquerySchemas.ENTITY_INTEGRITY_ALERTS_FIELD_MESSAGE;
|
||||
import static com.google.domain.registry.bigquery.BigquerySchemas.ENTITY_INTEGRITY_ALERTS_FIELD_SCANTIME;
|
||||
import static com.google.domain.registry.bigquery.BigquerySchemas.ENTITY_INTEGRITY_ALERTS_FIELD_SOURCE;
|
||||
import static com.google.domain.registry.bigquery.BigquerySchemas.ENTITY_INTEGRITY_ALERTS_FIELD_TARGET;
|
||||
import static com.google.domain.registry.bigquery.BigquerySchemas.ENTITY_INTEGRITY_ALERTS_TABLE_ID;
|
||||
import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.DATASET;
|
||||
import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.FIELD_MESSAGE;
|
||||
import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.FIELD_SCANTIME;
|
||||
import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.FIELD_SOURCE;
|
||||
import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.FIELD_TARGET;
|
||||
import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.TABLE_ID;
|
||||
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.Bigquery.Tabledata.InsertAll;
|
||||
|
|
@ -54,8 +55,6 @@ import javax.annotation.Nullable;
|
|||
@AutoFactory(allowSubclasses = true)
|
||||
public class VerifyEntityIntegrityStreamer {
|
||||
|
||||
private static final String DATASET = "entity_integrity";
|
||||
|
||||
private final DateTime scanTime;
|
||||
private Bigquery bigquery;
|
||||
|
||||
|
|
@ -84,7 +83,7 @@ public class VerifyEntityIntegrityStreamer {
|
|||
if (bigquery == null) {
|
||||
bigquery =
|
||||
bigqueryFactory.create(
|
||||
environment.config().getProjectId(), DATASET, ENTITY_INTEGRITY_ALERTS_TABLE_ID);
|
||||
environment.config().getProjectId(), DATASET, TABLE_ID);
|
||||
}
|
||||
return bigquery;
|
||||
}
|
||||
|
|
@ -157,16 +156,16 @@ public class VerifyEntityIntegrityStreamer {
|
|||
Map<String, Object> rowData =
|
||||
new ImmutableMap.Builder<String, Object>()
|
||||
.put(
|
||||
ENTITY_INTEGRITY_ALERTS_FIELD_SCANTIME,
|
||||
FIELD_SCANTIME,
|
||||
new com.google.api.client.util.DateTime(scanTime.toDate()))
|
||||
.put(
|
||||
ENTITY_INTEGRITY_ALERTS_FIELD_SOURCE,
|
||||
FIELD_SOURCE,
|
||||
source.toString())
|
||||
.put(
|
||||
ENTITY_INTEGRITY_ALERTS_FIELD_TARGET,
|
||||
FIELD_TARGET,
|
||||
target.toString())
|
||||
.put(
|
||||
ENTITY_INTEGRITY_ALERTS_FIELD_MESSAGE,
|
||||
FIELD_MESSAGE,
|
||||
(message == null) ? NULL_STRING : message)
|
||||
.build();
|
||||
rows.add(
|
||||
|
|
@ -185,7 +184,7 @@ public class VerifyEntityIntegrityStreamer {
|
|||
.insertAll(
|
||||
environment.config().getProjectId(),
|
||||
DATASET,
|
||||
ENTITY_INTEGRITY_ALERTS_TABLE_ID,
|
||||
TABLE_ID,
|
||||
new TableDataInsertAllRequest().setRows(rows));
|
||||
|
||||
Callable<Void> callable =
|
||||
|
|
|
|||
|
|
@ -14,13 +14,21 @@
|
|||
|
||||
package com.google.domain.registry.monitoring.whitebox;
|
||||
|
||||
import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.ENTITY_INTEGRITY_ALERTS_SCHEMA_FIELDS;
|
||||
import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.TABLE_ID;
|
||||
import static com.google.domain.registry.monitoring.whitebox.EppMetrics.EPPMETRICS_SCHEMA_FIELDS;
|
||||
import static com.google.domain.registry.monitoring.whitebox.EppMetrics.EPPMETRICS_TABLE_ID;
|
||||
import static com.google.domain.registry.request.RequestParameters.extractRequiredParameter;
|
||||
import static dagger.Provides.Type.MAP;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.common.base.Supplier;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.domain.registry.request.Parameter;
|
||||
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import dagger.multibindings.StringKey;
|
||||
|
||||
import java.util.UUID;
|
||||
|
||||
|
|
@ -32,6 +40,18 @@ import javax.servlet.http.HttpServletRequest;
|
|||
@Module
|
||||
public class WhiteboxModule {
|
||||
|
||||
@Provides(type = MAP)
|
||||
@StringKey(EPPMETRICS_TABLE_ID)
|
||||
static ImmutableList<TableFieldSchema> provideEppMetricsSchema() {
|
||||
return EPPMETRICS_SCHEMA_FIELDS;
|
||||
}
|
||||
|
||||
@Provides(type = MAP)
|
||||
@StringKey(TABLE_ID)
|
||||
static ImmutableList<TableFieldSchema> provideEntityIntegrityAlertsSchema() {
|
||||
return ENTITY_INTEGRITY_ALERTS_SCHEMA_FIELDS;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter("tableId")
|
||||
static String provideTableId(HttpServletRequest req) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue