mirror of
https://github.com/google/nomulus.git
synced 2025-07-24 03:30:46 +02:00
Use @DefaultCredential for Cloud API access in GAE
This change completes the switch to @DefaultCredential for all use cases in GAE. Impacted modules: - IcannReporting - CreateCdnsTld command - LoadSnapshot command. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=213511730
This commit is contained in:
weiminyu
Ben McIlwain
parent
9bcd5579ef
commit
961e5cc7c7
20 changed files with 184 additions and 226 deletions
|
|
@ -26,12 +26,8 @@ import static google.registry.bigquery.BigqueryUtils.toJobReferenceString;
|
|||
import static google.registry.config.RegistryConfig.getProjectId;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
||||
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
|
||||
import com.google.api.client.http.AbstractInputStreamContent;
|
||||
import com.google.api.client.http.HttpRequestInitializer;
|
||||
import com.google.api.client.http.HttpTransport;
|
||||
import com.google.api.client.json.JsonFactory;
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.model.Dataset;
|
||||
import com.google.api.services.bigquery.model.DatasetReference;
|
||||
|
|
@ -51,6 +47,7 @@ import com.google.api.services.bigquery.model.TableFieldSchema;
|
|||
import com.google.api.services.bigquery.model.TableReference;
|
||||
import com.google.api.services.bigquery.model.TableRow;
|
||||
import com.google.api.services.bigquery.model.ViewDefinition;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableTable;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
|
|
@ -73,6 +70,7 @@ import java.util.List;
|
|||
import java.util.Random;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
|
|
@ -96,20 +94,11 @@ public class BigqueryConnection implements AutoCloseable {
|
|||
private static final Duration TEMP_TABLE_TTL = Duration.standardHours(24);
|
||||
|
||||
/** Bigquery client instance wrapped by this class. */
|
||||
private Bigquery bigquery;
|
||||
private final Bigquery bigquery;
|
||||
|
||||
/** Executor service for bigquery jobs. */
|
||||
private ListeningExecutorService service;
|
||||
|
||||
/** Credential object to use for initializing HTTP requests to the bigquery API. */
|
||||
private HttpRequestInitializer credential;
|
||||
|
||||
/** HTTP transport object to use for accessing bigquery API. */
|
||||
private HttpTransport httpTransport;
|
||||
|
||||
/** JSON factory object to use for accessing bigquery API. */
|
||||
private JsonFactory jsonFactory;
|
||||
|
||||
/** Pseudo-randomness source to use for creating random table names. */
|
||||
private Random random = new Random();
|
||||
|
||||
|
|
@ -122,12 +111,17 @@ public class BigqueryConnection implements AutoCloseable {
|
|||
/** Duration to wait between polls for job status. */
|
||||
private Duration pollInterval = Duration.millis(1000);
|
||||
|
||||
BigqueryConnection(Bigquery bigquery) {
|
||||
this.bigquery = bigquery;
|
||||
}
|
||||
|
||||
/** Builder for a {@link BigqueryConnection}, since the latter is immutable once created. */
|
||||
public static class Builder {
|
||||
private BigqueryConnection instance;
|
||||
|
||||
public Builder() {
|
||||
instance = new BigqueryConnection();
|
||||
@Inject
|
||||
Builder(Bigquery bigquery) {
|
||||
instance = new BigqueryConnection(bigquery);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -139,13 +133,6 @@ public class BigqueryConnection implements AutoCloseable {
|
|||
return this;
|
||||
}
|
||||
|
||||
public Builder setCredential(GoogleCredential credential) {
|
||||
instance.credential = checkNotNull(credential);
|
||||
instance.httpTransport = credential.getTransport();
|
||||
instance.jsonFactory = credential.getJsonFactory();
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDatasetId(String datasetId) {
|
||||
instance.datasetId = checkNotNull(datasetId);
|
||||
return this;
|
||||
|
|
@ -167,7 +154,11 @@ public class BigqueryConnection implements AutoCloseable {
|
|||
public BigqueryConnection build() {
|
||||
try {
|
||||
checkNotNull(instance.service, "Must provide executor service");
|
||||
instance.initialize();
|
||||
return instance;
|
||||
} catch (Throwable e) {
|
||||
Throwables.throwIfUnchecked(e);
|
||||
throw new RuntimeException("Cannot initialize BigqueryConnection", e);
|
||||
} finally {
|
||||
// Clear the internal instance so you can't accidentally mutate it through this builder.
|
||||
instance = null;
|
||||
|
|
@ -306,13 +297,10 @@ public class BigqueryConnection implements AutoCloseable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Initializes the BigqueryConnection object by setting up the API client and creating the
|
||||
* default dataset if it doesn't exist.
|
||||
* Initializes the BigqueryConnection object by setting up the API client and creating the default
|
||||
* dataset if it doesn't exist.
|
||||
*/
|
||||
public BigqueryConnection initialize() throws Exception {
|
||||
bigquery = new Bigquery.Builder(httpTransport, jsonFactory, credential)
|
||||
.setApplicationName(getClass().getSimpleName())
|
||||
.build();
|
||||
private BigqueryConnection initialize() throws Exception {
|
||||
createDatasetIfNeeded(datasetId);
|
||||
createDatasetIfNeeded(TEMP_DATASET_NAME);
|
||||
return this;
|
||||
|
|
|
|||
|
|
@ -14,20 +14,16 @@
|
|||
|
||||
package google.registry.bigquery;
|
||||
|
||||
import com.google.api.client.http.HttpRequestInitializer;
|
||||
import com.google.api.client.http.HttpTransport;
|
||||
import com.google.api.client.json.JsonFactory;
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.BigqueryScopes;
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import dagger.multibindings.Multibinds;
|
||||
import google.registry.config.CredentialModule.DefaultCredential;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
|
||||
/** Dagger module for Google {@link Bigquery} connection objects. */
|
||||
@Module
|
||||
|
|
@ -39,11 +35,8 @@ public abstract class BigqueryModule {
|
|||
|
||||
@Provides
|
||||
static Bigquery provideBigquery(
|
||||
HttpTransport transport,
|
||||
JsonFactory jsonFactory,
|
||||
Function<Set<String>, ? extends HttpRequestInitializer> credential,
|
||||
@Config("projectId") String projectId) {
|
||||
return new Bigquery.Builder(transport, jsonFactory, credential.apply(BigqueryScopes.all()))
|
||||
@DefaultCredential GoogleCredential credential, @Config("projectId") String projectId) {
|
||||
return new Bigquery.Builder(credential.getTransport(), credential.getJsonFactory(), credential)
|
||||
.setApplicationName(projectId)
|
||||
.build();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
|
@ -17,14 +17,7 @@ package google.registry.bigquery;
|
|||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.collect.Sets.newConcurrentHashSet;
|
||||
|
||||
import com.google.api.client.extensions.appengine.http.UrlFetchTransport;
|
||||
import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential;
|
||||
import com.google.api.client.http.HttpRequestInitializer;
|
||||
import com.google.api.client.http.HttpTransport;
|
||||
import com.google.api.client.json.JsonFactory;
|
||||
import com.google.api.client.json.jackson2.JacksonFactory;
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.BigqueryScopes;
|
||||
import com.google.api.services.bigquery.model.Dataset;
|
||||
import com.google.api.services.bigquery.model.DatasetReference;
|
||||
import com.google.api.services.bigquery.model.Table;
|
||||
|
|
@ -39,8 +32,8 @@ import java.util.Map;
|
|||
import java.util.Set;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/** Factory for creating {@link Bigquery} connections. */
|
||||
public class BigqueryFactory {
|
||||
/** Wrapper of {@link Bigquery} with validation helpers. */
|
||||
public class CheckedBigquery {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
|
|
@ -49,45 +42,16 @@ public class BigqueryFactory {
|
|||
private static Set<String> knownExistingTables = newConcurrentHashSet();
|
||||
|
||||
@Inject Map<String, ImmutableList<TableFieldSchema>> bigquerySchemas;
|
||||
@Inject Subfactory subfactory;
|
||||
@Inject BigqueryFactory() {}
|
||||
@Inject Bigquery bigquery;
|
||||
|
||||
/** This class is broken out solely so that it can be mocked inside of tests. */
|
||||
static class Subfactory {
|
||||
|
||||
@Inject Subfactory() {}
|
||||
|
||||
public Bigquery create(
|
||||
String applicationName,
|
||||
HttpTransport transport,
|
||||
JsonFactory jsonFactory,
|
||||
HttpRequestInitializer httpRequestInitializer) {
|
||||
return new Bigquery.Builder(transport, jsonFactory, httpRequestInitializer)
|
||||
.setApplicationName(applicationName)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns a new connection to BigQuery. */
|
||||
public Bigquery create(
|
||||
String applicationName,
|
||||
HttpTransport transport,
|
||||
JsonFactory jsonFactory,
|
||||
HttpRequestInitializer httpRequestInitializer) {
|
||||
return subfactory.create(applicationName, transport, jsonFactory, httpRequestInitializer);
|
||||
}
|
||||
@Inject
|
||||
CheckedBigquery() {}
|
||||
|
||||
/**
|
||||
* Returns a new connection to Bigquery, first ensuring that the given dataset exists in the
|
||||
* project with the given id, creating it if required.
|
||||
*/
|
||||
public Bigquery create(String projectId, String datasetId) throws IOException {
|
||||
Bigquery bigquery = create(
|
||||
getClass().getSimpleName(),
|
||||
new UrlFetchTransport(),
|
||||
new JacksonFactory(),
|
||||
new AppIdentityCredential(BigqueryScopes.all()));
|
||||
|
||||
public Bigquery ensureDataSetExists(String projectId, String datasetId) throws IOException {
|
||||
// Note: it's safe for multiple threads to call this as the dataset will only be created once.
|
||||
if (!knownExistingDatasets.contains(datasetId)) {
|
||||
ensureDataset(bigquery, projectId, datasetId);
|
||||
|
|
@ -101,9 +65,9 @@ public class BigqueryFactory {
|
|||
* Returns a new connection to Bigquery, first ensuring that the given dataset and table exist in
|
||||
* project with the given id, creating them if required.
|
||||
*/
|
||||
public Bigquery create(String projectId, String datasetId, String tableId)
|
||||
public Bigquery ensureDataSetAndTableExist(String projectId, String datasetId, String tableId)
|
||||
throws IOException {
|
||||
Bigquery bigquery = create(projectId, datasetId);
|
||||
ensureDataSetExists(projectId, datasetId);
|
||||
checkArgument(bigquerySchemas.containsKey(tableId), "Unknown table ID: %s", tableId);
|
||||
|
||||
if (!knownExistingTables.contains(tableId)) {
|
||||
Loading…
Add table
Add a link
Reference in a new issue