From a612e9bf66d12fe7da6febbaffa24b2c63cb6f82 Mon Sep 17 00:00:00 2001 From: jianglai Date: Wed, 5 Dec 2018 12:20:27 -0800 Subject: [PATCH] Use local credential to deploy beam pipelines We are moving away from using Application Default Credentials generated by "gcloud auth application-default login" in our code base and consolidate on using self-managed credentials provided from AuthModule. One of the remaining dependencies on the ADCs is from beam pipeline deployment commands, which by default use the ADCs to talk to GCS and upload the jar files and templates. In this CL, we explicitly provide the locally created credential to the Options used in deployments. Also moved all credential qualifiers to CredentialModule, and removed @AppEngineAdminApiCredential, which is no longer used. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=224199812 --- gradle/core/build.gradle | 6 +-- java/google/registry/beam/invoicing/BUILD | 1 + .../beam/invoicing/InvoicingPipeline.java | 15 +++++++ java/google/registry/beam/spec11/BUILD | 1 + .../registry/beam/spec11/Spec11Pipeline.java | 13 ++++++ .../registry/config/CredentialModule.java | 36 ++++++++-------- .../tools/AppEngineAdminApiModule.java | 2 +- java/google/registry/tools/AuthModule.java | 43 ++++++------------- java/google/registry/tools/RegistryCli.java | 4 +- .../registry/tools/RegistryToolComponent.java | 8 ++-- .../registry/tools/RequestFactoryModule.java | 2 +- .../google/registry/tools/AuthModuleTest.java | 13 ++---- 12 files changed, 77 insertions(+), 67 deletions(-) diff --git a/gradle/core/build.gradle b/gradle/core/build.gradle index 59e1e7187..00d4bf8c3 100644 --- a/gradle/core/build.gradle +++ b/gradle/core/build.gradle @@ -83,7 +83,7 @@ dependencies { maybeRuntime 'com.fasterxml.jackson.core:jackson-annotations:2.8.0' maybeRuntime 'com.fasterxml.jackson.core:jackson-databind:2.8.5' compile 'com.google.api-client:google-api-client:1.22.0' - compile 'com.google.api-client:google-api-client-appengine:1.22.0' + maybeRuntime 'com.google.api-client:google-api-client-appengine:1.22.0' maybeRuntime 'com.google.api-client:google-api-client-jackson2:1.20.0' compile 'com.google.monitoring-client:metrics:1.0.4' compile 'com.google.monitoring-client:stackdriver:1.0.4' @@ -113,8 +113,8 @@ dependencies { compile 'com.google.appengine.tools:appengine-pipeline:0.2.13' compile 'com.google.appengine:appengine-remote-api:1.9.48' maybeRuntime 'com.google.appengine:appengine-tools-sdk:1.9.48' - maybeRuntime 'com.google.auth:google-auth-library-credentials:0.7.1' - maybeRuntime 'com.google.auth:google-auth-library-oauth2-http:0.7.1' + compile 'com.google.auth:google-auth-library-credentials:0.7.1' + compile 'com.google.auth:google-auth-library-oauth2-http:0.7.1' maybeRuntime 'com.google.auto:auto-common:0.8' maybeRuntime 'com.google.auto.factory:auto-factory:1.0-beta3' compile 'com.google.code.gson:gson:2.8.5' diff --git a/java/google/registry/beam/invoicing/BUILD b/java/google/registry/beam/invoicing/BUILD index af2e4b32b..d94f93f0c 100644 --- a/java/google/registry/beam/invoicing/BUILD +++ b/java/google/registry/beam/invoicing/BUILD @@ -15,6 +15,7 @@ java_library( "//java/google/registry/reporting/billing", "//java/google/registry/util", "@com_google_apis_google_api_services_bigquery", + "@com_google_auth_library_oauth2_http", "@com_google_auto_value", "@com_google_dagger", "@com_google_flogger", diff --git a/java/google/registry/beam/invoicing/InvoicingPipeline.java b/java/google/registry/beam/invoicing/InvoicingPipeline.java index 0f70ff631..f439b5290 100644 --- a/java/google/registry/beam/invoicing/InvoicingPipeline.java +++ b/java/google/registry/beam/invoicing/InvoicingPipeline.java @@ -14,11 +14,17 @@ package google.registry.beam.invoicing; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.auth.oauth2.GoogleCredentials; import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey; import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder; +import google.registry.config.CredentialModule.LocalCredentialJson; import google.registry.config.RegistryConfig.Config; import google.registry.reporting.billing.BillingModule; import google.registry.reporting.billing.GenerateInvoicesAction; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.Serializable; import javax.inject.Inject; import org.apache.beam.runners.dataflow.DataflowRunner; @@ -79,6 +85,8 @@ public class InvoicingPipeline implements Serializable { @Config("invoiceFilePrefix") String invoiceFilePrefix; + @Inject @LocalCredentialJson String credentialJson; + @Inject InvoicingPipeline() {} @@ -100,6 +108,13 @@ public class InvoicingPipeline implements Serializable { public void deploy() { // We can't store options as a member variable due to serialization concerns. InvoicingPipelineOptions options = PipelineOptionsFactory.as(InvoicingPipelineOptions.class); + try { + options.setGcpCredential( + GoogleCredentials.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8)))); + } catch (IOException e) { + throw new RuntimeException( + "Cannot obtain local credential to deploy the invoicing pipeline", e); + } options.setProject(projectId); options.setRunner(DataflowRunner.class); // This causes p.run() to stage the pipeline as a template on GCS, as opposed to running it. diff --git a/java/google/registry/beam/spec11/BUILD b/java/google/registry/beam/spec11/BUILD index eefeb4c59..74696060c 100644 --- a/java/google/registry/beam/spec11/BUILD +++ b/java/google/registry/beam/spec11/BUILD @@ -12,6 +12,7 @@ java_library( "//java/google/registry/beam", "//java/google/registry/config", "//java/google/registry/util", + "@com_google_auth_library_oauth2_http", "@com_google_auto_value", "@com_google_dagger", "@com_google_flogger", diff --git a/java/google/registry/beam/spec11/Spec11Pipeline.java b/java/google/registry/beam/spec11/Spec11Pipeline.java index 3b80b0ae6..b96d8aef0 100644 --- a/java/google/registry/beam/spec11/Spec11Pipeline.java +++ b/java/google/registry/beam/spec11/Spec11Pipeline.java @@ -15,11 +15,16 @@ package google.registry.beam.spec11; import static google.registry.beam.BeamUtils.getQueryFromFile; +import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.auth.oauth2.GoogleCredentials; import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn; +import google.registry.config.CredentialModule.LocalCredentialJson; import google.registry.config.RegistryConfig.Config; import google.registry.util.Retrier; import google.registry.util.SqlTemplate; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.Serializable; import javax.inject.Inject; import org.apache.beam.runners.dataflow.DataflowRunner; @@ -89,6 +94,8 @@ public class Spec11Pipeline implements Serializable { @Inject Retrier retrier; + @Inject @LocalCredentialJson String credentialJson; + @Inject Spec11Pipeline() {} @@ -123,6 +130,12 @@ public class Spec11Pipeline implements Serializable { public void deploy() { // We can't store options as a member variable due to serialization concerns. Spec11PipelineOptions options = PipelineOptionsFactory.as(Spec11PipelineOptions.class); + try { + options.setGcpCredential( + GoogleCredentials.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8)))); + } catch (IOException e) { + throw new RuntimeException("Cannot obtain local credential to deploy the spec11 pipeline", e); + } options.setProject(projectId); options.setRunner(DataflowRunner.class); // This causes p.run() to stage the pipeline as a template on GCS, as opposed to running it. diff --git a/java/google/registry/config/CredentialModule.java b/java/google/registry/config/CredentialModule.java index c20911869..5544870ae 100644 --- a/java/google/registry/config/CredentialModule.java +++ b/java/google/registry/config/CredentialModule.java @@ -17,7 +17,6 @@ package google.registry.config; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.googleapis.util.Utils; import com.google.common.collect.ImmutableList; @@ -27,6 +26,9 @@ import google.registry.config.RegistryConfig.Config; import google.registry.keyring.api.KeyModule.Key; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; import java.security.GeneralSecurityException; import javax.inject.Qualifier; import javax.inject.Singleton; @@ -108,22 +110,10 @@ public abstract class CredentialModule { .build(); } - /** - * Provides a {@link AppIdentityCredential} with access for App Engine Admin API. - * - *

{@link AppIdentityCredential} is an OAuth 2.0 credential in which a client Google App Engine - * application needs to access data that it owns. - */ - @AppEngineAdminApiCredential - @Provides - @Singleton - public static AppIdentityCredential provideAppEngineAdminApiCredential( - @Config("appEngineAdminApiCredentialOauthScopes") ImmutableList requiredScopes) { - return new AppIdentityCredential(requiredScopes); - } - /** Dagger qualifier for the Application Default Credential. */ @Qualifier + @Documented + @Retention(RetentionPolicy.RUNTIME) public @interface DefaultCredential {} /** @@ -131,6 +121,8 @@ public abstract class CredentialModule { * threads. */ @Qualifier + @Documented + @Retention(RetentionPolicy.RUNTIME) public @interface JsonCredential {} /** @@ -138,9 +130,19 @@ public abstract class CredentialModule { * Suite). */ @Qualifier + @Documented + @Retention(RetentionPolicy.RUNTIME) public @interface DelegatedCredential {} - /** Dagger qualifier for a credential with access for App Engine Admin API. */ + /** Dagger qualifier for the local credential used in the nomulus tool. */ @Qualifier - public @interface AppEngineAdminApiCredential {} + @Documented + @Retention(RetentionPolicy.RUNTIME) + public @interface LocalCredential {} + + /** Dagger qualifier for the JSON string used to create the local credential. */ + @Qualifier + @Documented + @Retention(RetentionPolicy.RUNTIME) + public @interface LocalCredentialJson {} } diff --git a/java/google/registry/tools/AppEngineAdminApiModule.java b/java/google/registry/tools/AppEngineAdminApiModule.java index d7beac24c..dedbf4142 100644 --- a/java/google/registry/tools/AppEngineAdminApiModule.java +++ b/java/google/registry/tools/AppEngineAdminApiModule.java @@ -19,8 +19,8 @@ import com.google.api.client.googleapis.util.Utils; import com.google.api.services.appengine.v1.Appengine; import dagger.Module; import dagger.Provides; +import google.registry.config.CredentialModule.LocalCredential; import google.registry.config.RegistryConfig.Config; -import google.registry.tools.AuthModule.LocalCredential; import javax.inject.Singleton; /** Module providing the instance of {@link Appengine} to access App Engine Admin Api. */ diff --git a/java/google/registry/tools/AuthModule.java b/java/google/registry/tools/AuthModule.java index 3d4138360..a6767d2be 100644 --- a/java/google/registry/tools/AuthModule.java +++ b/java/google/registry/tools/AuthModule.java @@ -28,7 +28,6 @@ import com.google.api.client.json.JsonFactory; import com.google.api.client.util.store.AbstractDataStoreFactory; import com.google.api.client.util.store.FileDataStoreFactory; import com.google.common.base.Joiner; -import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; @@ -37,11 +36,12 @@ import dagger.Binds; import dagger.Module; import dagger.Provides; import google.registry.config.CredentialModule.DefaultCredential; +import google.registry.config.CredentialModule.LocalCredential; +import google.registry.config.CredentialModule.LocalCredentialJson; import google.registry.config.RegistryConfig.Config; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -84,9 +84,9 @@ public class AuthModule { @Provides @LocalCredential public static GoogleCredential provideLocalCredential( - @LocalCredentialStream Supplier credentialStream) { + @LocalCredentialJson String credentialJson) { try { - return GoogleCredential.fromStream(credentialStream.get()); + return GoogleCredential.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8))); } catch (IOException e) { throw new RuntimeException(e); } @@ -132,20 +132,17 @@ public class AuthModule { } @Provides - @LocalCredentialStream - public static Supplier provideLocalCredentialStream( + @LocalCredentialJson + public static String provideLocalCredentialJson( GoogleClientSecrets clientSecrets, @StoredCredential Credential credential) { - String json = - new Gson() - .toJson( - ImmutableMap.builder() - .put("type", "authorized_user") - .put("client_id", clientSecrets.getDetails().getClientId()) - .put("client_secret", clientSecrets.getDetails().getClientSecret()) - .put("refresh_token", credential.getRefreshToken()) - .build()); - // A supplier is provided so that each binding gets a fresh stream, to avoid contention. - return () -> new ByteArrayInputStream(json.getBytes(UTF_8)); + return new Gson() + .toJson( + ImmutableMap.builder() + .put("type", "authorized_user") + .put("client_id", clientSecrets.getDetails().getClientId()) + .put("client_secret", clientSecrets.getDetails().getClientSecret()) + .put("refresh_token", credential.getRefreshToken()) + .build()); } @Provides @@ -189,18 +186,6 @@ public class AuthModule { @Retention(RetentionPolicy.RUNTIME) private @interface StoredCredential {} - /** Dagger qualifier for the local credential used in the nomulus tool. */ - @Qualifier - @Documented - @Retention(RetentionPolicy.RUNTIME) - @interface LocalCredential {} - - /** Dagger qualifier for the JSON stream used to create the local credential. */ - @Qualifier - @Documented - @Retention(RetentionPolicy.RUNTIME) - @interface LocalCredentialStream {} - /** Dagger qualifier for the credential qualifier consisting of client and scopes. */ @Qualifier @Documented diff --git a/java/google/registry/tools/RegistryCli.java b/java/google/registry/tools/RegistryCli.java index f3eef71ac..412d0f472 100644 --- a/java/google/registry/tools/RegistryCli.java +++ b/java/google/registry/tools/RegistryCli.java @@ -17,6 +17,7 @@ package google.registry.tools; import static com.google.common.base.Preconditions.checkState; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.tools.Injector.injectReflectively; +import static java.nio.charset.StandardCharsets.UTF_8; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -32,6 +33,7 @@ import google.registry.config.RegistryConfig; import google.registry.model.ofy.ObjectifyService; import google.registry.tools.AuthModule.LoginRequiredException; import google.registry.tools.params.ParameterFactory; +import java.io.ByteArrayInputStream; import java.net.URL; import java.security.Security; import java.util.Map; @@ -211,7 +213,7 @@ final class RegistryCli implements AutoCloseable, CommandRunner { options.useDevelopmentServerCredential(); } else { RemoteApiOptionsUtil.useGoogleCredentialStream( - options, component.googleCredentialStream().get()); + options, new ByteArrayInputStream(component.googleCredentialJson().getBytes(UTF_8))); } installer.install(options); } diff --git a/java/google/registry/tools/RegistryToolComponent.java b/java/google/registry/tools/RegistryToolComponent.java index bad763893..1590a9910 100644 --- a/java/google/registry/tools/RegistryToolComponent.java +++ b/java/google/registry/tools/RegistryToolComponent.java @@ -14,9 +14,9 @@ package google.registry.tools; -import com.google.common.base.Supplier; import dagger.Component; import google.registry.bigquery.BigqueryModule; +import google.registry.config.CredentialModule.LocalCredentialJson; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.dns.writer.VoidDnsWriterModule; import google.registry.dns.writer.clouddns.CloudDnsWriterModule; @@ -32,12 +32,10 @@ import google.registry.request.Modules.URLFetchServiceModule; import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UserServiceModule; import google.registry.tools.AuthModule.LocalCredentialModule; -import google.registry.tools.AuthModule.LocalCredentialStream; import google.registry.util.AppEngineServiceUtilsImpl.AppEngineServiceUtilsModule; import google.registry.util.SystemClock.SystemClockModule; import google.registry.util.SystemSleeper.SystemSleeperModule; import google.registry.whois.WhoisModule; -import java.io.InputStream; import javax.inject.Singleton; /** @@ -112,7 +110,7 @@ interface RegistryToolComponent { AppEngineConnection appEngineConnection(); - @LocalCredentialStream - Supplier googleCredentialStream(); + @LocalCredentialJson + String googleCredentialJson(); } diff --git a/java/google/registry/tools/RequestFactoryModule.java b/java/google/registry/tools/RequestFactoryModule.java index dbc6674ac..e72c355f5 100644 --- a/java/google/registry/tools/RequestFactoryModule.java +++ b/java/google/registry/tools/RequestFactoryModule.java @@ -19,8 +19,8 @@ import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.javanet.NetHttpTransport; import dagger.Module; import dagger.Provides; +import google.registry.config.CredentialModule.LocalCredential; import google.registry.config.RegistryConfig; -import google.registry.tools.AuthModule.LocalCredential; /** * Module for providing the HttpRequestFactory. diff --git a/javatests/google/registry/tools/AuthModuleTest.java b/javatests/google/registry/tools/AuthModuleTest.java index affa4dae5..b58763500 100644 --- a/javatests/google/registry/tools/AuthModuleTest.java +++ b/javatests/google/registry/tools/AuthModuleTest.java @@ -16,7 +16,6 @@ package google.registry.tools; import static com.google.common.truth.Truth.assertThat; import static google.registry.testing.JUnitBackports.assertThrows; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -34,8 +33,6 @@ import com.google.common.collect.ImmutableList; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.io.Serializable; import java.util.Map; import org.junit.Before; @@ -156,14 +153,10 @@ public class AuthModuleTest { } @Test - public void test_provideLocalCredentialStream() { - InputStream jsonStream = - AuthModule.provideLocalCredentialStream(getSecrets(), getCredential()).get(); + public void test_provideLocalCredentialJson() { + String credentialJson = AuthModule.provideLocalCredentialJson(getSecrets(), getCredential()); Map jsonMap = - new Gson() - .fromJson( - new InputStreamReader(jsonStream, UTF_8), - new TypeToken>() {}.getType()); + new Gson().fromJson(credentialJson, new TypeToken>() {}.getType()); assertThat(jsonMap.get("type")).isEqualTo("authorized_user"); assertThat(jsonMap.get("client_secret")).isEqualTo(CLIENT_SECRET); assertThat(jsonMap.get("client_id")).isEqualTo(CLIENT_ID);