Use local credential to deploy beam pipelines

We are moving away from using Application Default Credentials generated by "gcloud auth application-default login" in our code base and consolidate on using self-managed credentials provided from AuthModule.

One of the remaining dependencies on the ADCs is from beam pipeline deployment commands, which by default use the ADCs to talk to GCS and upload the jar files and templates. In this CL, we explicitly provide the locally created credential to the Options used in deployments.

Also moved all credential qualifiers to CredentialModule, and removed @AppEngineAdminApiCredential, which is no longer used.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=224199812

gradle/core/build.gradle

@@ -83,7 +83,7 @@ dependencies {
   maybeRuntime 'com.fasterxml.jackson.core:jackson-annotations:2.8.0'
   maybeRuntime 'com.fasterxml.jackson.core:jackson-databind:2.8.5'
   compile 'com.google.api-client:google-api-client:1.22.0'
-  compile 'com.google.api-client:google-api-client-appengine:1.22.0'
+  maybeRuntime 'com.google.api-client:google-api-client-appengine:1.22.0'
   maybeRuntime 'com.google.api-client:google-api-client-jackson2:1.20.0'
   compile 'com.google.monitoring-client:metrics:1.0.4'
   compile 'com.google.monitoring-client:stackdriver:1.0.4'
@@ -113,8 +113,8 @@ dependencies {
   compile 'com.google.appengine.tools:appengine-pipeline:0.2.13'
   compile 'com.google.appengine:appengine-remote-api:1.9.48'
   maybeRuntime 'com.google.appengine:appengine-tools-sdk:1.9.48'
-  maybeRuntime 'com.google.auth:google-auth-library-credentials:0.7.1'
-  maybeRuntime 'com.google.auth:google-auth-library-oauth2-http:0.7.1'
+  compile 'com.google.auth:google-auth-library-credentials:0.7.1'
+  compile 'com.google.auth:google-auth-library-oauth2-http:0.7.1'
   maybeRuntime 'com.google.auto:auto-common:0.8'
   maybeRuntime 'com.google.auto.factory:auto-factory:1.0-beta3'
   compile 'com.google.code.gson:gson:2.8.5'

java/google/registry/beam/invoicing/BUILD

@@ -15,6 +15,7 @@ java_library(
         "//java/google/registry/reporting/billing",
         "//java/google/registry/util",
         "@com_google_apis_google_api_services_bigquery",
+        "@com_google_auth_library_oauth2_http",
         "@com_google_auto_value",
         "@com_google_dagger",
         "@com_google_flogger",

java/google/registry/beam/invoicing/InvoicingPipeline.java

@@ -14,11 +14,17 @@
 
 package google.registry.beam.invoicing;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.auth.oauth2.GoogleCredentials;
 import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey;
 import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder;
+import google.registry.config.CredentialModule.LocalCredentialJson;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.reporting.billing.BillingModule;
 import google.registry.reporting.billing.GenerateInvoicesAction;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.Serializable;
 import javax.inject.Inject;
 import org.apache.beam.runners.dataflow.DataflowRunner;
@@ -79,6 +85,8 @@ public class InvoicingPipeline implements Serializable {
   @Config("invoiceFilePrefix")
   String invoiceFilePrefix;
 
+  @Inject @LocalCredentialJson String credentialJson;
+
   @Inject
   InvoicingPipeline() {}
 
@@ -100,6 +108,13 @@ public class InvoicingPipeline implements Serializable {
   public void deploy() {
     // We can't store options as a member variable due to serialization concerns.
     InvoicingPipelineOptions options = PipelineOptionsFactory.as(InvoicingPipelineOptions.class);
+    try {
+      options.setGcpCredential(
+          GoogleCredentials.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8))));
+    } catch (IOException e) {
+      throw new RuntimeException(
+          "Cannot obtain local credential to deploy the invoicing pipeline", e);
+    }
     options.setProject(projectId);
     options.setRunner(DataflowRunner.class);
     // This causes p.run() to stage the pipeline as a template on GCS, as opposed to running it.

java/google/registry/beam/spec11/BUILD

@@ -12,6 +12,7 @@ java_library(
         "//java/google/registry/beam",
         "//java/google/registry/config",
         "//java/google/registry/util",
+        "@com_google_auth_library_oauth2_http",
         "@com_google_auto_value",
         "@com_google_dagger",
         "@com_google_flogger",

java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -15,11 +15,16 @@
 package google.registry.beam.spec11;
 
 import static google.registry.beam.BeamUtils.getQueryFromFile;
+import static java.nio.charset.StandardCharsets.UTF_8;
 
+import com.google.auth.oauth2.GoogleCredentials;
 import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn;
+import google.registry.config.CredentialModule.LocalCredentialJson;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.util.Retrier;
 import google.registry.util.SqlTemplate;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.Serializable;
 import javax.inject.Inject;
 import org.apache.beam.runners.dataflow.DataflowRunner;
@@ -89,6 +94,8 @@ public class Spec11Pipeline implements Serializable {
   @Inject
   Retrier retrier;
 
+  @Inject @LocalCredentialJson String credentialJson;
+
   @Inject
   Spec11Pipeline() {}
 
@@ -123,6 +130,12 @@ public class Spec11Pipeline implements Serializable {
   public void deploy() {
     // We can't store options as a member variable due to serialization concerns.
     Spec11PipelineOptions options = PipelineOptionsFactory.as(Spec11PipelineOptions.class);
+    try {
+      options.setGcpCredential(
+          GoogleCredentials.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8))));
+    } catch (IOException e) {
+      throw new RuntimeException("Cannot obtain local credential to deploy the spec11 pipeline", e);
+    }
     options.setProject(projectId);
     options.setRunner(DataflowRunner.class);
     // This causes p.run() to stage the pipeline as a template on GCS, as opposed to running it.

java/google/registry/config/CredentialModule.java

@@ -17,7 +17,6 @@ package google.registry.config;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential;
 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
 import com.google.api.client.googleapis.util.Utils;
 import com.google.common.collect.ImmutableList;
@@ -27,6 +26,9 @@ import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
 import java.security.GeneralSecurityException;
 import javax.inject.Qualifier;
 import javax.inject.Singleton;
@@ -108,22 +110,10 @@ public abstract class CredentialModule {
         .build();
   }
 
-  /**
-   * Provides a {@link AppIdentityCredential} with access for App Engine Admin API.
-   *
-   * <p>{@link AppIdentityCredential} is an OAuth 2.0 credential in which a client Google App Engine
-   * application needs to access data that it owns.
-   */
-  @AppEngineAdminApiCredential
-  @Provides
-  @Singleton
-  public static AppIdentityCredential provideAppEngineAdminApiCredential(
-      @Config("appEngineAdminApiCredentialOauthScopes") ImmutableList<String> requiredScopes) {
-    return new AppIdentityCredential(requiredScopes);
-  }
-
   /** Dagger qualifier for the Application Default Credential. */
   @Qualifier
+  @Documented
+  @Retention(RetentionPolicy.RUNTIME)
   public @interface DefaultCredential {}
 
   /**
@@ -131,6 +121,8 @@ public abstract class CredentialModule {
    * threads.
    */
   @Qualifier
+  @Documented
+  @Retention(RetentionPolicy.RUNTIME)
   public @interface JsonCredential {}
 
   /**
@@ -138,9 +130,19 @@ public abstract class CredentialModule {
    * Suite).
    */
   @Qualifier
+  @Documented
+  @Retention(RetentionPolicy.RUNTIME)
   public @interface DelegatedCredential {}
 
-  /** Dagger qualifier for a credential with access for App Engine Admin API. */
+  /** Dagger qualifier for the local credential used in the nomulus tool. */
   @Qualifier
-  public @interface AppEngineAdminApiCredential {}
+  @Documented
+  @Retention(RetentionPolicy.RUNTIME)
+  public @interface LocalCredential {}
+
+  /** Dagger qualifier for the JSON string used to create the local credential. */
+  @Qualifier
+  @Documented
+  @Retention(RetentionPolicy.RUNTIME)
+  public @interface LocalCredentialJson {}
 }

java/google/registry/tools/AppEngineAdminApiModule.java

@@ -19,8 +19,8 @@ import com.google.api.client.googleapis.util.Utils;
 import com.google.api.services.appengine.v1.Appengine;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.config.CredentialModule.LocalCredential;
 import google.registry.config.RegistryConfig.Config;
-import google.registry.tools.AuthModule.LocalCredential;
 import javax.inject.Singleton;
 
 /** Module providing the instance of {@link Appengine} to access App Engine Admin Api. */

java/google/registry/tools/AuthModule.java

@@ -28,7 +28,6 @@ import com.google.api.client.json.JsonFactory;
 import com.google.api.client.util.store.AbstractDataStoreFactory;
 import com.google.api.client.util.store.FileDataStoreFactory;
 import com.google.common.base.Joiner;
-import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Ordering;
@@ -37,11 +36,12 @@ import dagger.Binds;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.config.CredentialModule.DefaultCredential;
+import google.registry.config.CredentialModule.LocalCredential;
+import google.registry.config.CredentialModule.LocalCredentialJson;
 import google.registry.config.RegistryConfig.Config;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
 import java.lang.annotation.Documented;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
@@ -84,9 +84,9 @@ public class AuthModule {
   @Provides
   @LocalCredential
   public static GoogleCredential provideLocalCredential(
-      @LocalCredentialStream Supplier<InputStream> credentialStream) {
+      @LocalCredentialJson String credentialJson) {
     try {
-      return GoogleCredential.fromStream(credentialStream.get());
+      return GoogleCredential.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8)));
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
@@ -132,11 +132,10 @@ public class AuthModule {
   }
 
   @Provides
-  @LocalCredentialStream
-  public static Supplier<InputStream> provideLocalCredentialStream(
+  @LocalCredentialJson
+  public static String provideLocalCredentialJson(
       GoogleClientSecrets clientSecrets, @StoredCredential Credential credential) {
-    String json =
-        new Gson()
+    return new Gson()
         .toJson(
             ImmutableMap.<String, String>builder()
                 .put("type", "authorized_user")
@@ -144,8 +143,6 @@ public class AuthModule {
                 .put("client_secret", clientSecrets.getDetails().getClientSecret())
                 .put("refresh_token", credential.getRefreshToken())
                 .build());
-    // A supplier is provided so that each binding gets a fresh stream, to avoid contention.
-    return () -> new ByteArrayInputStream(json.getBytes(UTF_8));
   }
 
   @Provides
@@ -189,18 +186,6 @@ public class AuthModule {
   @Retention(RetentionPolicy.RUNTIME)
   private @interface StoredCredential {}
 
-  /** Dagger qualifier for the local credential used in the nomulus tool. */
-  @Qualifier
-  @Documented
-  @Retention(RetentionPolicy.RUNTIME)
-  @interface LocalCredential {}
-
-  /** Dagger qualifier for the JSON stream used to create the local credential. */
-  @Qualifier
-  @Documented
-  @Retention(RetentionPolicy.RUNTIME)
-  @interface LocalCredentialStream {}
-
   /** Dagger qualifier for the credential qualifier consisting of client and scopes. */
   @Qualifier
   @Documented

java/google/registry/tools/RegistryCli.java

@@ -17,6 +17,7 @@ package google.registry.tools;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.tools.Injector.injectReflectively;
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
@@ -32,6 +33,7 @@ import google.registry.config.RegistryConfig;
 import google.registry.model.ofy.ObjectifyService;
 import google.registry.tools.AuthModule.LoginRequiredException;
 import google.registry.tools.params.ParameterFactory;
+import java.io.ByteArrayInputStream;
 import java.net.URL;
 import java.security.Security;
 import java.util.Map;
@@ -211,7 +213,7 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
           options.useDevelopmentServerCredential();
         } else {
           RemoteApiOptionsUtil.useGoogleCredentialStream(
-              options, component.googleCredentialStream().get());
+              options, new ByteArrayInputStream(component.googleCredentialJson().getBytes(UTF_8)));
         }
         installer.install(options);
       }

java/google/registry/tools/RegistryToolComponent.java

@@ -14,9 +14,9 @@
 
 package google.registry.tools;
 
-import com.google.common.base.Supplier;
 import dagger.Component;
 import google.registry.bigquery.BigqueryModule;
+import google.registry.config.CredentialModule.LocalCredentialJson;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.dns.writer.VoidDnsWriterModule;
 import google.registry.dns.writer.clouddns.CloudDnsWriterModule;
@@ -32,12 +32,10 @@ import google.registry.request.Modules.URLFetchServiceModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
-import google.registry.tools.AuthModule.LocalCredentialStream;
 import google.registry.util.AppEngineServiceUtilsImpl.AppEngineServiceUtilsModule;
 import google.registry.util.SystemClock.SystemClockModule;
 import google.registry.util.SystemSleeper.SystemSleeperModule;
 import google.registry.whois.WhoisModule;
-import java.io.InputStream;
 import javax.inject.Singleton;
 
 /**
@@ -112,7 +110,7 @@ interface RegistryToolComponent {
 
   AppEngineConnection appEngineConnection();
 
-  @LocalCredentialStream
-  Supplier<InputStream> googleCredentialStream();
+  @LocalCredentialJson
+  String googleCredentialJson();
 }
 

java/google/registry/tools/RequestFactoryModule.java

@@ -19,8 +19,8 @@ import com.google.api.client.http.HttpRequestFactory;
 import com.google.api.client.http.javanet.NetHttpTransport;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.config.CredentialModule.LocalCredential;
 import google.registry.config.RegistryConfig;
-import google.registry.tools.AuthModule.LocalCredential;
 
 /**
  * Module for providing the HttpRequestFactory.

javatests/google/registry/tools/AuthModuleTest.java

@@ -16,7 +16,6 @@ package google.registry.tools;
 
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.testing.JUnitBackports.assertThrows;
-import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -34,8 +33,6 @@ import com.google.common.collect.ImmutableList;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.io.Serializable;
 import java.util.Map;
 import org.junit.Before;
@@ -156,14 +153,10 @@ public class AuthModuleTest {
   }
 
   @Test
-  public void test_provideLocalCredentialStream() {
-    InputStream jsonStream =
-        AuthModule.provideLocalCredentialStream(getSecrets(), getCredential()).get();
+  public void test_provideLocalCredentialJson() {
+    String credentialJson = AuthModule.provideLocalCredentialJson(getSecrets(), getCredential());
     Map<String, String> jsonMap =
-        new Gson()
-            .fromJson(
-                new InputStreamReader(jsonStream, UTF_8),
-                new TypeToken<Map<String, String>>() {}.getType());
+        new Gson().fromJson(credentialJson, new TypeToken<Map<String, String>>() {}.getType());
     assertThat(jsonMap.get("type")).isEqualTo("authorized_user");
     assertThat(jsonMap.get("client_secret")).isEqualTo(CLIENT_SECRET);
     assertThat(jsonMap.get("client_id")).isEqualTo(CLIENT_ID);