From ece6ae1d780abc6726fb10d9d6acc1e6152f0e5f Mon Sep 17 00:00:00 2001 From: Shicong Huang Date: Thu, 13 Jun 2019 13:59:14 -0400 Subject: [PATCH] Add a CloudBuild task to deploy Beam pipeline --- .../beam/invoicing/InvoicingPipeline.java | 17 ++---- .../google/registry/tools/AuthModule.java | 25 +++++++++ release/cloudbuild-deploy-beam.yaml | 55 +++++++++++++++++++ 3 files changed, 84 insertions(+), 13 deletions(-) create mode 100644 release/cloudbuild-deploy-beam.yaml diff --git a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java index f439b5290..71c170981 100644 --- a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java +++ b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java @@ -14,17 +14,13 @@ package google.registry.beam.invoicing; -import static java.nio.charset.StandardCharsets.UTF_8; - import com.google.auth.oauth2.GoogleCredentials; import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey; import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder; -import google.registry.config.CredentialModule.LocalCredentialJson; import google.registry.config.RegistryConfig.Config; import google.registry.reporting.billing.BillingModule; import google.registry.reporting.billing.GenerateInvoicesAction; -import java.io.ByteArrayInputStream; -import java.io.IOException; +import google.registry.tools.AuthModule.LocalOAuth2Credentials; import java.io.Serializable; import javax.inject.Inject; import org.apache.beam.runners.dataflow.DataflowRunner; @@ -85,7 +81,8 @@ public class InvoicingPipeline implements Serializable { @Config("invoiceFilePrefix") String invoiceFilePrefix; - @Inject @LocalCredentialJson String credentialJson; + @Inject @LocalOAuth2Credentials + GoogleCredentials credentials; @Inject InvoicingPipeline() {} @@ -108,13 +105,7 @@ public class InvoicingPipeline implements Serializable { public void deploy() { // We can't store options as a member variable due to serialization concerns. InvoicingPipelineOptions options = PipelineOptionsFactory.as(InvoicingPipelineOptions.class); - try { - options.setGcpCredential( - GoogleCredentials.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8)))); - } catch (IOException e) { - throw new RuntimeException( - "Cannot obtain local credential to deploy the invoicing pipeline", e); - } + options.setGcpCredential(credentials); options.setProject(projectId); options.setRunner(DataflowRunner.class); // This causes p.run() to stage the pipeline as a template on GCS, as opposed to running it. diff --git a/core/src/main/java/google/registry/tools/AuthModule.java b/core/src/main/java/google/registry/tools/AuthModule.java index 61286a187..88b90dcb7 100644 --- a/core/src/main/java/google/registry/tools/AuthModule.java +++ b/core/src/main/java/google/registry/tools/AuthModule.java @@ -27,6 +27,7 @@ import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.util.store.AbstractDataStoreFactory; import com.google.api.client.util.store.FileDataStoreFactory; +import com.google.auth.oauth2.GoogleCredentials; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -43,6 +44,7 @@ import google.registry.config.RegistryConfig.Config; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -101,6 +103,23 @@ public class AuthModule { } } + @Provides + @LocalOAuth2Credentials + public static GoogleCredentials provideLocalOAuth2Credentials( + @LocalCredentialJson String credentialJson, + @Config("localCredentialOauthScopes") ImmutableList scopes) { + try { + GoogleCredentials credentials = + GoogleCredentials.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8))); + if (credentials.createScopedRequired()) { + credentials = credentials.createScoped(scopes); + } + return credentials; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + @Provides public static GoogleAuthorizationCodeFlow provideAuthorizationCodeFlow( JsonFactory jsonFactory, @@ -215,4 +234,10 @@ public class AuthModule { @Documented @Retention(RetentionPolicy.RUNTIME) @interface OAuthClientId {} + + /** Dagger qualifier for the local OAuth2 Credentials. */ + @Qualifier + @Documented + @Retention(RetentionPolicy.RUNTIME) + public @interface LocalOAuth2Credentials {} } diff --git a/release/cloudbuild-deploy-beam.yaml b/release/cloudbuild-deploy-beam.yaml new file mode 100644 index 000000000..aa84402d1 --- /dev/null +++ b/release/cloudbuild-deploy-beam.yaml @@ -0,0 +1,55 @@ +# To run the build locally, install cloud-build-local first. +# Then run: +# cloud-build-local --config=cloudbuild-deploy-beam.yaml --dryrun=false \ +# --substitutions=TAG_NAME=[TAG],_CREDENTIAL_JSON=[CREDENTIAL_JSON],\ +# _CREDENTIAL_KEYRING=[CREDENTIAL_KEYRING],_CREDENTIAL_KEY=[CREDENTIAL_KEY],\ +# _ENV=[ENV] . +# +# This will deploy Beam pipelines to GCS for the PROJECT_ID defined in gcloud +# tool. +# +# To manually trigger a build on GCB, run: +# gcloud builds submit --config=cloudbuild-deploy-beam.yaml --substitutions=TAG_NAME=[TAG],\ +# _CREDENTIAL_JSON=[CREDENTIAL_JSON],_CREDENTIAL_KEYRING=[CREDENTIAL_KEYRING],\ +# _CREDENTIAL_KEY=[CREDENTIAL_KEY],_ENV=[ENV] . +# +# To trigger a build automatically, follow the instructions below and add a trigger: +# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds +steps: +# Pull the latest nomulus.jar to local +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + args: + - gsutil + - cp + - gs://${PROJECT_ID}-deploy/${TAG_NAME}/nomulus.jar + - . +# Pull the credential for nomulus tool +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + args: + - gsutil + - cp + - gs://${PROJECT_ID}-deploy/nomulus-credential/${_CREDENTIAL_JSON} + - . +# Decrypt the credential +- name: 'gcr.io/cloud-builders/gcloud' + args: + - kms + - decrypt + - --ciphertext-file=${_CREDENTIAL_JSON} + - --plaintext-file=nomulus-credential.json + - --location=global + - --keyring=${_CREDENTIAL_KEYRING} + - --key=${_CREDENTIAL_KEY} +# Deploy spec11 and invoicing pipeline to GCS +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + entrypoint: /bin/bash + args: + - -c + - | + java -jar nomulus.jar -e ${_ENV} --credential nomulus-credential.json \ + deploy_spec11_pipeline + java -jar nomulus.jar -e ${_ENV} --credential nomulus-credential.json \ + deploy_invoicing_pipeline +timeout: 3600s +options: + machineType: 'N1_HIGHCPU_8'