diff --git a/java/google/registry/config/BUILD b/java/google/registry/config/BUILD index a1793bd8f..275a144d3 100644 --- a/java/google/registry/config/BUILD +++ b/java/google/registry/config/BUILD @@ -10,6 +10,7 @@ java_library( resources = glob(["files/*.yaml"]), deps = [ "//java/google/registry/util", + "@com_google_api_client", "@com_google_appengine_api_1_0_sdk", "@com_google_auto_value", "@com_google_code_findbugs_jsr305", diff --git a/java/google/registry/config/CredentialModule.java b/java/google/registry/config/CredentialModule.java new file mode 100644 index 000000000..a14aacc97 --- /dev/null +++ b/java/google/registry/config/CredentialModule.java @@ -0,0 +1,68 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.config; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.common.collect.ImmutableList; +import dagger.Module; +import dagger.Provides; +import google.registry.config.RegistryConfig.Config; +import java.io.IOException; +import javax.inject.Qualifier; +import javax.inject.Singleton; + +/** + * Dagger module that provides all {@link GoogleCredential GoogleCredentials} used in the + * application. + */ +@Module +public abstract class CredentialModule { + + /** Provides the default {@link GoogleCredential} from the Google Cloud runtime. */ + @DefaultCredential + @Provides + @Singleton + public static GoogleCredential provideDefaultCredential( + @Config("credentialOauthScopes") ImmutableList requiredScopes) { + GoogleCredential credential; + try { + credential = GoogleCredential.getApplicationDefault(); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (credential.createScopedRequired()) { + return credential.createScoped(requiredScopes); + } + return credential; + } + + /** Dagger qualifier for the Application Default Credential. */ + @Qualifier + public @interface DefaultCredential {} + + /** + * Dagger qualifier for a credential from a service account's JSON key, to be used in non-request + * threads. + */ + @Qualifier + public @interface JsonCredential {} + + /** + * Dagger qualifier for a credential with delegated admin access for a dasher domain (for G + * Suite). + */ + @Qualifier + public @interface DelegatedCredential {} +} diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index e6780253d..5d0fae842 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -1135,6 +1135,14 @@ public final class RegistryConfig { return ImmutableSet.copyOf(config.oAuth.allowedOauthClientIds); } + /** Provides the OAuth scopes required for accessing Google APIs. */ + @Provides + @Config("credentialOauthScopes") + public static ImmutableList provideCredentialOauthScopes( + RegistryConfigSettings config) { + return ImmutableList.copyOf(config.credentialOAuth.credentialOauthScopes); + } + /** * Returns the help path for the RDAP terms of service. * diff --git a/java/google/registry/config/RegistryConfigSettings.java b/java/google/registry/config/RegistryConfigSettings.java index 812c8af94..c2045b510 100644 --- a/java/google/registry/config/RegistryConfigSettings.java +++ b/java/google/registry/config/RegistryConfigSettings.java @@ -22,6 +22,7 @@ public class RegistryConfigSettings { public AppEngine appEngine; public GSuite gSuite; public OAuth oAuth; + public CredentialOAuth credentialOAuth; public RegistryPolicy registryPolicy; public Datastore datastore; public CloudDns cloudDns; @@ -48,13 +49,18 @@ public class RegistryConfigSettings { } } - /** Configuration options for OAuth settings. */ + /** Configuration options for OAuth settings for authenticating users. */ public static class OAuth { public List availableOauthScopes; public List requiredOauthScopes; public List allowedOauthClientIds; } + /** Configuration options for accessing Google APIs. */ + public static class CredentialOAuth { + public List credentialOauthScopes; + } + /** Configuration options for the G Suite account used by Nomulus. */ public static class GSuite { public String domainName; diff --git a/java/google/registry/config/files/default-config.yaml b/java/google/registry/config/files/default-config.yaml index dcdc7fa55..e0a2796f0 100644 --- a/java/google/registry/config/files/default-config.yaml +++ b/java/google/registry/config/files/default-config.yaml @@ -176,6 +176,21 @@ oAuth: # numbers-alphanumerics.apps.googleusercontent.com allowedOauthClientIds: [] +credentialOAuth: + # OAuth scopes required for accessing Google APIs. + credentialOauthScopes: + # View and manage data in all Google Cloud APIs. + - https://www.googleapis.com/auth/cloud-platform + # View and manage files in Google Drive. + - https://www.googleapis.com/auth/drive + # View and manage groups on your domain in Directory API. + - https://www.googleapis.com/auth/admin.directory.group + # Inherited from current code. + # TODO(weiminyu): verify if the scope above is sufficient by itself. + - https://www.googleapis.com/auth/admin.directory.group.member + # View and manage the settings of a Google Apps Group. + - https://www.googleapis.com/auth/apps.groups.settings + icannReporting: # URL we PUT monthly ICANN transactions reports to. icannTransactionsReportingUploadUrl: https://ry-api.icann.org/report/registrar-transactions diff --git a/java/google/registry/export/BUILD b/java/google/registry/export/BUILD index f2d7a4160..5ad934b5d 100644 --- a/java/google/registry/export/BUILD +++ b/java/google/registry/export/BUILD @@ -16,7 +16,6 @@ java_library( "//java/google/registry/mapreduce/inputs", "//java/google/registry/model", "//java/google/registry/request", - "//java/google/registry/request:modules", "//java/google/registry/request/auth", "//java/google/registry/storage/drive", "//java/google/registry/util", diff --git a/java/google/registry/export/DriveModule.java b/java/google/registry/export/DriveModule.java index 72889417a..984cbda0f 100644 --- a/java/google/registry/export/DriveModule.java +++ b/java/google/registry/export/DriveModule.java @@ -14,23 +14,16 @@ package google.registry.export; -import com.google.api.client.http.HttpRequestInitializer; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.json.JsonFactory; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.services.drive.Drive; -import com.google.api.services.drive.DriveScopes; import dagger.Component; import dagger.Module; import dagger.Provides; +import google.registry.config.CredentialModule; +import google.registry.config.CredentialModule.DefaultCredential; import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryConfig.ConfigModule; -import google.registry.request.Modules.AppIdentityCredentialModule; -import google.registry.request.Modules.Jackson2Module; -import google.registry.request.Modules.UrlFetchTransportModule; -import google.registry.request.Modules.UseAppIdentityCredentialForGoogleApisModule; import google.registry.storage.drive.DriveConnection; -import java.util.Set; -import java.util.function.Function; import javax.inject.Singleton; /** Dagger module for Google {@link Drive} service connection objects. */ @@ -39,25 +32,14 @@ public final class DriveModule { @Provides static Drive provideDrive( - HttpTransport transport, - JsonFactory jsonFactory, - Function, ? extends HttpRequestInitializer> credential, - @Config("projectId") String projectId) { - return new Drive.Builder(transport, jsonFactory, credential.apply(DriveScopes.all())) + @DefaultCredential GoogleCredential credential, @Config("projectId") String projectId) { + return new Drive.Builder(credential.getTransport(), credential.getJsonFactory(), credential) .setApplicationName(projectId) .build(); } @Singleton - @Component( - modules = { - DriveModule.class, - UrlFetchTransportModule.class, - Jackson2Module.class, - AppIdentityCredentialModule.class, - UseAppIdentityCredentialForGoogleApisModule.class, - ConfigModule.class - }) + @Component(modules = {DriveModule.class, ConfigModule.class, CredentialModule.class}) interface DriveComponent { DriveConnection driveConnection(); } diff --git a/java/google/registry/export/ExportDomainListsAction.java b/java/google/registry/export/ExportDomainListsAction.java index 43b7a2a19..c42afc04a 100644 --- a/java/google/registry/export/ExportDomainListsAction.java +++ b/java/google/registry/export/ExportDomainListsAction.java @@ -15,6 +15,7 @@ package google.registry.export; import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; +import static com.google.common.base.Verify.verifyNotNull; import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput; import static google.registry.model.EppResourceUtils.isActive; import static google.registry.model.registry.Registries.getTldsOfType; @@ -28,7 +29,9 @@ import com.google.appengine.tools.cloudstorage.RetryParams; import com.google.appengine.tools.mapreduce.Mapper; import com.google.appengine.tools.mapreduce.Reducer; import com.google.appengine.tools.mapreduce.ReducerInput; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.flogger.FluentLogger; @@ -45,9 +48,11 @@ import google.registry.request.auth.Auth; import google.registry.storage.drive.DriveConnection; import google.registry.util.NonFinalForTesting; import java.io.IOException; +import java.io.ObjectInputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; +import java.util.function.Supplier; import javax.inject.Inject; import org.joda.time.DateTime; @@ -108,9 +113,10 @@ public class ExportDomainListsAction implements Runnable { private static final long serialVersionUID = 7035260977259119087L; + /** Allows overriding the default {@link DriveConnection} in tests. */ @NonFinalForTesting - private static DriveConnection driveConnection = - DaggerDriveModule_DriveComponent.create().driveConnection(); + private static Supplier driveConnectionSupplier = + Suppliers.memoize(() -> DaggerDriveModule_DriveComponent.create().driveConnection()); static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt"; static final MediaType EXPORT_MIME_TYPE = MediaType.PLAIN_TEXT_UTF_8; @@ -118,16 +124,27 @@ public class ExportDomainListsAction implements Runnable { private final String gcsBucket; private final int gcsBufferSize; - static void setDriveConnectionForTesting(DriveConnection driveConnection) { - ExportDomainListsReducer.driveConnection = driveConnection; - } + /** + * Non-serializable {@link DriveConnection} that will be created when an instance of {@link + * ExportDomainListsReducer} is deserialized in a MR pipeline worker. + * + *

See {@link #readObject(ObjectInputStream)}. + */ + private transient DriveConnection driveConnection; public ExportDomainListsReducer(String gcsBucket, int gcsBufferSize) { this.gcsBucket = gcsBucket; this.gcsBufferSize = gcsBufferSize; } + @SuppressWarnings("unused") + private void readObject(ObjectInputStream is) throws IOException, ClassNotFoundException { + is.defaultReadObject(); + driveConnection = driveConnectionSupplier.get(); + } + private void exportToDrive(String tld, String domains) { + verifyNotNull(driveConnection, "expecting non-null driveConnection"); try { Registry registry = Registry.get(tld); if (registry.getDriveFolderId() == null) { @@ -174,5 +191,11 @@ public class ExportDomainListsAction implements Runnable { exportToGcs(tld, domainsList); exportToDrive(tld, domainsList); } + + @VisibleForTesting + static void setDriveConnectionForTesting( + Supplier testDriveConnectionSupplier) { + driveConnectionSupplier = testDriveConnectionSupplier; + } } } diff --git a/java/google/registry/keyring/kms/KmsModule.java b/java/google/registry/keyring/kms/KmsModule.java index 3e3291e66..1b63fff76 100644 --- a/java/google/registry/keyring/kms/KmsModule.java +++ b/java/google/registry/keyring/kms/KmsModule.java @@ -14,17 +14,13 @@ package google.registry.keyring.kms; -import com.google.api.client.http.HttpRequestInitializer; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.json.JsonFactory; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.services.cloudkms.v1.CloudKMS; -import com.google.api.services.cloudkms.v1.CloudKMSScopes; import dagger.Binds; import dagger.Module; import dagger.Provides; +import google.registry.config.CredentialModule.DefaultCredential; import google.registry.config.RegistryConfig.Config; -import java.util.Set; -import java.util.function.Function; /** Dagger module for Cloud KMS connection objects. */ @Module @@ -32,11 +28,9 @@ public abstract class KmsModule { @Provides static CloudKMS provideKms( - HttpTransport transport, - JsonFactory jsonFactory, - Function, ? extends HttpRequestInitializer> credential, + @DefaultCredential GoogleCredential credential, @Config("cloudKmsProjectId") String projectId) { - return new CloudKMS.Builder(transport, jsonFactory, credential.apply(CloudKMSScopes.all())) + return new CloudKMS.Builder(credential.getTransport(), credential.getJsonFactory(), credential) .setApplicationName(projectId) .build(); } diff --git a/java/google/registry/module/backend/BackendComponent.java b/java/google/registry/module/backend/BackendComponent.java index 5a715c964..e30d93401 100644 --- a/java/google/registry/module/backend/BackendComponent.java +++ b/java/google/registry/module/backend/BackendComponent.java @@ -18,6 +18,7 @@ import com.google.monitoring.metrics.MetricReporter; import dagger.Component; import dagger.Lazy; import google.registry.bigquery.BigqueryModule; +import google.registry.config.CredentialModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.export.DriveModule; import google.registry.export.sheet.SheetsServiceModule; @@ -55,6 +56,7 @@ import javax.inject.Singleton; BackendRequestComponentModule.class, BigqueryModule.class, ConfigModule.class, + CredentialModule.class, DatastoreServiceModule.class, DirectoryModule.class, google.registry.keyring.api.DummyKeyringModule.class, diff --git a/java/google/registry/module/frontend/FrontendComponent.java b/java/google/registry/module/frontend/FrontendComponent.java index 27a249dd4..263f4b6b7 100644 --- a/java/google/registry/module/frontend/FrontendComponent.java +++ b/java/google/registry/module/frontend/FrontendComponent.java @@ -17,6 +17,7 @@ package google.registry.module.frontend; import com.google.monitoring.metrics.MetricReporter; import dagger.Component; import dagger.Lazy; +import google.registry.config.CredentialModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.flows.ServerTridProviderModule; import google.registry.flows.custom.CustomLogicFactoryModule; @@ -47,6 +48,7 @@ import javax.inject.Singleton; AuthModule.class, ConfigModule.class, ConsoleConfigModule.class, + CredentialModule.class, CustomLogicFactoryModule.class, google.registry.keyring.api.DummyKeyringModule.class, FrontendRequestComponentModule.class, diff --git a/java/google/registry/module/pubapi/PubApiComponent.java b/java/google/registry/module/pubapi/PubApiComponent.java index 0240870d4..da606f81b 100644 --- a/java/google/registry/module/pubapi/PubApiComponent.java +++ b/java/google/registry/module/pubapi/PubApiComponent.java @@ -17,6 +17,7 @@ package google.registry.module.pubapi; import com.google.monitoring.metrics.MetricReporter; import dagger.Component; import dagger.Lazy; +import google.registry.config.CredentialModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.flows.ServerTridProviderModule; import google.registry.flows.custom.CustomLogicFactoryModule; @@ -45,6 +46,7 @@ import javax.inject.Singleton; AppIdentityCredentialModule.class, AuthModule.class, ConfigModule.class, + CredentialModule.class, CustomLogicFactoryModule.class, google.registry.keyring.api.DummyKeyringModule.class, PubApiRequestComponentModule.class, diff --git a/java/google/registry/module/tools/ToolsComponent.java b/java/google/registry/module/tools/ToolsComponent.java index 68dfd2aca..26495260a 100644 --- a/java/google/registry/module/tools/ToolsComponent.java +++ b/java/google/registry/module/tools/ToolsComponent.java @@ -15,6 +15,7 @@ package google.registry.module.tools; import dagger.Component; +import google.registry.config.CredentialModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.export.DriveModule; import google.registry.flows.ServerTridProviderModule; @@ -48,6 +49,7 @@ import javax.inject.Singleton; AppIdentityCredentialModule.class, AuthModule.class, ConfigModule.class, + CredentialModule.class, CustomLogicFactoryModule.class, DatastoreServiceModule.class, DirectoryModule.class, diff --git a/java/google/registry/tools/RegistryToolComponent.java b/java/google/registry/tools/RegistryToolComponent.java index 7cff301a8..b25a9cf0b 100644 --- a/java/google/registry/tools/RegistryToolComponent.java +++ b/java/google/registry/tools/RegistryToolComponent.java @@ -15,6 +15,7 @@ package google.registry.tools; import dagger.Component; +import google.registry.config.CredentialModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.dns.writer.VoidDnsWriterModule; import google.registry.dns.writer.clouddns.CloudDnsWriterModule; @@ -51,6 +52,7 @@ import javax.inject.Singleton; AppIdentityCredentialModule.class, AuthModule.class, ConfigModule.class, + CredentialModule.class, DatastoreServiceModule.class, google.registry.keyring.api.DummyKeyringModule.class, CloudDnsWriterModule.class, diff --git a/javatests/google/registry/export/ExportDomainListsActionTest.java b/javatests/google/registry/export/ExportDomainListsActionTest.java index e08ba14cd..ab4bb49a3 100644 --- a/javatests/google/registry/export/ExportDomainListsActionTest.java +++ b/javatests/google/registry/export/ExportDomainListsActionTest.java @@ -64,7 +64,7 @@ public class ExportDomainListsActionTest extends MapreduceTestCase driveConnection); action = new ExportDomainListsAction(); action.mrRunner = makeDefaultRunner();