diff --git a/gradle/core/build.gradle b/gradle/core/build.gradle index ad7045cee..79d0cd66e 100644 --- a/gradle/core/build.gradle +++ b/gradle/core/build.gradle @@ -79,17 +79,18 @@ dependencies { testImplementation project(':third_party') compile 'com.beust:jcommander:1.48' - maybeRuntime 'com.fasterxml.jackson.core:jackson-core:2.8.5' + maybeRuntime 'com.fasterxml.jackson.core:jackson-core:2.9.6' maybeRuntime 'com.fasterxml.jackson.core:jackson-annotations:2.8.0' maybeRuntime 'com.fasterxml.jackson.core:jackson-databind:2.8.5' compile 'com.google.api-client:google-api-client:1.22.0' - maybeRuntime 'com.google.api-client:google-api-client-appengine:1.22.0' + compile 'com.google.api-client:google-api-client-appengine:1.22.0' maybeRuntime 'com.google.api-client:google-api-client-jackson2:1.20.0' compile 'com.google.monitoring-client:metrics:1.0.4' compile 'com.google.monitoring-client:stackdriver:1.0.4' maybeRuntime 'com.google.api-client:google-api-client-java6:1.20.0' maybeRuntime 'com.google.api-client:google-api-client-servlet:1.22.0' compile 'com.google.apis:google-api-services-admin-directory:directory_v1-rev72-1.22.0' + compile 'com.google.apis:google-api-services-appengine:v1-rev85-1.25.0' compile 'com.google.apis:google-api-services-bigquery:v2-rev325-1.22.0' maybeRuntime 'com.google.apis:google-api-services-clouddebugger:v2-rev8-1.22.0' compile 'com.google.apis:google-api-services-cloudkms:v1-rev12-1.22.0' @@ -132,10 +133,10 @@ dependencies { gradleLint.ignore('unused-dependency') { compile 'com.google.gwt:gwt-user:2.8.2' } - compile 'com.google.http-client:google-http-client:1.22.0' + compile 'com.google.http-client:google-http-client:1.25.0' compile 'com.google.http-client:google-http-client-appengine:1.22.0' - compile 'com.google.http-client:google-http-client-jackson2:1.22.0' - compile 'com.google.oauth-client:google-oauth-client:1.22.0' + compile 'com.google.http-client:google-http-client-jackson2:1.25.0' + compile 'com.google.oauth-client:google-oauth-client:1.25.0' maybeRuntime 'com.google.oauth-client:google-oauth-client-appengine:1.22.0' compile 'com.google.oauth-client:google-oauth-client-java6:1.22.0' compile 'com.google.oauth-client:google-oauth-client-jetty:1.22.0' @@ -151,8 +152,8 @@ dependencies { maybeRuntime 'com.squareup:javawriter:2.5.1' maybeRuntime 'com.sun.activation:javax.activation:1.2.0' maybeRuntime 'com.thoughtworks.paranamer:paranamer:2.7' - maybeRuntime 'commons-codec:commons-codec:1.6' - maybeRuntime 'commons-logging:commons-logging:1.1.1' + maybeRuntime 'commons-codec:commons-codec:1.10' + maybeRuntime 'commons-logging:commons-logging:1.2' compile 'dnsjava:dnsjava:2.1.7' maybeRuntime 'io.netty:netty-buffer:4.1.28.Final' maybeRuntime 'io.netty:netty-codec:4.1.28.Final' diff --git a/java/google/registry/config/BUILD b/java/google/registry/config/BUILD index d9bd096eb..c0c22deac 100644 --- a/java/google/registry/config/BUILD +++ b/java/google/registry/config/BUILD @@ -12,6 +12,7 @@ java_library( "//java/google/registry/keyring/api", "//java/google/registry/util", "@com_google_api_client", + "@com_google_api_client_appengine", "@com_google_appengine_api_1_0_sdk", "@com_google_auto_value", "@com_google_code_findbugs_jsr305", diff --git a/java/google/registry/config/CredentialModule.java b/java/google/registry/config/CredentialModule.java index 446280be3..c20911869 100644 --- a/java/google/registry/config/CredentialModule.java +++ b/java/google/registry/config/CredentialModule.java @@ -17,6 +17,7 @@ package google.registry.config; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.googleapis.util.Utils; import com.google.common.collect.ImmutableList; @@ -107,6 +108,20 @@ public abstract class CredentialModule { .build(); } + /** + * Provides a {@link AppIdentityCredential} with access for App Engine Admin API. + * + *

{@link AppIdentityCredential} is an OAuth 2.0 credential in which a client Google App Engine + * application needs to access data that it owns. + */ + @AppEngineAdminApiCredential + @Provides + @Singleton + public static AppIdentityCredential provideAppEngineAdminApiCredential( + @Config("appEngineAdminApiCredentialOauthScopes") ImmutableList requiredScopes) { + return new AppIdentityCredential(requiredScopes); + } + /** Dagger qualifier for the Application Default Credential. */ @Qualifier public @interface DefaultCredential {} @@ -124,4 +139,8 @@ public abstract class CredentialModule { */ @Qualifier public @interface DelegatedCredential {} + + /** Dagger qualifier for a credential with access for App Engine Admin API. */ + @Qualifier + public @interface AppEngineAdminApiCredential {} } diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index f2fe2d2ce..851d24840 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -1229,6 +1229,14 @@ public final class RegistryConfig { return ImmutableList.copyOf(config.credentialOAuth.delegatedCredentialOauthScopes); } + /** Provides the OAuth scopes required for access to App Engine Admin API. */ + @Provides + @Config("appEngineAdminApiCredentialOauthScopes") + public static ImmutableList provideAppEngineAdminApiCredentialOauthScopes( + RegistryConfigSettings config) { + return ImmutableList.copyOf(config.credentialOAuth.appEngineAdminApiCredentialOauthScopes); + } + /** * Returns the help path for the RDAP terms of service. * diff --git a/java/google/registry/config/RegistryConfigSettings.java b/java/google/registry/config/RegistryConfigSettings.java index 5a39013e1..ab1b2c790 100644 --- a/java/google/registry/config/RegistryConfigSettings.java +++ b/java/google/registry/config/RegistryConfigSettings.java @@ -58,6 +58,7 @@ public class RegistryConfigSettings { public static class CredentialOAuth { public List defaultCredentialOauthScopes; public List delegatedCredentialOauthScopes; + public List appEngineAdminApiCredentialOauthScopes; } /** Configuration options for the G Suite account used by Nomulus. */ diff --git a/java/google/registry/config/files/default-config.yaml b/java/google/registry/config/files/default-config.yaml index 2e329cad7..e8e700484 100644 --- a/java/google/registry/config/files/default-config.yaml +++ b/java/google/registry/config/files/default-config.yaml @@ -282,6 +282,11 @@ credentialOAuth: - https://www.googleapis.com/auth/admin.directory.group # View and manage group settings in Group Settings API. - https://www.googleapis.com/auth/apps.groups.settings + # OAuth scopes required for accessing App Engine Admin API using the + # AppIdentityCredential. + appEngineAdminApiCredentialOauthScopes: + # View and manage your applications deployed on Google App Engine + - https://www.googleapis.com/auth/appengine.admin icannReporting: diff --git a/java/google/registry/repositories.bzl b/java/google/registry/repositories.bzl index d05fb830a..401631c51 100644 --- a/java/google/registry/repositories.bzl +++ b/java/google/registry/repositories.bzl @@ -31,6 +31,7 @@ def domain_registry_repositories( omit_com_google_api_client_jackson2 = False, omit_com_google_api_client_java6 = False, omit_com_google_api_client_servlet = False, + omit_com_google_apis_google_api_services_appengine = False, omit_com_google_apis_google_api_services_admin_directory = False, omit_com_google_apis_google_api_services_bigquery = False, omit_com_google_apis_google_api_services_clouddebugger = False, @@ -187,6 +188,8 @@ def domain_registry_repositories( com_google_api_client_servlet() if not omit_com_google_apis_google_api_services_admin_directory: com_google_apis_google_api_services_admin_directory() + if not omit_com_google_apis_google_api_services_appengine: + com_google_apis_google_api_services_appengine() if not omit_com_google_apis_google_api_services_bigquery: com_google_apis_google_api_services_bigquery() if not omit_com_google_apis_google_api_services_clouddebugger: @@ -466,12 +469,12 @@ def com_beust_jcommander(): def com_fasterxml_jackson_core(): java_import_external( name = "com_fasterxml_jackson_core", - jar_sha256 = "85b48d80d0ff36eecdc61ab57fe211a266b9fc326d5e172764d150e29fc99e21", - jar_urls = [ - "http://maven.ibiblio.org/maven2/com/fasterxml/jackson/core/jackson-core/2.8.5/jackson-core-2.8.5.jar", - "http://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.8.5/jackson-core-2.8.5.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "fab8746aedd6427788ee390ea04d438ec141bff7eb3476f8bdd5d9110fb2718a", + jar_urls = [ + "http://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.9.6/jackson-core-2.9.6.jar", + "http://maven.ibiblio.org/maven2/com/fasterxml/jackson/core/jackson-core/2.9.6/jackson-core-2.9.6.jar", + ], ) def com_fasterxml_jackson_core_jackson_annotations(): @@ -503,12 +506,12 @@ def com_fasterxml_jackson_core_jackson_databind(): def com_google_api_client(): java_import_external( name = "com_google_api_client", - jar_sha256 = "47c625c83a8cf97b8bbdff2acde923ff8fd3174e62aabcfc5d1b86692594ffba", - jar_urls = [ - "http://maven.ibiblio.org/maven2/com/google/api-client/google-api-client/1.22.0/google-api-client-1.22.0.jar", - "http://repo1.maven.org/maven2/com/google/api-client/google-api-client/1.22.0/google-api-client-1.22.0.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "24e1a69d6c04e6e72e3e16757d46d32daa7dd43cb32c3895f832f25358be1402", + jar_urls = [ + "http://maven.ibiblio.org/maven2/com/google/api-client/google-api-client/1.25.0/google-api-client-1.25.0.jar", + "http://repo1.maven.org/maven2/com/google/api-client/google-api-client/1.25.0/google-api-client-1.25.0.jar", + ], deps = [ "@com_google_oauth_client", "@com_google_http_client_jackson2", @@ -642,6 +645,18 @@ def com_google_apis_google_api_services_admin_directory(): deps = ["@com_google_api_client"], ) +def com_google_apis_google_api_services_appengine(): + java_import_external( + name = "com_google_apis_google_api_services_appengine", + licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "15329545770163aec4f2bb0c37949a03667f06e012e2204ede22a0c2fb8f9f21", + jar_urls = [ + "http://repo1.maven.org/maven2/com/google/apis/google-api-services-appengine/v1-rev85-1.25.0/google-api-services-appengine-v1-rev85-1.25.0.jar", + "http://maven.ibiblio.org/maven2/com/google/apis/google-api-services-appengine/v1-rev85-1.25.0/google-api-services-appengine-v1-rev85-1.25.0.jar", + ], + deps = ["@com_google_api_client"], + ) + def com_google_apis_google_api_services_bigquery(): java_import_external( name = "com_google_apis_google_api_services_bigquery", @@ -1377,12 +1392,12 @@ def com_google_gwt_user(): def com_google_http_client(): java_import_external( name = "com_google_http_client", - jar_sha256 = "f88ffa329ac52fb4f2ff0eb877ef7318423ac9b791a107f886ed5c7a00e77e11", - jar_urls = [ - "http://maven.ibiblio.org/maven2/com/google/http-client/google-http-client/1.22.0/google-http-client-1.22.0.jar", - "http://repo1.maven.org/maven2/com/google/http-client/google-http-client/1.22.0/google-http-client-1.22.0.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "fb7d80a515da4618e2b402e1fef96999e07621b381a5889ef091482c5a3e961d", + jar_urls = [ + "http://repo1.maven.org/maven2/com/google/http-client/google-http-client/1.25.0/google-http-client-1.25.0.jar", + "http://maven.ibiblio.org/maven2/com/google/http-client/google-http-client/1.25.0/google-http-client-1.25.0.jar", + ], deps = [ "@com_google_code_findbugs_jsr305", "@com_google_guava", @@ -1409,12 +1424,12 @@ def com_google_http_client_appengine(): def com_google_http_client_jackson2(): java_import_external( name = "com_google_http_client_jackson2", - jar_sha256 = "45b1e34b2dcef5cb496ef25a1223d19cf102b8c2ea4abf96491631b2faf4611c", - jar_urls = [ - "http://maven.ibiblio.org/maven2/com/google/http-client/google-http-client-jackson2/1.22.0/google-http-client-jackson2-1.22.0.jar", - "http://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2/1.22.0/google-http-client-jackson2-1.22.0.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "f9e7e0d318860a2092d70b56331976280c4e9348a065ede3b99c92aa032fd853", + jar_urls = [ + "http://maven.ibiblio.org/maven2/com/google/http-client/google-http-client-jackson2/1.25.0/google-http-client-jackson2-1.25.0.jar", + "http://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2/1.25.0/google-http-client-jackson2-1.25.0.jar", + ], deps = [ "@com_google_http_client", "@com_fasterxml_jackson_core", @@ -1424,12 +1439,12 @@ def com_google_http_client_jackson2(): def com_google_oauth_client(): java_import_external( name = "com_google_oauth_client", - jar_sha256 = "a4c56168b3e042105d68cf136e40e74f6e27f63ed0a948df966b332678e19022", - jar_urls = [ - "http://maven.ibiblio.org/maven2/com/google/oauth-client/google-oauth-client/1.22.0/google-oauth-client-1.22.0.jar", - "http://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client/1.22.0/google-oauth-client-1.22.0.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "7e2929133d4231e702b5956a7e5dc8347a352acc1e97082b40c3585b81cd3501", + jar_urls = [ + "http://maven.ibiblio.org/maven2/com/google/oauth-client/google-oauth-client/1.25.0/google-oauth-client-1.25.0.jar", + "http://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client/1.25.0/google-oauth-client-1.25.0.jar", + ], deps = [ "@com_google_http_client", "@com_google_code_findbugs_jsr305", @@ -1747,23 +1762,23 @@ def com_thoughtworks_paranamer(): def commons_codec(): java_import_external( name = "commons_codec", - jar_sha256 = "54b34e941b8e1414bd3e40d736efd3481772dc26db3296f6aa45cec9f6203d86", + licenses = ["notice"], # Apache License, Version 2.0 + jar_sha256 = "4241dfa94e711d435f29a4604a3e2de5c4aa3c165e23bd066be6fc1fc4309569", jar_urls = [ - "http://maven.ibiblio.org/maven2/commons-codec/commons-codec/1.6/commons-codec-1.6.jar", - "http://repo1.maven.org/maven2/commons-codec/commons-codec/1.6/commons-codec-1.6.jar", + "http://repo1.maven.org/maven2/commons-codec/commons-codec/1.10/commons-codec-1.10.jar", + "http://maven.ibiblio.org/maven2/commons-codec/commons-codec/1.10/commons-codec-1.10.jar", ], - licenses = ["notice"], # The Apache Software License, Version 2.0 ) def commons_logging(): java_import_external( name = "commons_logging", - jar_sha256 = "ce6f913cad1f0db3aad70186d65c5bc7ffcc9a99e3fe8e0b137312819f7c362f", - jar_urls = [ - "http://maven.ibiblio.org/maven2/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar", - "http://repo1.maven.org/maven2/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "daddea1ea0be0f56978ab3006b8ac92834afeefbd9b7e4e6316fca57df0fa636", + jar_urls = [ + "http://repo1.maven.org/maven2/commons-logging/commons-logging/1.2/commons-logging-1.2.jar", + "http://maven.ibiblio.org/maven2/commons-logging/commons-logging/1.2/commons-logging-1.2.jar", + ], ) def dnsjava(): @@ -2212,12 +2227,12 @@ def org_apache_ftpserver_core(): def org_apache_httpcomponents_httpclient(): java_import_external( name = "org_apache_httpcomponents_httpclient", - jar_sha256 = "0dffc621400d6c632f55787d996b8aeca36b30746a716e079a985f24d8074057", + licenses = ["notice"], # Apache License, Version 2.0 + jar_sha256 = "7e97724443ad2a25ad8c73183431d47cc7946271bcbbdfa91a8a17522a566573", jar_urls = [ - "http://maven.ibiblio.org/maven2/org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar", - "http://repo1.maven.org/maven2/org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar", + "http://repo1.maven.org/maven2/org/apache/httpcomponents/httpclient/4.5.5/httpclient-4.5.5.jar", + "http://maven.ibiblio.org/maven2/org/apache/httpcomponents/httpclient/4.5.5/httpclient-4.5.5.jar", ], - licenses = ["notice"], # Apache License deps = [ "@org_apache_httpcomponents_httpcore", "@commons_logging", @@ -2228,12 +2243,12 @@ def org_apache_httpcomponents_httpclient(): def org_apache_httpcomponents_httpcore(): java_import_external( name = "org_apache_httpcomponents_httpcore", - jar_sha256 = "f7bc09dc8a7003822d109634ffd3845d579d12e725ae54673e323a7ce7f5e325", + licenses = ["notice"], # Apache License, Version 2.0 + jar_sha256 = "1b4a1c0b9b4222eda70108d3c6e2befd4a6be3d9f78ff53dd7a94966fdf51fc5", jar_urls = [ - "http://maven.ibiblio.org/maven2/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar", - "http://repo1.maven.org/maven2/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar", + "http://repo1.maven.org/maven2/org/apache/httpcomponents/httpcore/4.4.9/httpcore-4.4.9.jar", + "http://maven.ibiblio.org/maven2/org/apache/httpcomponents/httpcore/4.4.9/httpcore-4.4.9.jar", ], - licenses = ["notice"], # Apache License ) def org_apache_mina_core(): @@ -2570,7 +2585,7 @@ def org_yaml_snakeyaml(): def xerces_xmlParserAPIs(): java_import_external( name = "xerces_xmlParserAPIs", - licenses = ["TODO"], # NO LICENSE DECLARED + licenses = ["notice"], # Apache License, Version 2.0 jar_sha256 = "1c2867be1faa73c67e9232631241eb1df4cd0763048646e7bb575a9980e9d7e5", jar_urls = [ "http://repo1.maven.org/maven2/xerces/xmlParserAPIs/2.6.2/xmlParserAPIs-2.6.2.jar", @@ -2587,7 +2602,7 @@ def xpp3(): # http://creativecommons.org/licenses/publicdomain # Apache Software License, version 1.1 # http://www.apache.org/licenses/LICENSE-1.1 - licenses = ["TODO"], + licenses = ["notice"], jar_sha256 = "0341395a481bb887803957145a6a37879853dd625e9244c2ea2509d9bb7531b9", jar_urls = [ "http://maven.ibiblio.org/maven2/xpp3/xpp3/1.1.4c/xpp3-1.1.4c.jar", diff --git a/java/google/registry/tools/AppEngineAdminApiModule.java b/java/google/registry/tools/AppEngineAdminApiModule.java new file mode 100644 index 000000000..8a67a7365 --- /dev/null +++ b/java/google/registry/tools/AppEngineAdminApiModule.java @@ -0,0 +1,40 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential; +import com.google.api.client.googleapis.util.Utils; +import com.google.api.services.appengine.v1.Appengine; +import dagger.Module; +import dagger.Provides; +import google.registry.config.CredentialModule.AppEngineAdminApiCredential; +import google.registry.config.RegistryConfig.Config; +import javax.inject.Singleton; + +/** Module providing the instance of {@link Appengine} to access App Engine Admin Api. */ +@Module +public abstract class AppEngineAdminApiModule { + + @Provides + @Singleton + public static Appengine provideAppengine( + @AppEngineAdminApiCredential AppIdentityCredential appIdentityCredential, + @Config("projectId") String projectId) { + return new Appengine.Builder( + Utils.getDefaultTransport(), Utils.getDefaultJsonFactory(), appIdentityCredential) + .setApplicationName(projectId) + .build(); + } +} diff --git a/java/google/registry/tools/AppEngineConnection.java b/java/google/registry/tools/AppEngineConnection.java index 68ab47ddb..a98e847cb 100644 --- a/java/google/registry/tools/AppEngineConnection.java +++ b/java/google/registry/tools/AppEngineConnection.java @@ -66,10 +66,21 @@ class AppEngineConnection { } enum Service { - DEFAULT, - TOOLS, - BACKEND, - PUBAPI + DEFAULT("default"), + TOOLS("tools"), + BACKEND("backend"), + PUBAPI("pubapi"); + + private final String serviceId; + + Service(String serviceId) { + this.serviceId = serviceId; + } + + /** Returns the actual service id in App Engine. */ + String getServiceId() { + return serviceId; + } } /** Returns a copy of this connection that talks to a different service. */ diff --git a/java/google/registry/tools/BUILD b/java/google/registry/tools/BUILD index d4671b9b9..22708ee0d 100644 --- a/java/google/registry/tools/BUILD +++ b/java/google/registry/tools/BUILD @@ -68,6 +68,8 @@ java_library( "//third_party/objectify:objectify-v4_1", "@com_beust_jcommander", "@com_google_api_client", + "@com_google_api_client_appengine", + "@com_google_apis_google_api_services_appengine", "@com_google_apis_google_api_services_bigquery", "@com_google_apis_google_api_services_dns", "@com_google_appengine_api_1_0_sdk", diff --git a/java/google/registry/tools/RegistryToolComponent.java b/java/google/registry/tools/RegistryToolComponent.java index 3b56dd4a9..44d728fd8 100644 --- a/java/google/registry/tools/RegistryToolComponent.java +++ b/java/google/registry/tools/RegistryToolComponent.java @@ -46,6 +46,7 @@ import javax.inject.Singleton; @Singleton @Component( modules = { + AppEngineAdminApiModule.class, AppEngineServiceUtilsModule.class, // TODO(b/36866706): Find a way to replace this with a command-line friendly version AuthModule.class, diff --git a/java/google/registry/tools/SetNumInstancesCommand.java b/java/google/registry/tools/SetNumInstancesCommand.java index 9e2ef091d..5df9c2f62 100644 --- a/java/google/registry/tools/SetNumInstancesCommand.java +++ b/java/google/registry/tools/SetNumInstancesCommand.java @@ -15,11 +15,28 @@ package google.registry.tools; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import com.google.api.services.appengine.v1.Appengine; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.common.collect.MultimapBuilder; +import com.google.common.collect.Multimaps; +import com.google.common.collect.Sets; import com.google.common.flogger.FluentLogger; +import google.registry.config.RegistryConfig.Config; +import google.registry.tools.AppEngineConnection.Service; import google.registry.util.AppEngineServiceUtils; +import java.io.IOException; +import java.util.AbstractMap.SimpleEntry; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; import javax.inject.Inject; /** A command to set the number of instances for an App Engine service. */ @@ -32,35 +49,147 @@ final class SetNumInstancesCommand implements CommandWithRemoteApi { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final ImmutableSet ALL_VALID_SERVICES = + Arrays.stream(Service.values()).map(Service::name).collect(toImmutableSet()); + + private static final ImmutableSet ALL_DEPLOYED_SERVICE_IDS = + Arrays.stream(Service.values()).map(Service::getServiceId).collect(toImmutableSet()); + + // TODO(b/119629679): Use List after upgrading jcommander to latest version. @Parameter( - names = "--service", - description = "Name of the App Engine service, e.g., default.", - required = true) - private String service; + names = "--services", + description = + "Comma-delimited list of App Engine services to set. " + + "Allowed values: [DEFAULT, TOOLS, BACKEND, PUBAPI]") + private List services = ImmutableList.of(); @Parameter( - names = "--version", - description = "Name of the service's version, e.g., canary.", - required = true) - private String version; + names = "--versions", + description = + "Comma-delimited list of App Engine versions to set, e.g., canary. " + + "Cannot be set if --non_live_versions is set.") + private List versions = ImmutableList.of(); @Parameter( - names = "--numInstances", - description = "The new number of instances for the version.", + names = "--num_instances", + description = + "The new number of instances for the given versions " + + "or for all non-live versions if --non_live_versions is set.", required = true) private Long numInstances; + @Parameter( + names = "--non_live_versions", + description = "Whether to set number of instances for all non-live versions.", + arity = 1) + private Boolean nonLiveVersions = false; + @Inject AppEngineServiceUtils appEngineServiceUtils; + @Inject Appengine appengine; + + @Inject + @Config("projectId") + String projectId; @Override public void run() throws Exception { - checkArgument(!service.isEmpty(), "Service must be specified"); - checkArgument(!version.isEmpty(), "Version must be specified"); - checkArgument(numInstances > 0, "Number of instances must be greater than zero"); + Set invalidServiceIds = + Sets.difference(ImmutableSet.copyOf(services), ALL_VALID_SERVICES); + checkArgument(invalidServiceIds.isEmpty(), "Invalid service(s): %s", invalidServiceIds); + Set serviceIds = + services.stream() + .map(service -> Service.valueOf(service).getServiceId()) + .collect(toImmutableSet()); + + if (nonLiveVersions) { + checkArgument(versions.isEmpty(), "--versions cannot be set if --non_live_versions is set"); + + serviceIds = serviceIds.isEmpty() ? ALL_DEPLOYED_SERVICE_IDS : serviceIds; + Multimap allLiveVersionsMap = getAllLiveVersionsMap(serviceIds); + Multimap manualScalingVersionsMap = getManualScalingVersionsMap(serviceIds); + + // Set number of instances for versions which are manual scaling and non-live + manualScalingVersionsMap.forEach( + (serviceId, versionId) -> { + if (!allLiveVersionsMap.containsEntry(serviceId, versionId)) { + setNumInstances(serviceId, versionId, numInstances); + } + }); + } else { + checkArgument(!serviceIds.isEmpty(), "Service must be specified"); + checkArgument(!versions.isEmpty(), "Version must be specified"); + checkArgument(numInstances > 0, "Number of instances must be greater than zero"); + + Multimap manualScalingVersionsMap = getManualScalingVersionsMap(serviceIds); + + for (String serviceId : serviceIds) { + for (String versionId : versions) { + checkArgument( + manualScalingVersionsMap.containsEntry(serviceId, versionId), + "Version %s of service %s is not managed through manual scaling", + versionId, + serviceId); + setNumInstances(serviceId, versionId, numInstances); + } + } + } + } + + private void setNumInstances(String service, String version, long numInstances) { appEngineServiceUtils.setNumInstances(service, version, numInstances); logger.atInfo().log( "Successfully set version %s of service %s to %d instances.", version, service, numInstances); } + + private Multimap getAllLiveVersionsMap(Set services) { + try { + return Stream.of(appengine.apps().services().list(projectId).execute().getServices()) + .flatMap(Collection::stream) + .filter(service -> services.contains(service.getId())) + .flatMap( + service -> + // getAllocations returns only live versions or null + Stream.of(service.getSplit().getAllocations()) + .flatMap( + allocations -> + allocations.keySet().stream() + .map(versionId -> new SimpleEntry<>(service.getId(), versionId)))) + .collect( + Multimaps.toMultimap( + SimpleEntry::getKey, + SimpleEntry::getValue, + MultimapBuilder.treeKeys().arrayListValues()::build)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private Multimap getManualScalingVersionsMap(Set services) { + return services.stream() + .flatMap( + serviceId -> { + try { + return Stream.of( + appengine + .apps() + .services() + .versions() + .list(projectId, serviceId) + .execute() + .getVersions()) + .flatMap(Collection::stream) + .filter(version -> version.getManualScaling() != null) + .map(version -> new SimpleEntry<>(serviceId, version.getId())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect( + Multimaps.toMultimap( + SimpleEntry::getKey, + SimpleEntry::getValue, + MultimapBuilder.treeKeys().arrayListValues()::build)); + } } diff --git a/javatests/google/registry/testing/AppEngineAdminApiHelper.java b/javatests/google/registry/testing/AppEngineAdminApiHelper.java new file mode 100644 index 000000000..e4e767b45 --- /dev/null +++ b/javatests/google/registry/testing/AppEngineAdminApiHelper.java @@ -0,0 +1,146 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.testing; + +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.api.services.appengine.v1.Appengine; +import com.google.api.services.appengine.v1.model.ListServicesResponse; +import com.google.api.services.appengine.v1.model.ListVersionsResponse; +import com.google.api.services.appengine.v1.model.ManualScaling; +import com.google.api.services.appengine.v1.model.Service; +import com.google.api.services.appengine.v1.model.TrafficSplit; +import com.google.api.services.appengine.v1.model.Version; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.Multimap; +import google.registry.model.Buildable; +import google.registry.model.ImmutableObject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.stream.Collectors; + +/** Helper class to provide a builder to construct {@link Appengine} object for testing. */ +public class AppEngineAdminApiHelper extends ImmutableObject implements Buildable { + + private Appengine appengine; + private String appId = "domain-registry-test"; + private Multimap liveVersionsMap = ImmutableMultimap.of(); + private Multimap manualScalingVersionsMap = ImmutableMultimap.of(); + + /** Returns the {@link Appengine} object. */ + public Appengine getAppengine() { + return appengine; + } + + @Override + public Builder asBuilder() { + return new Builder(clone(this)); + } + + /** A builder for constructing {@link Appengine} object, since it is immutable. */ + public static class Builder extends Buildable.Builder { + public Builder() {} + + private Builder(AppEngineAdminApiHelper instance) { + super(instance); + } + + public Builder setAppId(String appId) { + getInstance().appId = appId; + return this; + } + + public Builder setLiveVersionsMap(Multimap liveVersionsMap) { + getInstance().liveVersionsMap = liveVersionsMap; + return this; + } + + public Builder setManualScalingVersionsMap(Multimap manualScalingVersionsMap) { + getInstance().manualScalingVersionsMap = manualScalingVersionsMap; + return this; + } + + @Override + public AppEngineAdminApiHelper build() { + getInstance().appengine = mock(Appengine.class, RETURNS_DEEP_STUBS); + + // Mockito cannot mock ListServicesResponse as it is a final class + ListServicesResponse listServicesResponse = new ListServicesResponse(); + try { + when((Object) getInstance().appengine.apps().services().list(getInstance().appId).execute()) + .thenReturn(listServicesResponse); + } catch (IOException e) { + throw new RuntimeException(e); + } + + // Add all given live versions to mocked Appengine object + java.util.List serviceList = new ArrayList<>(); + getInstance() + .liveVersionsMap + .asMap() + .forEach( + (serviceId, versionList) -> { + Service service = new Service(); + TrafficSplit trafficSplit = new TrafficSplit(); + trafficSplit.setAllocations( + versionList.stream() + .collect(Collectors.toMap(version -> version, version -> 1.0))); + + service.setId(serviceId); + service.setSplit(trafficSplit); + serviceList.add(service); + }); + listServicesResponse.setServices(serviceList); + + // Add all given manual scaling versions to mocked Appengine object + getInstance() + .manualScalingVersionsMap + .asMap() + .forEach( + (service, versionList) -> { + // Mockito cannot mock ListVersionsResponse as it is a final class + ListVersionsResponse listVersionsResponse = new ListVersionsResponse(); + try { + when((Object) + getInstance() + .appengine + .apps() + .services() + .versions() + .list(getInstance().appId, service) + .execute()) + .thenReturn(listVersionsResponse); + } catch (IOException e) { + throw new RuntimeException(e); + } + listVersionsResponse.setVersions( + versionList.stream() + .map( + versionId -> { + Version version = new Version(); + ManualScaling manualScaling = new ManualScaling(); + version.setManualScaling(manualScaling); + version.setId(versionId); + return version; + }) + .collect(Collectors.toList())); + }); + + return getInstance(); + } + } +} diff --git a/javatests/google/registry/testing/BUILD b/javatests/google/registry/testing/BUILD index 9e5e555f2..906f6d0a1 100644 --- a/javatests/google/registry/testing/BUILD +++ b/javatests/google/registry/testing/BUILD @@ -31,6 +31,7 @@ java_library( "//java/google/registry/util", "//java/google/registry/xml", "//third_party/objectify:objectify-v4_1", + "@com_google_apis_google_api_services_appengine", "@com_google_appengine_api_1_0_sdk", "@com_google_appengine_api_stubs", "@com_google_appengine_testing", diff --git a/javatests/google/registry/tools/BUILD b/javatests/google/registry/tools/BUILD index 9add124a4..8cc9724e2 100644 --- a/javatests/google/registry/tools/BUILD +++ b/javatests/google/registry/tools/BUILD @@ -39,6 +39,7 @@ java_library( "//third_party/objectify:objectify-v4_1", "@com_beust_jcommander", "@com_google_api_client", + "@com_google_apis_google_api_services_appengine", "@com_google_apis_google_api_services_dns", "@com_google_appengine_api_1_0_sdk", "@com_google_appengine_remote_api", diff --git a/javatests/google/registry/tools/SetNumInstancesCommandTest.java b/javatests/google/registry/tools/SetNumInstancesCommandTest.java index 543fe066a..6b159666d 100644 --- a/javatests/google/registry/tools/SetNumInstancesCommandTest.java +++ b/javatests/google/registry/tools/SetNumInstancesCommandTest.java @@ -20,6 +20,8 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import com.beust.jcommander.ParameterException; +import com.google.common.collect.ImmutableMultimap; +import google.registry.testing.AppEngineAdminApiHelper; import google.registry.testing.InjectRule; import google.registry.util.AppEngineServiceUtils; import org.junit.Before; @@ -34,18 +36,22 @@ public class SetNumInstancesCommandTest extends CommandTestCase runCommand("--version=version", "--numInstances=5")); - assertThat(thrown).hasMessageThat().contains("The following option is required: --service"); + IllegalArgumentException.class, + () -> runCommand("--versions=version", "--num_instances=5")); + assertThat(thrown).hasMessageThat().contains("Service must be specified"); } @Test @@ -53,35 +59,47 @@ public class SetNumInstancesCommandTest extends CommandTestCase runCommand("--service=", "--version=version", "--numInstances=5")); - assertThat(thrown).hasMessageThat().contains("Service must be specified"); + () -> runCommand("--services=", "--versions=version", "--num_instances=5")); + assertThat(thrown).hasMessageThat().contains("Invalid service(s): []"); + } + + @Test + public void test_invalidService_throwsException() { + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> + runCommand( + "--services=INVALID,DEFAULT", "--versions=version", "--num_instances=5")); + assertThat(thrown).hasMessageThat().contains("Invalid service(s): [INVALID]"); } @Test public void test_missingVersion_throwsException() { - ParameterException thrown = + IllegalArgumentException thrown = assertThrows( - ParameterException.class, () -> runCommand("--service=service", "--numInstances=5")); - assertThat(thrown).hasMessageThat().contains("The following option is required: --version"); + IllegalArgumentException.class, + () -> runCommand("--services=DEFAULT", "--num_instances=5")); + assertThat(thrown).hasMessageThat().contains("Version must be specified"); } @Test public void test_emptyVersion_throwsException() { - IllegalArgumentException thrown = + ParameterException thrown = assertThrows( - IllegalArgumentException.class, - () -> runCommand("--service=service", "--version=", "--numInstances=5")); - assertThat(thrown).hasMessageThat().contains("Version must be specified"); + ParameterException.class, + () -> runCommand("--services=DEFAULT", "--num_instances=5", "--versions")); + assertThat(thrown).hasMessageThat().contains("Expected a value after parameter --versions"); } @Test public void test_missingNumInstances_throwsException() { ParameterException thrown = assertThrows( - ParameterException.class, () -> runCommand("--service=service", "--version=version")); + ParameterException.class, () -> runCommand("--services=DEFAULT", "--versions=version")); assertThat(thrown) .hasMessageThat() - .contains("The following option is required: --numInstances"); + .contains("The following option is required: --num_instances"); } @Test @@ -89,13 +107,111 @@ public class SetNumInstancesCommandTest extends CommandTestCase runCommand("--service=service", "--version=version", "--numInstances=-5")); + () -> runCommand("--services=DEFAULT", "--versions=version", "--num_instances=-5")); assertThat(thrown).hasMessageThat().contains("Number of instances must be greater than zero"); } + @Test + public void test_versionNotNullWhenSettingAllNonLiveVersions_throwsException() { + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> runCommand("--services=DEFAULT", "--versions=version", "--num_instances=-5")); + assertThat(thrown).hasMessageThat().contains("Number of instances must be greater than zero"); + } + + @Test + public void test_settingNonManualScalingVersions_throwsException() { + command.appengine = + new AppEngineAdminApiHelper.Builder() + .setAppId(projectId) + .setManualScalingVersionsMap(ImmutableMultimap.of("default", "version1")) + .build() + .getAppengine(); + + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> + runCommand( + "--non_live_versions=true", + "--services=DEFAULT", + "--versions=version", + "--num_instances=10")); + assertThat(thrown) + .hasMessageThat() + .contains("--versions cannot be set if --non_live_versions is set"); + } + @Test public void test_validParameters_succeeds() throws Exception { - runCommand("--service=service", "--version=version", "--numInstances=10"); - verify(appEngineServiceUtils, times(1)).setNumInstances("service", "version", 10L); + command.appengine = + new AppEngineAdminApiHelper.Builder() + .setAppId(projectId) + .setManualScalingVersionsMap(ImmutableMultimap.of("default", "version")) + .build() + .getAppengine(); + + runCommand("--services=DEFAULT", "--versions=version", "--num_instances=10"); + verify(appEngineServiceUtils, times(1)).setNumInstances("default", "version", 10L); + } + + @Test + public void test_settingMultipleServicesAndVersions_succeeds() throws Exception { + command.appengine = + new AppEngineAdminApiHelper.Builder() + .setAppId(projectId) + .setManualScalingVersionsMap( + ImmutableMultimap.of( + "default", "version1", + "default", "version2", + "backend", "version1", + "backend", "version2")) + .build() + .getAppengine(); + + runCommand("--services=DEFAULT,BACKEND", "--versions=version1,version2", "--num_instances=10"); + verify(appEngineServiceUtils, times(1)).setNumInstances("default", "version1", 10L); + verify(appEngineServiceUtils, times(1)).setNumInstances("default", "version2", 10L); + verify(appEngineServiceUtils, times(1)).setNumInstances("backend", "version1", 10L); + verify(appEngineServiceUtils, times(1)).setNumInstances("backend", "version2", 10L); + } + + @Test + public void test_settingAllNonLiveVersions_succeeds() throws Exception { + command.appengine = + new AppEngineAdminApiHelper.Builder() + .setAppId(projectId) + .setManualScalingVersionsMap( + ImmutableMultimap.of( + "default", "version1", "default", "version2", "default", "version3")) + .setLiveVersionsMap(ImmutableMultimap.of("default", "version2")) + .build() + .getAppengine(); + + runCommand("--non_live_versions=true", "--services=DEFAULT", "--num_instances=10"); + verify(appEngineServiceUtils, times(1)).setNumInstances("default", "version1", 10L); + verify(appEngineServiceUtils, times(0)).setNumInstances("default", "version2", 10L); + verify(appEngineServiceUtils, times(1)).setNumInstances("default", "version3", 10L); + } + + @Test + public void test_noNonLiveVersions_succeeds() throws Exception { + command.appengine = + new AppEngineAdminApiHelper.Builder() + .setAppId(projectId) + .setManualScalingVersionsMap( + ImmutableMultimap.of( + "default", "version1", "default", "version2", "default", "version3")) + .setLiveVersionsMap( + ImmutableMultimap.of( + "default", "version1", "default", "version2", "default", "version3")) + .build() + .getAppengine(); + + runCommand("--non_live_versions=true", "--services=DEFAULT", "--num_instances=10"); + verify(appEngineServiceUtils, times(0)).setNumInstances("default", "version1", 10L); + verify(appEngineServiceUtils, times(0)).setNumInstances("default", "version2", 10L); + verify(appEngineServiceUtils, times(0)).setNumInstances("default", "version3", 10L); } }