Use standard java thread factory instead of the AppEngine flavor

With Java 8 in GAE standard environment, we can now use standard java thread factory to run the metric reporter in the background in daemon mode, which would not interfere with basic scaling idle timeout as App Engine thread would.

Because the thread is not created by ThreadManager, no App Engine APIs can be called from it. We therefore use GoogleCredential instead of AppIdentityCredential as HttpRequestInitializer, and NetHttpTransport instead of UlrFetchTransport as HttpTransport.

MetricReporter is lazy injected because it depends on jsonCredential retrieved from CloudKms, which is not available in a test environment, causing FrontendServletTest and BackendServletTest to fail.

Some minor re-formatting with google-java-format on edited files.

Lastly removed moe comments in import statement, which makes the linter unhappy.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=172896227
This commit is contained in:
jianglai 2017-10-20 09:58:14 -07:00
parent b01fa6b4c9
commit c702b4486c
8 changed files with 151 additions and 118 deletions

View file

@ -15,9 +15,9 @@
package google.registry.module.backend; package google.registry.module.backend;
import dagger.Component; import dagger.Component;
import dagger.Lazy;
import google.registry.bigquery.BigqueryModule; import google.registry.bigquery.BigqueryModule;
import google.registry.config.RegistryConfig.ConfigModule; import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.dns.writer.VoidDnsWriterModule;
import google.registry.export.DriveModule; import google.registry.export.DriveModule;
import google.registry.export.sheet.SheetsServiceModule; import google.registry.export.sheet.SheetsServiceModule;
import google.registry.gcs.GcsServiceModule; import google.registry.gcs.GcsServiceModule;
@ -36,6 +36,7 @@ import google.registry.request.Modules.DatastoreServiceModule;
import google.registry.request.Modules.GoogleCredentialModule; import google.registry.request.Modules.GoogleCredentialModule;
import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.ModulesServiceModule; import google.registry.request.Modules.ModulesServiceModule;
import google.registry.request.Modules.NetHttpTransportModule;
import google.registry.request.Modules.URLFetchServiceModule; import google.registry.request.Modules.URLFetchServiceModule;
import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UrlFetchTransportModule;
import google.registry.request.Modules.UseAppIdentityCredentialForGoogleApisModule; import google.registry.request.Modules.UseAppIdentityCredentialForGoogleApisModule;
@ -48,36 +49,39 @@ import javax.inject.Singleton;
/** Dagger component with instance lifetime for "backend" App Engine module. */ /** Dagger component with instance lifetime for "backend" App Engine module. */
@Singleton @Singleton
@Component( @Component(
modules = { modules = {
AppIdentityCredentialModule.class, AppIdentityCredentialModule.class,
AuthModule.class, AuthModule.class,
BackendRequestComponentModule.class, BackendRequestComponentModule.class,
BigqueryModule.class, BigqueryModule.class,
ConfigModule.class, ConfigModule.class,
DatastoreServiceModule.class, DatastoreServiceModule.class,
DirectoryModule.class, DirectoryModule.class,
DriveModule.class, DriveModule.class,
GcsServiceModule.class, GcsServiceModule.class,
GoogleCredentialModule.class, GoogleCredentialModule.class,
GroupsModule.class, GroupsModule.class,
GroupssettingsModule.class, GroupssettingsModule.class,
JSchModule.class, JSchModule.class,
Jackson2Module.class, Jackson2Module.class,
KeyModule.class, KeyModule.class,
KeyringModule.class, KeyringModule.class,
KmsModule.class, KmsModule.class,
ModulesServiceModule.class, ModulesServiceModule.class,
SheetsServiceModule.class, NetHttpTransportModule.class,
StackdriverModule.class, SheetsServiceModule.class,
SystemClockModule.class, StackdriverModule.class,
SystemSleeperModule.class, SystemClockModule.class,
URLFetchServiceModule.class, SystemSleeperModule.class,
UrlFetchTransportModule.class, URLFetchServiceModule.class,
UseAppIdentityCredentialForGoogleApisModule.class, UrlFetchTransportModule.class,
UserServiceModule.class, UseAppIdentityCredentialForGoogleApisModule.class,
VoidDnsWriterModule.class, UserServiceModule.class,
}) google.registry.dns.writer.VoidDnsWriterModule.class,
}
)
interface BackendComponent { interface BackendComponent {
BackendRequestHandler requestHandler(); BackendRequestHandler requestHandler();
MetricReporter metricReporter();
Lazy<MetricReporter> metricReporter();
} }

View file

@ -15,7 +15,7 @@
package google.registry.module.backend; package google.registry.module.backend;
import com.google.appengine.api.LifecycleManager; import com.google.appengine.api.LifecycleManager;
import com.google.appengine.api.LifecycleManager.ShutdownHook; import dagger.Lazy;
import google.registry.monitoring.metrics.MetricReporter; import google.registry.monitoring.metrics.MetricReporter;
import google.registry.util.FormattingLogger; import google.registry.util.FormattingLogger;
import java.io.IOException; import java.io.IOException;
@ -32,7 +32,7 @@ public final class BackendServlet extends HttpServlet {
private static final BackendComponent component = DaggerBackendComponent.create(); private static final BackendComponent component = DaggerBackendComponent.create();
private static final BackendRequestHandler requestHandler = component.requestHandler(); private static final BackendRequestHandler requestHandler = component.requestHandler();
private static final MetricReporter metricReporter = component.metricReporter(); private static final Lazy<MetricReporter> metricReporter = component.metricReporter();
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Override @Override
@ -40,7 +40,7 @@ public final class BackendServlet extends HttpServlet {
Security.addProvider(new BouncyCastleProvider()); Security.addProvider(new BouncyCastleProvider());
try { try {
metricReporter.startAsync().awaitRunning(10, TimeUnit.SECONDS); metricReporter.get().startAsync().awaitRunning(10, TimeUnit.SECONDS);
logger.info("Started up MetricReporter"); logger.info("Started up MetricReporter");
} catch (TimeoutException timeoutException) { } catch (TimeoutException timeoutException) {
logger.severefmt("Failed to initialize MetricReporter: %s", timeoutException); logger.severefmt("Failed to initialize MetricReporter: %s", timeoutException);
@ -48,15 +48,12 @@ public final class BackendServlet extends HttpServlet {
LifecycleManager.getInstance() LifecycleManager.getInstance()
.setShutdownHook( .setShutdownHook(
new ShutdownHook() { () -> {
@Override try {
public void shutdown() { metricReporter.get().stopAsync().awaitTerminated(10, TimeUnit.SECONDS);
try { logger.info("Shut down MetricReporter");
metricReporter.stopAsync().awaitTerminated(10, TimeUnit.SECONDS); } catch (TimeoutException timeoutException) {
logger.info("Shut down MetricReporter"); logger.severefmt("Failed to stop MetricReporter: %s", timeoutException);
} catch (TimeoutException timeoutException) {
logger.severefmt("Failed to stop MetricReporter: %s", timeoutException);
}
} }
}); });
} }

View file

@ -15,6 +15,7 @@
package google.registry.module.frontend; package google.registry.module.frontend;
import dagger.Component; import dagger.Component;
import dagger.Lazy;
import google.registry.braintree.BraintreeModule; import google.registry.braintree.BraintreeModule;
import google.registry.config.RegistryConfig.ConfigModule; import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.flows.ServerTridProviderModule; import google.registry.flows.ServerTridProviderModule;
@ -26,8 +27,10 @@ import google.registry.module.frontend.FrontendRequestComponent.FrontendRequestC
import google.registry.monitoring.metrics.MetricReporter; import google.registry.monitoring.metrics.MetricReporter;
import google.registry.monitoring.whitebox.StackdriverModule; import google.registry.monitoring.whitebox.StackdriverModule;
import google.registry.request.Modules.AppIdentityCredentialModule; import google.registry.request.Modules.AppIdentityCredentialModule;
import google.registry.request.Modules.GoogleCredentialModule;
import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.ModulesServiceModule; import google.registry.request.Modules.ModulesServiceModule;
import google.registry.request.Modules.NetHttpTransportModule;
import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UrlFetchTransportModule;
import google.registry.request.Modules.UseAppIdentityCredentialForGoogleApisModule; import google.registry.request.Modules.UseAppIdentityCredentialForGoogleApisModule;
import google.registry.request.Modules.UserServiceModule; import google.registry.request.Modules.UserServiceModule;
@ -40,29 +43,33 @@ import javax.inject.Singleton;
/** Dagger component with instance lifetime for "default" App Engine module. */ /** Dagger component with instance lifetime for "default" App Engine module. */
@Singleton @Singleton
@Component( @Component(
modules = { modules = {
AppIdentityCredentialModule.class, AppIdentityCredentialModule.class,
AuthModule.class, AuthModule.class,
BraintreeModule.class, BraintreeModule.class,
ConfigModule.class, ConfigModule.class,
ConsoleConfigModule.class, ConsoleConfigModule.class,
CustomLogicFactoryModule.class, CustomLogicFactoryModule.class,
FrontendMetricsModule.class, FrontendMetricsModule.class,
FrontendRequestComponentModule.class, FrontendRequestComponentModule.class,
Jackson2Module.class, GoogleCredentialModule.class,
KeyModule.class, Jackson2Module.class,
KeyringModule.class, KeyModule.class,
KmsModule.class, KeyringModule.class,
ModulesServiceModule.class, KmsModule.class,
ServerTridProviderModule.class, ModulesServiceModule.class,
StackdriverModule.class, NetHttpTransportModule.class,
SystemClockModule.class, ServerTridProviderModule.class,
SystemSleeperModule.class, StackdriverModule.class,
UrlFetchTransportModule.class, SystemClockModule.class,
UseAppIdentityCredentialForGoogleApisModule.class, SystemSleeperModule.class,
UserServiceModule.class, UrlFetchTransportModule.class,
}) UseAppIdentityCredentialForGoogleApisModule.class,
UserServiceModule.class,
}
)
interface FrontendComponent { interface FrontendComponent {
FrontendRequestHandler requestHandler(); FrontendRequestHandler requestHandler();
MetricReporter metricReporter();
Lazy<MetricReporter> metricReporter();
} }

View file

@ -15,7 +15,7 @@
package google.registry.module.frontend; package google.registry.module.frontend;
import com.google.appengine.api.LifecycleManager; import com.google.appengine.api.LifecycleManager;
import com.google.appengine.api.LifecycleManager.ShutdownHook; import dagger.Lazy;
import google.registry.monitoring.metrics.MetricReporter; import google.registry.monitoring.metrics.MetricReporter;
import google.registry.util.FormattingLogger; import google.registry.util.FormattingLogger;
import java.io.IOException; import java.io.IOException;
@ -32,7 +32,7 @@ public final class FrontendServlet extends HttpServlet {
private static final FrontendComponent component = DaggerFrontendComponent.create(); private static final FrontendComponent component = DaggerFrontendComponent.create();
private static final FrontendRequestHandler requestHandler = component.requestHandler(); private static final FrontendRequestHandler requestHandler = component.requestHandler();
private static final MetricReporter metricReporter = component.metricReporter(); private static final Lazy<MetricReporter> metricReporter = component.metricReporter();
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Override @Override
@ -40,7 +40,7 @@ public final class FrontendServlet extends HttpServlet {
Security.addProvider(new BouncyCastleProvider()); Security.addProvider(new BouncyCastleProvider());
try { try {
metricReporter.startAsync().awaitRunning(10, TimeUnit.SECONDS); metricReporter.get().startAsync().awaitRunning(10, TimeUnit.SECONDS);
logger.info("Started up MetricReporter"); logger.info("Started up MetricReporter");
} catch (TimeoutException timeoutException) { } catch (TimeoutException timeoutException) {
logger.severefmt("Failed to initialize MetricReporter: %s", timeoutException); logger.severefmt("Failed to initialize MetricReporter: %s", timeoutException);
@ -48,15 +48,12 @@ public final class FrontendServlet extends HttpServlet {
LifecycleManager.getInstance() LifecycleManager.getInstance()
.setShutdownHook( .setShutdownHook(
new ShutdownHook() { () -> {
@Override try {
public void shutdown() { metricReporter.get().stopAsync().awaitTerminated(10, TimeUnit.SECONDS);
try { logger.info("Shut down MetricReporter");
metricReporter.stopAsync().awaitTerminated(10, TimeUnit.SECONDS); } catch (TimeoutException timeoutException) {
logger.info("Shut down MetricReporter"); logger.severefmt("Failed to stop MetricReporter: %s", timeoutException);
} catch (TimeoutException timeoutException) {
logger.severefmt("Failed to stop MetricReporter: %s", timeoutException);
}
} }
}); });
} }

View file

@ -32,6 +32,7 @@ import google.registry.request.Modules.DatastoreServiceModule;
import google.registry.request.Modules.GoogleCredentialModule; import google.registry.request.Modules.GoogleCredentialModule;
import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.ModulesServiceModule; import google.registry.request.Modules.ModulesServiceModule;
import google.registry.request.Modules.NetHttpTransportModule;
import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UrlFetchTransportModule;
import google.registry.request.Modules.UseAppIdentityCredentialForGoogleApisModule; import google.registry.request.Modules.UseAppIdentityCredentialForGoogleApisModule;
import google.registry.request.Modules.UserServiceModule; import google.registry.request.Modules.UserServiceModule;
@ -43,31 +44,33 @@ import javax.inject.Singleton;
/** Dagger component with instance lifetime for "tools" App Engine module. */ /** Dagger component with instance lifetime for "tools" App Engine module. */
@Singleton @Singleton
@Component( @Component(
modules = { modules = {
AppIdentityCredentialModule.class, AppIdentityCredentialModule.class,
AuthModule.class, AuthModule.class,
ConfigModule.class, ConfigModule.class,
CustomLogicFactoryModule.class, CustomLogicFactoryModule.class,
DatastoreServiceModule.class, DatastoreServiceModule.class,
DirectoryModule.class, DirectoryModule.class,
DriveModule.class, DriveModule.class,
GcsServiceModule.class, GcsServiceModule.class,
GoogleCredentialModule.class, GoogleCredentialModule.class,
GroupsModule.class, GroupsModule.class,
GroupssettingsModule.class, GroupssettingsModule.class,
Jackson2Module.class, Jackson2Module.class,
KeyModule.class, KeyModule.class,
KeyringModule.class, KeyringModule.class,
KmsModule.class, KmsModule.class,
ModulesServiceModule.class, ModulesServiceModule.class,
ServerTridProviderModule.class, NetHttpTransportModule.class,
SystemClockModule.class, ServerTridProviderModule.class,
SystemSleeperModule.class, SystemClockModule.class,
ToolsRequestComponentModule.class, SystemSleeperModule.class,
UrlFetchTransportModule.class, ToolsRequestComponentModule.class,
UseAppIdentityCredentialForGoogleApisModule.class, UrlFetchTransportModule.class,
UserServiceModule.class, UseAppIdentityCredentialForGoogleApisModule.class,
}) UserServiceModule.class,
}
)
interface ToolsComponent { interface ToolsComponent {
ToolsRequestHandler requestHandler(); ToolsRequestHandler requestHandler();
} }

View file

@ -17,6 +17,7 @@ java_library(
"//java/google/registry/request/auth", "//java/google/registry/request/auth",
"//java/google/registry/util", "//java/google/registry/util",
"//third_party/java/objectify:objectify-v4_1", "//third_party/java/objectify:objectify-v4_1",
"@com_google_api_client",
"@com_google_apis_google_api_services_bigquery", "@com_google_apis_google_api_services_bigquery",
"@com_google_apis_google_api_services_monitoring", "@com_google_apis_google_api_services_monitoring",
"@com_google_appengine_api_1_0_sdk", "@com_google_appengine_api_1_0_sdk",

View file

@ -14,16 +14,16 @@
package google.registry.monitoring.whitebox; package google.registry.monitoring.whitebox;
import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory; import com.google.api.client.json.JsonFactory;
import com.google.api.services.monitoring.v3.Monitoring; import com.google.api.services.monitoring.v3.Monitoring;
import com.google.api.services.monitoring.v3.MonitoringScopes; import com.google.api.services.monitoring.v3.MonitoringScopes;
import com.google.api.services.monitoring.v3.model.MonitoredResource; import com.google.api.services.monitoring.v3.model.MonitoredResource;
import com.google.appengine.api.ThreadManager;
import com.google.appengine.api.modules.ModulesService; import com.google.appengine.api.modules.ModulesService;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import dagger.Module; import dagger.Module;
import dagger.Provides; import dagger.Provides;
import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryConfig.Config;
@ -43,9 +43,9 @@ public final class StackdriverModule {
@Provides @Provides
static Monitoring provideMonitoring( static Monitoring provideMonitoring(
HttpTransport transport, NetHttpTransport transport,
JsonFactory jsonFactory, JsonFactory jsonFactory,
Function<Set<String>, ? extends HttpRequestInitializer> credential, Function<Set<String>, GoogleCredential> credential,
@Config("projectId") String projectId) { @Config("projectId") String projectId) {
return new Monitoring.Builder(transport, jsonFactory, credential.apply(MonitoringScopes.all())) return new Monitoring.Builder(transport, jsonFactory, credential.apply(MonitoringScopes.all()))
.setApplicationName(projectId) .setApplicationName(projectId)
@ -87,6 +87,8 @@ public final class StackdriverModule {
static MetricReporter provideMetricReporter( static MetricReporter provideMetricReporter(
MetricWriter metricWriter, @Config("metricsWriteInterval") Duration writeInterval) { MetricWriter metricWriter, @Config("metricsWriteInterval") Duration writeInterval) {
return new MetricReporter( return new MetricReporter(
metricWriter, writeInterval.getStandardSeconds(), ThreadManager.backgroundThreadFactory()); metricWriter,
writeInterval.getStandardSeconds(),
new ThreadFactoryBuilder().setDaemon(true).build());
} }
} }

View file

@ -20,8 +20,10 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.api.client.extensions.appengine.http.UrlFetchTransport; import com.google.api.client.extensions.appengine.http.UrlFetchTransport;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential; import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport; import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory; import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.appengine.api.datastore.DatastoreService; import com.google.appengine.api.datastore.DatastoreService;
@ -117,6 +119,24 @@ public final class Modules {
} }
} }
/**
* Dagger module that provides standard {@link NetHttpTransport}. Used in non App Engine
* environment.
*/
@Module
public static final class NetHttpTransportModule {
@Provides
@Singleton
static NetHttpTransport provideNetHttpTransport() {
try {
return GoogleNetHttpTransport.newTrustedTransport();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
/** /**
* Dagger module providing {@link AppIdentityCredential}. * Dagger module providing {@link AppIdentityCredential}.
* *
@ -140,8 +160,8 @@ public final class Modules {
@Module @Module
public abstract static class UseAppIdentityCredentialForGoogleApisModule { public abstract static class UseAppIdentityCredentialForGoogleApisModule {
@Binds @Binds
abstract Function<Set<String>, ? extends HttpRequestInitializer> abstract Function<Set<String>, ? extends HttpRequestInitializer> provideHttpRequestInitializer(
provideHttpRequestInitializer(Function<Set<String>, AppIdentityCredential> credential); Function<Set<String>, AppIdentityCredential> credential);
} }
/** /**
@ -155,8 +175,8 @@ public final class Modules {
@Module @Module
public abstract static class UseGoogleCredentialForGoogleApisModule { public abstract static class UseGoogleCredentialForGoogleApisModule {
@Binds @Binds
abstract Function<Set<String>, ? extends HttpRequestInitializer> abstract Function<Set<String>, ? extends HttpRequestInitializer> provideHttpRequestInitializer(
provideHttpRequestInitializer(Function<Set<String>, GoogleCredential> credential); Function<Set<String>, GoogleCredential> credential);
} }
/** /**
@ -169,8 +189,8 @@ public final class Modules {
* this authentication method should only be used for the following situations: * this authentication method should only be used for the following situations:
* *
* <ol> * <ol>
* <li>Locally-running programs (which aren't executing on the App Engine platform) * <li>Locally-running programs (which aren't executing on the App Engine platform)
* <li>Spreadsheet service (which can't use {@link AppIdentityCredential} due to an old library) * <li>Spreadsheet service (which can't use {@link AppIdentityCredential} due to an old library)
* </ol> * </ol>
* *
* @see google.registry.keyring.api.Keyring#getJsonCredential() * @see google.registry.keyring.api.Keyring#getJsonCredential()
@ -181,12 +201,14 @@ public final class Modules {
@Provides @Provides
@Singleton @Singleton
static GoogleCredential provideGoogleCredential( static GoogleCredential provideGoogleCredential(
HttpTransport httpTransport, NetHttpTransport netHttpTransport,
JsonFactory jsonFactory, JsonFactory jsonFactory,
@Key("jsonCredential") String jsonCredential) { @Key("jsonCredential") String jsonCredential) {
try { try {
return GoogleCredential.fromStream( return GoogleCredential.fromStream(
new ByteArrayInputStream(jsonCredential.getBytes(UTF_8)), httpTransport, jsonFactory); new ByteArrayInputStream(jsonCredential.getBytes(UTF_8)),
netHttpTransport,
jsonFactory);
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
@ -199,7 +221,7 @@ public final class Modules {
} }
/** /**
* Provides a GoogleCredential that will connect to GAE using delegated admin access. This is * Provides a GoogleCredential that will connect to GAE using delegated admin access. This is
* needed for API calls requiring domain admin access to the relevant GAFYD using delegated * needed for API calls requiring domain admin access to the relevant GAFYD using delegated
* scopes, e.g. the Directory API and the Groupssettings API. * scopes, e.g. the Directory API and the Groupssettings API.
* *