Refactor AppEngineConnection

AppEngineConnection can now connect to all services and not just the tools.

The default is still the tools.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=218734983
This commit is contained in:
guyben 2018-10-25 12:57:16 -07:00 committed by jianglai
parent 97aa98eb35
commit b48061b792
31 changed files with 269 additions and 329 deletions

View file

@ -24,10 +24,8 @@ import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.net.HostAndPort;
import dagger.Module;
import dagger.Provides;
import google.registry.config.RegistryConfigSettings.AppEngine.ToolsServiceUrl;
import google.registry.util.RandomStringGenerator;
import google.registry.util.StringGenerator;
import google.registry.util.TaskQueueUtils;
@ -1275,7 +1273,7 @@ public final class RegistryConfig {
@Config("insecureRandom")
public static Random provideInsecureRandom() {
return new Random();
};
}
/** Returns a singleton secure random number generator this is slow. */
@Singleton
@ -1351,14 +1349,44 @@ public final class RegistryConfig {
return Duration.standardDays(30);
}
public static boolean areServersLocal() {
return CONFIG_SETTINGS.get().appEngine.isLocal;
}
/**
* Returns the address of the Nomulus app HTTP server.
* Returns the address of the Nomulus app default HTTP server.
*
* <p>This is used by the {@code nomulus} tool to connect to the App Engine remote API.
*/
public static HostAndPort getServer() {
ToolsServiceUrl url = CONFIG_SETTINGS.get().appEngine.toolsServiceUrl;
return HostAndPort.fromParts(url.hostName, url.port);
public static URL getDefaultServer() {
return makeUrl(CONFIG_SETTINGS.get().appEngine.defaultServiceUrl);
}
/**
* Returns the address of the Nomulus app backend HTTP server.
*
* <p>This is used by the {@code nomulus} tool to connect to the App Engine remote API.
*/
public static URL getBackendServer() {
return makeUrl(CONFIG_SETTINGS.get().appEngine.backendServiceUrl);
}
/**
* Returns the address of the Nomulus app tools HTTP server.
*
* <p>This is used by the {@code nomulus} tool to connect to the App Engine remote API.
*/
public static URL getToolsServer() {
return makeUrl(CONFIG_SETTINGS.get().appEngine.toolsServiceUrl);
}
/**
* Returns the address of the Nomulus app pubapi HTTP server.
*
* <p>This is used by the {@code nomulus} tool to connect to the App Engine remote API.
*/
public static URL getPubapiServer() {
return makeUrl(CONFIG_SETTINGS.get().appEngine.pubapiServiceUrl);
}
/** Returns the amount of time a singleton should be cached, before expiring. */
@ -1466,7 +1494,7 @@ public final class RegistryConfig {
* change the contents of the YAML config files.
*/
@VisibleForTesting
static final Supplier<RegistryConfigSettings> CONFIG_SETTINGS =
public static final Supplier<RegistryConfigSettings> CONFIG_SETTINGS =
memoize(YamlUtils::getConfigSettings);
private static String formatComments(String text) {

View file

@ -40,13 +40,11 @@ public class RegistryConfigSettings {
/** Configuration options that apply to the entire App Engine project. */
public static class AppEngine {
public String projectId;
public ToolsServiceUrl toolsServiceUrl;
/** Configuration options for the tools service URL. */
public static class ToolsServiceUrl {
public String hostName;
public int port;
}
public boolean isLocal;
public String defaultServiceUrl;
public String backendServiceUrl;
public String toolsServiceUrl;
public String pubapiServiceUrl;
}
/** Configuration options for OAuth settings for authenticating users. */

View file

@ -9,10 +9,13 @@ appEngine:
# Globally unique App Engine project ID
projectId: registry-project-id
# Hostname and port of the tools service for the project.
toolsServiceUrl:
hostName: localhost
port: 443
# whether to use local/test credentials when connecting to the servers
isLocal: true
# URLs of the services for the project.
defaultServiceUrl: https://localhost
backendServiceUrl: https://localhost
toolsServiceUrl: https://localhost
pubapiServiceUrl: https://localhost
gSuite:
# Publicly accessible domain name of the running G Suite instance.

View file

@ -4,11 +4,14 @@
appEngine:
projectId: placeholder
# The "tools-dot-" prefix is used on the project ID in this URL in order to
# get around an issue with double-wildcard SSL certs.
toolsServiceUrl:
hostName: tools-dot-placeholder.appspot.com
port: 443
# Set to true if running against local servers (localhost)
isLocal: false
# The "<service>-dot-" prefix is used on the project ID in this URL in order
# to get around an issue with double-wildcard SSL certs.
defaultServiceUrl: https://domain-registry-placeholder.appspot.com
backendServiceUrl: https://backend-dot-domain-registry-placeholder.appspot.com
toolsServiceUrl: https://tools-dot-domain-registry-placeholder.appspot.com
pubapiServiceUrl: https://pubapi-dot-domain-registry-placeholder.appspot.com
gSuite:
domainName: placeholder

View file

@ -15,7 +15,6 @@
package google.registry.tools;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Suppliers.memoize;
import static com.google.common.net.HttpHeaders.X_REQUESTED_WITH;
import static com.google.common.net.MediaType.JSON_UTF_8;
import static google.registry.security.JsonHttp.JSON_SAFETY_PREFIX;
@ -27,48 +26,55 @@ import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpResponse;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.CharStreams;
import com.google.common.net.HostAndPort;
import com.google.common.net.MediaType;
import com.google.re2j.Matcher;
import com.google.re2j.Pattern;
import google.registry.security.XsrfTokenManager;
import google.registry.tools.CommandWithConnection.Connection;
import google.registry.config.RegistryConfig;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.Map;
import javax.annotation.Nullable;
import javax.inject.Inject;
import org.json.simple.JSONValue;
/** An http connection to the appengine server. */
class AppEngineConnection implements Connection {
/**
* An http connection to an appengine server.
*
* <p>By default - connects to the TOOLS service. To create a Connection to another service, call
* the {@link #withService} function.
*/
class AppEngineConnection {
/** Pattern to heuristically extract title tag contents in HTML responses. */
private static final Pattern HTML_TITLE_TAG_PATTERN = Pattern.compile("<title>(.*?)</title>");
@Inject HttpRequestFactory requestFactory;
@Inject AppEngineConnectionFlags flags;
@Inject XsrfTokenManager xsrfTokenManager;
private final Service service;
@Inject
AppEngineConnection() {}
AppEngineConnection() {
service = Service.TOOLS;
}
/**
* Memoized XSRF security token.
*
* <p>Computing this is expensive since it needs to load {@code ServerSecret} so do it once.
*/
private final Supplier<String> xsrfToken =
memoize(() -> xsrfTokenManager.generateToken(getUserId()));
private AppEngineConnection(Service service, HttpRequestFactory requestFactory) {
this.service = service;
this.requestFactory = requestFactory;
}
@Override
public void prefetchXsrfToken() {
// Cause XSRF token to be fetched, and then stay resident in cache (since it's memoized).
xsrfToken.get();
enum Service {
DEFAULT,
TOOLS,
BACKEND,
PUBAPI
}
/** Returns a copy of this connection that talks to a different service. */
public AppEngineConnection withService(Service service) {
return new AppEngineConnection(service, requestFactory);
}
/** Returns the contents of the title tag in the given HTML, or null if not found. */
@ -85,7 +91,8 @@ class AppEngineConnection implements Connection {
private String internalSend(
String endpoint, Map<String, ?> params, MediaType contentType, @Nullable byte[] payload)
throws IOException {
GenericUrl url = new GenericUrl(String.format("%s%s", getServerUrl(), endpoint));
GenericUrl url = new GenericUrl(getServer());
url.setRawPath(endpoint);
url.putAll(params);
HttpRequest request =
(payload != null)
@ -120,23 +127,20 @@ class AppEngineConnection implements Connection {
}
}
// TODO(b/111123862): Rename this to sendPostRequest()
@Override
public String send(String endpoint, Map<String, ?> params, MediaType contentType, byte[] payload)
public String sendPostRequest(
String endpoint, Map<String, ?> params, MediaType contentType, byte[] payload)
throws IOException {
return internalSend(endpoint, params, contentType, checkNotNull(payload, "payload"));
}
@Override
public String sendGetRequest(String endpoint, Map<String, ?> params) throws IOException {
return internalSend(endpoint, params, MediaType.PLAIN_TEXT_UTF_8, null);
}
@Override
@SuppressWarnings("unchecked")
public Map<String, Object> sendJson(String endpoint, Map<String, ?> object) throws IOException {
String response =
send(
sendPostRequest(
endpoint,
ImmutableMap.of(),
JSON_UTF_8,
@ -144,22 +148,17 @@ class AppEngineConnection implements Connection {
return (Map<String, Object>) JSONValue.parse(response.substring(JSON_SAFETY_PREFIX.length()));
}
@Override
public String getServerUrl() {
return (isLocalhost() ? "http://" : "https://") + getServer().toString();
}
HostAndPort getServer() {
return flags.getServer().withDefaultPort(443); // Default to HTTPS port if unspecified.
}
boolean isLocalhost() {
return flags.getServer().getHost().equals("localhost");
}
private String getUserId() {
return isLocalhost()
? UserIdProvider.getTestUserId()
: UserIdProvider.getProdUserId();
public URL getServer() {
switch (service) {
case DEFAULT:
return RegistryConfig.getDefaultServer();
case TOOLS:
return RegistryConfig.getToolsServer();
case BACKEND:
return RegistryConfig.getBackendServer();
case PUBAPI:
return RegistryConfig.getPubapiServer();
}
throw new IllegalStateException("Unknown service: " + service);
}
}

View file

@ -1,63 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.net.HostAndPort;
import dagger.Module;
import dagger.Provides;
import google.registry.config.RegistryConfig;
/**
* Class to contain the configuration flags for AppEngineConnection.
*
* <p>This is broken out into its own class to make it cleaner to extract these from the dagger
* module, where these values are injected.
*/
@Parameters(separators = " =")
class AppEngineConnectionFlags {
@Parameter(names = "--server", description = "HOST[:PORT] to which remote commands are sent.")
private HostAndPort server = RegistryConfig.getServer();
/** Provided for testing. */
@VisibleForTesting
AppEngineConnectionFlags(HostAndPort server) {
this.server = server;
}
AppEngineConnectionFlags() {}
HostAndPort getServer() {
return server;
}
@Module
static class FlagsModule {
AppEngineConnectionFlags flags;
FlagsModule(AppEngineConnectionFlags flags) {
this.flags = flags;
}
@Provides
AppEngineConnectionFlags provideAppEngineConnectionFlags() {
return flags;
}
}
}

View file

@ -14,30 +14,7 @@
package google.registry.tools;
import com.google.common.net.MediaType;
import java.io.IOException;
import java.util.Map;
import javax.annotation.Nullable;
/** A command that can send HTTP requests to a backend module. */
interface CommandWithConnection extends Command {
/** An http connection to AppEngine. */
interface Connection {
void prefetchXsrfToken();
/** Send a POST request. TODO(mmuller): change to sendPostRequest() */
String send(
String endpoint, Map<String, ?> params, MediaType contentType, @Nullable byte[] payload)
throws IOException;
String sendGetRequest(String endpoint, Map<String, ?> params) throws IOException;
Map<String, Object> sendJson(String endpoint, Map<String, ?> object) throws IOException;
String getServerUrl();
}
void setConnection(Connection connection);
void setConnection(AppEngineConnection connection);
}

View file

@ -57,11 +57,11 @@ abstract class CreateOrUpdatePremiumListCommand extends ConfirmingCommand
required = true)
Path inputFile;
protected Connection connection;
protected AppEngineConnection connection;
protected int inputLineCount;
@Override
public void setConnection(Connection connection) {
public void setConnection(AppEngineConnection connection) {
this.connection = connection;
}
@ -101,11 +101,9 @@ abstract class CreateOrUpdatePremiumListCommand extends ConfirmingCommand
}
// Call the server and get the response data
String response = connection.send(
getCommandPath(),
params.build(),
MediaType.FORM_DATA,
requestBody.getBytes(UTF_8));
String response =
connection.sendPostRequest(
getCommandPath(), params.build(), MediaType.FORM_DATA, requestBody.getBytes(UTF_8));
return extractServerResponse(response);
}

View file

@ -50,10 +50,10 @@ final class CreateRegistrarCommand extends CreateOrUpdateRegistrarCommand
arity = 1)
boolean createGoogleGroups = true;
private Connection connection;
private AppEngineConnection connection;
@Override
public void setConnection(Connection connection) {
public void setConnection(AppEngineConnection connection) {
this.connection = connection;
}

View file

@ -41,10 +41,10 @@ public class CreateRegistrarGroupsCommand extends ConfirmingCommand
private List<Registrar> registrars = new ArrayList<>();
private Connection connection;
private AppEngineConnection connection;
@Override
public void setConnection(Connection connection) {
public void setConnection(AppEngineConnection connection) {
this.connection = connection;
}
@ -66,8 +66,8 @@ public class CreateRegistrarGroupsCommand extends ConfirmingCommand
}
/** Calls the server endpoint to create groups for the specified registrar client id. */
static void executeOnServer(Connection connection, String clientId) throws IOException {
connection.send(
static void executeOnServer(AppEngineConnection connection, String clientId) throws IOException {
connection.sendPostRequest(
CreateGroupsAction.PATH,
ImmutableMap.of(CreateGroupsAction.CLIENT_ID_PARAM, clientId),
MediaType.PLAIN_TEXT_UTF_8,
@ -77,7 +77,7 @@ public class CreateRegistrarGroupsCommand extends ConfirmingCommand
@Override
protected String execute() throws IOException {
for (Registrar registrar : registrars) {
connection.send(
connection.sendPostRequest(
CreateGroupsAction.PATH,
ImmutableMap.of(CreateGroupsAction.CLIENT_ID_PARAM, registrar.getClientId()),
MediaType.PLAIN_TEXT_UTF_8,

View file

@ -22,11 +22,12 @@ import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.net.MediaType;
import google.registry.tools.AppEngineConnection.Service;
import java.util.List;
@Parameters(separators = " =", commandDescription = "Send an HTTP command to the nomulus server.")
class CurlCommand implements CommandWithConnection {
private Connection connection;
private AppEngineConnection connection;
// HTTP Methods that are acceptable for use as values for --method.
public enum Method {
@ -62,8 +63,14 @@ class CurlCommand implements CommandWithConnection {
+ "absent, a GET request is sent.")
private List<String> data;
@Parameter(
names = {"--service"},
description = "Which service to connect to",
required = true)
private Service service;
@Override
public void setConnection(Connection connection) {
public void setConnection(AppEngineConnection connection) {
this.connection = connection;
}
@ -77,12 +84,14 @@ class CurlCommand implements CommandWithConnection {
throw new IllegalArgumentException("You may not specify a body for a get method.");
}
// TODO(b/112315418): Make it possible to address any backend.
AppEngineConnection connectionToService = connection.withService(service);
String response =
(method == Method.GET)
? connection.sendGetRequest(path, ImmutableMap.<String, String>of())
: connection.send(
path, ImmutableMap.<String, String>of(), mimeType,
? connectionToService.sendGetRequest(path, ImmutableMap.<String, String>of())
: connectionToService.sendPostRequest(
path,
ImmutableMap.<String, String>of(),
mimeType,
Joiner.on("&").join(data).getBytes(UTF_8));
System.out.println(response);
}

View file

@ -20,6 +20,7 @@ import com.google.api.client.http.javanet.NetHttpTransport;
import dagger.Binds;
import dagger.Module;
import dagger.Provides;
import google.registry.config.RegistryConfig;
import javax.inject.Named;
import javax.inject.Provider;
@ -43,9 +44,8 @@ class DefaultRequestFactoryModule {
@Provides
@Named("default")
public HttpRequestFactory provideHttpRequestFactory(
AppEngineConnectionFlags connectionFlags,
Provider<Credential> credentialProvider) {
if (connectionFlags.getServer().getHost().equals("localhost")) {
if (RegistryConfig.areServersLocal()) {
return new NetHttpTransport()
.createRequestFactory(
request -> request

View file

@ -59,7 +59,7 @@ abstract class EppToolCommand extends ConfirmingCommand
private List<XmlEppParameters> commands = new ArrayList<>();
private Connection connection;
private AppEngineConnection connection;
static class XmlEppParameters {
final String clientId;
@ -95,7 +95,7 @@ abstract class EppToolCommand extends ConfirmingCommand
}
@Override
public void setConnection(Connection connection) {
public void setConnection(AppEngineConnection connection) {
this.connection = connection;
}
@ -145,11 +145,13 @@ abstract class EppToolCommand extends ConfirmingCommand
params.put("xml", URLEncoder.encode(command.xml, UTF_8.toString()));
String requestBody =
Joiner.on('&').withKeyValueSeparator("=").join(filterValues(params, Objects::nonNull));
responses.add(nullToEmpty(connection.send(
"/_dr/epptool",
ImmutableMap.<String, String>of(),
MediaType.FORM_DATA,
requestBody.getBytes(UTF_8))));
responses.add(
nullToEmpty(
connection.sendPostRequest(
"/_dr/epptool",
ImmutableMap.<String, String>of(),
MediaType.FORM_DATA,
requestBody.getBytes(UTF_8))));
}
return responses.build();
}

View file

@ -45,10 +45,10 @@ final class GenerateZoneFilesCommand implements CommandWithConnection, CommandWi
validateWith = DateParameter.class)
private DateTime exportDate = DateTime.now(UTC).minus(standardMinutes(2)).withTimeAtStartOfDay();
private Connection connection;
private AppEngineConnection connection;
@Override
public void setConnection(Connection connection) {
public void setConnection(AppEngineConnection connection) {
this.connection = connection;
}
@ -59,10 +59,7 @@ final class GenerateZoneFilesCommand implements CommandWithConnection, CommandWi
"tlds", mainParameters,
"exportTime", exportDate.toString());
Map<String, Object> response = connection.sendJson(GenerateZoneFilesAction.PATH, params);
System.out.printf(
"Job started at %s%s\n",
connection.getServerUrl(),
response.get("jobPath"));
System.out.printf("Job started at %s %s\n", connection.getServer(), response.get("jobPath"));
System.out.println("Output files:");
@SuppressWarnings("unchecked")
List<String> filenames = (List<String>) response.get("filenames");

View file

@ -54,10 +54,10 @@ abstract class ListObjectsCommand implements CommandWithConnection, CommandWithR
description = "Whether to print full field names in header row (as opposed to aliases)")
private boolean fullFieldNames = false;
private Connection connection;
private AppEngineConnection connection;
@Override
public void setConnection(Connection connection) {
public void setConnection(AppEngineConnection connection) {
this.connection = connection;
}
@ -83,11 +83,9 @@ abstract class ListObjectsCommand implements CommandWithConnection, CommandWithR
}
params.putAll(getParameterMap());
// Call the server and get the response data.
String response = connection.send(
getCommandPath(),
params.build(),
MediaType.PLAIN_TEXT_UTF_8,
new byte[0]);
String response =
connection.sendPostRequest(
getCommandPath(), params.build(), MediaType.PLAIN_TEXT_UTF_8, new byte[0]);
// Parse the returned JSON and make sure it's a map.
Object obj = JSONValue.parse(response.substring(JSON_SAFETY_PREFIX.length()));
if (!(obj instanceof Map<?, ?>)) {

View file

@ -77,10 +77,10 @@ class LoadTestCommand extends ConfirmingCommand
description = "Time to run the load test in seconds.")
int runSeconds = DEFAULT_RUN_SECONDS;
private Connection connection;
private AppEngineConnection connection;
@Override
public void setConnection(Connection connection) {
public void setConnection(AppEngineConnection connection) {
this.connection = connection;
}
@ -127,10 +127,7 @@ class LoadTestCommand extends ConfirmingCommand
.put("runSeconds", runSeconds)
.build();
return connection.send(
LoadTestAction.PATH,
params,
MediaType.PLAIN_TEXT_UTF_8,
new byte[0]);
return connection.sendPostRequest(
LoadTestAction.PATH, params, MediaType.PLAIN_TEXT_UTF_8, new byte[0]);
}
}

View file

@ -28,6 +28,7 @@ import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import google.registry.config.RegistryConfig;
import google.registry.model.ofy.ObjectifyService;
import google.registry.tools.params.ParameterFactory;
import java.security.Security;
@ -53,11 +54,6 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
description = "Returns all command names.")
private boolean showAllCommands;
// Do not make this final - compile-time constant inlining may interfere with JCommander.
@ParametersDelegate
private AppEngineConnectionFlags appEngineConnectionFlags =
new AppEngineConnectionFlags();
// Do not make this final - compile-time constant inlining may interfere with JCommander.
@ParametersDelegate
@ -84,7 +80,6 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
Security.addProvider(new BouncyCastleProvider());
component = DaggerRegistryToolComponent.builder()
.flagsModule(new AppEngineConnectionFlags.FlagsModule(appEngineConnectionFlags))
.build();
}
@ -197,7 +192,7 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
RemoteApiOptions options = new RemoteApiOptions();
options.server(
getConnection().getServer().getHost(), getConnection().getServer().getPort());
if (getConnection().isLocalhost()) {
if (RegistryConfig.areServersLocal()) {
// Use dev credentials for localhost.
options.useDevelopmentServerCredential();
} else {

View file

@ -46,7 +46,6 @@ import javax.inject.Singleton;
@Singleton
@Component(
modules = {
AppEngineConnectionFlags.FlagsModule.class,
AppEngineServiceUtilsModule.class,
// TODO(b/36866706): Find a way to replace this with a command-line friendly version
AuthModule.class,

View file

@ -57,10 +57,10 @@ final class VerifyOteCommand implements CommandWithConnection, CommandWithRemote
description = "Only show a summary of information")
private boolean summarize;
private Connection connection;
private AppEngineConnection connection;
@Override
public void setConnection(Connection connection) {
public void setConnection(AppEngineConnection connection) {
this.connection = connection;
}