From a45d3d3bc73b7a3deaa0fba5ff8b64cf539c4408 Mon Sep 17 00:00:00 2001 From: guyben Date: Tue, 30 Oct 2018 09:06:03 -0700 Subject: [PATCH 001/134] Add a log statement at the very end of our code Having a log at the very begining of "our" code helped us find issues with App-Engine's dispatcher, where we could clearly see in the logs that "our" code started many seconds after the request came in. We now suspect there is something similar going on after the end of "our" code - where the reply is sent back many seconds after our code finished running. To make sure - we add a log statement at the very last line of "our" code, so we know exactly when it ended. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219309962 --- .../registry/module/backend/BackendServlet.java | 12 +++++++++++- java/google/registry/module/frontend/BUILD | 1 + .../registry/module/frontend/FrontendServlet.java | 12 +++++++++++- java/google/registry/module/pubapi/BUILD | 1 + .../registry/module/pubapi/PubApiServlet.java | 14 ++++++++++++-- java/google/registry/module/tools/BUILD | 1 + .../google/registry/module/tools/ToolsServlet.java | 12 +++++++++++- 7 files changed, 48 insertions(+), 5 deletions(-) diff --git a/java/google/registry/module/backend/BackendServlet.java b/java/google/registry/module/backend/BackendServlet.java index 328775f13..5a7f7f4fa 100644 --- a/java/google/registry/module/backend/BackendServlet.java +++ b/java/google/registry/module/backend/BackendServlet.java @@ -18,6 +18,7 @@ import com.google.appengine.api.LifecycleManager; import com.google.common.flogger.FluentLogger; import com.google.monitoring.metrics.MetricReporter; import dagger.Lazy; +import google.registry.util.SystemClock; import java.io.IOException; import java.security.Security; import java.util.concurrent.TimeUnit; @@ -26,6 +27,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.joda.time.DateTime; /** Servlet that should handle all requests to our "backend" App Engine module. */ public final class BackendServlet extends HttpServlet { @@ -34,6 +36,7 @@ public final class BackendServlet extends HttpServlet { private static final BackendRequestHandler requestHandler = component.requestHandler(); private static final Lazy metricReporter = component.metricReporter(); private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final SystemClock clock = new SystemClock(); @Override public void init() { @@ -64,6 +67,13 @@ public final class BackendServlet extends HttpServlet { @Override public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { logger.atInfo().log("Received backend request"); - requestHandler.handleRequest(req, rsp); + DateTime startTime = clock.nowUtc(); + try { + requestHandler.handleRequest(req, rsp); + } finally { + logger.atInfo().log( + "Finished backend request. Latency: %.3fs", + (clock.nowUtc().getMillis() - startTime.getMillis()) / 1000d); + } } } diff --git a/java/google/registry/module/frontend/BUILD b/java/google/registry/module/frontend/BUILD index 589a82287..246b6c426 100644 --- a/java/google/registry/module/frontend/BUILD +++ b/java/google/registry/module/frontend/BUILD @@ -29,6 +29,7 @@ java_library( "@com_google_monitoring_client_metrics", "@javax_inject", "@javax_servlet_api", + "@joda_time", "@org_bouncycastle_bcpkix_jdk15on", ], ) diff --git a/java/google/registry/module/frontend/FrontendServlet.java b/java/google/registry/module/frontend/FrontendServlet.java index 455c410e5..3043392eb 100644 --- a/java/google/registry/module/frontend/FrontendServlet.java +++ b/java/google/registry/module/frontend/FrontendServlet.java @@ -18,6 +18,7 @@ import com.google.appengine.api.LifecycleManager; import com.google.common.flogger.FluentLogger; import com.google.monitoring.metrics.MetricReporter; import dagger.Lazy; +import google.registry.util.SystemClock; import java.io.IOException; import java.security.Security; import java.util.concurrent.TimeUnit; @@ -26,6 +27,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.joda.time.DateTime; /** Servlet that should handle all requests to our "default" App Engine module. */ public final class FrontendServlet extends HttpServlet { @@ -34,6 +36,7 @@ public final class FrontendServlet extends HttpServlet { private static final FrontendRequestHandler requestHandler = component.requestHandler(); private static final Lazy metricReporter = component.metricReporter(); private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final SystemClock clock = new SystemClock(); @Override public void init() { @@ -63,6 +66,13 @@ public final class FrontendServlet extends HttpServlet { @Override public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { logger.atInfo().log("Received frontend request"); - requestHandler.handleRequest(req, rsp); + DateTime startTime = clock.nowUtc(); + try { + requestHandler.handleRequest(req, rsp); + } finally { + logger.atInfo().log( + "Finished frontend request. Latency: %.3fs", + (clock.nowUtc().getMillis() - startTime.getMillis()) / 1000d); + } } } diff --git a/java/google/registry/module/pubapi/BUILD b/java/google/registry/module/pubapi/BUILD index e3d388311..be2d79e5e 100644 --- a/java/google/registry/module/pubapi/BUILD +++ b/java/google/registry/module/pubapi/BUILD @@ -29,6 +29,7 @@ java_library( "@com_google_monitoring_client_metrics", "@javax_inject", "@javax_servlet_api", + "@joda_time", "@org_bouncycastle_bcpkix_jdk15on", ], ) diff --git a/java/google/registry/module/pubapi/PubApiServlet.java b/java/google/registry/module/pubapi/PubApiServlet.java index 15d48d9a9..47df81a93 100644 --- a/java/google/registry/module/pubapi/PubApiServlet.java +++ b/java/google/registry/module/pubapi/PubApiServlet.java @@ -18,6 +18,7 @@ import com.google.appengine.api.LifecycleManager; import com.google.common.flogger.FluentLogger; import com.google.monitoring.metrics.MetricReporter; import dagger.Lazy; +import google.registry.util.SystemClock; import java.io.IOException; import java.security.Security; import java.util.concurrent.TimeUnit; @@ -26,6 +27,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.joda.time.DateTime; /** Servlet that should handle all requests to our "default" App Engine module. */ public final class PubApiServlet extends HttpServlet { @@ -34,6 +36,7 @@ public final class PubApiServlet extends HttpServlet { private static final PubApiRequestHandler requestHandler = component.requestHandler(); private static final Lazy metricReporter = component.metricReporter(); private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final SystemClock clock = new SystemClock(); @Override public void init() { @@ -62,7 +65,14 @@ public final class PubApiServlet extends HttpServlet { @Override public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - logger.atInfo().log("Received frontend request"); - requestHandler.handleRequest(req, rsp); + logger.atInfo().log("Received pubapi request"); + DateTime startTime = clock.nowUtc(); + try { + requestHandler.handleRequest(req, rsp); + } finally { + logger.atInfo().log( + "Finished pubapi request. Latency: %.3fs", + (clock.nowUtc().getMillis() - startTime.getMillis()) / 1000d); + } } } diff --git a/java/google/registry/module/tools/BUILD b/java/google/registry/module/tools/BUILD index a46e09e90..6b0767ba8 100644 --- a/java/google/registry/module/tools/BUILD +++ b/java/google/registry/module/tools/BUILD @@ -32,6 +32,7 @@ java_library( "@com_google_flogger_system_backend", "@javax_inject", "@javax_servlet_api", + "@joda_time", "@org_bouncycastle_bcpkix_jdk15on", ], ) diff --git a/java/google/registry/module/tools/ToolsServlet.java b/java/google/registry/module/tools/ToolsServlet.java index 79dd085b6..5914dcdb7 100644 --- a/java/google/registry/module/tools/ToolsServlet.java +++ b/java/google/registry/module/tools/ToolsServlet.java @@ -15,12 +15,14 @@ package google.registry.module.tools; import com.google.common.flogger.FluentLogger; +import google.registry.util.SystemClock; import java.io.IOException; import java.security.Security; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.joda.time.DateTime; /** Servlet that should handle all requests to our "tools" App Engine module. */ public final class ToolsServlet extends HttpServlet { @@ -28,6 +30,7 @@ public final class ToolsServlet extends HttpServlet { private static final ToolsComponent component = DaggerToolsComponent.create(); private static final ToolsRequestHandler requestHandler = component.requestHandler(); private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final SystemClock clock = new SystemClock(); @Override public void init() { @@ -37,6 +40,13 @@ public final class ToolsServlet extends HttpServlet { @Override public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { logger.atInfo().log("Received tools request"); - requestHandler.handleRequest(req, rsp); + DateTime startTime = clock.nowUtc(); + try { + requestHandler.handleRequest(req, rsp); + } finally { + logger.atInfo().log( + "Finished tools request. Latency: %.3fs", + (clock.nowUtc().getMillis() - startTime.getMillis()) / 1000d); + } } } From 57f06258d35db6e6d5630b81e6c7039d2a56e4ec Mon Sep 17 00:00:00 2001 From: guyben Date: Tue, 30 Oct 2018 09:23:55 -0700 Subject: [PATCH 002/134] Add metrics measuring all request processing times The cardinality of this new metric is: buckets - 16 path + method - around 100 (the number of Actions we have) authLevel - at most 3 success - 2 Total: 16*100*3*2 = 9,600 This is still low, especially for the value it could give in understanding our system (graphs of all endpoints, how often are they called, how long they take, how often do they fail) Instead of "success true/false", we might want to give the actual status code. This can be a bit annoying because HttpServletResponse doesn't have a getStatus. But it's possible, and worth considering. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219312400 --- java/google/registry/request/BUILD | 1 + .../registry/request/RequestHandler.java | 18 ++++++ .../registry/request/RequestMetrics.java | 61 +++++++++++++++++++ javatests/google/registry/request/BUILD | 1 + .../registry/request/RequestHandlerTest.java | 25 +++++++- 5 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 java/google/registry/request/RequestMetrics.java diff --git a/java/google/registry/request/BUILD b/java/google/registry/request/BUILD index 3b9eee131..870cb29f9 100644 --- a/java/google/registry/request/BUILD +++ b/java/google/registry/request/BUILD @@ -21,6 +21,7 @@ java_library( "@com_google_flogger", "@com_google_flogger_system_backend", "@com_google_guava", + "@com_google_monitoring_client_metrics", "@com_googlecode_json_simple", "@javax_inject", "@javax_servlet_api", diff --git a/java/google/registry/request/RequestHandler.java b/java/google/registry/request/RequestHandler.java index c2748faa2..a2f8421fb 100644 --- a/java/google/registry/request/RequestHandler.java +++ b/java/google/registry/request/RequestHandler.java @@ -23,6 +23,8 @@ import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; import com.google.common.flogger.FluentLogger; import google.registry.request.auth.AuthResult; import google.registry.request.auth.RequestAuthenticator; +import google.registry.util.NonFinalForTesting; +import google.registry.util.SystemClock; import google.registry.util.TypeUtils.TypeInstantiator; import java.io.IOException; import java.util.Optional; @@ -30,6 +32,8 @@ import javax.annotation.Nullable; import javax.inject.Provider; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.joda.time.DateTime; +import org.joda.time.Duration; /** * Dagger-based request processor. @@ -64,6 +68,10 @@ public class RequestHandler { private final Router router; private final Provider> requestComponentBuilderProvider; private final RequestAuthenticator requestAuthenticator; + private final SystemClock clock = new SystemClock(); + + @NonFinalForTesting + RequestMetrics requestMetrics = new RequestMetrics(); /** * Constructor for subclasses to create a new request handler for a specific request component. @@ -143,6 +151,8 @@ public class RequestHandler { .requestModule(new RequestModule(req, rsp, authResult.get())) .build(); // Apply the selected Route to the component to produce an Action instance, and run it. + boolean success = true; + DateTime startTime = clock.nowUtc(); try { route.get().instantiator().apply(component).run(); if (route.get().action().automaticallyPrintOk()) { @@ -151,6 +161,14 @@ public class RequestHandler { } } catch (HttpException e) { e.send(rsp); + success = false; + } finally { + requestMetrics.record( + new Duration(startTime, clock.nowUtc()), + path, + method, + authResult.get().authLevel(), + success); } } } diff --git a/java/google/registry/request/RequestMetrics.java b/java/google/registry/request/RequestMetrics.java new file mode 100644 index 000000000..e7f88d954 --- /dev/null +++ b/java/google/registry/request/RequestMetrics.java @@ -0,0 +1,61 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.request; + +import static com.google.monitoring.metrics.EventMetric.DEFAULT_FITTER; + +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.FluentLogger; +import com.google.monitoring.metrics.EventMetric; +import com.google.monitoring.metrics.LabelDescriptor; +import com.google.monitoring.metrics.MetricRegistryImpl; +import google.registry.request.auth.AuthLevel; +import org.joda.time.Duration; + +class RequestMetrics { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private static final ImmutableSet REQUEST_LABEL_DESCRIPTORS = + ImmutableSet.of( + LabelDescriptor.create("path", "target path"), + LabelDescriptor.create("method", "request method"), + LabelDescriptor.create("authLevel", "how the user was authenticated"), + LabelDescriptor.create("success", "whether the request succeeded")); + + static final EventMetric requestDurationMetric = + MetricRegistryImpl.getDefault() + .newEventMetric( + "/request/processing_time", + "Action processing time", + "milliseconds", + REQUEST_LABEL_DESCRIPTORS, + DEFAULT_FITTER); + + public RequestMetrics() {} + + public void record( + Duration duration, String path, Action.Method method, AuthLevel authLevel, boolean success) { + requestDurationMetric.record( + duration.getMillis(), + path, + String.valueOf(method), + String.valueOf(authLevel), + String.valueOf(success)); + logger.atInfo().log( + "Action called for path=%s, method=%s, authLevel=%s, success=%s. Took: %.3fs", + path, method, authLevel, success, duration.getMillis() / 1000d); + } +} diff --git a/javatests/google/registry/request/BUILD b/javatests/google/registry/request/BUILD index 270178661..e4ed50602 100644 --- a/javatests/google/registry/request/BUILD +++ b/javatests/google/registry/request/BUILD @@ -17,6 +17,7 @@ java_library( "@com_google_appengine_api_1_0_sdk", "@com_google_guava", "@com_google_guava_testlib", + "@com_google_monitoring_client_contrib", "@com_google_truth", "@com_google_truth_extensions_truth_java8_extension", "@com_googlecode_json_simple", diff --git a/javatests/google/registry/request/RequestHandlerTest.java b/javatests/google/registry/request/RequestHandlerTest.java index 9a3bdb0aa..93fb43de8 100644 --- a/javatests/google/registry/request/RequestHandlerTest.java +++ b/javatests/google/registry/request/RequestHandlerTest.java @@ -20,6 +20,8 @@ import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.POST; import static google.registry.request.auth.Auth.AUTH_INTERNAL_OR_ADMIN; import static google.registry.request.auth.Auth.AUTH_PUBLIC; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -126,7 +128,7 @@ public final class RequestHandlerTest { @Action( path = "/auth/none", auth = AUTH_PUBLIC, - method = Action.Method.GET + method = GET ) public class AuthNoneAction extends AuthBase { AuthNoneAction(AuthResult authResult) { @@ -137,7 +139,7 @@ public final class RequestHandlerTest { @Action( path = "/auth/adminUser", auth = AUTH_INTERNAL_OR_ADMIN, - method = Action.Method.GET) + method = GET) public class AuthAdminUserAction extends AuthBase { AuthAdminUserAction(AuthResult authResult) { super(authResult); @@ -200,6 +202,7 @@ public final class RequestHandlerTest { private final SlothTask slothTask = mock(SlothTask.class); private final SafeSlothTask safeSlothTask = mock(SafeSlothTask.class); private final RequestAuthenticator requestAuthenticator = mock(RequestAuthenticator.class); + private final RequestMetrics requestMetrics = mock(RequestMetrics.class); private final Component component = new Component(); private final StringWriter httpOutput = new StringWriter(); @@ -223,11 +226,17 @@ public final class RequestHandlerTest { }, requestAuthenticator); when(rsp.getWriter()).thenReturn(new PrintWriter(httpOutput)); + handler.requestMetrics = requestMetrics; } @After public void after() { - verifyNoMoreInteractions(rsp, bumblebeeTask, slothTask, safeSlothTask); + verifyNoMoreInteractions(rsp, bumblebeeTask, slothTask, safeSlothTask, requestMetrics); + } + + private void assertMetric( + String path, Action.Method method, AuthLevel authLevel, boolean success) { + verify(requestMetrics).record(any(), eq(path), eq(method), eq(authLevel), eq(success)); } @Test @@ -241,6 +250,7 @@ public final class RequestHandlerTest { verifyZeroInteractions(rsp); verify(bumblebeeTask).run(); + assertMetric("/bumblebee", GET, AuthLevel.NONE, true); } @Test @@ -253,6 +263,7 @@ public final class RequestHandlerTest { handler.handleRequest(req, rsp); verify(bumblebeeTask).run(); + assertMetric("/bumblebee", POST, AuthLevel.NONE, true); } @Test @@ -265,6 +276,7 @@ public final class RequestHandlerTest { handler.handleRequest(req, rsp); verify(bumblebeeTask).run(); + assertMetric("/bumblebee/hive", GET, AuthLevel.NONE, true); } @Test @@ -280,6 +292,7 @@ public final class RequestHandlerTest { verify(rsp).setContentType("text/plain; charset=utf-8"); verify(rsp).getWriter(); assertThat(httpOutput.toString()).isEqualTo("OK\n"); + assertMetric("/sloth", POST, AuthLevel.NONE, true); } @Test @@ -304,6 +317,7 @@ public final class RequestHandlerTest { handler.handleRequest(req, rsp); verify(rsp).sendError(503, "Set sail for fail"); + assertMetric("/fail", GET, AuthLevel.NONE, false); } /** Test for a regression of the issue in b/21377705. */ @@ -318,6 +332,7 @@ public final class RequestHandlerTest { handler.handleRequest(req, rsp); verify(rsp).sendError(503, "Fail at construction"); + assertMetric("/failAtConstruction", GET, AuthLevel.NONE, false); } @Test @@ -389,6 +404,7 @@ public final class RequestHandlerTest { handler.handleRequest(req, rsp); verify(safeSlothTask).run(); + assertMetric("/safe-sloth", POST, AuthLevel.NONE, true); } @Test @@ -401,6 +417,7 @@ public final class RequestHandlerTest { handler.handleRequest(req, rsp); verify(safeSlothTask).run(); + assertMetric("/safe-sloth", GET, AuthLevel.NONE, true); } @Test @@ -415,6 +432,7 @@ public final class RequestHandlerTest { assertThat(providedAuthResult).isNotNull(); assertThat(providedAuthResult.authLevel()).isEqualTo(AuthLevel.NONE); assertThat(providedAuthResult.userAuthInfo()).isEmpty(); + assertMetric("/auth/none", GET, AuthLevel.NONE, true); } @Test @@ -445,6 +463,7 @@ public final class RequestHandlerTest { assertThat(providedAuthResult.userAuthInfo()).isPresent(); assertThat(providedAuthResult.userAuthInfo().get().user()).isEqualTo(testUser); assertThat(providedAuthResult.userAuthInfo().get().oauthTokenInfo()).isEmpty(); + assertMetric("/auth/adminUser", GET, AuthLevel.USER, true); } } From 09202562c723fee1e63a14d0f91f358e720fa363 Mon Sep 17 00:00:00 2001 From: mountford Date: Tue, 30 Oct 2018 11:17:40 -0700 Subject: [PATCH 003/134] RDAP: Add link to static TOS page ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219332975 --- .../registry/config/RegistryConfig.java | 66 ++++++++++++------- .../config/RegistryConfigSettings.java | 1 + .../registry/config/files/default-config.yaml | 4 ++ 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index 65f9883d5..813946e75 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -1225,6 +1225,18 @@ public final class RegistryConfig { return ImmutableList.copyOf(Splitter.on('\n').split(config.registryPolicy.rdapTos)); } + /** + * Link to static Web page with RDAP terms of service. Displayed in RDAP responses. + * + * @see google.registry.rdap.RdapJsonFormatter + */ + @Provides + @Config("rdapTosStaticUrl") + @Nullable + public static String provideRdapTosStaticUrl(RegistryConfigSettings config) { + return config.registryPolicy.rdapTosStaticUrl; + } + /** * Returns the help text to be used by RDAP. * @@ -1235,30 +1247,38 @@ public final class RegistryConfig { @Provides @Config("rdapHelpMap") public static ImmutableMap provideRdapHelpMap( - @Config("rdapTos") ImmutableList rdapTos) { + @Config("rdapTos") ImmutableList rdapTos, + @Config("rdapTosStaticUrl") @Nullable String rdapTosStaticUrl) { return new ImmutableMap.Builder() - .put("/", RdapNoticeDescriptor.builder() - .setTitle("RDAP Help") - .setDescription(ImmutableList.of( - "domain/XXXX", - "nameserver/XXXX", - "entity/XXXX", - "domains?name=XXXX", - "domains?nsLdhName=XXXX", - "domains?nsIp=XXXX", - "nameservers?name=XXXX", - "nameservers?ip=XXXX", - "entities?fn=XXXX", - "entities?handle=XXXX", - "help/XXXX")) - .setLinkValueSuffix("help/") - .setLinkHrefUrlString("https://github.com/google/nomulus/blob/master/docs/rdap.md") - .build()) - .put("/tos", RdapNoticeDescriptor.builder() - .setTitle("RDAP Terms of Service") - .setDescription(rdapTos) - .setLinkValueSuffix("help/tos") - .build()) + .put( + "/", + RdapNoticeDescriptor.builder() + .setTitle("RDAP Help") + .setDescription( + ImmutableList.of( + "domain/XXXX", + "nameserver/XXXX", + "entity/XXXX", + "domains?name=XXXX", + "domains?nsLdhName=XXXX", + "domains?nsIp=XXXX", + "nameservers?name=XXXX", + "nameservers?ip=XXXX", + "entities?fn=XXXX", + "entities?handle=XXXX", + "help/XXXX")) + .setLinkValueSuffix("help/") + .setLinkHrefUrlString( + "https://github.com/google/nomulus/blob/master/docs/rdap.md") + .build()) + .put( + "/tos", + RdapNoticeDescriptor.builder() + .setTitle("RDAP Terms of Service") + .setDescription(rdapTos) + .setLinkValueSuffix("help/tos") + .setLinkHrefUrlString(rdapTosStaticUrl) + .build()) .build(); } diff --git a/java/google/registry/config/RegistryConfigSettings.java b/java/google/registry/config/RegistryConfigSettings.java index 58561f0fa..2449c1c58 100644 --- a/java/google/registry/config/RegistryConfigSettings.java +++ b/java/google/registry/config/RegistryConfigSettings.java @@ -88,6 +88,7 @@ public class RegistryConfigSettings { public String reservedTermsExportDisclaimer; public String whoisDisclaimer; public String rdapTos; + public String rdapTosStaticUrl; public String spec11EmailBodyTemplate; } diff --git a/java/google/registry/config/files/default-config.yaml b/java/google/registry/config/files/default-config.yaml index 9c4a37628..85fe99625 100644 --- a/java/google/registry/config/files/default-config.yaml +++ b/java/google/registry/config/files/default-config.yaml @@ -145,6 +145,10 @@ registryPolicy: We reserve the right to modify this agreement at any time. + # Link to static Web page with RDAP terms of service. Displayed in RDAP + # responses. If null, no static Web page link is generated. + rdapTosStaticUrl: null + # Body of the spec 11 email sent to registrars. # Items in braces are to be replaced. spec11EmailBodyTemplate: | From 00d0284d5b173965cc605618e16cc413208a1c90 Mon Sep 17 00:00:00 2001 From: weiminyu Date: Tue, 30 Oct 2018 19:05:13 -0700 Subject: [PATCH 004/134] Add nebula-lint plugin to Gradle scripts Updated the project config so that the plugin can work. Reverted to the deprecated compile/testCompile labels so that the plugin can work. This plugin provides valuable checks that we do not want to give up. Added undeclared-dependency check. Still need to add unused-dependency and one-version check. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219406600 --- gradle/build.gradle | 37 ++-- gradle/core/build.gradle | 356 +++++++++++++++++++++------------------ 2 files changed, 211 insertions(+), 182 deletions(-) diff --git a/gradle/build.gradle b/gradle/build.gradle index 9c8b11b2c..645505a82 100644 --- a/gradle/build.gradle +++ b/gradle/build.gradle @@ -1,29 +1,36 @@ -buildscript { - repositories { - jcenter() - mavenCentral() - - maven { - url 'https://plugins.gradle.org/m2/' - } - } +plugins { + id 'java' + id 'nebula.lint' version '10.1.2' } allprojects { repositories { jcenter() mavenCentral() - flatDir { - // The objectify jar that comes with Nomulus. - dirs "${rootDir}/../third_party/objectify/v4_1" - } } - // Single version across all projects for now. version = '1.0' - // Java plugin: apply plugin: 'java' + + apply plugin: 'nebula.lint' + gradleLint.rules = [ + // Checks if Gradle wrapper is up-to-date + 'archaic-wrapper', + // Checks for indirect dependencies with dynamic version spec. Best + // practice calls for declaring them with specific versions. + 'undeclared-dependency' + // TODO(weiminyu): enable more dependency check + ] } +project(':core') { + dependencies { + // Custom-built objectify jar at commit ecd5165, included in Nomulus + // release. + implementation files( + "${rootDir}/../third_party/objectify/v4_1/objectify-4.1.3.jar") + testImplementation project(':third_party') + } +} diff --git a/gradle/core/build.gradle b/gradle/core/build.gradle index c9745215c..98ef87c2c 100644 --- a/gradle/core/build.gradle +++ b/gradle/core/build.gradle @@ -45,162 +45,171 @@ configurations { } -// Relevant canned dependency labels: -// - implementation: Dependencies to be included in release distribution. -// - compileOnly: Dependencies used at compile time only for production code. They will not be -// included in release. -// - testImplementation: Dependencies needed for testing only. +// TODO(weiminyu): switch to api/implementation labels. +// Gradle has deprecated the compile/testCompile labels, but nebula-lint has +// not caught up. The plugin provides valuable dependency checks that we do not +// want to give up. +// See https://github.com/nebula-plugins/gradle-lint-plugin/issues/130 for +// issue status. dependencies { - implementation 'com.beust:jcommander:1.48' - implementation 'com.fasterxml.jackson.core:jackson-core:2.8.5' - implementation 'com.fasterxml.jackson.core:jackson-annotations:2.8.0' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.8.5' - implementation 'com.google.api-client:google-api-client:1.22.0' - implementation 'com.google.api-client:google-api-client-appengine:1.22.0' - implementation 'com.google.api-client:google-api-client-jackson2:1.20.0' - implementation 'com.google.monitoring-client:metrics:1.0.4' - implementation 'com.google.monitoring-client:stackdriver:1.0.4' - implementation 'com.google.api-client:google-api-client-java6:1.20.0' - implementation 'com.google.api-client:google-api-client-servlet:1.22.0' - implementation 'com.google.apis:google-api-services-admin-directory:directory_v1-rev72-1.22.0' - implementation 'com.google.apis:google-api-services-bigquery:v2-rev325-1.22.0' - implementation 'com.google.apis:google-api-services-clouddebugger:v2-rev8-1.22.0' - implementation 'com.google.apis:google-api-services-cloudkms:v1-rev12-1.22.0' - implementation 'com.google.apis:google-api-services-cloudresourcemanager:v1-rev6-1.22.0' - implementation 'com.google.apis:google-api-services-dataflow:v1b3-rev196-1.22.0' - implementation 'com.google.apis:google-api-services-dns:v2beta1-rev6-1.22.0' - implementation 'com.google.apis:google-api-services-drive:v2-rev160-1.19.1' - implementation 'com.google.apis:google-api-services-groupssettings:v1-rev60-1.22.0' - implementation 'com.google.apis:google-api-services-monitoring:v3-rev11-1.22.0' - implementation 'com.google.apis:google-api-services-sheets:v4-rev483-1.22.0' - implementation 'com.google.apis:google-api-services-storage:v1-rev86-1.22.0' - // TODO(b/71631624): change appengine:appengine-api-1.0-sdk to testCompileOnly after - // BillingEmailUtilsTest.java is fixed. - implementation 'com.google.appengine:appengine-api-1.0-sdk:1.9.48' - implementation 'com.google.appengine:appengine-api-labs:1.9.48' - implementation 'com.google.appengine:appengine-api-stubs:1.9.48' - implementation 'com.google.appengine.tools:appengine-gcs-client:0.6' - implementation 'com.google.appengine.tools:appengine-mapreduce:0.8.5' - implementation 'com.google.appengine.tools:appengine-pipeline:0.2.13' - implementation 'com.google.appengine:appengine-tools-sdk:1.9.48' - implementation 'com.google.auth:google-auth-library-credentials:0.7.1' - implementation 'com.google.auth:google-auth-library-oauth2-http:0.7.1' - implementation 'com.google.auto:auto-common:0.8' - implementation 'com.google.auto.factory:auto-factory:1.0-beta3' - implementation 'com.google.auto.value:auto-value-annotations:1.6.2' - implementation 'com.google.cloud.bigdataoss:gcsio:1.4.5' - implementation 'com.google.cloud.bigdataoss:util:1.4.5' - implementation 'com.google.code.findbugs:jsr305:3.0.2' - implementation 'com.google.dagger:dagger:2.15' - implementation 'com.google.dagger:dagger-producers:2.15' - implementation 'com.google.errorprone:error_prone_annotations:2.1.3' - implementation 'com.google.errorprone:javac-shaded:9-dev-r4023-3' - implementation 'com.google.flogger:flogger:0.1' - implementation 'com.google.flogger:flogger-system-backend:0.1' - implementation 'com.google.gdata:core:1.47.1' - implementation 'com.google.googlejavaformat:google-java-format:1.4' - implementation 'com.google.guava:guava:25.1-jre' - implementation 'com.google.http-client:google-http-client:1.22.0' - implementation 'com.google.http-client:google-http-client-appengine:1.22.0' - implementation 'com.google.http-client:google-http-client-jackson2:1.22.0' - implementation 'com.google.oauth-client:google-oauth-client:1.22.0' - implementation 'com.google.oauth-client:google-oauth-client-appengine:1.22.0' - implementation 'com.google.oauth-client:google-oauth-client-java6:1.22.0' - implementation 'com.google.oauth-client:google-oauth-client-jetty:1.22.0' - implementation 'com.google.oauth-client:google-oauth-client-servlet:1.22.0' - implementation 'com.google.protobuf:protobuf-java:2.6.0' - implementation 'com.google.re2j:re2j:1.1' - implementation 'com.google.template:soy:2018-03-14' - implementation 'com.googlecode.charts4j:charts4j:1.3' - implementation 'com.googlecode.json-simple:json-simple:1.1.1' - implementation 'com.ibm.icu:icu4j:57.1' - implementation 'com.jcraft:jsch:0.1.53' - implementation 'com.jcraft:jzlib:1.1.3' - implementation 'com.squareup:javapoet:1.8.0' - implementation 'com.squareup:javawriter:2.5.1' - implementation 'com.sun.activation:javax.activation:1.2.0' - implementation 'com.thoughtworks.paranamer:paranamer:2.7' - implementation 'commons-codec:commons-codec:1.6' - implementation 'commons-logging:commons-logging:1.1.1' - implementation 'dnsjava:dnsjava:2.1.7' - implementation 'io.netty:netty-buffer:4.1.28.Final' - implementation 'io.netty:netty-codec:4.1.28.Final' - implementation 'io.netty:netty-codec-http:4.1.28.Final' - implementation 'io.netty:netty-common:4.1.28.Final' - implementation 'io.netty:netty-handler:4.1.28.Final' - implementation 'io.netty:netty-resolver:4.1.28.Final' - implementation 'io.netty:netty-tcnative:2.0.12.Final' - implementation 'io.netty:netty-tcnative-boringssl-static:2.0.12.Final' - implementation 'io.netty:netty-transport:4.1.28.Final' - implementation 'it.unimi.dsi:fastutil:6.5.16' - implementation 'javax.annotation:jsr250-api:1.0' - implementation 'javax.inject:javax.inject:1' - implementation 'javax.mail:mail:1.4' - implementation 'javax.servlet:servlet-api:2.5' - implementation 'javax.xml.bind:jaxb-api:2.3.0' - implementation 'javax.xml.soap:javax.xml.soap-api:1.4.0' - implementation 'jline:jline:1.0' - implementation 'joda-time:joda-time:2.3' - implementation 'org.apache.avro:avro:1.8.2' - implementation 'org.apache.beam:beam-runners-direct-java:2.2.0' - implementation 'org.apache.beam:beam-runners-google-cloud-dataflow-java:2.1.0' - implementation 'org.apache.beam:beam-sdks-common-runner-api:2.1.0' - implementation 'org.apache.beam:beam-sdks-java-core:2.2.0' - implementation 'org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.1.0' - implementation 'org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.2.0' - implementation 'org.apache.commons:commons-compress:1.8.1' - implementation 'org.apache.ftpserver:ftpserver-core:1.0.6' - implementation 'org.apache.httpcomponents:httpclient:4.5.2' - implementation 'org.apache.httpcomponents:httpcore:4.4.4' - implementation 'org.apache.mina:mina-core:2.0.4' - implementation 'org.apache.sshd:sshd-core:2.0.0' - implementation 'org.apache.sshd:sshd-scp:2.0.0' - implementation 'org.apache.sshd:sshd-sftp:2.0.0' - implementation 'org.apache.tomcat:servlet-api:6.0.45' - implementation 'org.apache.tomcat:tomcat-annotations-api:8.0.5' - implementation 'org.bouncycastle:bcpg-jdk15on:1.52' - implementation 'org.bouncycastle:bcpkix-jdk15on:1.52' - implementation 'org.bouncycastle:bcprov-jdk15on:1.52' - implementation 'org.codehaus.jackson:jackson-core-asl:1.9.13' - implementation 'org.codehaus.jackson:jackson-mapper-asl:1.9.13' - implementation 'org.joda:joda-money:0.10.0' - implementation 'org.json:json:20160810' - implementation 'org.khronos:opengl-api:gl1.1-android-2.1_r1' - implementation 'org.mortbay.jetty:jetty:6.1.26' - implementation 'org.mortbay.jetty:servlet-api:2.5-20081211' - implementation 'org.mortbay.jetty:jetty-util:6.1.26' - implementation 'org.slf4j:slf4j-api:1.7.16' - implementation 'org.tukaani:xz:1.5' - implementation 'org.xerial.snappy:snappy-java:1.1.4-M3' - implementation 'org.yaml:snakeyaml:1.17' - implementation 'xerces:xmlParserAPIs:2.6.2' - implementation 'xpp3:xpp3:1.1.4c' - // Custom-built objectify jar at commit ecd5165, included in Nomulus release. - implementation name: 'objectify-4.1.3' + compile 'com.beust:jcommander:1.48' + compile 'com.fasterxml.jackson.core:jackson-core:2.8.5' + compile 'com.fasterxml.jackson.core:jackson-annotations:2.8.0' + compile 'com.fasterxml.jackson.core:jackson-databind:2.8.5' + compile 'com.google.api-client:google-api-client:1.22.0' + compile 'com.google.api-client:google-api-client-appengine:1.22.0' + compile 'com.google.api-client:google-api-client-jackson2:1.20.0' + compile 'com.google.monitoring-client:metrics:1.0.4' + compile 'com.google.monitoring-client:stackdriver:1.0.4' + compile 'com.google.api-client:google-api-client-java6:1.20.0' + compile 'com.google.api-client:google-api-client-servlet:1.22.0' + compile 'com.google.apis:google-api-services-admin-directory:directory_v1-rev72-1.22.0' + compile 'com.google.apis:google-api-services-bigquery:v2-rev325-1.22.0' + compile 'com.google.apis:google-api-services-clouddebugger:v2-rev8-1.22.0' + compile 'com.google.apis:google-api-services-cloudkms:v1-rev12-1.22.0' + compile 'com.google.apis:google-api-services-cloudresourcemanager:v1-rev6-1.22.0' + compile 'com.google.apis:google-api-services-dataflow:v1b3-rev196-1.22.0' + compile 'com.google.apis:google-api-services-dns:v2beta1-rev6-1.22.0' + compile 'com.google.apis:google-api-services-drive:v2-rev160-1.19.1' + compile 'com.google.apis:google-api-services-groupssettings:v1-rev60-1.22.0' + compile 'com.google.apis:google-api-services-monitoring:v3-rev11-1.22.0' + compile 'com.google.apis:google-api-services-sheets:v4-rev483-1.22.0' + compile 'com.google.apis:google-api-services-storage:v1-rev86-1.22.0' + // TODO(b/71631624): change appengine:appengine-api-1.0-sdk to + // testCompileOnly after BillingEmailUtilsTest.java is fixed. + compile 'com.google.appengine:appengine-api-1.0-sdk:1.9.48' + compile 'com.google.appengine:appengine-api-labs:1.9.48' + compile 'com.google.appengine:appengine-api-stubs:1.9.48' + compile 'com.google.appengine.tools:appengine-gcs-client:0.6' + compile 'com.google.appengine.tools:appengine-mapreduce:0.8.5' + compile 'com.google.appengine.tools:appengine-pipeline:0.2.13' + compile 'com.google.appengine:appengine-remote-api:1.9.48' + compile 'com.google.appengine:appengine-tools-sdk:1.9.48' + compile 'com.google.auth:google-auth-library-credentials:0.7.1' + compile 'com.google.auth:google-auth-library-oauth2-http:0.7.1' + compile 'com.google.auto:auto-common:0.8' + compile 'com.google.auto.factory:auto-factory:1.0-beta3' + compile 'com.google.auto.value:auto-value-annotations:1.6.2' + compile 'com.google.cloud.bigdataoss:gcsio:1.4.5' + compile 'com.google.cloud.bigdataoss:util:1.4.5' + compile 'com.google.code.findbugs:jsr305:3.0.2' + compile 'com.google.dagger:dagger:2.15' + compile 'com.google.dagger:dagger-producers:2.15' + compile 'com.google.errorprone:error_prone_annotations:2.3.1' + compile 'com.google.errorprone:javac-shaded:9-dev-r4023-3' + compile 'com.google.flogger:flogger:0.1' + compile 'com.google.flogger:flogger-system-backend:0.1' + compile 'com.google.gdata:core:1.47.1' + compile 'com.google.googlejavaformat:google-java-format:1.4' + compile 'com.google.guava:guava-jdk5:17.0' + compile 'com.google.guava:guava:25.1-jre' + compile 'com.google.gwt:gwt-user:2.8.2' + compile 'com.google.http-client:google-http-client:1.22.0' + compile 'com.google.http-client:google-http-client-appengine:1.22.0' + compile 'com.google.http-client:google-http-client-jackson2:1.22.0' + compile 'com.google.oauth-client:google-oauth-client:1.22.0' + compile 'com.google.oauth-client:google-oauth-client-appengine:1.22.0' + compile 'com.google.oauth-client:google-oauth-client-java6:1.22.0' + compile 'com.google.oauth-client:google-oauth-client-jetty:1.22.0' + compile 'com.google.oauth-client:google-oauth-client-servlet:1.22.0' + compile 'com.google.protobuf:protobuf-java:2.6.0' + compile 'com.google.re2j:re2j:1.1' + compile 'com.google.template:soy:2018-03-14' + compile 'com.googlecode.charts4j:charts4j:1.3' + compile 'com.googlecode.json-simple:json-simple:1.1.1' + compile 'com.ibm.icu:icu4j:57.1' + compile 'com.jcraft:jsch:0.1.53' + compile 'com.jcraft:jzlib:1.1.3' + compile 'com.squareup:javapoet:1.8.0' + compile 'com.squareup:javawriter:2.5.1' + compile 'com.sun.activation:javax.activation:1.2.0' + compile 'com.thoughtworks.paranamer:paranamer:2.7' + compile 'commons-codec:commons-codec:1.6' + compile 'commons-logging:commons-logging:1.1.1' + compile 'dnsjava:dnsjava:2.1.7' + compile 'io.netty:netty-buffer:4.1.28.Final' + compile 'io.netty:netty-codec:4.1.28.Final' + compile 'io.netty:netty-codec-http:4.1.28.Final' + compile 'io.netty:netty-common:4.1.28.Final' + compile 'io.netty:netty-handler:4.1.28.Final' + compile 'io.netty:netty-resolver:4.1.28.Final' + compile 'io.netty:netty-tcnative:2.0.12.Final' + compile 'io.netty:netty-tcnative-boringssl-static:2.0.12.Final' + compile 'io.netty:netty-transport:4.1.28.Final' + compile 'it.unimi.dsi:fastutil:6.5.16' + compile 'javax.annotation:jsr250-api:1.0' + compile 'javax.inject:javax.inject:1' + compile 'javax.mail:mail:1.4' + compile 'javax.servlet:servlet-api:2.5' + compile 'javax.xml.bind:jaxb-api:2.3.0' + compile 'javax.xml.soap:javax.xml.soap-api:1.4.0' + compile 'jline:jline:1.0' + compile 'joda-time:joda-time:2.3' + compile 'org.apache.avro:avro:1.8.2' + compile 'org.apache.beam:beam-runners-direct-java:2.2.0' + compile 'org.apache.beam:beam-runners-google-cloud-dataflow-java:2.1.0' + compile 'org.apache.beam:beam-sdks-common-runner-api:2.1.0' + compile 'org.apache.beam:beam-sdks-java-core:2.2.0' + compile 'org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.1.0' + compile 'org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.2.0' + compile 'org.apache.commons:commons-compress:1.8.1' + compile 'org.apache.ftpserver:ftplet-api:1.0.6' + compile 'org.apache.ftpserver:ftpserver-core:1.0.6' + compile 'org.apache.httpcomponents:httpclient:4.5.2' + compile 'org.apache.httpcomponents:httpcore:4.4.4' + compile 'org.apache.mina:mina-core:2.0.4' + compile 'org.apache.sshd:sshd-core:2.0.0' + compile 'org.apache.sshd:sshd-scp:2.0.0' + compile 'org.apache.sshd:sshd-sftp:2.0.0' + compile 'org.apache.tomcat:servlet-api:6.0.45' + compile 'org.apache.tomcat:tomcat-annotations-api:8.0.5' + compile 'org.bouncycastle:bcpg-jdk15on:1.52' + compile 'org.bouncycastle:bcpkix-jdk15on:1.52' + compile 'org.bouncycastle:bcprov-jdk15on:1.52' + compile 'org.codehaus.jackson:jackson-core-asl:1.9.13' + compile 'org.codehaus.jackson:jackson-mapper-asl:1.9.13' + compile 'org.joda:joda-money:0.10.0' + compile 'org.json:json:20160810' + compile 'org.khronos:opengl-api:gl1.1-android-2.1_r1' + compile 'org.mortbay.jetty:jetty:6.1.26' + compile 'org.mortbay.jetty:servlet-api:2.5-20081211' + compile 'org.mortbay.jetty:jetty-util:6.1.26' + compile 'org.slf4j:slf4j-api:1.7.16' + compile 'org.tukaani:xz:1.5' + compile 'org.xerial.snappy:snappy-java:1.1.4-M3' + compile 'org.yaml:snakeyaml:1.17' + compile 'xerces:xmlParserAPIs:2.6.2' + compile 'xpp3:xpp3:1.1.4c' - compileOnly 'com.google.appengine:appengine-remote-api:1.9.48' // Also testImplementation - compileOnly 'com.google.auto.service:auto-service:1.0-rc4' - compileOnly 'org.osgi:org.osgi.core:4.3.0' + // Known issue: nebula-lint misses inherited dependency. + compile project(':third_party') + // Include auto-value in compile until nebula-lint understands + // annotationProcessor + compile 'com.google.auto.value:auto-value:1.6.2' annotationProcessor 'com.google.auto.value:auto-value:1.6.2' testAnnotationProcessor 'com.google.auto.value:auto-value:1.6.2' annotationProcessor 'com.google.dagger:dagger-compiler:2.15' testAnnotationProcessor 'com.google.dagger:dagger-compiler:2.15' - testImplementation 'com.google.appengine:appengine-remote-api:1.9.48' // Also compileOnly - testImplementation 'com.google.appengine:appengine-testing:1.9.58' - testImplementation 'com.google.guava:guava-testlib:25.0-jre' - testImplementation 'com.google.monitoring-client:contrib:1.0.4' - testImplementation 'com.google.truth:truth:0.42' - testImplementation 'com.google.truth.extensions:truth-java8-extension:0.39' - testImplementation 'org.hamcrest:hamcrest-all:1.3' - testImplementation 'org.hamcrest:hamcrest-core:1.3' - testImplementation 'org.hamcrest:hamcrest-library:1.3' - testImplementation 'junit:junit:4.12' - testImplementation 'org.mockito:mockito-all:1.9.5' - testImplementation 'org.slf4j:slf4j-simple:1.7.16' // Not needed by Bazel + // Use compile to work around nebula bug. This should be testCompile. + compile 'com.google.appengine:appengine-testing:1.9.58' - testImplementation project(':third_party') + testCompile 'com.google.guava:guava-testlib:25.0-jre' + testCompile 'com.google.monitoring-client:contrib:1.0.4' + testCompile 'com.google.truth:truth:0.42' + testCompile 'com.google.truth.extensions:truth-java8-extension:0.39' + testCompile 'org.hamcrest:hamcrest-all:1.3' + testCompile 'org.hamcrest:hamcrest-core:1.3' + testCompile 'org.hamcrest:hamcrest-library:1.3' + testCompile 'junit:junit:4.12' + testCompile 'org.mockito:mockito-all:1.9.5' + testCompile 'org.slf4j:slf4j-simple:1.7.16' // Not needed by Bazel + + // Indirect dependency found by undeclared-dependency check. Such + // dependencies should go after all other compile and testCompile + // dependencies to avoid overriding them accidentally. + compile 'javax.servlet:javax.servlet-api:3.1.0' // google-api-client-appeng // Dependencies needed for jaxb compilation. // Use jaxb 2.2.11 because 2.3 is known to break the Ant task we use. @@ -219,12 +228,13 @@ dependencies { css 'args4j:args4j:2.0.26' } -task jaxbToJava() { +task jaxbToJava { doLast { file(generatedDir).mkdirs() - // Temp dir to hold schema and bindings files. Files must be in the same directory because - // the bindings (.xjb) file does not declare relative paths to schema (.xsd) files. + // Temp dir to hold schema and bindings files. Files must be in the same + // directory because the bindings (.xjb) file does not declare relative + // paths to schema (.xsd) files. def xjcTempSourceDir = file("${temporaryDir}/xjc") xjcTempSourceDir.mkdirs() ant.copy( @@ -252,7 +262,8 @@ task jaxbToJava() { dir: new File("$xjcTempSourceDir"), include: ['**/*.xsd']) .addToAntBuilder(ant, 'schema', FileCollection.AntType.FileSet) - // -npa: do not generate package-info.java files. They will be generated below. + // -npa: do not generate package-info.java files. They will be generated + // below. arg(line: '-npa -quiet -extension') } exec { @@ -265,7 +276,7 @@ task jaxbToJava() { } } -task soyToJava() { +task soyToJava { ext.soyToJava = { javaPackage, outputDirectory, soyFiles -> javaexec { main = "com.google.template.soy.SoyParseInfoGenerator" @@ -280,12 +291,17 @@ task soyToJava() { doLast { - soyToJava('google.registry.tools.soy', "${generatedDir}/google/registry/tools/soy", - fileTree(dir: "${javaDir}/google/registry/tools/soy", include: ['**/*.soy'])) + soyToJava('google.registry.tools.soy', + "${generatedDir}/google/registry/tools/soy", + fileTree( + dir: "${javaDir}/google/registry/tools/soy", + include: ['**/*.soy'])) soyToJava('google.registry.ui.soy.registrar', "${generatedDir}/google/registry/ui/soy/registrar", - fileTree(dir: "${javaDir}/google/registry/ui/soy/registrar", include: ['**/*.soy'])) + fileTree( + dir: "${javaDir}/google/registry/ui/soy/registrar", + include: ['**/*.soy'])) soyToJava('google.registry.ui.soy', "${generatedDir}/google/registry/ui/soy", @@ -327,11 +343,16 @@ task stylesheetsToJavascript { def outputDir = "${project.buildDir}/resources/main/google/registry/ui/css" file("${outputDir}").mkdirs() def srcFiles = [ - "${cssSourceDir}/console.css", "${cssSourceDir}/contact-settings.css", - "${cssSourceDir}/contact-us.css", "${cssSourceDir}/dashboard.css", - "${cssSourceDir}/epp.css", "${cssSourceDir}/forms.css", - "${cssSourceDir}/kd_components.css", "${cssSourceDir}/registry.css", - "${cssSourceDir}/resources.css", "${cssSourceDir}/security-settings.css" + "${cssSourceDir}/console.css", + "${cssSourceDir}/contact-settings.css", + "${cssSourceDir}/contact-us.css", + "${cssSourceDir}/dashboard.css", + "${cssSourceDir}/epp.css", + "${cssSourceDir}/forms.css", + "${cssSourceDir}/kd_components.css", + "${cssSourceDir}/registry.css", + "${cssSourceDir}/resources.css", + "${cssSourceDir}/security-settings.css" ] cssCompile("${outputDir}/registrar_bin", false, srcFiles) cssCompile("${outputDir}/registrar_dbg", true, srcFiles) @@ -341,8 +362,8 @@ task stylesheetsToJavascript { compileJava.dependsOn jaxbToJava compileJava.dependsOn soyToJava -// stylesheetsToJavascript must happen after processResources, which wipes the resources folder -// before copying data into it. +// stylesheetsToJavascript must happen after processResources, which wipes the +// resources folder before copying data into it. stylesheetsToJavascript.dependsOn processResources classes.dependsOn stylesheetsToJavascript @@ -350,15 +371,16 @@ classes.dependsOn stylesheetsToJavascript test { // Test exclusion patterns: // - *TestCase.java are inherited by concrete test classes. - // - *TestSuite.java are excluded to avoid duplicate execution of suite members. See README - // in this directory for more information. + // - *TestSuite.java are excluded to avoid duplicate execution of suite + // members. See README in this directory for more information. exclude "**/*TestCase.*", "**/*TestSuite.*" - // Use a single JVM to execute all tests. See README in this directory for more information. + // Use a single JVM to execute all tests. See README in this directory for + // more information. maxParallelForks 1 - // Use a single thread to execute all tests in a JVM. See README in this directory for more - // information. + // Use a single thread to execute all tests in a JVM. See README in this + // directory for more information. forkEvery 1 // Uncomment to see test outputs in stdout. From 08290e6b87f6aeb2b262feff0ccc79b4f0e29fb1 Mon Sep 17 00:00:00 2001 From: guyben Date: Thu, 1 Nov 2018 07:15:39 -0700 Subject: [PATCH 005/134] Fix the tool that was broken in [] ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219629901 --- java/google/registry/tools/RegistryCli.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/java/google/registry/tools/RegistryCli.java b/java/google/registry/tools/RegistryCli.java index 2de69bd38..f9dbe7f1d 100644 --- a/java/google/registry/tools/RegistryCli.java +++ b/java/google/registry/tools/RegistryCli.java @@ -31,6 +31,7 @@ import com.google.common.collect.Iterables; import google.registry.config.RegistryConfig; import google.registry.model.ofy.ObjectifyService; import google.registry.tools.params.ParameterFactory; +import java.net.URL; import java.security.Security; import java.util.Map; import org.bouncycastle.jce.provider.BouncyCastleProvider; @@ -191,7 +192,7 @@ final class RegistryCli implements AutoCloseable, CommandRunner { installer = new RemoteApiInstaller(); RemoteApiOptions options = new RemoteApiOptions(); options.server( - getConnection().getServer().getHost(), getConnection().getServer().getPort()); + getConnection().getServer().getHost(), getPort(getConnection().getServer())); if (RegistryConfig.areServersLocal()) { // Use dev credentials for localhost. options.useDevelopmentServerCredential(); @@ -211,6 +212,10 @@ final class RegistryCli implements AutoCloseable, CommandRunner { command.run(); } + private int getPort(URL url) { + return url.getPort() == -1 ? url.getDefaultPort() : url.getPort(); + } + void setEnvironment(RegistryToolEnvironment environment) { this.environment = environment; } From 545b68ad9a07fe16e19045095f8e58e7b0df656c Mon Sep 17 00:00:00 2001 From: mmuller Date: Thu, 1 Nov 2018 12:25:59 -0700 Subject: [PATCH 006/134] Add a .travis.yml file Add the Travis-CI configuration file, which will allow our Gradle build to be tested from Github with travis. TESTED: 1) Repeatedly tweaked this on own local fork of google/nomulus until I was able to successfully build. 2) Verified that this shows up in the tarball after "blaze build :opensource-staging" ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219678030 --- .travis.yml | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..eb0915070 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,45 @@ +# Copyright 2018 The Nomulus Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +language: java +install: true +jdk: + # Our builds fail against Oracle Java for reasons yet unknown. + - openjdk8 + +# Caching options suggested by a random article. +before_cache: + - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock + - rm -f $HOME/.gradle/caches/*/plugin-resolution/ + +cache: + directories: + - $HOME/.gradle/caches/ + - $HOME/.gradle/wrapper/ + +env: + # Disable fancy status information (looks bad on travis and exceeds logfile + # quota) + TERM=dumb + +install: + # Install a specific gradle version first, default gradle can't deal with + # our gradle build scripts. + - wget http://services.gradle.org/distributions/gradle-4.10.2-bin.zip && unzip gradle-4.10.2-bin.zip + +# Specialize gradle build to use an up-to-date gradle and the /gradle +# directory. +# The "travis_wait 45" lets our build spend up to 45 minutes without writing +# output, instead of the default 10. +script: cd gradle && ../gradle-4.10.2/bin/gradle wrapper && travis_wait 45 ./gradlew build From 3f6585fccc3806112536acaac26f2087b8f598d3 Mon Sep 17 00:00:00 2001 From: guyben Date: Fri, 2 Nov 2018 06:59:50 -0700 Subject: [PATCH 007/134] Reduce duplicate code in the servlets Currently, all 4 servlets (backend, frontend, pubapi, tools) have duplicates of the same exact code. That's an anti-pattern! Created a ServletBase they can all extend which has the duplicate code. As a bonus, the tools servlet now runs the metric reporter, meaning tool related metrics will now be reported! ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219792176 --- java/google/registry/module/backend/BUILD | 1 + .../module/backend/BackendServlet.java | 56 ++----------------- java/google/registry/module/frontend/BUILD | 1 + .../module/frontend/FrontendServlet.java | 55 ++---------------- java/google/registry/module/pubapi/BUILD | 1 + .../module/pubapi/PubApiComponent.java | 2 +- .../registry/module/pubapi/PubApiServlet.java | 55 ++---------------- java/google/registry/module/tools/BUILD | 2 + .../registry/module/tools/ToolsComponent.java | 6 ++ .../registry/module/tools/ToolsServlet.java | 35 +++--------- 10 files changed, 31 insertions(+), 183 deletions(-) diff --git a/java/google/registry/module/backend/BUILD b/java/google/registry/module/backend/BUILD index 75aa46b18..52c375bfd 100644 --- a/java/google/registry/module/backend/BUILD +++ b/java/google/registry/module/backend/BUILD @@ -27,6 +27,7 @@ java_library( "//java/google/registry/keyring/kms", "//java/google/registry/mapreduce", "//java/google/registry/model", + "//java/google/registry/module", "//java/google/registry/monitoring/whitebox", "//java/google/registry/rde", "//java/google/registry/rde/imports", diff --git a/java/google/registry/module/backend/BackendServlet.java b/java/google/registry/module/backend/BackendServlet.java index 5a7f7f4fa..896840937 100644 --- a/java/google/registry/module/backend/BackendServlet.java +++ b/java/google/registry/module/backend/BackendServlet.java @@ -14,66 +14,18 @@ package google.registry.module.backend; -import com.google.appengine.api.LifecycleManager; -import com.google.common.flogger.FluentLogger; import com.google.monitoring.metrics.MetricReporter; import dagger.Lazy; -import google.registry.util.SystemClock; -import java.io.IOException; -import java.security.Security; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.joda.time.DateTime; +import google.registry.module.ServletBase; /** Servlet that should handle all requests to our "backend" App Engine module. */ -public final class BackendServlet extends HttpServlet { +public final class BackendServlet extends ServletBase { private static final BackendComponent component = DaggerBackendComponent.create(); private static final BackendRequestHandler requestHandler = component.requestHandler(); private static final Lazy metricReporter = component.metricReporter(); - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final SystemClock clock = new SystemClock(); - @Override - public void init() { - Security.addProvider(new BouncyCastleProvider()); - - // If metric reporter failed to instantiate for any reason (bad keyring, bad json credential, - // etc), we log the error but keep the main thread running. Also the shutdown hook will only be - // registered if metric reporter starts up correctly. - try { - metricReporter.get().startAsync().awaitRunning(10, TimeUnit.SECONDS); - logger.atInfo().log("Started up MetricReporter"); - LifecycleManager.getInstance() - .setShutdownHook( - () -> { - try { - metricReporter.get().stopAsync().awaitTerminated(10, TimeUnit.SECONDS); - logger.atInfo().log("Shut down MetricReporter"); - } catch (TimeoutException timeoutException) { - logger.atSevere().withCause(timeoutException).log( - "Failed to stop MetricReporter: %s", timeoutException); - } - }); - } catch (Exception e) { - logger.atSevere().withCause(e).log("Failed to initialize MetricReporter."); - } - } - - @Override - public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - logger.atInfo().log("Received backend request"); - DateTime startTime = clock.nowUtc(); - try { - requestHandler.handleRequest(req, rsp); - } finally { - logger.atInfo().log( - "Finished backend request. Latency: %.3fs", - (clock.nowUtc().getMillis() - startTime.getMillis()) / 1000d); - } + public BackendServlet() { + super(requestHandler, metricReporter); } } diff --git a/java/google/registry/module/frontend/BUILD b/java/google/registry/module/frontend/BUILD index 246b6c426..88f243784 100644 --- a/java/google/registry/module/frontend/BUILD +++ b/java/google/registry/module/frontend/BUILD @@ -14,6 +14,7 @@ java_library( "//java/google/registry/keyring", "//java/google/registry/keyring/api", "//java/google/registry/keyring/kms", + "//java/google/registry/module", "//java/google/registry/monitoring/whitebox", "//java/google/registry/request", "//java/google/registry/request:modules", diff --git a/java/google/registry/module/frontend/FrontendServlet.java b/java/google/registry/module/frontend/FrontendServlet.java index 3043392eb..2b2b72350 100644 --- a/java/google/registry/module/frontend/FrontendServlet.java +++ b/java/google/registry/module/frontend/FrontendServlet.java @@ -14,65 +14,18 @@ package google.registry.module.frontend; -import com.google.appengine.api.LifecycleManager; -import com.google.common.flogger.FluentLogger; import com.google.monitoring.metrics.MetricReporter; import dagger.Lazy; -import google.registry.util.SystemClock; -import java.io.IOException; -import java.security.Security; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.joda.time.DateTime; +import google.registry.module.ServletBase; /** Servlet that should handle all requests to our "default" App Engine module. */ -public final class FrontendServlet extends HttpServlet { +public final class FrontendServlet extends ServletBase { private static final FrontendComponent component = DaggerFrontendComponent.create(); private static final FrontendRequestHandler requestHandler = component.requestHandler(); private static final Lazy metricReporter = component.metricReporter(); - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final SystemClock clock = new SystemClock(); - @Override - public void init() { - Security.addProvider(new BouncyCastleProvider()); - - // If metric reporter failed to instantiate for any reason (bad keyring, bad json credential, - // etc), we log the error but keep the main thread running. Also the shutdown hook will only be - // registered if metric reporter starts up correctly. - try { - metricReporter.get().startAsync().awaitRunning(10, TimeUnit.SECONDS); - logger.atInfo().log("Started up MetricReporter"); - LifecycleManager.getInstance() - .setShutdownHook( - () -> { - try { - metricReporter.get().stopAsync().awaitTerminated(10, TimeUnit.SECONDS); - logger.atInfo().log("Shut down MetricReporter"); - } catch (TimeoutException e) { - logger.atSevere().withCause(e).log("Failed to stop MetricReporter."); - } - }); - } catch (Exception e) { - logger.atSevere().withCause(e).log("Failed to initialize MetricReporter."); - } - } - - @Override - public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - logger.atInfo().log("Received frontend request"); - DateTime startTime = clock.nowUtc(); - try { - requestHandler.handleRequest(req, rsp); - } finally { - logger.atInfo().log( - "Finished frontend request. Latency: %.3fs", - (clock.nowUtc().getMillis() - startTime.getMillis()) / 1000d); - } + public FrontendServlet() { + super(requestHandler, metricReporter); } } diff --git a/java/google/registry/module/pubapi/BUILD b/java/google/registry/module/pubapi/BUILD index be2d79e5e..40d4b233b 100644 --- a/java/google/registry/module/pubapi/BUILD +++ b/java/google/registry/module/pubapi/BUILD @@ -14,6 +14,7 @@ java_library( "//java/google/registry/keyring", "//java/google/registry/keyring/api", "//java/google/registry/keyring/kms", + "//java/google/registry/module", "//java/google/registry/monitoring/whitebox", "//java/google/registry/rdap", "//java/google/registry/request", diff --git a/java/google/registry/module/pubapi/PubApiComponent.java b/java/google/registry/module/pubapi/PubApiComponent.java index ef5ffbc65..966ad54f0 100644 --- a/java/google/registry/module/pubapi/PubApiComponent.java +++ b/java/google/registry/module/pubapi/PubApiComponent.java @@ -47,12 +47,12 @@ import javax.inject.Singleton; CredentialModule.class, CustomLogicFactoryModule.class, DummyKeyringModule.class, - PubApiRequestComponentModule.class, Jackson2Module.class, KeyModule.class, KeyringModule.class, KmsModule.class, NetHttpTransportModule.class, + PubApiRequestComponentModule.class, ServerTridProviderModule.class, StackdriverModule.class, SystemClockModule.class, diff --git a/java/google/registry/module/pubapi/PubApiServlet.java b/java/google/registry/module/pubapi/PubApiServlet.java index 47df81a93..b907f9042 100644 --- a/java/google/registry/module/pubapi/PubApiServlet.java +++ b/java/google/registry/module/pubapi/PubApiServlet.java @@ -14,65 +14,18 @@ package google.registry.module.pubapi; -import com.google.appengine.api.LifecycleManager; -import com.google.common.flogger.FluentLogger; import com.google.monitoring.metrics.MetricReporter; import dagger.Lazy; -import google.registry.util.SystemClock; -import java.io.IOException; -import java.security.Security; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.joda.time.DateTime; +import google.registry.module.ServletBase; /** Servlet that should handle all requests to our "default" App Engine module. */ -public final class PubApiServlet extends HttpServlet { +public final class PubApiServlet extends ServletBase { private static final PubApiComponent component = DaggerPubApiComponent.create(); private static final PubApiRequestHandler requestHandler = component.requestHandler(); private static final Lazy metricReporter = component.metricReporter(); - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final SystemClock clock = new SystemClock(); - @Override - public void init() { - Security.addProvider(new BouncyCastleProvider()); - - // If metric reporter failed to instantiate for any reason (bad keyring, bad json credential, - // etc), we log the error but keep the main thread running. Also the shutdown hook will only be - // registered if metric reporter starts up correctly. - try { - metricReporter.get().startAsync().awaitRunning(10, TimeUnit.SECONDS); - logger.atInfo().log("Started up MetricReporter"); - LifecycleManager.getInstance() - .setShutdownHook( - () -> { - try { - metricReporter.get().stopAsync().awaitTerminated(10, TimeUnit.SECONDS); - logger.atInfo().log("Shut down MetricReporter"); - } catch (TimeoutException e) { - logger.atSevere().withCause(e).log("Failed to stop MetricReporter."); - } - }); - } catch (Exception e) { - logger.atSevere().withCause(e).log("Failed to initialize MetricReporter."); - } - } - - @Override - public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - logger.atInfo().log("Received pubapi request"); - DateTime startTime = clock.nowUtc(); - try { - requestHandler.handleRequest(req, rsp); - } finally { - logger.atInfo().log( - "Finished pubapi request. Latency: %.3fs", - (clock.nowUtc().getMillis() - startTime.getMillis()) / 1000d); - } + public PubApiServlet() { + super(requestHandler, metricReporter); } } diff --git a/java/google/registry/module/tools/BUILD b/java/google/registry/module/tools/BUILD index 6b0767ba8..39d5dc545 100644 --- a/java/google/registry/module/tools/BUILD +++ b/java/google/registry/module/tools/BUILD @@ -20,6 +20,7 @@ java_library( "//java/google/registry/keyring/kms", "//java/google/registry/loadtest", "//java/google/registry/mapreduce", + "//java/google/registry/module", "//java/google/registry/monitoring/whitebox", "//java/google/registry/request", "//java/google/registry/request:modules", @@ -30,6 +31,7 @@ java_library( "@com_google_dagger", "@com_google_flogger", "@com_google_flogger_system_backend", + "@com_google_monitoring_client_metrics", "@javax_inject", "@javax_servlet_api", "@joda_time", diff --git a/java/google/registry/module/tools/ToolsComponent.java b/java/google/registry/module/tools/ToolsComponent.java index 605ddc95a..b7dc1747a 100644 --- a/java/google/registry/module/tools/ToolsComponent.java +++ b/java/google/registry/module/tools/ToolsComponent.java @@ -14,7 +14,9 @@ package google.registry.module.tools; +import com.google.monitoring.metrics.MetricReporter; import dagger.Component; +import dagger.Lazy; import google.registry.config.CredentialModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.export.DriveModule; @@ -29,6 +31,7 @@ import google.registry.keyring.api.DummyKeyringModule; import google.registry.keyring.api.KeyModule; import google.registry.keyring.kms.KmsModule; import google.registry.module.tools.ToolsRequestComponent.ToolsRequestComponentModule; +import google.registry.monitoring.whitebox.StackdriverModule; import google.registry.request.Modules.DatastoreServiceModule; import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.NetHttpTransportModule; @@ -62,6 +65,7 @@ import javax.inject.Singleton; KmsModule.class, NetHttpTransportModule.class, ServerTridProviderModule.class, + StackdriverModule.class, SystemClockModule.class, SystemSleeperModule.class, ToolsRequestComponentModule.class, @@ -70,4 +74,6 @@ import javax.inject.Singleton; }) interface ToolsComponent { ToolsRequestHandler requestHandler(); + + Lazy metricReporter(); } diff --git a/java/google/registry/module/tools/ToolsServlet.java b/java/google/registry/module/tools/ToolsServlet.java index 5914dcdb7..023c51164 100644 --- a/java/google/registry/module/tools/ToolsServlet.java +++ b/java/google/registry/module/tools/ToolsServlet.java @@ -14,39 +14,18 @@ package google.registry.module.tools; -import com.google.common.flogger.FluentLogger; -import google.registry.util.SystemClock; -import java.io.IOException; -import java.security.Security; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.joda.time.DateTime; +import com.google.monitoring.metrics.MetricReporter; +import dagger.Lazy; +import google.registry.module.ServletBase; /** Servlet that should handle all requests to our "tools" App Engine module. */ -public final class ToolsServlet extends HttpServlet { +public final class ToolsServlet extends ServletBase { private static final ToolsComponent component = DaggerToolsComponent.create(); private static final ToolsRequestHandler requestHandler = component.requestHandler(); - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final SystemClock clock = new SystemClock(); + private static final Lazy metricReporter = component.metricReporter(); - @Override - public void init() { - Security.addProvider(new BouncyCastleProvider()); - } - - @Override - public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - logger.atInfo().log("Received tools request"); - DateTime startTime = clock.nowUtc(); - try { - requestHandler.handleRequest(req, rsp); - } finally { - logger.atInfo().log( - "Finished tools request. Latency: %.3fs", - (clock.nowUtc().getMillis() - startTime.getMillis()) / 1000d); - } + public ToolsServlet() { + super(requestHandler, metricReporter); } } From b5856a14673e2f844d6f5bcc08bfff8b2a83d022 Mon Sep 17 00:00:00 2001 From: jianglai Date: Fri, 2 Nov 2018 07:24:26 -0700 Subject: [PATCH 008/134] Fix FOSS build ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219794867 --- java/google/registry/module/BUILD | 23 ++++++ java/google/registry/module/ServletBase.java | 83 ++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 java/google/registry/module/BUILD create mode 100644 java/google/registry/module/ServletBase.java diff --git a/java/google/registry/module/BUILD b/java/google/registry/module/BUILD new file mode 100644 index 000000000..84346bf7f --- /dev/null +++ b/java/google/registry/module/BUILD @@ -0,0 +1,23 @@ +package( + default_visibility = ["//java/google/registry:registry_project"], +) + +licenses(["notice"]) # Apache 2.0 + +java_library( + name = "module", + srcs = glob(["*.java"]), + deps = [ + "//java/google/registry/request", + "//java/google/registry/util", + "@com_google_appengine_api_1_0_sdk", + "@com_google_dagger", + "@com_google_flogger", + "@com_google_flogger_system_backend", + "@com_google_monitoring_client_metrics", + "@javax_inject", + "@javax_servlet_api", + "@joda_time", + "@org_bouncycastle_bcpkix_jdk15on", + ], +) diff --git a/java/google/registry/module/ServletBase.java b/java/google/registry/module/ServletBase.java new file mode 100644 index 000000000..99b2a0dcc --- /dev/null +++ b/java/google/registry/module/ServletBase.java @@ -0,0 +1,83 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.module; + +import com.google.appengine.api.LifecycleManager; +import com.google.common.flogger.FluentLogger; +import com.google.monitoring.metrics.MetricReporter; +import dagger.Lazy; +import google.registry.request.RequestHandler; +import google.registry.util.SystemClock; +import java.io.IOException; +import java.security.Security; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.joda.time.DateTime; + +/** Base for Servlets that handle all requests to our App Engine modules. */ +public class ServletBase extends HttpServlet { + + private final RequestHandler requestHandler; + private final Lazy metricReporter; + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final SystemClock clock = new SystemClock(); + + public ServletBase(RequestHandler requestHandler, Lazy metricReporter) { + this.requestHandler = requestHandler; + this.metricReporter = metricReporter; + } + + @Override + public void init() { + Security.addProvider(new BouncyCastleProvider()); + + // If metric reporter failed to instantiate for any reason (bad keyring, bad json credential, + // etc), we log the error but keep the main thread running. Also the shutdown hook will only be + // registered if metric reporter starts up correctly. + try { + metricReporter.get().startAsync().awaitRunning(10, TimeUnit.SECONDS); + logger.atInfo().log("Started up MetricReporter"); + LifecycleManager.getInstance() + .setShutdownHook( + () -> { + try { + metricReporter.get().stopAsync().awaitTerminated(10, TimeUnit.SECONDS); + logger.atInfo().log("Shut down MetricReporter"); + } catch (TimeoutException e) { + logger.atSevere().withCause(e).log("Failed to stop MetricReporter."); + } + }); + } catch (Exception e) { + logger.atSevere().withCause(e).log("Failed to initialize MetricReporter."); + } + } + + @Override + public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { + logger.atInfo().log("Received %s request", getClass().getSimpleName()); + DateTime startTime = clock.nowUtc(); + try { + requestHandler.handleRequest(req, rsp); + } finally { + logger.atInfo().log( + "Finished %s request. Latency: %.3fs", + getClass().getSimpleName(), (clock.nowUtc().getMillis() - startTime.getMillis()) / 1000d); + } + } +} From 6a870e582003190c8cd6aa77e1d036e4d773b229 Mon Sep 17 00:00:00 2001 From: weiminyu Date: Fri, 2 Nov 2018 08:36:46 -0700 Subject: [PATCH 009/134] Check for unused dependencies Enabled unused-dependency check using nebula-lint. Dependencies that are not used by compile or testing are labeled with 'maybe_runtime". We leave these dependencies in the script for easy reference. Before launching Gradle-based release process we must determine which of these should be removed and which should be relabeled as runtime. Label assignment: - All dependencies recommended for removal from 'compile' are changed to maybe_runtime - All dependencies recommended for move from 'compile' to testCompile are split into two lines, one with testCompile, the other maybe_runtime Incidentally, Gradle 4.10.2 needs a groovy upgrade before it can work with Oracle JDK 11. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219803797 --- gradle/build.gradle | 5 +- gradle/core/build.gradle | 152 ++++++++++++++++++++++----------------- 2 files changed, 90 insertions(+), 67 deletions(-) diff --git a/gradle/build.gradle b/gradle/build.gradle index 645505a82..c0dde73d5 100644 --- a/gradle/build.gradle +++ b/gradle/build.gradle @@ -19,8 +19,9 @@ allprojects { 'archaic-wrapper', // Checks for indirect dependencies with dynamic version spec. Best // practice calls for declaring them with specific versions. - 'undeclared-dependency' - // TODO(weiminyu): enable more dependency check + 'undeclared-dependency', + 'unused-dependency' + // TODO(weiminyu): enable more dependency checks ] } diff --git a/gradle/core/build.gradle b/gradle/core/build.gradle index 98ef87c2c..fe3a0cfb5 100644 --- a/gradle/core/build.gradle +++ b/gradle/core/build.gradle @@ -42,32 +42,41 @@ configurations { css jaxb soy + // Label for all dependencies inherited from Bazel build but not used in + // either compile or testRuntime. However, they may be needed at runtime. + // TODO(weiminyu): identify runtime dependencies and remove the rest. + maybe_runtime } - -// TODO(weiminyu): switch to api/implementation labels. -// Gradle has deprecated the compile/testCompile labels, but nebula-lint has -// not caught up. The plugin provides valuable dependency checks that we do not -// want to give up. -// See https://github.com/nebula-plugins/gradle-lint-plugin/issues/130 for -// issue status. +// Known issues: +// - The (test/)compile/runtime labels are deprecated. We continue using these +// labels due to nebula-lint. +// TODO(weiminyu): switch to api/implementation labels. +// See https://github.com/nebula-plugins/gradle-lint-plugin/issues/130 for +// issue status. +// - Nebula-lint's conflict between unused and undeclared dependency check. +// If an undeclared dependency is added, the unused-dependency check will flag +// it. For now we wrap affected dependency in gradleLint.ignore block. +// TODO(weiminyu): drop gradleLint.ignore block when issue is fixed. +// See https://github.com/nebula-plugins/gradle-lint-plugin/issues/181 for +// issue status. dependencies { compile 'com.beust:jcommander:1.48' compile 'com.fasterxml.jackson.core:jackson-core:2.8.5' compile 'com.fasterxml.jackson.core:jackson-annotations:2.8.0' compile 'com.fasterxml.jackson.core:jackson-databind:2.8.5' compile 'com.google.api-client:google-api-client:1.22.0' - compile 'com.google.api-client:google-api-client-appengine:1.22.0' - compile 'com.google.api-client:google-api-client-jackson2:1.20.0' + maybe_runtime 'com.google.api-client:google-api-client-appengine:1.22.0' + maybe_runtime 'com.google.api-client:google-api-client-jackson2:1.20.0' compile 'com.google.monitoring-client:metrics:1.0.4' compile 'com.google.monitoring-client:stackdriver:1.0.4' - compile 'com.google.api-client:google-api-client-java6:1.20.0' - compile 'com.google.api-client:google-api-client-servlet:1.22.0' + maybe_runtime 'com.google.api-client:google-api-client-java6:1.20.0' + maybe_runtime 'com.google.api-client:google-api-client-servlet:1.22.0' compile 'com.google.apis:google-api-services-admin-directory:directory_v1-rev72-1.22.0' compile 'com.google.apis:google-api-services-bigquery:v2-rev325-1.22.0' - compile 'com.google.apis:google-api-services-clouddebugger:v2-rev8-1.22.0' + maybe_runtime 'com.google.apis:google-api-services-clouddebugger:v2-rev8-1.22.0' compile 'com.google.apis:google-api-services-cloudkms:v1-rev12-1.22.0' - compile 'com.google.apis:google-api-services-cloudresourcemanager:v1-rev6-1.22.0' + maybe_runtime 'com.google.apis:google-api-services-cloudresourcemanager:v1-rev6-1.22.0' compile 'com.google.apis:google-api-services-dataflow:v1b3-rev196-1.22.0' compile 'com.google.apis:google-api-services-dns:v2beta1-rev6-1.22.0' compile 'com.google.apis:google-api-services-drive:v2-rev160-1.19.1' @@ -78,105 +87,117 @@ dependencies { // TODO(b/71631624): change appengine:appengine-api-1.0-sdk to // testCompileOnly after BillingEmailUtilsTest.java is fixed. compile 'com.google.appengine:appengine-api-1.0-sdk:1.9.48' - compile 'com.google.appengine:appengine-api-labs:1.9.48' - compile 'com.google.appengine:appengine-api-stubs:1.9.48' + maybe_runtime 'com.google.appengine:appengine-api-labs:1.9.48' + maybe_runtime 'com.google.appengine:appengine-api-stubs:1.9.48' + testCompile 'com.google.appengine:appengine-api-stubs:1.9.48' compile 'com.google.appengine.tools:appengine-gcs-client:0.6' compile 'com.google.appengine.tools:appengine-mapreduce:0.8.5' compile 'com.google.appengine.tools:appengine-pipeline:0.2.13' compile 'com.google.appengine:appengine-remote-api:1.9.48' - compile 'com.google.appengine:appengine-tools-sdk:1.9.48' - compile 'com.google.auth:google-auth-library-credentials:0.7.1' - compile 'com.google.auth:google-auth-library-oauth2-http:0.7.1' - compile 'com.google.auto:auto-common:0.8' - compile 'com.google.auto.factory:auto-factory:1.0-beta3' + maybe_runtime 'com.google.appengine:appengine-tools-sdk:1.9.48' + maybe_runtime 'com.google.auth:google-auth-library-credentials:0.7.1' + maybe_runtime 'com.google.auth:google-auth-library-oauth2-http:0.7.1' + maybe_runtime 'com.google.auto:auto-common:0.8' + maybe_runtime 'com.google.auto.factory:auto-factory:1.0-beta3' compile 'com.google.auto.value:auto-value-annotations:1.6.2' - compile 'com.google.cloud.bigdataoss:gcsio:1.4.5' - compile 'com.google.cloud.bigdataoss:util:1.4.5' + maybe_runtime 'com.google.cloud.bigdataoss:gcsio:1.4.5' + maybe_runtime 'com.google.cloud.bigdataoss:util:1.4.5' compile 'com.google.code.findbugs:jsr305:3.0.2' compile 'com.google.dagger:dagger:2.15' - compile 'com.google.dagger:dagger-producers:2.15' + maybe_runtime 'com.google.dagger:dagger-producers:2.15' compile 'com.google.errorprone:error_prone_annotations:2.3.1' - compile 'com.google.errorprone:javac-shaded:9-dev-r4023-3' + maybe_runtime 'com.google.errorprone:javac-shaded:9-dev-r4023-3' compile 'com.google.flogger:flogger:0.1' - compile 'com.google.flogger:flogger-system-backend:0.1' - compile 'com.google.gdata:core:1.47.1' - compile 'com.google.googlejavaformat:google-java-format:1.4' + runtime 'com.google.flogger:flogger-system-backend:0.1' + maybe_runtime 'com.google.gdata:core:1.47.1' + maybe_runtime 'com.google.googlejavaformat:google-java-format:1.4' compile 'com.google.guava:guava-jdk5:17.0' compile 'com.google.guava:guava:25.1-jre' - compile 'com.google.gwt:gwt-user:2.8.2' + gradleLint.ignore('unused-dependency') { + compile 'com.google.gwt:gwt-user:2.8.2' + } compile 'com.google.http-client:google-http-client:1.22.0' compile 'com.google.http-client:google-http-client-appengine:1.22.0' compile 'com.google.http-client:google-http-client-jackson2:1.22.0' compile 'com.google.oauth-client:google-oauth-client:1.22.0' - compile 'com.google.oauth-client:google-oauth-client-appengine:1.22.0' + maybe_runtime 'com.google.oauth-client:google-oauth-client-appengine:1.22.0' compile 'com.google.oauth-client:google-oauth-client-java6:1.22.0' compile 'com.google.oauth-client:google-oauth-client-jetty:1.22.0' - compile 'com.google.oauth-client:google-oauth-client-servlet:1.22.0' - compile 'com.google.protobuf:protobuf-java:2.6.0' + maybe_runtime 'com.google.oauth-client:google-oauth-client-servlet:1.22.0' + maybe_runtime 'com.google.protobuf:protobuf-java:2.6.0' compile 'com.google.re2j:re2j:1.1' compile 'com.google.template:soy:2018-03-14' - compile 'com.googlecode.charts4j:charts4j:1.3' + maybe_runtime 'com.googlecode.charts4j:charts4j:1.3' compile 'com.googlecode.json-simple:json-simple:1.1.1' compile 'com.ibm.icu:icu4j:57.1' compile 'com.jcraft:jsch:0.1.53' - compile 'com.jcraft:jzlib:1.1.3' - compile 'com.squareup:javapoet:1.8.0' - compile 'com.squareup:javawriter:2.5.1' - compile 'com.sun.activation:javax.activation:1.2.0' - compile 'com.thoughtworks.paranamer:paranamer:2.7' - compile 'commons-codec:commons-codec:1.6' - compile 'commons-logging:commons-logging:1.1.1' + maybe_runtime 'com.jcraft:jzlib:1.1.3' + maybe_runtime 'com.squareup:javapoet:1.8.0' + maybe_runtime 'com.squareup:javawriter:2.5.1' + maybe_runtime 'com.sun.activation:javax.activation:1.2.0' + maybe_runtime 'com.thoughtworks.paranamer:paranamer:2.7' + maybe_runtime 'commons-codec:commons-codec:1.6' + maybe_runtime 'commons-logging:commons-logging:1.1.1' compile 'dnsjava:dnsjava:2.1.7' compile 'io.netty:netty-buffer:4.1.28.Final' compile 'io.netty:netty-codec:4.1.28.Final' compile 'io.netty:netty-codec-http:4.1.28.Final' compile 'io.netty:netty-common:4.1.28.Final' compile 'io.netty:netty-handler:4.1.28.Final' - compile 'io.netty:netty-resolver:4.1.28.Final' - compile 'io.netty:netty-tcnative:2.0.12.Final' - compile 'io.netty:netty-tcnative-boringssl-static:2.0.12.Final' + maybe_runtime 'io.netty:netty-resolver:4.1.28.Final' + maybe_runtime 'io.netty:netty-tcnative:2.0.12.Final' + maybe_runtime 'io.netty:netty-tcnative-boringssl-static:2.0.12.Final' compile 'io.netty:netty-transport:4.1.28.Final' - compile 'it.unimi.dsi:fastutil:6.5.16' - compile 'javax.annotation:jsr250-api:1.0' + maybe_runtime 'it.unimi.dsi:fastutil:6.5.16' + maybe_runtime 'javax.annotation:jsr250-api:1.0' + testCompile 'javax.annotation:jsr250-api:1.0' compile 'javax.inject:javax.inject:1' compile 'javax.mail:mail:1.4' compile 'javax.servlet:servlet-api:2.5' compile 'javax.xml.bind:jaxb-api:2.3.0' - compile 'javax.xml.soap:javax.xml.soap-api:1.4.0' + maybe_runtime 'javax.xml.soap:javax.xml.soap-api:1.4.0' compile 'jline:jline:1.0' compile 'joda-time:joda-time:2.3' compile 'org.apache.avro:avro:1.8.2' - compile 'org.apache.beam:beam-runners-direct-java:2.2.0' + maybe_runtime 'org.apache.beam:beam-runners-direct-java:2.2.0' + testCompile 'org.apache.beam:beam-runners-direct-java:2.2.0' compile 'org.apache.beam:beam-runners-google-cloud-dataflow-java:2.1.0' - compile 'org.apache.beam:beam-sdks-common-runner-api:2.1.0' + maybe_runtime 'org.apache.beam:beam-sdks-common-runner-api:2.1.0' compile 'org.apache.beam:beam-sdks-java-core:2.2.0' compile 'org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.1.0' compile 'org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.2.0' - compile 'org.apache.commons:commons-compress:1.8.1' - compile 'org.apache.ftpserver:ftplet-api:1.0.6' - compile 'org.apache.ftpserver:ftpserver-core:1.0.6' + maybe_runtime 'org.apache.commons:commons-compress:1.8.1' + maybe_runtime 'org.apache.ftpserver:ftplet-api:1.0.6' + testCompile 'org.apache.ftpserver:ftplet-api:1.0.6' + maybe_runtime 'org.apache.ftpserver:ftpserver-core:1.0.6' + testCompile 'org.apache.ftpserver:ftpserver-core:1.0.6' compile 'org.apache.httpcomponents:httpclient:4.5.2' compile 'org.apache.httpcomponents:httpcore:4.4.4' - compile 'org.apache.mina:mina-core:2.0.4' - compile 'org.apache.sshd:sshd-core:2.0.0' - compile 'org.apache.sshd:sshd-scp:2.0.0' - compile 'org.apache.sshd:sshd-sftp:2.0.0' + maybe_runtime 'org.apache.mina:mina-core:2.0.4' + maybe_runtime 'org.apache.sshd:sshd-core:2.0.0' + testCompile 'org.apache.sshd:sshd-core:2.0.0' + maybe_runtime 'org.apache.sshd:sshd-scp:2.0.0' + testCompile 'org.apache.sshd:sshd-scp:2.0.0' + maybe_runtime 'org.apache.sshd:sshd-sftp:2.0.0' + testCompile 'org.apache.sshd:sshd-sftp:2.0.0' compile 'org.apache.tomcat:servlet-api:6.0.45' - compile 'org.apache.tomcat:tomcat-annotations-api:8.0.5' + maybe_runtime 'org.apache.tomcat:tomcat-annotations-api:8.0.5' + testCompile 'org.apache.tomcat:tomcat-annotations-api:8.0.5' compile 'org.bouncycastle:bcpg-jdk15on:1.52' compile 'org.bouncycastle:bcpkix-jdk15on:1.52' compile 'org.bouncycastle:bcprov-jdk15on:1.52' - compile 'org.codehaus.jackson:jackson-core-asl:1.9.13' - compile 'org.codehaus.jackson:jackson-mapper-asl:1.9.13' + maybe_runtime 'org.codehaus.jackson:jackson-core-asl:1.9.13' + maybe_runtime 'org.codehaus.jackson:jackson-mapper-asl:1.9.13' compile 'org.joda:joda-money:0.10.0' compile 'org.json:json:20160810' - compile 'org.khronos:opengl-api:gl1.1-android-2.1_r1' - compile 'org.mortbay.jetty:jetty:6.1.26' + maybe_runtime 'org.khronos:opengl-api:gl1.1-android-2.1_r1' + maybe_runtime 'org.mortbay.jetty:jetty:6.1.26' + testCompile 'org.mortbay.jetty:jetty:6.1.26' compile 'org.mortbay.jetty:servlet-api:2.5-20081211' - compile 'org.mortbay.jetty:jetty-util:6.1.26' - compile 'org.slf4j:slf4j-api:1.7.16' - compile 'org.tukaani:xz:1.5' - compile 'org.xerial.snappy:snappy-java:1.1.4-M3' + maybe_runtime 'org.mortbay.jetty:jetty-util:6.1.26' + maybe_runtime 'org.slf4j:slf4j-api:1.7.16' + maybe_runtime 'org.tukaani:xz:1.5' + maybe_runtime 'org.xerial.snappy:snappy-java:1.1.4-M3' compile 'org.yaml:snakeyaml:1.17' compile 'xerces:xmlParserAPIs:2.6.2' compile 'xpp3:xpp3:1.1.4c' @@ -186,7 +207,9 @@ dependencies { // Include auto-value in compile until nebula-lint understands // annotationProcessor - compile 'com.google.auto.value:auto-value:1.6.2' + gradleLint.ignore('unused-dependency') { + compile 'com.google.auto.value:auto-value:1.6.2' + } annotationProcessor 'com.google.auto.value:auto-value:1.6.2' testAnnotationProcessor 'com.google.auto.value:auto-value:1.6.2' annotationProcessor 'com.google.dagger:dagger-compiler:2.15' @@ -204,7 +227,6 @@ dependencies { testCompile 'org.hamcrest:hamcrest-library:1.3' testCompile 'junit:junit:4.12' testCompile 'org.mockito:mockito-all:1.9.5' - testCompile 'org.slf4j:slf4j-simple:1.7.16' // Not needed by Bazel // Indirect dependency found by undeclared-dependency check. Such // dependencies should go after all other compile and testCompile From f59005ad352234f7020f0ef3d0d7f45e67b8118e Mon Sep 17 00:00:00 2001 From: mcilwain Date: Fri, 2 Nov 2018 09:36:50 -0700 Subject: [PATCH 010/134] Fix sender email address for invoicing alerts It was failing to send alert emails because the email address it was constructing did not have permission through GAE to send emails. This switches it over to using the send from email address already in use elsewhere in the app that does successfully send emails. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219812019 --- java/google/registry/config/RegistryConfig.java | 14 -------------- .../registry/config/RegistryConfigSettings.java | 1 - .../registry/config/files/default-config.yaml | 6 +++--- .../reporting/billing/BillingEmailUtils.java | 10 +++++----- .../reporting/icann/ReportingEmailUtils.java | 2 +- .../reporting/spec11/Spec11EmailUtils.java | 10 +++++----- 6 files changed, 14 insertions(+), 29 deletions(-) diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index 813946e75..551a7d184 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -827,20 +827,6 @@ public final class RegistryConfig { return config.misc.alertRecipientEmailAddress; } - /** - * Returns the email address we send emails from. - * - * @see google.registry.reporting.icann.ReportingEmailUtils - * @see google.registry.reporting.billing.BillingEmailUtils - * @see google.registry.reporting.spec11.Spec11EmailUtils - */ - @Provides - @Config("alertSenderEmailAddress") - public static String provideAlertSenderEmailAddress( - @Config("projectId") String projectId, RegistryConfigSettings config) { - return String.format("%s-no-reply@%s", projectId, config.misc.alertEmailSenderDomain); - } - /** * Returns the email address to which spec 11 email should be replied. * diff --git a/java/google/registry/config/RegistryConfigSettings.java b/java/google/registry/config/RegistryConfigSettings.java index 2449c1c58..404ce1254 100644 --- a/java/google/registry/config/RegistryConfigSettings.java +++ b/java/google/registry/config/RegistryConfigSettings.java @@ -161,7 +161,6 @@ public class RegistryConfigSettings { public String sheetExportId; public String alertRecipientEmailAddress; public String spec11ReplyToEmailAddress; - public String alertEmailSenderDomain; public int asyncDeleteDelaySeconds; } diff --git a/java/google/registry/config/files/default-config.yaml b/java/google/registry/config/files/default-config.yaml index 85fe99625..b5deec22f 100644 --- a/java/google/registry/config/files/default-config.yaml +++ b/java/google/registry/config/files/default-config.yaml @@ -22,6 +22,9 @@ gSuite: domainName: domain-registry.example # Display name and email address used on outgoing emails through G Suite. + # The email address must be valid and have permission in the GAE app to send + # emails. For more info see: + # https://cloud.google.com/appengine/docs/standard/java/mail/#who_can_send_mail outgoingEmailDisplayName: Example Registry outgoingEmailAddress: noreply@project-id.appspotmail.com @@ -343,9 +346,6 @@ misc: # to be a deliverable email address in case the registrars want to contact us. spec11ReplyToEmailAddress: reply-to@example.com - # Domain for the email address we send alert summary emails from. - alertEmailSenderDomain: appspotmail.com - # How long to delay processing of asynchronous deletions. This should always # be longer than eppResourceCachingSeconds, to prevent deleted contacts or # hosts from being used on domains. diff --git a/java/google/registry/reporting/billing/BillingEmailUtils.java b/java/google/registry/reporting/billing/BillingEmailUtils.java index ae91c7981..918074949 100644 --- a/java/google/registry/reporting/billing/BillingEmailUtils.java +++ b/java/google/registry/reporting/billing/BillingEmailUtils.java @@ -44,7 +44,7 @@ class BillingEmailUtils { private final SendEmailService emailService; private final YearMonth yearMonth; - private final String alertSenderAddress; + private final String outgoingEmailAddress; private final String alertRecipientAddress; private final ImmutableList invoiceEmailRecipients; private final String billingBucket; @@ -56,7 +56,7 @@ class BillingEmailUtils { BillingEmailUtils( SendEmailService emailService, YearMonth yearMonth, - @Config("alertSenderEmailAddress") String alertSenderAddress, + @Config("gSuiteOutgoingEmailAddress") String outgoingEmailAddress, @Config("alertRecipientEmailAddress") String alertRecipientAddress, @Config("invoiceEmailRecipients") ImmutableList invoiceEmailRecipients, @Config("billingBucket") String billingBucket, @@ -65,7 +65,7 @@ class BillingEmailUtils { Retrier retrier) { this.emailService = emailService; this.yearMonth = yearMonth; - this.alertSenderAddress = alertSenderAddress; + this.outgoingEmailAddress = outgoingEmailAddress; this.alertRecipientAddress = alertRecipientAddress; this.invoiceEmailRecipients = invoiceEmailRecipients; this.billingBucket = billingBucket; @@ -86,7 +86,7 @@ class BillingEmailUtils { new GcsFilename(billingBucket, invoiceDirectoryPrefix + invoiceFile); try (InputStream in = gcsUtils.openInputStream(invoiceFilename)) { Message msg = emailService.createMessage(); - msg.setFrom(new InternetAddress(alertSenderAddress)); + msg.setFrom(new InternetAddress(outgoingEmailAddress)); for (String recipient : invoiceEmailRecipients) { msg.addRecipient(RecipientType.TO, new InternetAddress(recipient)); } @@ -126,7 +126,7 @@ class BillingEmailUtils { retrier.callWithRetry( () -> { Message msg = emailService.createMessage(); - msg.setFrom(new InternetAddress(alertSenderAddress)); + msg.setFrom(new InternetAddress(outgoingEmailAddress)); msg.addRecipient(RecipientType.TO, new InternetAddress(alertRecipientAddress)); msg.setSubject(String.format("Billing Pipeline Alert: %s", yearMonth.toString())); msg.setText(body); diff --git a/java/google/registry/reporting/icann/ReportingEmailUtils.java b/java/google/registry/reporting/icann/ReportingEmailUtils.java index 9b248f94d..c011b8bfa 100644 --- a/java/google/registry/reporting/icann/ReportingEmailUtils.java +++ b/java/google/registry/reporting/icann/ReportingEmailUtils.java @@ -25,7 +25,7 @@ import javax.mail.internet.InternetAddress; /** Static utils for emailing reporting results. */ public class ReportingEmailUtils { - @Inject @Config("alertSenderEmailAddress") String sender; + @Inject @Config("gSuiteOutgoingEmailAddress") String sender; @Inject @Config("alertRecipientEmailAddress") String recipient; @Inject SendEmailService emailService; @Inject ReportingEmailUtils() {} diff --git a/java/google/registry/reporting/spec11/Spec11EmailUtils.java b/java/google/registry/reporting/spec11/Spec11EmailUtils.java index 76b716446..c29e2003b 100644 --- a/java/google/registry/reporting/spec11/Spec11EmailUtils.java +++ b/java/google/registry/reporting/spec11/Spec11EmailUtils.java @@ -45,7 +45,7 @@ public class Spec11EmailUtils { private final SendEmailService emailService; private final YearMonth yearMonth; - private final String alertSenderAddress; + private final String outgoingEmailAddress; private final String alertRecipientAddress; private final String spec11ReplyToAddress; private final String reportingBucket; @@ -58,7 +58,7 @@ public class Spec11EmailUtils { Spec11EmailUtils( SendEmailService emailService, YearMonth yearMonth, - @Config("alertSenderEmailAddress") String alertSenderAddress, + @Config("gSuiteOutgoingEmailAddress") String outgoingEmailAddress, @Config("alertRecipientEmailAddress") String alertRecipientAddress, @Config("spec11ReplyToEmailAddress") String spec11ReplyToAddress, @Config("spec11EmailBodyTemplate") String spec11EmailBodyTemplate, @@ -68,7 +68,7 @@ public class Spec11EmailUtils { Retrier retrier) { this.emailService = emailService; this.yearMonth = yearMonth; - this.alertSenderAddress = alertSenderAddress; + this.outgoingEmailAddress = outgoingEmailAddress; this.alertRecipientAddress = alertRecipientAddress; this.spec11ReplyToAddress = spec11ReplyToAddress; this.reportingBucket = reportingBucket; @@ -135,7 +135,7 @@ public class Spec11EmailUtils { msg.setSubject( String.format("Google Registry Monthly Threat Detector [%s]", yearMonth.toString())); msg.setText(body.toString()); - msg.setFrom(new InternetAddress(alertSenderAddress)); + msg.setFrom(new InternetAddress(outgoingEmailAddress)); msg.addRecipient(RecipientType.TO, new InternetAddress(registrarEmail)); msg.addRecipient(RecipientType.BCC, new InternetAddress(spec11ReplyToAddress)); emailService.sendMessage(msg); @@ -147,7 +147,7 @@ public class Spec11EmailUtils { retrier.callWithRetry( () -> { Message msg = emailService.createMessage(); - msg.setFrom(new InternetAddress(alertSenderAddress)); + msg.setFrom(new InternetAddress(outgoingEmailAddress)); msg.addRecipient(RecipientType.TO, new InternetAddress(alertRecipientAddress)); msg.setSubject(subject); msg.setText(body); From 9ce07db38a7a0bd14a42c37a4d887fc080033a11 Mon Sep 17 00:00:00 2001 From: guyben Date: Fri, 2 Nov 2018 11:59:58 -0700 Subject: [PATCH 011/134] Make OutputEncapsulator a CommandRunner This is in preparation for having other "command changing things" like redirecting to file and maybe variable substitutions in the arguments. "On the way" added a RUNNING "some_command" "--some_flag" "some_value" to the output encapsulator so that if we run multiple commands, we know what command was called where. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219837452 --- java/google/registry/tools/ShellCommand.java | 139 ++++++++++-------- .../registry/tools/ShellCommandTest.java | 38 ++++- 2 files changed, 107 insertions(+), 70 deletions(-) diff --git a/java/google/registry/tools/ShellCommand.java b/java/google/registry/tools/ShellCommand.java index eb6abe680..8c57bb329 100644 --- a/java/google/registry/tools/ShellCommand.java +++ b/java/google/registry/tools/ShellCommand.java @@ -28,6 +28,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableTable; +import com.google.common.escape.Escaper; +import com.google.common.escape.Escapers; import google.registry.util.Clock; import google.registry.util.SystemClock; import java.io.BufferedReader; @@ -70,10 +72,25 @@ public class ShellCommand implements Command { private static final String ALERT_COLOR = "\u001b[1;41;97m"; // red background private static final Duration IDLE_THRESHOLD = Duration.standardHours(1); private static final String SUCCESS = "SUCCESS"; - private static final String FAILURE = "FAILURE "; + private static final String FAILURE = "FAILURE"; + private static final String RUNNING = "RUNNING"; + private static final Escaper STRING_ESCAPER = + Escapers.builder() + .addEscape('\\', "\\\\") + .addEscape('"', "\\\"") + .addEscape('\n', "\\n") + .addEscape('\r', "\\r") + .addEscape('\t', "\\t") + .build(); + /** + * The runner we received in the constructor. + * + *

We might want to update this runner based on flags (e.g. --encapsulate_output), but these + * flags aren't available in the constructor so we have to do it in the {@link #run} function. + */ + private final CommandRunner originalRunner; - private final CommandRunner runner; private final BufferedReader lineReader; private final ConsoleReader consoleReader; private final Clock clock; @@ -96,7 +113,7 @@ public class ShellCommand implements Command { boolean encapsulateOutput = false; public ShellCommand(CommandRunner runner) throws IOException { - this.runner = runner; + this.originalRunner = runner; InputStream in = System.in; if (System.console() != null) { consoleReader = new ConsoleReader(); @@ -114,7 +131,7 @@ public class ShellCommand implements Command { @VisibleForTesting ShellCommand(BufferedReader bufferedReader, Clock clock, CommandRunner runner) { - this.runner = runner; + this.originalRunner = runner; this.lineReader = bufferedReader; this.clock = clock; this.consoleReader = null; @@ -145,42 +162,11 @@ public class ShellCommand implements Command { return this; } - private static class OutputEncapsulator { - private PrintStream orgStdout; - private PrintStream orgStderr; + private static class OutputEncapsulator implements CommandRunner { + private final CommandRunner runner; - private EncapsulatingOutputStream encapsulatedOutputStream = null; - private EncapsulatingOutputStream encapsulatedErrorStream = null; - - private Exception error; - - private OutputEncapsulator() { - orgStdout = System.out; - orgStderr = System.err; - encapsulatedOutputStream = new EncapsulatingOutputStream(System.out, "out: "); - encapsulatedErrorStream = new EncapsulatingOutputStream(System.out, "err: "); - System.setOut(new PrintStream(encapsulatedOutputStream)); - System.setErr(new PrintStream(encapsulatedErrorStream)); - } - - void setError(Exception e) { - error = e; - } - - private void restoreOriginalStreams() { - try { - encapsulatedOutputStream.dumpLastLine(); - encapsulatedErrorStream.dumpLastLine(); - System.setOut(orgStdout); - System.setErr(orgStderr); - if (error != null) { - emitFailure(error); - } else { - emitSuccess(); - } - } catch (IOException e) { - throw new RuntimeException(e); - } + private OutputEncapsulator(CommandRunner runner) { + this.runner = runner; } /** @@ -188,7 +174,7 @@ public class ShellCommand implements Command { * *

Dumps the last line of output prior to doing this. */ - private void emitSuccess() { + private static void emitSuccess() { System.out.println(SUCCESS); System.out.flush(); } @@ -198,23 +184,45 @@ public class ShellCommand implements Command { * *

Dumps the last line of output prior to doing this. */ - private void emitFailure(Throwable e) { - System.out.println( - FAILURE - + e.getClass().getName() - + " " - + e.getMessage().replace("\\", "\\\\").replace("\n", "\\n")); + private static void emitFailure(Throwable e) { + System.out.format( + "%s %s %s\n", FAILURE, e.getClass().getName(), STRING_ESCAPER.escape(e.getMessage())); + System.out.flush(); + } + + private static void emitArguments(String[] args) { + System.out.print(RUNNING); + Arrays.stream(args).forEach(arg -> System.out.format(" \"%s\"", STRING_ESCAPER.escape(arg))); + System.out.println(); + System.out.flush(); + } + + private void encapsulatedRun(String[] args) throws Exception { + PrintStream orgOut = System.out; + PrintStream orgErr = System.err; + try (PrintStream newOut = + new PrintStream(new EncapsulatingOutputStream(System.out, "out: ")); + PrintStream newErr = + new PrintStream(new EncapsulatingOutputStream(System.out, "err: "))) { + System.setOut(newOut); + System.setErr(newErr); + runner.run(args); + } finally { + System.setOut(orgOut); + System.setErr(orgErr); + } } /** Run "func" with output encapsulation. */ - static void run(CommandRunner runner, String[] args) { - OutputEncapsulator encapsulator = new OutputEncapsulator(); + @Override + public void run(String[] args) { + try { - runner.run(args); + emitArguments(args); + encapsulatedRun(args); + emitSuccess(); } catch (Exception e) { - encapsulator.setError(e); - } finally { - encapsulator.restoreOriginalStreams(); + emitFailure(e); } } } @@ -222,6 +230,10 @@ public class ShellCommand implements Command { /** Run the shell until the user presses "Ctrl-D". */ @Override public void run() { + // Wrap standard output and error if requested. We have to do so here in run because the flags + // haven't been processed in the constructor. + CommandRunner runner = + encapsulateOutput ? new OutputEncapsulator(originalRunner) : originalRunner; // On Production we want to be extra careful - to prevent accidental use. boolean beExtraCareful = (RegistryToolEnvironment.get() == RegistryToolEnvironment.PRODUCTION); setPrompt(RegistryToolEnvironment.get(), beExtraCareful); @@ -242,16 +254,11 @@ public class ShellCommand implements Command { continue; } - // Wrap standard output and error if requested. We have to do so here in run because the flags - // haven't been processed in the constructor. - if (encapsulateOutput) { - OutputEncapsulator.run(runner, lineArgs); - } else { - try { - runner.run(lineArgs); - } catch (Exception e) { - System.err.println("Got an exception:\n" + e); - } + try { + runner.run(lineArgs); + } catch (Exception e) { + System.err.println("Got an exception:\n" + e); + e.printStackTrace(); } } if (!encapsulateOutput) { @@ -566,6 +573,14 @@ public class ShellCommand implements Command { @Override public void flush() throws IOException { dumpLastLine(); + super.flush(); + } + + @Override + public void close() throws IOException { + dumpLastLine(); + // We do NOT want to call super.close as that would close the original outputStream + // (System.out) } /** Dump the accumulated last line of output, if there was one. */ diff --git a/javatests/google/registry/tools/ShellCommandTest.java b/javatests/google/registry/tools/ShellCommandTest.java index e32a46f08..e99ec6d2b 100644 --- a/javatests/google/registry/tools/ShellCommandTest.java +++ b/javatests/google/registry/tools/ShellCommandTest.java @@ -253,10 +253,11 @@ public class ShellCommandTest { @Test public void testEncapsulatedOutputStream_basicFuncionality() { ByteArrayOutputStream backing = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(new ShellCommand.EncapsulatingOutputStream(backing, "out: ")); - out.println("first line"); - out.print("second line\ntrailing data"); - out.flush(); + try (PrintStream out = + new PrintStream(new ShellCommand.EncapsulatingOutputStream(backing, "out: "))) { + out.println("first line"); + out.print("second line\ntrailing data"); + } assertThat(backing.toString()) .isEqualTo("out: first line\nout: second line\nout: trailing data\n"); } @@ -264,8 +265,8 @@ public class ShellCommandTest { @Test public void testEncapsulatedOutputStream_emptyStream() { ByteArrayOutputStream backing = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(new ShellCommand.EncapsulatingOutputStream(backing, "out: ")); - out.flush(); + try (PrintStream out = + new PrintStream(new ShellCommand.EncapsulatingOutputStream(backing, "out: "))) {} assertThat(backing.toString()).isEqualTo(""); } @@ -288,10 +289,31 @@ public class ShellCommandTest { assertThat(stderr.toString()).isEmpty(); assertThat(stdout.toString()) .isEqualTo( - "out: first line\nerr: second line\nerr: surprise!\nout: fragmented line\n" + "RUNNING \"command1\"\n" + + "out: first line\nerr: second line\nerr: surprise!\nout: fragmented line\n" + "SUCCESS\n"); } + @Test + public void testEncapsulatedOutput_throws() throws Exception { + RegistryToolEnvironment.ALPHA.setup(); + captureOutput(); + ShellCommand shellCommand = + new ShellCommand( + args -> { + System.out.println("first line"); + throw new Exception("some error!"); + }); + shellCommand.encapsulateOutput = true; + shellCommand.run(); + assertThat(stderr.toString()).isEmpty(); + assertThat(stdout.toString()) + .isEqualTo( + "RUNNING \"command1\"\n" + + "out: first line\n" + + "FAILURE java.lang.Exception some error!\n"); + } + @Test public void testEncapsulatedOutput_noCommand() throws Exception { captureOutput(); @@ -307,7 +329,7 @@ public class ShellCommandTest { shellCommand.run(); assertThat(stderr.toString()).isEmpty(); assertThat(stdout.toString()) - .isEqualTo("out: first line\nSUCCESS\n"); + .isEqualTo("RUNNING \"do\" \"something\"\nout: first line\nSUCCESS\n"); } void captureOutput() { From 9b10c116f35d135bc16b1b5510a68aadaa2c3da9 Mon Sep 17 00:00:00 2001 From: jianglai Date: Fri, 2 Nov 2018 14:18:48 -0700 Subject: [PATCH 012/134] Do not create a logger during initialization in CidrAddressBlock This is patched from [] We should have done this when we migrated to Flogger. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=219860046 --- java/google/registry/util/CidrAddressBlock.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/java/google/registry/util/CidrAddressBlock.java b/java/google/registry/util/CidrAddressBlock.java index 0737aa8d9..8323fcb97 100644 --- a/java/google/registry/util/CidrAddressBlock.java +++ b/java/google/registry/util/CidrAddressBlock.java @@ -41,7 +41,16 @@ import javax.annotation.Nullable; // TODO(b/21870796): Migrate to Guava version when this is open-sourced. public class CidrAddressBlock implements Iterable, Serializable { - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + /** + * Wrapper class around a logger instance for {@link CidrAddressBlock}. + * + *

We don't want to have a static instance of {@link Logger} in {@link CidrAddressBlock}, + * because that can cause a race condition, since the logging subsystem might not yet be + * initialized. With this wrapper, the {@link Logger} will be initialized on first use. + */ + static class CidrAddressBlockLogger { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + } private final InetAddress ip; @@ -343,7 +352,7 @@ public class CidrAddressBlock implements Iterable, Serializable { // not have been created with an invalid netmask and a valid // netmask should have been successfully applied to "ipAddr" as long // as it represents an address of the same family as "this.ip". - logger.atWarning().withCause(e).log("Error while applying netmask."); + CidrAddressBlockLogger.logger.atWarning().withCause(e).log("Error while applying netmask."); return false; } } From 61a5cf307eba4b826f0178d95da0e947aff43ec2 Mon Sep 17 00:00:00 2001 From: guyben Date: Tue, 6 Nov 2018 16:23:46 -0800 Subject: [PATCH 013/134] Add "Admin" tab to the registrar console This tab will set the "allowedTlds", but might have other functionality in the future. It is based on (branches from) the security-settings tab, because I'm copying the functionality of the "whitelisted IPs" to the "allowed TLDs": they are both lists of "arbitrary" strings that you can remove from and add to. There are a lot of moving parts in this CL, because of how all the different elements need to interact, and how intertwined they are (for example, we need to disable the admin-settings view for non admins both in the soy and in the JS code) It's really time to refactor the console given all we've learned... :/ ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=220373443 --- java/google/registry/ui/css/BUILD | 1 + .../google/registry/ui/css/admin-settings.css | 58 ++++++++++ java/google/registry/ui/externs/json.js | 1 + .../ui/js/registrar/admin_settings.js | 106 ++++++++++++++++++ .../registry/ui/js/registrar/console.js | 26 ++++- java/google/registry/ui/js/registrar/main.js | 4 +- .../ui/js/registrar/security_settings.js | 1 - .../ui/server/registrar/ConsoleUiAction.java | 2 + .../ui/soy/registrar/AdminSettings.soy | 69 ++++++++++++ .../registry/ui/soy/registrar/Console.soy | 7 ++ .../ui/js/registrar/console_test_util.js | 3 +- 11 files changed, 274 insertions(+), 4 deletions(-) create mode 100644 java/google/registry/ui/css/admin-settings.css create mode 100644 java/google/registry/ui/js/registrar/admin_settings.js create mode 100644 java/google/registry/ui/soy/registrar/AdminSettings.soy diff --git a/java/google/registry/ui/css/BUILD b/java/google/registry/ui/css/BUILD index 4f02ea3ee..fc479cf07 100644 --- a/java/google/registry/ui/css/BUILD +++ b/java/google/registry/ui/css/BUILD @@ -24,6 +24,7 @@ closure_css_library( closure_css_library( name = "registrar_lib", srcs = [ + "admin-settings.css", "contact-settings.css", "contact-us.css", "dashboard.css", diff --git a/java/google/registry/ui/css/admin-settings.css b/java/google/registry/ui/css/admin-settings.css new file mode 100644 index 000000000..52df054af --- /dev/null +++ b/java/google/registry/ui/css/admin-settings.css @@ -0,0 +1,58 @@ +/** Admin Settings */ + +div#tlds div.tld { + width: 209px; +} + +#newTld { + width: 187px; + margin-left: 0.5em; +} + +div#tlds div.tld input, +div#tlds div.tld button[type=button] { + height: 27px; + line-height: 27px; + background: #ebebeb; + vertical-align: top; + border: none; + border-bottom: solid 3px white; +} + +div#tlds div.tld input { + width: 169px; + margin: 0; + padding: 0; + color: #555; + padding-left: 5px ! important; +} + +div#tlds.editing div.tld input[readonly] { + margin-left: 0.5em; +} + +div#tlds.editing div.tld button[type=button] { + display: inline-block; + float: right; + margin-left: -2px; + width: 30px; + min-width: 30px; + height: 30px; + color: grey; + font-size: 1.1em; +} + +div#tlds.editing div.tld button[type=button]:hover { + -webkit-box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; +} + +div#tlds.editing div.tld button[type=button] i { + font-style: normal; +} + +div#tlds.editing .kd-errormessage { + margin-left: 0.5em; +} + diff --git a/java/google/registry/ui/externs/json.js b/java/google/registry/ui/externs/json.js index b5624949a..7f99b66a8 100644 --- a/java/google/registry/ui/externs/json.js +++ b/java/google/registry/ui/externs/json.js @@ -68,6 +68,7 @@ registry.json.Response.prototype.results; // XXX: Might not need undefineds here. /** * @typedef {{ + * allowedTlds: !Array, * clientIdentifier: string, * clientCertificate: string?, * clientCertificateHash: string?, diff --git a/java/google/registry/ui/js/registrar/admin_settings.js b/java/google/registry/ui/js/registrar/admin_settings.js new file mode 100644 index 000000000..2fa6f696c --- /dev/null +++ b/java/google/registry/ui/js/registrar/admin_settings.js @@ -0,0 +1,106 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +goog.provide('registry.registrar.AdminSettings'); + +goog.require('goog.array'); +goog.require('goog.dom'); +goog.require('goog.dom.classlist'); +goog.require('goog.events'); +goog.require('goog.events.EventType'); +goog.require('goog.soy'); +goog.require('registry.Resource'); +goog.require('registry.ResourceComponent'); +goog.require('registry.soy.registrar.admin'); + +goog.forwardDeclare('registry.registrar.Console'); + + + +/** + * Admin Settings page, such as allowed TLDs for this registrar. + * @param {!registry.registrar.Console} console + * @param {!registry.Resource} resource the RESTful resource for the registrar. + * @constructor + * @extends {registry.ResourceComponent} + * @final + */ +registry.registrar.AdminSettings = function(console, resource) { + registry.registrar.AdminSettings.base( + this, 'constructor', console, resource, + registry.soy.registrar.admin.settings, null); +}; +goog.inherits(registry.registrar.AdminSettings, registry.ResourceComponent); + + +/** @override */ +registry.registrar.AdminSettings.prototype.bindToDom = function(id) { + registry.registrar.AdminSettings.base(this, 'bindToDom', 'fake'); + goog.dom.removeNode(goog.dom.getRequiredElement('reg-app-btn-back')); +}; + + +/** @override */ +registry.registrar.AdminSettings.prototype.setupEditor = + function(objArgs) { + goog.dom.classlist.add(goog.dom.getRequiredElement('tlds'), + goog.getCssName('editing')); + var tlds = goog.dom.getElementsByClass(goog.getCssName('tld'), + goog.dom.getRequiredElement('tlds')); + goog.array.forEach(tlds, function(tld) { + var remBtn = goog.dom.getChildren(tld)[0]; + goog.events.listen(remBtn, + goog.events.EventType.CLICK, + goog.bind(this.onTldRemove_, this, remBtn)); + }, this); + this.typeCounts['reg-tlds'] = objArgs.allowedTlds ? + objArgs.allowedTlds.length : 0; + + goog.events.listen(goog.dom.getRequiredElement('btn-add-tld'), + goog.events.EventType.CLICK, + this.onTldAdd_, + false, + this); +}; + + +/** + * Click handler for TLD add button. + * @private + */ +registry.registrar.AdminSettings.prototype.onTldAdd_ = function() { + const tldInputElt = goog.dom.getRequiredElement('newTld'); + const tldElt = goog.soy.renderAsFragment(registry.soy.registrar.admin.tld, { + name: 'allowedTlds[' + this.typeCounts['reg-tlds'] + ']', + tld: tldInputElt.value, + }); + goog.dom.appendChild(goog.dom.getRequiredElement('tlds'), tldElt); + var remBtn = goog.dom.getFirstElementChild(tldElt); + goog.dom.classlist.remove(remBtn, goog.getCssName('hidden')); + goog.events.listen(remBtn, goog.events.EventType.CLICK, + goog.bind(this.onTldRemove_, this, remBtn)); + this.typeCounts['reg-tlds']++; + tldInputElt.value = ''; +}; + + +/** + * Click handler for TLD remove button. + * @param {!Element} remBtn The remove button. + * @private + */ +registry.registrar.AdminSettings.prototype.onTldRemove_ = + function(remBtn) { + goog.dom.removeNode(goog.dom.getParentElement(remBtn)); +}; diff --git a/java/google/registry/ui/js/registrar/console.js b/java/google/registry/ui/js/registrar/console.js index 9c8d639da..6542c03ea 100644 --- a/java/google/registry/ui/js/registrar/console.js +++ b/java/google/registry/ui/js/registrar/console.js @@ -21,6 +21,7 @@ goog.require('goog.dom.classlist'); goog.require('goog.net.XhrIo'); goog.require('registry.Console'); goog.require('registry.Resource'); +goog.require('registry.registrar.AdminSettings'); goog.require('registry.registrar.Contact'); goog.require('registry.registrar.ContactSettings'); goog.require('registry.registrar.ContactUs'); @@ -76,20 +77,43 @@ registry.registrar.Console = function(params) { this.lastActiveNavElt; /** + * A map from the URL fragment to the component to show. + * * @type {!Object.} */ this.pageMap = {}; + // Homepage. Displayed when there's no fragment, or when the fragment doesn't + // correspond to any view + this.pageMap[''] = registry.registrar.Dashboard; + // Updating the Registrar settings this.pageMap['security-settings'] = registry.registrar.SecuritySettings; this.pageMap['contact-settings'] = registry.registrar.ContactSettings; this.pageMap['whois-settings'] = registry.registrar.WhoisSettings; this.pageMap['contact-us'] = registry.registrar.ContactUs; this.pageMap['resources'] = registry.registrar.Resources; + // For admin use. The relevant tab is only shown in Console.soy for admins, + // but we also need to remove it here, otherwise it'd still be accessible if + // the user manually puts '#admin-settings' in the URL. + // + // Both the Console.soy and here, the "hiding the admin console for non + // admins" is purely for "aesthetic / design" reasons and have NO security + // implications. + // + // The security implications are only in the backend where we make sure all + // changes are made by users with the correct access (in other words - we + // don't trust the client-side to secure our application anyway) + if (this.params.isAdmin) { + this.pageMap['admin-settings'] = registry.registrar.AdminSettings; + } + + // sending EPPs through the console. Currently hidden (doesn't have a "tab") + // but still accessible if the user manually puts #domain (or other) in the + // fragment this.pageMap['contact'] = registry.registrar.Contact; this.pageMap['domain'] = registry.registrar.Domain; this.pageMap['host'] = registry.registrar.Host; - this.pageMap[''] = registry.registrar.Dashboard; }; goog.inherits(registry.registrar.Console, registry.Console); diff --git a/java/google/registry/ui/js/registrar/main.js b/java/google/registry/ui/js/registrar/main.js index dfc5817e9..dc44a4d1b 100644 --- a/java/google/registry/ui/js/registrar/main.js +++ b/java/google/registry/ui/js/registrar/main.js @@ -28,6 +28,7 @@ goog.require('registry.registrar.Console'); * * @param {string} xsrfToken populated by server-side soy template. * @param {string} clientId The registrar clientId. + * @param {boolean} isAdmin * @param {string} productName the product name displayed by the UI. * @param {string} integrationEmail * @param {string} supportEmail @@ -36,13 +37,14 @@ goog.require('registry.registrar.Console'); * @param {string} technicalDocsUrl * @export */ -registry.registrar.main = function(xsrfToken, clientId, productName, +registry.registrar.main = function(xsrfToken, clientId, isAdmin, productName, integrationEmail, supportEmail, announcementsEmail, supportPhoneNumber, technicalDocsUrl) { new registry.registrar.Console({ xsrfToken: xsrfToken, clientId: clientId, + isAdmin: isAdmin, productName: productName, integrationEmail: integrationEmail, supportEmail: supportEmail, diff --git a/java/google/registry/ui/js/registrar/security_settings.js b/java/google/registry/ui/js/registrar/security_settings.js index 1a3f2ed34..31a2c95ba 100644 --- a/java/google/registry/ui/js/registrar/security_settings.js +++ b/java/google/registry/ui/js/registrar/security_settings.js @@ -103,5 +103,4 @@ registry.registrar.SecuritySettings.prototype.onIpAdd_ = function() { registry.registrar.SecuritySettings.prototype.onIpRemove_ = function(remBtn) { goog.dom.removeNode(goog.dom.getParentElement(remBtn)); - this.typeCounts['reg-ips']--; }; diff --git a/java/google/registry/ui/server/registrar/ConsoleUiAction.java b/java/google/registry/ui/server/registrar/ConsoleUiAction.java index be45d805a..34add2820 100644 --- a/java/google/registry/ui/server/registrar/ConsoleUiAction.java +++ b/java/google/registry/ui/server/registrar/ConsoleUiAction.java @@ -16,6 +16,7 @@ package google.registry.ui.server.registrar; import static com.google.common.net.HttpHeaders.LOCATION; import static com.google.common.net.HttpHeaders.X_FRAME_OPTIONS; +import static google.registry.ui.server.registrar.AuthenticatedRegistrarAccessor.Role.ADMIN; import static google.registry.ui.server.registrar.RegistrarConsoleModule.PARAM_CLIENT_ID; import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; import static javax.servlet.http.HttpServletResponse.SC_MOVED_TEMPORARILY; @@ -139,6 +140,7 @@ public final class ConsoleUiAction implements Runnable { try { clientId = paramClientId.orElse(registrarAccessor.guessClientId()); data.put("clientId", clientId); + data.put("isAdmin", roleMap.containsEntry(clientId, ADMIN)); // We want to load the registrar even if we won't use it later (even if we remove the // requireFeeExtension) - to make sure the user indeed has access to the guessed registrar. diff --git a/java/google/registry/ui/soy/registrar/AdminSettings.soy b/java/google/registry/ui/soy/registrar/AdminSettings.soy new file mode 100644 index 000000000..5bc82d9a0 --- /dev/null +++ b/java/google/registry/ui/soy/registrar/AdminSettings.soy @@ -0,0 +1,69 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{namespace registry.soy.registrar.admin} + + + +/** Registrar admin settings page for view and edit. */ +{template .settings} + {@param clientId: string} + {@param allowedTlds: list} + {@param readonly: bool} +

+

Administrator settings for {$clientId}

+ {if $readonly} +

Use the 'Edit' button above to switch to enable editing the information below. + {/if} + + + + + +
+ + set or remove TLDs this + client is allowed access to. + +
+
+ {for $tld in $allowedTlds} + {call .tld} + {param name: 'allowedTlds[' + index($tld) + ']' /} + {param tld: $tld /} + {/call} + {/for} +
+
+ + +
+
+
+

+{/template} + + +/** TLD form input. */ +{template .tld} + {@param name: string} + {@param tld: string} +
+ + +
+{/template} diff --git a/java/google/registry/ui/soy/registrar/Console.soy b/java/google/registry/ui/soy/registrar/Console.soy index 65ef43d21..0eea97c15 100644 --- a/java/google/registry/ui/soy/registrar/Console.soy +++ b/java/google/registry/ui/soy/registrar/Console.soy @@ -24,6 +24,7 @@ {@param xsrfToken: string} /** Security token. */ {@param clientId: string} /** Registrar client identifier. */ {@param allClientIds: list} /** All registrar client identifiers for the user. */ + {@param isAdmin: bool} {@param username: string} /** Arbitrary username to display. */ {@param logoutUrl: string} /** Generated URL for logging out of Google. */ {@param productName: string} /** Name to display for this software product. */ @@ -63,6 +64,7 @@ {case google.registry.ui.ConsoleDebug.RAW} - + {/switch} From 2ec824609733d586e19b9be7d4889cb887c17fac Mon Sep 17 00:00:00 2001 From: mmuller Date: Fri, 14 Dec 2018 10:38:22 -0800 Subject: [PATCH 099/134] Add a dependency on the local tools.jar for doc tool generation. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=225565675 --- gradle/core/build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gradle/core/build.gradle b/gradle/core/build.gradle index 829f987ec..a61894bbf 100644 --- a/gradle/core/build.gradle +++ b/gradle/core/build.gradle @@ -262,6 +262,9 @@ dependencies { // Dependencies needed for compiling stylesheets to javascript css 'com.google.closure-stylesheets:closure-stylesheets:1.5.0' css 'args4j:args4j:2.0.26' + + // Tool dependencies. used for doc generation. + compile files("${System.properties['java.home']}/../lib/tools.jar") } task jaxbToJava { From 1004ef5621eb70db72de1659180a7b9dbaa20384 Mon Sep 17 00:00:00 2001 From: guyben Date: Fri, 14 Dec 2018 12:52:25 -0800 Subject: [PATCH 100/134] Allow unsetting of the support email group, disabling "support users" In addition to just making good sense to not have support group for some environments (local? unittest? crash?) - connecting with G Suit requires additional permissions that are harder to find. Specifically, it requires the Json Credentials that just aren't set in the Dummy Keyring used on some environments. So we make sure to not even *try* to create the credentials if the support email isn't set ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=225589255 --- .../registry/config/RegistryConfig.java | 6 +- .../config/files/nomulus-config-unittest.yaml | 5 ++ .../auth/AuthenticatedRegistrarAccessor.java | 87 +++++++++++-------- .../AuthenticatedRegistrarAccessorTest.java | 57 ++++++++---- javatests/google/registry/request/auth/BUILD | 1 + 5 files changed, 100 insertions(+), 56 deletions(-) diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index 5dd547e26..b153eff2c 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -22,6 +22,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ascii; import com.google.common.base.Splitter; +import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -424,8 +425,9 @@ public final class RegistryConfig { */ @Provides @Config("gSuiteSupportGroupEmailAddress") - public static String provideGSuiteSupportGroupEmailAddress(RegistryConfigSettings config) { - return config.gSuite.supportGroupEmailAddress; + public static Optional provideGSuiteSupportGroupEmailAddress( + RegistryConfigSettings config) { + return Optional.ofNullable(Strings.emptyToNull(config.gSuite.supportGroupEmailAddress)); } /** diff --git a/java/google/registry/config/files/nomulus-config-unittest.yaml b/java/google/registry/config/files/nomulus-config-unittest.yaml index a41fd9cf6..00501f26e 100644 --- a/java/google/registry/config/files/nomulus-config-unittest.yaml +++ b/java/google/registry/config/files/nomulus-config-unittest.yaml @@ -22,3 +22,8 @@ caching: staticPremiumListMaxCachedEntries: 50 eppResourceCachingEnabled: true eppResourceCachingSeconds: 0 + +# Remove the support G Suite group, because we don't want to try connecting to G Suite servers from +# tests +gSuite: + supportGroupEmailAddress: diff --git a/java/google/registry/request/auth/AuthenticatedRegistrarAccessor.java b/java/google/registry/request/auth/AuthenticatedRegistrarAccessor.java index b49b5f3d4..b04475a65 100644 --- a/java/google/registry/request/auth/AuthenticatedRegistrarAccessor.java +++ b/java/google/registry/request/auth/AuthenticatedRegistrarAccessor.java @@ -29,6 +29,7 @@ import google.registry.config.RegistryConfig.Config; import google.registry.groups.GroupsConnection; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.RegistrarContact; +import java.util.Optional; import javax.annotation.concurrent.Immutable; import javax.inject.Inject; @@ -43,8 +44,16 @@ import javax.inject.Inject; * *

An "admin" also has ADMIN role on ALL registrars. * - *

A user is an "admin" if they are a GAE-admin, or if their email is in the "Support" G-Suite + *

A user is an "admin" if they are a GAE-admin, or if their email is in the "Support" G Suite * group. + * + *

NOTE: to check whether the user is in the "Support" G Suite group, we need a connection to + * G Suite. This in turn requires we have valid JsonCredentials, which not all environments have set + * up. This connection will be created lazily (only if needed). + * + *

Specifically, we don't instantiate the connection if: (a) gSuiteSupportGroupEmailAddress isn't + * defined, or (b) the user is logged out, or (c) the user is a GAE-admin, or (d) bypassAdminCheck + * is true. */ @Immutable public class AuthenticatedRegistrarAccessor { @@ -69,40 +78,38 @@ public class AuthenticatedRegistrarAccessor { private final ImmutableSetMultimap roleMap; /** - * Overriding the injected {@link GroupsConnection} for tests. + * Bypass the "isAdmin" check making all users NOT admins. * - *

{@link GroupsConnection} needs the injected DelegatedCredential GoogleCredential. However, - * this can't be initialized in the test environment. + *

Currently our test server doesn't let you change the user after the test server was created. + * This means we'd need multiple test files to test the same actions as both a "regular" user and + * an admin. * - *

The test server used in the javatests/google/registry/webdriver/ tests will hang if we try - * to instantiate the {@link GroupsConnection}. So instead we inject a {@link Lazy} version and - * allow tests to override the injected instace with (presumabley) a mock insteance. + *

To overcome this - we add a flag that lets you dynamically choose whether a user is an admin + * or not by creating a fake "GAE-admin" user and then bypassing the admin check if they want to + * fake a "regular" user. + * + *

The reason we don't do it the other way around (have a flag that makes anyone an admin) is + * that such a flag would be a security risk, especially since VisibleForTesting is unenforced + * (and you could set it with reflection anyway). + * + *

Instead of having a test flag that elevates permissions (which has security concerns) we add + * this flag that reduces permissions. */ - @VisibleForTesting public static GroupsConnection overrideGroupsConnection = null; + @VisibleForTesting public static boolean bypassAdminCheck = false; @Inject public AuthenticatedRegistrarAccessor( AuthResult authResult, @Config("registryAdminClientId") String registryAdminClientId, - @Config("gSuiteSupportGroupEmailAddress") String gSuiteSupportGroupEmailAddress, - Lazy groupsConnection) { - this( - authResult, - registryAdminClientId, - gSuiteSupportGroupEmailAddress, - overrideGroupsConnection != null ? overrideGroupsConnection : groupsConnection.get()); - } - - @VisibleForTesting - AuthenticatedRegistrarAccessor( - AuthResult authResult, - String registryAdminClientId, - String gSuiteSupportGroupEmailAddress, - GroupsConnection groupsConnection) { + @Config("gSuiteSupportGroupEmailAddress") Optional gSuiteSupportGroupEmailAddress, + Lazy lazyGroupsConnection) { this( authResult.userIdForLogging(), createRoleMap( - authResult, registryAdminClientId, groupsConnection, gSuiteSupportGroupEmailAddress)); + authResult, + registryAdminClientId, + lazyGroupsConnection, + gSuiteSupportGroupEmailAddress)); logger.atInfo().log( "%s has the following roles: %s", authResult.userIdForLogging(), roleMap); @@ -235,17 +242,21 @@ public class AuthenticatedRegistrarAccessor { } private static boolean checkIsSupport( - GroupsConnection groupsConnection, String userEmail, String supportEmail) { - if (Strings.isNullOrEmpty(supportEmail)) { + Lazy lazyGroupsConnection, + String userEmail, + Optional gSuiteSupportGroupEmailAddress) { + if (!gSuiteSupportGroupEmailAddress.isPresent()) { return false; } try { - return groupsConnection.isMemberOfGroup(userEmail, supportEmail); + return lazyGroupsConnection + .get() + .isMemberOfGroup(userEmail, gSuiteSupportGroupEmailAddress.get()); } catch (RuntimeException e) { logger.atSevere().withCause(e).log( "Error checking whether email %s belongs to support group %s." + " Skipping support role check", - userEmail, supportEmail); + userEmail, gSuiteSupportGroupEmailAddress); return false; } } @@ -253,8 +264,8 @@ public class AuthenticatedRegistrarAccessor { private static ImmutableSetMultimap createRoleMap( AuthResult authResult, String registryAdminClientId, - GroupsConnection groupsConnection, - String gSuiteSupportGroupEmailAddress) { + Lazy lazyGroupsConnection, + Optional gSuiteSupportGroupEmailAddress) { if (!authResult.userAuthInfo().isPresent()) { return ImmutableSetMultimap.of(); @@ -263,9 +274,14 @@ public class AuthenticatedRegistrarAccessor { UserAuthInfo userAuthInfo = authResult.userAuthInfo().get(); User user = userAuthInfo.user(); - boolean isAdmin = userAuthInfo.isUserAdmin(); - boolean isSupport = - checkIsSupport(groupsConnection, user.getEmail(), gSuiteSupportGroupEmailAddress); + // both GAE project admin and members of the gSuiteSupportGroupEmailAddress are considered + // admins for the RegistrarConsole. + boolean isAdmin = + bypassAdminCheck + ? false + : userAuthInfo.isUserAdmin() + || checkIsSupport( + lazyGroupsConnection, user.getEmail(), gSuiteSupportGroupEmailAddress); ImmutableSetMultimap.Builder builder = new ImmutableSetMultimap.Builder<>(); @@ -278,9 +294,8 @@ public class AuthenticatedRegistrarAccessor { builder.put(registryAdminClientId, Role.OWNER); } - if (isAdmin || isSupport) { - // Admins and support have ADMIN access to all registrars, and OWNER access to all non-REAL - // registrars + if (isAdmin) { + // Admins have ADMIN access to all registrars, and OWNER access to all non-REAL registrars ofy() .load() .type(Registrar.class) diff --git a/javatests/google/registry/request/auth/AuthenticatedRegistrarAccessorTest.java b/javatests/google/registry/request/auth/AuthenticatedRegistrarAccessorTest.java index 4845b4f61..199614e01 100644 --- a/javatests/google/registry/request/auth/AuthenticatedRegistrarAccessorTest.java +++ b/javatests/google/registry/request/auth/AuthenticatedRegistrarAccessorTest.java @@ -23,9 +23,8 @@ import static google.registry.testing.DatastoreHelper.persistResource; import static google.registry.testing.JUnitBackports.assertThrows; import static google.registry.testing.LogsSubject.assertAboutLogs; import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import com.google.appengine.api.users.User; @@ -33,11 +32,14 @@ import com.google.common.collect.ImmutableSetMultimap; import com.google.common.flogger.LoggerConfig; import com.google.common.testing.NullPointerTester; import com.google.common.testing.TestLogHandler; +import dagger.Lazy; import google.registry.groups.GroupsConnection; import google.registry.model.registrar.Registrar; import google.registry.request.auth.AuthenticatedRegistrarAccessor.RegistrarAccessDeniedException; import google.registry.testing.AppEngineRule; import google.registry.testing.InjectRule; +import google.registry.testing.MockitoJUnitRule; +import java.util.Optional; import java.util.logging.Level; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -47,6 +49,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.mockito.Mock; /** Unit tests for {@link AuthenticatedRegistrarAccessor}. */ @RunWith(JUnit4.class) @@ -54,16 +57,19 @@ public class AuthenticatedRegistrarAccessorTest { @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build(); @Rule public final InjectRule inject = new InjectRule(); + @Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create(); + + @Mock private HttpServletRequest req; + @Mock private HttpServletResponse rsp; + @Mock private GroupsConnection groupsConnection; + @Mock private Lazy lazyGroupsConnection; - private final HttpServletRequest req = mock(HttpServletRequest.class); - private final HttpServletResponse rsp = mock(HttpServletResponse.class); - private final GroupsConnection groupsConnection = mock(GroupsConnection.class); private final TestLogHandler testLogHandler = new TestLogHandler(); private static final AuthResult USER = createAuthResult(false); private static final AuthResult GAE_ADMIN = createAuthResult(true); private static final AuthResult NO_USER = AuthResult.create(AuthLevel.NONE); - private static final String SUPPORT_GROUP = "support@registry.example"; + private static final Optional SUPPORT_GROUP = Optional.of("support@registry.example"); /** Client ID of a REAL registrar with a RegistrarContact for USER and GAE_ADMIN. */ private static final String CLIENT_ID_WITH_CONTACT = "TheRegistrar"; /** Client ID of a REAL registrar without a RegistrarContact. */ @@ -95,6 +101,7 @@ public class AuthenticatedRegistrarAccessorTest { @Before public void before() { + when(lazyGroupsConnection.get()).thenReturn(groupsConnection); LoggerConfig.getConfig(AuthenticatedRegistrarAccessor.class).addHandler(testLogHandler); // persistResource(loadRegistrar(ADMIN_CLIENT_ID)); persistResource( @@ -124,10 +131,11 @@ public class AuthenticatedRegistrarAccessorTest { public void getAllClientIdWithAccess_user() { AuthenticatedRegistrarAccessor registrarAccessor = new AuthenticatedRegistrarAccessor( - USER, ADMIN_CLIENT_ID, SUPPORT_GROUP, groupsConnection); + USER, ADMIN_CLIENT_ID, SUPPORT_GROUP, lazyGroupsConnection); assertThat(registrarAccessor.getAllClientIdWithRoles()) .containsExactly(CLIENT_ID_WITH_CONTACT, OWNER); + verify(lazyGroupsConnection).get(); } /** Logged out users don't have access to anything. */ @@ -135,9 +143,10 @@ public class AuthenticatedRegistrarAccessorTest { public void getAllClientIdWithAccess_loggedOutUser() { AuthenticatedRegistrarAccessor registrarAccessor = new AuthenticatedRegistrarAccessor( - NO_USER, ADMIN_CLIENT_ID, SUPPORT_GROUP, groupsConnection); + NO_USER, ADMIN_CLIENT_ID, SUPPORT_GROUP, lazyGroupsConnection); assertThat(registrarAccessor.getAllClientIdWithRoles()).isEmpty(); + verifyZeroInteractions(lazyGroupsConnection); } /** @@ -155,7 +164,7 @@ public class AuthenticatedRegistrarAccessorTest { public void getAllClientIdWithAccess_gaeAdmin() { AuthenticatedRegistrarAccessor registrarAccessor = new AuthenticatedRegistrarAccessor( - GAE_ADMIN, ADMIN_CLIENT_ID, SUPPORT_GROUP, groupsConnection); + GAE_ADMIN, ADMIN_CLIENT_ID, SUPPORT_GROUP, lazyGroupsConnection); assertThat(registrarAccessor.getAllClientIdWithRoles()) .containsExactly( @@ -169,6 +178,7 @@ public class AuthenticatedRegistrarAccessorTest { ADMIN_CLIENT_ID, ADMIN, ADMIN_CLIENT_ID, OWNER); + verifyZeroInteractions(lazyGroupsConnection); } /** @@ -184,10 +194,10 @@ public class AuthenticatedRegistrarAccessorTest { */ @Test public void getAllClientIdWithAccess_userInSupportGroup() { - when(groupsConnection.isMemberOfGroup("user@gmail.com", SUPPORT_GROUP)).thenReturn(true); + when(groupsConnection.isMemberOfGroup("user@gmail.com", SUPPORT_GROUP.get())).thenReturn(true); AuthenticatedRegistrarAccessor registrarAccessor = new AuthenticatedRegistrarAccessor( - USER, ADMIN_CLIENT_ID, SUPPORT_GROUP, groupsConnection); + USER, ADMIN_CLIENT_ID, SUPPORT_GROUP, lazyGroupsConnection); assertThat(registrarAccessor.getAllClientIdWithRoles()) .containsExactly( @@ -201,18 +211,20 @@ public class AuthenticatedRegistrarAccessorTest { ADMIN_CLIENT_ID, ADMIN, ADMIN_CLIENT_ID, OWNER); + verify(lazyGroupsConnection).get(); } - /** Empty Support group email - skips check. */ + /** Empty Support group email - skips check and doesn't generate the lazy. */ @Test public void getAllClientIdWithAccess_emptySupportEmail_works() { AuthenticatedRegistrarAccessor registrarAccessor = new AuthenticatedRegistrarAccessor( - USER, ADMIN_CLIENT_ID, "", groupsConnection); + USER, ADMIN_CLIENT_ID, Optional.empty(), lazyGroupsConnection); - verifyNoMoreInteractions(groupsConnection); assertThat(registrarAccessor.getAllClientIdWithRoles()) .containsExactly(CLIENT_ID_WITH_CONTACT, OWNER); + // Make sure we didn't instantiate the lazyGroupsConnection + verifyZeroInteractions(lazyGroupsConnection); } /** Support group check throws - continue anyway. */ @@ -221,11 +233,12 @@ public class AuthenticatedRegistrarAccessorTest { when(groupsConnection.isMemberOfGroup(any(), any())).thenThrow(new RuntimeException("blah")); AuthenticatedRegistrarAccessor registrarAccessor = new AuthenticatedRegistrarAccessor( - USER, ADMIN_CLIENT_ID, SUPPORT_GROUP, groupsConnection); + USER, ADMIN_CLIENT_ID, SUPPORT_GROUP, lazyGroupsConnection); - verify(groupsConnection).isMemberOfGroup("user@gmail.com", SUPPORT_GROUP); + verify(groupsConnection).isMemberOfGroup("user@gmail.com", SUPPORT_GROUP.get()); assertThat(registrarAccessor.getAllClientIdWithRoles()) .containsExactly(CLIENT_ID_WITH_CONTACT, OWNER); + verify(lazyGroupsConnection).get(); } /** Fail loading registrar if user doesn't have access to it. */ @@ -235,6 +248,7 @@ public class AuthenticatedRegistrarAccessorTest { REAL_CLIENT_ID_WITHOUT_CONTACT, USER, "user user@gmail.com doesn't have access to registrar NewRegistrar"); + verify(lazyGroupsConnection).get(); } /** Fail loading registrar if user doesn't have access to it, even if it's not REAL. */ @@ -244,6 +258,7 @@ public class AuthenticatedRegistrarAccessorTest { OTE_CLIENT_ID_WITHOUT_CONTACT, USER, "user user@gmail.com doesn't have access to registrar OteRegistrar"); + verify(lazyGroupsConnection).get(); } /** Fail loading registrar if there's no user associated with the request. */ @@ -253,6 +268,7 @@ public class AuthenticatedRegistrarAccessorTest { CLIENT_ID_WITH_CONTACT, NO_USER, " doesn't have access to registrar TheRegistrar"); + verifyZeroInteractions(lazyGroupsConnection); } /** Succeed loading registrar if user has access to it. */ @@ -262,6 +278,7 @@ public class AuthenticatedRegistrarAccessorTest { CLIENT_ID_WITH_CONTACT, USER, "user user@gmail.com has [OWNER] access to registrar TheRegistrar"); + verify(lazyGroupsConnection).get(); } /** Succeed loading registrar if admin with access. */ @@ -271,6 +288,7 @@ public class AuthenticatedRegistrarAccessorTest { CLIENT_ID_WITH_CONTACT, GAE_ADMIN, "admin admin@gmail.com has [OWNER, ADMIN] access to registrar TheRegistrar"); + verifyZeroInteractions(lazyGroupsConnection); } /** Succeed loading registrar for admin even if they aren't on the approved contacts list. */ @@ -280,6 +298,7 @@ public class AuthenticatedRegistrarAccessorTest { REAL_CLIENT_ID_WITHOUT_CONTACT, GAE_ADMIN, "admin admin@gmail.com has [ADMIN] access to registrar NewRegistrar."); + verifyZeroInteractions(lazyGroupsConnection); } /** Succeed loading non-REAL registrar for admin. */ @@ -289,6 +308,7 @@ public class AuthenticatedRegistrarAccessorTest { OTE_CLIENT_ID_WITHOUT_CONTACT, GAE_ADMIN, "admin admin@gmail.com has [OWNER, ADMIN] access to registrar OteRegistrar."); + verifyZeroInteractions(lazyGroupsConnection); } /** Fail loading registrar even if admin, if registrar doesn't exist. */ @@ -298,13 +318,14 @@ public class AuthenticatedRegistrarAccessorTest { "BadClientId", GAE_ADMIN, "admin admin@gmail.com doesn't have access to registrar BadClientId"); + verifyZeroInteractions(lazyGroupsConnection); } private void expectGetRegistrarSuccess(String clientId, AuthResult authResult, String message) throws Exception { AuthenticatedRegistrarAccessor registrarAccessor = new AuthenticatedRegistrarAccessor( - authResult, ADMIN_CLIENT_ID, SUPPORT_GROUP, groupsConnection); + authResult, ADMIN_CLIENT_ID, SUPPORT_GROUP, lazyGroupsConnection); // make sure loading the registrar succeeds and returns a value assertThat(registrarAccessor.getRegistrar(clientId)).isNotNull(); @@ -315,7 +336,7 @@ public class AuthenticatedRegistrarAccessorTest { String clientId, AuthResult authResult, String message) { AuthenticatedRegistrarAccessor registrarAccessor = new AuthenticatedRegistrarAccessor( - authResult, ADMIN_CLIENT_ID, SUPPORT_GROUP, groupsConnection); + authResult, ADMIN_CLIENT_ID, SUPPORT_GROUP, lazyGroupsConnection); // make sure getRegistrar fails RegistrarAccessDeniedException exception = diff --git a/javatests/google/registry/request/auth/BUILD b/javatests/google/registry/request/auth/BUILD index f048d9a2a..468154397 100644 --- a/javatests/google/registry/request/auth/BUILD +++ b/javatests/google/registry/request/auth/BUILD @@ -21,6 +21,7 @@ java_library( "@com_google_appengine_api_1_0_sdk", "@com_google_appengine_tools_appengine_gcs_client", "@com_google_appengine_tools_sdk", + "@com_google_dagger", "@com_google_flogger", "@com_google_flogger_system_backend", "@com_google_guava", From 9d6a7ef66a982e85e570455241a85f2c7050282b Mon Sep 17 00:00:00 2001 From: guyben Date: Mon, 17 Dec 2018 09:48:28 -0800 Subject: [PATCH 101/134] Create OT&E entities directly, instead of calling sub-commands This is in preparation for having a web-console endpoint to create OTE. In addition - we streamline the code: - we remove support for different premium lists - we remove support for different DNS writers - we never want a "real" DnsWriter for OTE - we remove support of --eap_only, because we don't need it anymore - We use a single password for all the Registrars ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=225841694 --- .../registry/model/OteAccountBuilder.java | 377 ++++++++++++++++++ .../registry/model/registrar/Registrar.java | 24 ++ .../registry/tools/SetupOteCommand.java | 316 +++------------ .../registry/model/OteAccountBuilderTest.java | 302 ++++++++++++++ .../model/registrar/RegistrarTest.java | 74 ++++ .../registry/tools/SetupOteCommandTest.java | 299 ++++---------- 6 files changed, 904 insertions(+), 488 deletions(-) create mode 100644 java/google/registry/model/OteAccountBuilder.java create mode 100644 javatests/google/registry/model/OteAccountBuilderTest.java diff --git a/java/google/registry/model/OteAccountBuilder.java b/java/google/registry/model/OteAccountBuilder.java new file mode 100644 index 000000000..03506889b --- /dev/null +++ b/java/google/registry/model/OteAccountBuilder.java @@ -0,0 +1,377 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.util.DateTimeUtils.START_OF_TIME; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.collect.Sets; +import com.google.common.collect.Streams; +import com.googlecode.objectify.Key; +import google.registry.config.RegistryEnvironment; +import google.registry.model.common.GaeUserIdConverter; +import google.registry.model.pricing.StaticPremiumListPricingEngine; +import google.registry.model.registrar.Registrar; +import google.registry.model.registrar.RegistrarAddress; +import google.registry.model.registrar.RegistrarContact; +import google.registry.model.registry.Registry; +import google.registry.model.registry.Registry.TldState; +import google.registry.model.registry.label.PremiumList; +import google.registry.util.CidrAddressBlock; +import java.util.Collection; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Pattern; +import org.joda.money.CurrencyUnit; +import org.joda.money.Money; +import org.joda.time.DateTime; +import org.joda.time.Duration; + +/** + * Class to help build and persist all the OT&E entities in Datastore. + * + *

This includes the TLDs (Registries), Registrars, and the RegistrarContacts that can access the + * web console. + * + * This class is basically a "builder" for the parameters needed to generate the OT&E entities. + * Nothing is created until you call {@link #buildAndPersist}. + * + * Usage example: + * + * 

   {@code
+ * OteAccountBuilder.forClientId("example")
+ *     .addContact("contact@email.com") // OPTIONAL
+ *     .setPassword("password") // OPTIONAL
+ *     .setCertificateHash(certificateHash) // OPTIONAL
+ *     .setIpWhitelist(ImmutableList.of("1.1.1.1", "2.2.2.0/24")) // OPTIONAL
+ *     .buildAndPersist();
+ * }
+ */ +public final class OteAccountBuilder { + + // Regex: 3-14 lower-case alphanumeric characters or hyphens, the first of which must be a letter. + private static final Pattern REGISTRAR_PATTERN = Pattern.compile("^[a-z][-a-z0-9]{2,13}$"); + + // Durations are short so that registrars can test with quick transfer (etc.) turnaround. + private static final Duration SHORT_ADD_GRACE_PERIOD = Duration.standardMinutes(60); + private static final Duration SHORT_REDEMPTION_GRACE_PERIOD = Duration.standardMinutes(10); + private static final Duration SHORT_PENDING_DELETE_LENGTH = Duration.standardMinutes(5); + + private static final String DEFAULT_PREMIUM_LIST = "default_sandbox_list"; + + private static final RegistrarAddress DEFAULT_ADDRESS = + new RegistrarAddress.Builder() + .setStreet(ImmutableList.of("e-street")) + .setCity("Neverland") + .setState("NY") + .setCountryCode("US") + .setZip("55555") + .build(); + + private static final ImmutableSortedMap EAP_FEE_SCHEDULE = + ImmutableSortedMap.of( + new DateTime(0), + Money.of(CurrencyUnit.USD, 0), + DateTime.parse("2018-03-01T00:00:00Z"), + Money.of(CurrencyUnit.USD, 100), + DateTime.parse("2030-03-01T00:00:00Z"), + Money.of(CurrencyUnit.USD, 0)); + + private final ImmutableMap clientIdToTld; + private final Registry sunriseTld; + private final Registry landrushTld; + private final Registry gaTld; + private final Registry eapTld; + private final ImmutableList.Builder contactsBuilder = + new ImmutableList.Builder<>(); + + private ImmutableList registrars; + private boolean replaceExisting = false; + + private OteAccountBuilder(String baseClientId) { + checkState( + RegistryEnvironment.get() != RegistryEnvironment.PRODUCTION, + "Can't setup OT&E in production"); + clientIdToTld = createClientIdToTldMap(baseClientId); + sunriseTld = + createTld( + baseClientId + "-sunrise", TldState.START_DATE_SUNRISE, null, null, null, false, 0); + landrushTld = + createTld(baseClientId + "-landrush", TldState.LANDRUSH, null, null, null, false, 1); + gaTld = + createTld( + baseClientId + "-ga", + TldState.GENERAL_AVAILABILITY, + SHORT_ADD_GRACE_PERIOD, + SHORT_REDEMPTION_GRACE_PERIOD, + SHORT_PENDING_DELETE_LENGTH, + false, + 2); + eapTld = + createTld( + baseClientId + "-eap", + TldState.GENERAL_AVAILABILITY, + SHORT_ADD_GRACE_PERIOD, + SHORT_REDEMPTION_GRACE_PERIOD, + SHORT_PENDING_DELETE_LENGTH, + true, + 3); + registrars = + clientIdToTld.keySet().stream() + .map(OteAccountBuilder::createRegistrar) + .collect(toImmutableList()); + } + + /** + * Creates an OteAccountBuilder for the given base client ID. + * + * @param baseClientId the base clientId which will help name all the entities we create. Normally + * is the same as the "prod" clientId designated for this registrar. + */ + public static OteAccountBuilder forClientId(String baseClientId) { + return new OteAccountBuilder(baseClientId); + } + + /** + * Set whether to replace any conflicting existing entities. + * + *

If true, any existing entity that conflicts with the entities we want to create will be + * replaced with the newly created data. + * + *

If false, encountering an existing entity that conflicts with one we want to create will + * throw an exception during {@link #buildAndPersist}. + * + *

NOTE that if we fail, no entities are created (the creation is atomic). + * + *

Default is false (failing if entities exist) + */ + public OteAccountBuilder setReplaceExisting(boolean replaceExisting) { + this.replaceExisting = replaceExisting; + return this; + } + + /** + * Adds a RegistrarContact with Web Console access. + * + *

NOTE: can be called more than once, adding multiple contacts. Each contact will have access + * to all OT&E Registrars. + * + * @param email the contact email that will have web-console access to all the Registrars. Must be + * from "our G Suite domain" (we have to be able to get its GaeUserId) + */ + public OteAccountBuilder addContact(String email) { + String gaeUserId = + checkNotNull( + GaeUserIdConverter.convertEmailAddressToGaeUserId(email), + "Email address %s is not associated with any GAE ID", + email); + registrars.forEach( + registrar -> contactsBuilder.add(createRegistrarContact(email, gaeUserId, registrar))); + return this; + } + + /** + * Apply a function on all the OT&E Registrars. + * + *

Use this to set up registrar fields. + * + *

NOTE: DO NOT change anything that would affect the {@link Key#create} result on Registrars. + * If you want to make this function public, add a check that the Key.create on the registrars + * hasn't changed. + * + * @param func a function setting the requested fields on Registrar Builders. Will be applied to + * all the Registrars. + */ + private OteAccountBuilder transformRegistrars( + Function func) { + registrars = + registrars.stream() + .map(Registrar::asBuilder) + .map(func) + .map(Registrar.Builder::build) + .collect(toImmutableList()); + return this; + } + + /** Sets the EPP login password for all the OT&E Registrars. */ + public OteAccountBuilder setPassword(String password) { + return transformRegistrars(builder -> builder.setPassword(password)); + } + + /** Sets the client certificate hash to all the OT&E Registrars. */ + public OteAccountBuilder setCertificateHash(String certHash) { + return transformRegistrars(builder -> builder.setClientCertificateHash(certHash)); + } + + /** Sets the client certificate to all the OT&E Registrars. */ + public OteAccountBuilder setCertificate(String asciiCert, DateTime now) { + return transformRegistrars(builder -> builder.setClientCertificate(asciiCert, now)); + } + + /** Sets the IP whitelist to all the OT&E Registrars. */ + public OteAccountBuilder setIpWhitelist(Collection ipWhitelist) { + ImmutableList ipAddressWhitelist = + ipWhitelist.stream().map(CidrAddressBlock::create).collect(toImmutableList()); + return transformRegistrars(builder -> builder.setIpAddressWhitelist(ipAddressWhitelist)); + } + + /** + * Persists all the OT&E entities to datastore. + * + * @return map from the new clientIds created to the new TLDs they have access to. Can be used to + * go over all the newly created Registrars / Registries / RegistrarContacts if any + * post-creation work is needed. + */ + public ImmutableMap buildAndPersist() { + // save all the entitiesl in a single transaction + ofy().transact(this::saveAllEntities); + return clientIdToTld; + } + + /** + * Return map from the OT&E clientIds we will create to the new TLDs they will have access to. + */ + public ImmutableMap getClientIdToTldMap() { + return clientIdToTld; + } + + /** Saves all the OT&E entities we created. */ + private void saveAllEntities() { + ofy().assertInTransaction(); + + ImmutableList registries = ImmutableList.of(sunriseTld, landrushTld, gaTld, eapTld); + ImmutableList contacts = contactsBuilder.build(); + + if (!replaceExisting) { + ImmutableList> keys = + Streams.concat(registries.stream(), registrars.stream(), contacts.stream()) + .map(Key::create) + .collect(toImmutableList()); + Set> existingKeys = ofy().load().keys(keys).keySet(); + checkState( + existingKeys.isEmpty(), + "Found existing object(s) conflicting with OT&E objects: %s", + existingKeys); + } + // Save the Registries (TLDs) first + ofy().save().entities(registries).now(); + // Now we can set the allowedTlds for the registrars + registrars = registrars.stream().map(this::addAllowedTld).collect(toImmutableList()); + // and we can save the registrars and contacts! + ofy().save().entities(registrars); + ofy().save().entities(contacts); + } + + private Registrar addAllowedTld(Registrar registrar) { + String tld = clientIdToTld.get(registrar.getClientId()); + if (registrar.getAllowedTlds().contains(tld)) { + return registrar; + } + return registrar + .asBuilder() + .setAllowedTldsUncached(Sets.union(registrar.getAllowedTlds(), ImmutableSet.of(tld))) + .build(); + } + + private static Registry createTld( + String tldName, + TldState initialTldState, + Duration addGracePeriod, + Duration redemptionGracePeriod, + Duration pendingDeleteLength, + boolean isEarlyAccess, + int roidSuffix) { + String tldNameAlphaNumerical = tldName.replaceAll("[^a-z0-9]", ""); + Optional premiumList = PremiumList.getUncached(DEFAULT_PREMIUM_LIST); + checkState(premiumList.isPresent(), "Couldn't find premium list %s.", DEFAULT_PREMIUM_LIST); + Registry.Builder builder = + new Registry.Builder() + .setTldStr(tldName) + .setPremiumPricingEngine(StaticPremiumListPricingEngine.NAME) + .setTldStateTransitions(ImmutableSortedMap.of(START_OF_TIME, initialTldState)) + .setDnsWriters(ImmutableSet.of("VoidDnsWriter")) + .setPremiumList(premiumList.get()) + .setRoidSuffix( + String.format( + "%S%X", + tldNameAlphaNumerical.substring(0, Math.min(tldNameAlphaNumerical.length(), 7)), + roidSuffix)); + if (addGracePeriod != null) { + builder.setAddGracePeriodLength(addGracePeriod); + } + if (pendingDeleteLength != null) { + builder.setPendingDeleteLength(pendingDeleteLength); + } + if (redemptionGracePeriod != null) { + builder.setRedemptionGracePeriodLength(redemptionGracePeriod); + } + if (isEarlyAccess) { + builder.setEapFeeSchedule(EAP_FEE_SCHEDULE); + } + return builder.build(); + } + + /** + * Creates the Registrar without the allowedTlds set - because we can't set allowedTlds before the + * TLD is saved. + */ + private static Registrar createRegistrar(String registrarName) { + return new Registrar.Builder() + .setClientId(registrarName) + .setRegistrarName(registrarName) + .setType(Registrar.Type.OTE) + .setLocalizedAddress(DEFAULT_ADDRESS) + .setEmailAddress("foo@neverland.com") + .setFaxNumber("+1.2125550100") + .setPhoneNumber("+1.2125550100") + .setIcannReferralEmail("nightmare@registrar.test") + .setState(Registrar.State.ACTIVE) + .build(); + } + + private static RegistrarContact createRegistrarContact( + String email, String gaeUserId, Registrar registrar) { + return new RegistrarContact.Builder() + .setParent(registrar) + .setName(email) + .setEmailAddress(email) + .setGaeUserId(gaeUserId) + .build(); + } + + /** Returns the ClientIds of the OT&E, with the TLDs each has access to. */ + private static ImmutableMap createClientIdToTldMap(String baseClientId) { + checkArgument( + REGISTRAR_PATTERN.matcher(baseClientId).matches(), + "Invalid registrar name: %s", + baseClientId); + return new ImmutableMap.Builder() + .put(baseClientId + "-1", baseClientId + "-sunrise") + .put(baseClientId + "-2", baseClientId + "-landrush") + .put(baseClientId + "-3", baseClientId + "-ga") + .put(baseClientId + "-4", baseClientId + "-ga") + .put(baseClientId + "-5", baseClientId + "-eap") + .build(); + } +} diff --git a/java/google/registry/model/registrar/Registrar.java b/java/google/registry/model/registrar/Registrar.java index b67910cbe..23f026159 100644 --- a/java/google/registry/model/registrar/Registrar.java +++ b/java/google/registry/model/registrar/Registrar.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.emptyToNull; import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap; import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet; import static com.google.common.collect.Ordering.natural; @@ -68,6 +69,7 @@ import google.registry.model.UpdateAutoTimestamp; import google.registry.model.annotations.ReportedOn; import google.registry.model.common.EntityGroupRoot; import google.registry.model.registrar.Registrar.BillingAccountEntry.CurrencyMapper; +import google.registry.model.registry.Registry; import google.registry.util.CidrAddressBlock; import google.registry.util.NonFinalForTesting; import java.security.MessageDigest; @@ -704,6 +706,28 @@ public class Registrar extends ImmutableObject implements Buildable, Jsonifiable return this; } + /** + * Same as {@link #setAllowedTlds}, but doesn't use the cache to check if the TLDs exist. + * + *

This should be used if the TLD we want to set is persisted in the same transaction - + * meaning its existence can't be cached before we need to save the Registrar. + * + *

We can still only set the allowedTld AFTER we saved the Registry entity. Make sure to call + * {@code .now()} when saving the Registry entity to make sure it's actually saved before trying + * to set the allowed TLDs. + */ + public Builder setAllowedTldsUncached(Set allowedTlds) { + ImmutableSet> newTldKeys = + Sets.difference(allowedTlds, getInstance().getAllowedTlds()).stream() + .map(tld -> Key.create(getCrossTldKey(), Registry.class, tld)) + .collect(toImmutableSet()); + Set> missingTldKeys = + Sets.difference(newTldKeys, ofy().load().keys(newTldKeys).keySet()); + checkArgument(missingTldKeys.isEmpty(), "Trying to set nonexisting TLDs: %s", missingTldKeys); + getInstance().allowedTlds = ImmutableSortedSet.copyOf(allowedTlds); + return this; + } + public Builder setClientCertificate(String clientCertificate, DateTime now) { clientCertificate = emptyToNull(clientCertificate); String clientCertificateHash = calculateHash(clientCertificate); diff --git a/java/google/registry/tools/SetupOteCommand.java b/java/google/registry/tools/SetupOteCommand.java index 8764103bb..b53bfcc7c 100644 --- a/java/google/registry/tools/SetupOteCommand.java +++ b/java/google/registry/tools/SetupOteCommand.java @@ -15,69 +15,30 @@ package google.registry.tools; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.tools.CommandUtilities.promptForYes; import static google.registry.util.X509Utils.loadCertificate; +import static java.nio.charset.StandardCharsets.US_ASCII; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSortedMap; -import com.google.re2j.Pattern; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.MoreFiles; import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryEnvironment; -import google.registry.model.common.GaeUserIdConverter; -import google.registry.model.registrar.Registrar; -import google.registry.model.registry.Registry.TldState; +import google.registry.model.OteAccountBuilder; import google.registry.tools.params.PathParameter; +import google.registry.util.Clock; import google.registry.util.StringGenerator; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.Optional; -import java.util.Set; import javax.inject.Inject; -import javax.inject.Named; -import org.joda.money.CurrencyUnit; -import org.joda.money.Money; -import org.joda.time.DateTime; -import org.joda.time.Duration; /** Composite command to set up OT&E TLDs and accounts. */ @Parameters(separators = " =", commandDescription = "Set up OT&E TLDs and registrars") final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemoteApi { - // Regex: 3-14 alphanumeric characters or hyphens, the first of which must be a letter. - private static final Pattern REGISTRAR_PATTERN = Pattern.compile("^[a-z][-a-z0-9]{2,13}$"); private static final int PASSWORD_LENGTH = 16; - // Durations are short so that registrars can test with quick transfer (etc.) turnaround. - private static final Duration SHORT_ADD_GRACE_PERIOD = Duration.standardMinutes(60); - private static final Duration SHORT_REDEMPTION_GRACE_PERIOD = Duration.standardMinutes(10); - private static final Duration SHORT_PENDING_DELETE_LENGTH = Duration.standardMinutes(5); - - // Whether to prompt the user on command failures. Set to false for testing of these failures. - @VisibleForTesting - static boolean interactive = true; - - private static final ImmutableSortedMap EAP_FEE_SCHEDULE = - ImmutableSortedMap.of( - new DateTime(0), - Money.of(CurrencyUnit.USD, 0), - DateTime.parse("2018-03-01T00:00:00Z"), - Money.of(CurrencyUnit.USD, 100), - DateTime.parse("2022-03-01T00:00:00Z"), - Money.of(CurrencyUnit.USD, 0)); - - private static final String DEFAULT_PREMIUM_LIST = "default_sandbox_list"; - - @Inject - @Named("dnsWriterNames") - Set validDnsWriterNames; - @Parameter( names = {"-r", "--registrar"}, description = @@ -101,7 +62,7 @@ final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemo names = {"--email"}, description = "the registrar's account to use for console access. " - + "Must be on the registry's G-Suite domain.", + + "Must be on the registry's G Suite domain.", required = true) private String email; @@ -121,252 +82,83 @@ final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemo private String certHash; @Parameter( - names = {"--dns_writers"}, - description = "comma separated list of DNS writers to use on all TLDs", - required = true + names = {"--overwrite"}, + description = "whether to replace existing entities if we encounter any, instead of failing" ) - private List dnsWriters; - - @Parameter( - names = {"--premium_list"}, - description = "premium list to apply to all TLDs" - ) - private String premiumList = DEFAULT_PREMIUM_LIST; - - // TODO: (b/74079782) remove this flag once OT&E for .app is complete. - @Parameter( - names = {"--eap_only"}, - description = "whether to only create EAP TLD and registrar" - ) - private boolean eapOnly = false; + private boolean overwrite = false; @Inject @Config("base64StringGenerator") StringGenerator passwordGenerator; - /** - * Long registrar names are truncated and then have an incrementing digit appended at the end so - * that unique ROID suffixes can be generated for all TLDs for the registrar. - */ - private int roidSuffixCounter = 0; + @Inject Clock clock; - /** Runs a command, clearing the cache before and prompting the user on failures. */ - private void runCommand(Command command) { - ofy().clearSessionCache(); - try { - command.run(); - } catch (Exception e) { - System.err.format("Command failed with error %s\n", e); - if (interactive && promptForYes("Continue to next command?")) { - return; - } - Throwables.throwIfUnchecked(e); - throw new RuntimeException(e); - } - } - - /** Constructs and runs a CreateTldCommand. */ - private void createTld( - String tldName, - TldState initialTldState, - Duration addGracePeriod, - Duration redemptionGracePeriod, - Duration pendingDeleteLength, - boolean isEarlyAccess) { - CreateTldCommand command = new CreateTldCommand(); - command.addGracePeriod = addGracePeriod; - command.dnsWriters = dnsWriters; - command.validDnsWriterNames = validDnsWriterNames; - command.force = force; - command.initialTldState = initialTldState; - command.mainParameters = ImmutableList.of(tldName); - command.pendingDeleteLength = pendingDeleteLength; - command.premiumListName = Optional.of(premiumList); - String tldNameAlphaNumerical = tldName.replaceAll("[^a-z0-9]", ""); - command.roidSuffix = - String.format( - "%S%X", - tldNameAlphaNumerical.substring(0, Math.min(tldNameAlphaNumerical.length(), 7)), - roidSuffixCounter++); - command.redemptionGracePeriod = redemptionGracePeriod; - if (isEarlyAccess) { - command.eapFeeSchedule = EAP_FEE_SCHEDULE; - } - runCommand(command); - } - - /** Constructs and runs a CreateRegistrarCommand */ - private void createRegistrar(String registrarName, String password, String tld) { - CreateRegistrarCommand command = new CreateRegistrarCommand(); - command.mainParameters = ImmutableList.of(registrarName); - command.createGoogleGroups = false; // Don't create Google Groups for OT&E registrars. - command.allowedTlds = ImmutableList.of(tld); - command.registrarName = registrarName; - command.registrarType = Registrar.Type.OTE; - command.password = password; - command.clientCertificateFilename = certFile; - command.clientCertificateHash = certHash; - command.ipWhitelist = ipWhitelist; - command.street = ImmutableList.of("e-street"); - command.city = "Neverland"; - command.state = "NY"; - command.countryCode = "US"; - command.zip = "55555"; - command.email = Optional.of("foo@neverland.com"); - command.fax = Optional.of("+1.2125550100"); - command.phone = Optional.of("+1.2125550100"); - command.icannReferralEmail = "nightmare@registrar.test"; - command.force = force; - runCommand(command); - } - - /** Constructs and runs a RegistrarContactCommand */ - private void createRegistrarContact(String registrarName) { - RegistrarContactCommand command = new RegistrarContactCommand(); - command.mainParameters = ImmutableList.of(registrarName); - command.mode = RegistrarContactCommand.Mode.CREATE; - command.name = email; - command.email = email; - command.allowConsoleAccess = true; - command.force = force; - runCommand(command); - } + OteAccountBuilder oteAccountBuilder; + String password; /** Run any pre-execute command checks */ @Override - protected boolean checkExecutionState() throws Exception { - checkArgument( - REGISTRAR_PATTERN.matcher(registrar).matches(), - "Registrar name is invalid (see usage text for requirements)."); - - // Make sure the email is "correct" - as in it's a valid email we can convert to gaeId - // There's no need to look at the result - it'll be converted again inside - // RegistrarContactCommand. - checkNotNull( - GaeUserIdConverter.convertEmailAddressToGaeUserId(email), - "Email address %s is not associated with any GAE ID", - email); - - boolean warned = false; - if (RegistryEnvironment.get() != RegistryEnvironment.SANDBOX - && RegistryEnvironment.get() != RegistryEnvironment.UNITTEST) { - System.err.printf( - "WARNING: Running against %s environment. Are " - + "you sure you didn\'t mean to run this against sandbox (e.g. \"-e SANDBOX\")?%n", - RegistryEnvironment.get()); - warned = true; - } - - if (warned && !promptForYes("Proceed despite warnings?")) { - System.out.println("Command aborted."); - return false; - } - + protected void init() throws Exception { checkArgument( certFile == null ^ certHash == null, "Must specify exactly one of client certificate file or client certificate hash."); - // Don't wait for create_registrar to fail if it's a bad certificate file. + password = passwordGenerator.createString(PASSWORD_LENGTH); + oteAccountBuilder = + OteAccountBuilder.forClientId(registrar) + .addContact(email) + .setPassword(password) + .setIpWhitelist(ipWhitelist) + .setReplaceExisting(overwrite); + if (certFile != null) { - loadCertificate(certFile.toAbsolutePath()); + String asciiCert = MoreFiles.asCharSource(certFile, US_ASCII).read(); + // Don't wait for create_registrar to fail if it's a bad certificate file. + loadCertificate(asciiCert); + oteAccountBuilder.setCertificate(asciiCert, clock.nowUtc()); + } + + if (certHash != null) { + oteAccountBuilder.setCertificateHash(certHash); } - return true; } @Override protected String prompt() { - // Each underlying command will confirm its own operation as well, so just provide - // a summary of the steps in this command. - if (eapOnly) { - return "Creating TLD:\n" - + " " + registrar + "-eap\n" - + "Creating registrar:\n" - + " " + registrar + "-5 (access to TLD " + registrar + "-eap)\n" - + "Giving contact access to this registrar:\n" - + " " + email; - } else { - return "Creating TLDs:\n" - + " " + registrar + "-sunrise\n" - + " " + registrar + "-landrush\n" - + " " + registrar + "-ga\n" - + " " + registrar + "-eap\n" - + "Creating registrars:\n" - + " " + registrar + "-1 (access to TLD " + registrar + "-sunrise)\n" - + " " + registrar + "-2 (access to TLD " + registrar + "-landrush)\n" - + " " + registrar + "-3 (access to TLD " + registrar + "-ga)\n" - + " " + registrar + "-4 (access to TLD " + registrar + "-ga)\n" - + " " + registrar + "-5 (access to TLD " + registrar + "-eap)\n" - + "Giving contact access to these registrars:\n" - + " " + email; + ImmutableMap registrarToTldMap = oteAccountBuilder.getClientIdToTldMap(); + StringBuilder builder = new StringBuilder(); + builder.append("Creating TLDs:"); + registrarToTldMap.values().forEach(tld -> builder.append("\n ").append(tld)); + builder.append("\nCreating registrars:"); + registrarToTldMap.forEach( + (clientId, tld) -> + builder.append(String.format("\n %s (with access to %s)", clientId, tld))); + builder.append("\nGiving contact access to these registrars:").append("\n ").append(email); + + if (RegistryEnvironment.get() != RegistryEnvironment.SANDBOX + && RegistryEnvironment.get() != RegistryEnvironment.UNITTEST) { + builder.append( + String.format( + "\n\nWARNING: Running against %s environment. Are " + + "you sure you didn\'t mean to run this against sandbox (e.g. \"-e SANDBOX\")?", + RegistryEnvironment.get())); } + + return builder.toString(); } @Override public String execute() throws Exception { - if (!eapOnly) { - createTld(registrar + "-sunrise", TldState.START_DATE_SUNRISE, null, null, null, false); - createTld(registrar + "-landrush", TldState.LANDRUSH, null, null, null, false); - createTld( - registrar + "-ga", - TldState.GENERAL_AVAILABILITY, - SHORT_ADD_GRACE_PERIOD, - SHORT_REDEMPTION_GRACE_PERIOD, - SHORT_PENDING_DELETE_LENGTH, - false); - } else { - // Increase ROID suffix counter to not collide with existing TLDs. - roidSuffixCounter = roidSuffixCounter + 3; - } - createTld( - registrar + "-eap", - TldState.GENERAL_AVAILABILITY, - SHORT_ADD_GRACE_PERIOD, - SHORT_REDEMPTION_GRACE_PERIOD, - SHORT_PENDING_DELETE_LENGTH, - true); - - // Storing names and credentials in a list of tuples for later play-back. - List> registrars = new ArrayList<>(); - if (!eapOnly) { - registrars.add( - ImmutableList.of( - registrar + "-1", - passwordGenerator.createString(PASSWORD_LENGTH), - registrar + "-sunrise")); - registrars.add( - ImmutableList.of( - registrar + "-2", - passwordGenerator.createString(PASSWORD_LENGTH), - registrar + "-landrush")); - registrars.add( - ImmutableList.of( - registrar + "-3", - passwordGenerator.createString(PASSWORD_LENGTH), - registrar + "-ga")); - registrars.add( - ImmutableList.of( - registrar + "-4", - passwordGenerator.createString(PASSWORD_LENGTH), - registrar + "-ga")); - } - registrars.add( - ImmutableList.of( - registrar + "-5", passwordGenerator.createString(PASSWORD_LENGTH), registrar + "-eap")); - - for (List r : registrars) { - createRegistrar(r.get(0), r.get(1), r.get(2)); - createRegistrarContact(r.get(0)); - } + ImmutableMap clientIdToTld = oteAccountBuilder.buildAndPersist(); StringBuilder output = new StringBuilder(); output.append("Copy these usernames/passwords back into the onboarding bug:\n\n"); - - for (List r : registrars) { - output.append("Login: " + r.get(0) + "\n"); - output.append("Password: " + r.get(1) + "\n"); - output.append("TLD: " + r.get(2) + "\n\n"); - } + clientIdToTld.forEach( + (clientId, tld) -> { + output.append( + String.format("Login: %s\nPassword: %s\nTLD: %s\n\n", clientId, password, tld)); + }); return output.toString(); } diff --git a/javatests/google/registry/model/OteAccountBuilderTest.java b/javatests/google/registry/model/OteAccountBuilderTest.java new file mode 100644 index 000000000..792f8cd84 --- /dev/null +++ b/javatests/google/registry/model/OteAccountBuilderTest.java @@ -0,0 +1,302 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static google.registry.testing.CertificateSamples.SAMPLE_CERT; +import static google.registry.testing.CertificateSamples.SAMPLE_CERT_HASH; +import static google.registry.testing.DatastoreHelper.persistPremiumList; +import static google.registry.testing.JUnitBackports.assertThrows; +import static google.registry.util.DateTimeUtils.START_OF_TIME; +import static org.joda.money.CurrencyUnit.USD; + +import com.google.common.collect.ImmutableList; +import google.registry.model.registrar.Registrar; +import google.registry.model.registrar.RegistrarContact; +import google.registry.model.registry.Registry; +import google.registry.model.registry.Registry.TldState; +import google.registry.testing.AppEngineRule; +import google.registry.testing.DatastoreHelper; +import google.registry.util.CidrAddressBlock; +import google.registry.util.SystemClock; +import org.joda.money.Money; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.Duration; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class OteAccountBuilderTest { + + @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build(); + + @Test + public void testGetRegistrarToTldMap() { + assertThat(OteAccountBuilder.forClientId("myclientid").getClientIdToTldMap()) + .containsExactly( + "myclientid-1", "myclientid-sunrise", + "myclientid-2", "myclientid-landrush", + "myclientid-3", "myclientid-ga", + "myclientid-4", "myclientid-ga", + "myclientid-5", "myclientid-eap"); + } + + @Before + public void setUp() { + persistPremiumList("default_sandbox_list", "sandbox,USD 1000"); + } + + private void assertTldExists(String tld, Registry.TldState tldState) { + Registry registry = Registry.get(tld); + assertThat(registry).isNotNull(); + assertThat(registry.getPremiumList().getName()).isEqualTo("default_sandbox_list"); + assertThat(registry.getTldStateTransitions()).containsExactly(START_OF_TIME, tldState); + assertThat(registry.getDnsWriters()).containsExactly("VoidDnsWriter"); + assertThat(registry.getAddGracePeriodLength()).isEqualTo(Duration.standardDays(5)); + assertThat(registry.getPendingDeleteLength()).isEqualTo(Duration.standardDays(5)); + assertThat(registry.getRedemptionGracePeriodLength()).isEqualTo(Duration.standardDays(30)); + assertThat(registry.getEapFeeScheduleAsMap()).containsExactly(START_OF_TIME, Money.of(USD, 0)); + } + + private void assertTldExistsGa(String tld, Money eapFee) { + Registry registry = Registry.get(tld); + assertThat(registry).isNotNull(); + assertThat(registry.getPremiumList().getName()).isEqualTo("default_sandbox_list"); + assertThat(registry.getTldStateTransitions()) + .containsExactly(START_OF_TIME, TldState.GENERAL_AVAILABILITY); + assertThat(registry.getDnsWriters()).containsExactly("VoidDnsWriter"); + assertThat(registry.getAddGracePeriodLength()).isEqualTo(Duration.standardHours(1)); + assertThat(registry.getPendingDeleteLength()).isEqualTo(Duration.standardMinutes(5)); + assertThat(registry.getRedemptionGracePeriodLength()).isEqualTo(Duration.standardMinutes(10)); + assertThat(registry.getCurrency()).isEqualTo(eapFee.getCurrencyUnit()); + // This uses "now" on purpose - so the test will break at 2022 when the current EapFee in OTE + // goes back to 0 + assertThat(registry.getEapFeeFor(DateTime.now(DateTimeZone.UTC)).getCost()) + .isEqualTo(eapFee.getAmount()); + } + + private void assertRegistrarExists(String clientId, String tld) { + Registrar registrar = Registrar.loadByClientId(clientId).orElse(null); + assertThat(registrar).isNotNull(); + assertThat(registrar.getType()).isEqualTo(Registrar.Type.OTE); + assertThat(registrar.getState()).isEqualTo(Registrar.State.ACTIVE); + assertThat(registrar.getAllowedTlds()).containsExactly(tld); + } + + private void assertContactExists(String clientId, String email) { + Registrar registrar = Registrar.loadByClientId(clientId).get(); + assertThat(registrar.getContacts().stream().map(RegistrarContact::getEmailAddress)) + .contains(email); + RegistrarContact contact = + registrar.getContacts().stream() + .filter(c -> email.equals(c.getEmailAddress())) + .findAny() + .get(); + assertThat(contact.getEmailAddress()).isEqualTo(email); + assertThat(contact.getGaeUserId()).isNotEmpty(); + } + + @Test + public void testCreateOteEntities_success() { + OteAccountBuilder.forClientId("myclientid").addContact("email@example.com").buildAndPersist(); + + assertTldExists("myclientid-sunrise", TldState.START_DATE_SUNRISE); + assertTldExists("myclientid-landrush", TldState.LANDRUSH); + assertTldExistsGa("myclientid-ga", Money.of(USD, 0)); + assertTldExistsGa("myclientid-eap", Money.of(USD, 100)); + assertRegistrarExists("myclientid-1", "myclientid-sunrise"); + assertRegistrarExists("myclientid-2", "myclientid-landrush"); + assertRegistrarExists("myclientid-3", "myclientid-ga"); + assertRegistrarExists("myclientid-4", "myclientid-ga"); + assertRegistrarExists("myclientid-5", "myclientid-eap"); + assertContactExists("myclientid-1", "email@example.com"); + assertContactExists("myclientid-2", "email@example.com"); + assertContactExists("myclientid-3", "email@example.com"); + assertContactExists("myclientid-4", "email@example.com"); + assertContactExists("myclientid-5", "email@example.com"); + } + + @Test + public void testCreateOteEntities_multipleContacts_success() { + OteAccountBuilder.forClientId("myclientid") + .addContact("email@example.com") + .addContact("other@example.com") + .addContact("someone@example.com") + .buildAndPersist(); + + assertTldExists("myclientid-sunrise", TldState.START_DATE_SUNRISE); + assertTldExists("myclientid-landrush", TldState.LANDRUSH); + assertTldExistsGa("myclientid-ga", Money.of(USD, 0)); + assertTldExistsGa("myclientid-eap", Money.of(USD, 100)); + assertRegistrarExists("myclientid-1", "myclientid-sunrise"); + assertRegistrarExists("myclientid-2", "myclientid-landrush"); + assertRegistrarExists("myclientid-3", "myclientid-ga"); + assertRegistrarExists("myclientid-4", "myclientid-ga"); + assertRegistrarExists("myclientid-5", "myclientid-eap"); + assertContactExists("myclientid-1", "email@example.com"); + assertContactExists("myclientid-2", "email@example.com"); + assertContactExists("myclientid-3", "email@example.com"); + assertContactExists("myclientid-4", "email@example.com"); + assertContactExists("myclientid-5", "email@example.com"); + assertContactExists("myclientid-1", "other@example.com"); + assertContactExists("myclientid-2", "other@example.com"); + assertContactExists("myclientid-3", "other@example.com"); + assertContactExists("myclientid-4", "other@example.com"); + assertContactExists("myclientid-5", "other@example.com"); + assertContactExists("myclientid-1", "someone@example.com"); + assertContactExists("myclientid-2", "someone@example.com"); + assertContactExists("myclientid-3", "someone@example.com"); + assertContactExists("myclientid-4", "someone@example.com"); + assertContactExists("myclientid-5", "someone@example.com"); + } + + @Test + public void testCreateOteEntities_setPassword() { + OteAccountBuilder.forClientId("myclientid").setPassword("myPassword").buildAndPersist(); + + assertThat(Registrar.loadByClientId("myclientid-3").get().testPassword("myPassword")).isTrue(); + } + + @Test + public void testCreateOteEntities_setCertificateHash() { + OteAccountBuilder.forClientId("myclientid") + .setCertificateHash(SAMPLE_CERT_HASH) + .buildAndPersist(); + + assertThat(Registrar.loadByClientId("myclientid-3").get().getClientCertificateHash()) + .isEqualTo(SAMPLE_CERT_HASH); + } + + @Test + public void testCreateOteEntities_setCertificate() { + OteAccountBuilder.forClientId("myclientid") + .setCertificate(SAMPLE_CERT, new SystemClock().nowUtc()) + .buildAndPersist(); + + assertThat(Registrar.loadByClientId("myclientid-3").get().getClientCertificateHash()) + .isEqualTo(SAMPLE_CERT_HASH); + assertThat(Registrar.loadByClientId("myclientid-3").get().getClientCertificate()) + .isEqualTo(SAMPLE_CERT); + } + + @Test + public void testCreateOteEntities_setIpWhitelist() { + OteAccountBuilder.forClientId("myclientid") + .setIpWhitelist(ImmutableList.of("1.1.1.0/24")) + .buildAndPersist(); + + assertThat(Registrar.loadByClientId("myclientid-3").get().getIpAddressWhitelist()) + .containsExactly(CidrAddressBlock.create("1.1.1.0/24")); + } + + @Test + public void testCreateOteEntities_invalidClientId_fails() { + assertThat( + assertThrows( + IllegalArgumentException.class, () -> OteAccountBuilder.forClientId("3blobio"))) + .hasMessageThat() + .isEqualTo("Invalid registrar name: 3blobio"); + } + + @Test + public void testCreateOteEntities_clientIdTooShort_fails() { + assertThat( + assertThrows(IllegalArgumentException.class, () -> OteAccountBuilder.forClientId("bl"))) + .hasMessageThat() + .isEqualTo("Invalid registrar name: bl"); + } + + @Test + public void testCreateOteEntities_clientIdTooLong_fails() { + assertThat( + assertThrows( + IllegalArgumentException.class, + () -> OteAccountBuilder.forClientId("blobiotoooolong"))) + .hasMessageThat() + .isEqualTo("Invalid registrar name: blobiotoooolong"); + } + + @Test + public void testCreateOteEntities_clientIdBadCharacter_fails() { + assertThat( + assertThrows( + IllegalArgumentException.class, () -> OteAccountBuilder.forClientId("blo#bio"))) + .hasMessageThat() + .isEqualTo("Invalid registrar name: blo#bio"); + } + + @Test + public void testCreateOteEntities_entityExists_failsWhenNotReplaceExisting() { + DatastoreHelper.persistSimpleResource( + AppEngineRule.makeRegistrar1().asBuilder().setClientId("myclientid-1").build()); + OteAccountBuilder oteSetupHelper = OteAccountBuilder.forClientId("myclientid"); + + assertThat(assertThrows(IllegalStateException.class, () -> oteSetupHelper.buildAndPersist())) + .hasMessageThat() + .contains("Found existing object(s) conflicting with OT&E objects"); + } + + @Test + public void testCreateOteEntities_entityExists_succeedsWhenReplaceExisting() { + DatastoreHelper.persistSimpleResource( + AppEngineRule.makeRegistrar1().asBuilder().setClientId("myclientid-1").build()); + DatastoreHelper.createTld("myclientid-landrush", Registry.TldState.SUNRUSH); + + OteAccountBuilder.forClientId("myclientid").setReplaceExisting(true).buildAndPersist(); + + assertTldExists("myclientid-landrush", TldState.LANDRUSH); + assertRegistrarExists("myclientid-3", "myclientid-ga"); + } + + @Test + public void testCreateOteEntities_doubleCreation_actuallyReplaces() { + OteAccountBuilder.forClientId("myclientid") + .setPassword("oldPassword") + .addContact("email@example.com") + .buildAndPersist(); + + assertThat(Registrar.loadByClientId("myclientid-3").get().testPassword("oldPassword")).isTrue(); + + OteAccountBuilder.forClientId("myclientid") + .setPassword("newPassword") + .addContact("email@example.com") + .setReplaceExisting(true) + .buildAndPersist(); + + assertThat(Registrar.loadByClientId("myclientid-3").get().testPassword("oldPassword")) + .isFalse(); + assertThat(Registrar.loadByClientId("myclientid-3").get().testPassword("newPassword")).isTrue(); + } + + @Test + public void testCreateOteEntities_doubleCreation_keepsOldContacts() { + OteAccountBuilder.forClientId("myclientid").addContact("email@example.com").buildAndPersist(); + + assertContactExists("myclientid-3", "email@example.com"); + + OteAccountBuilder.forClientId("myclientid") + .addContact("other@example.com") + .setReplaceExisting(true) + .buildAndPersist(); + + assertContactExists("myclientid-3", "other@example.com"); + assertContactExists("myclientid-3", "email@example.com"); + } +} diff --git a/javatests/google/registry/model/registrar/RegistrarTest.java b/javatests/google/registry/model/registrar/RegistrarTest.java index 066660683..ed095087f 100644 --- a/javatests/google/registry/model/registrar/RegistrarTest.java +++ b/javatests/google/registry/model/registrar/RegistrarTest.java @@ -23,6 +23,7 @@ import static google.registry.testing.CertificateSamples.SAMPLE_CERT2_HASH; import static google.registry.testing.CertificateSamples.SAMPLE_CERT_HASH; import static google.registry.testing.DatastoreHelper.cloneAndSetAutoTimestamps; import static google.registry.testing.DatastoreHelper.createTld; +import static google.registry.testing.DatastoreHelper.newRegistry; import static google.registry.testing.DatastoreHelper.persistResource; import static google.registry.testing.DatastoreHelper.persistSimpleResource; import static google.registry.testing.DatastoreHelper.persistSimpleResources; @@ -33,10 +34,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; import com.googlecode.objectify.Key; +import google.registry.config.RegistryConfig; import google.registry.model.EntityTestCase; import google.registry.model.common.EntityGroupRoot; import google.registry.model.registrar.Registrar.State; import google.registry.model.registrar.Registrar.Type; +import google.registry.model.registry.Registries; import google.registry.util.CidrAddressBlock; import org.joda.money.CurrencyUnit; import org.junit.Before; @@ -415,6 +418,77 @@ public class RegistrarTest extends EntityTestCase { IllegalArgumentException.class, () -> new Registrar.Builder().setPhonePasscode("code1")); } + @Test + public void testSuccess_setAllowedTlds() { + assertThat( + registrar.asBuilder() + .setAllowedTlds(ImmutableSet.of("xn--q9jyb4c")) + .build() + .getAllowedTlds()) + .containsExactly("xn--q9jyb4c"); + } + + @Test + public void testSuccess_setAllowedTldsUncached() { + assertThat( + registrar.asBuilder() + .setAllowedTldsUncached(ImmutableSet.of("xn--q9jyb4c")) + .build() + .getAllowedTlds()) + .containsExactly("xn--q9jyb4c"); + } + + @Test + public void testFailure_setAllowedTlds_nonexistentTld() { + assertThrows( + IllegalArgumentException.class, + () -> registrar.asBuilder().setAllowedTlds(ImmutableSet.of("bad"))); + } + + @Test + public void testFailure_setAllowedTldsUncached_nonexistentTld() { + assertThrows( + IllegalArgumentException.class, + () -> registrar.asBuilder().setAllowedTldsUncached(ImmutableSet.of("bad"))); + } + + @Test + public void testSuccess_setAllowedTldsUncached_newTldNotInCache() { + // Cache duration in tests is 0. To make sure the data isn't in the cache we have to set it to a + // higher value and reset the cache. + RegistryConfig.CONFIG_SETTINGS.get().caching.singletonCacheRefreshSeconds = 600; + Registries.resetCache(); + // Make sure the TLD we want to create doesn't exist yet. + // This is also important because getTlds fills out the cache when used. + assertThat(Registries.getTlds()).doesNotContain("newtld"); + // We can't use createTld here because it failes when the cache is used. + persistResource(newRegistry("newtld", "NEWTLD")); + // Make sure we set up the cache correctly, so the newly created TLD isn't in the cache + assertThat(Registries.getTlds()).doesNotContain("newtld"); + + // Test that the uncached version works + assertThat( + registrar.asBuilder() + .setAllowedTldsUncached(ImmutableSet.of("newtld")) + .build() + .getAllowedTlds()) + .containsExactly("newtld"); + + // Test that the "regular" cached version fails. If this doesn't throw - then we changed how the + // cached version works: + // - either we switched to a different cache type/duration, and we haven't actually set up that + // cache in the test + // - or we stopped using the cache entirely and we should rethink if the Uncached version is + // still needed + assertThrows( + IllegalArgumentException.class, + () -> registrar.asBuilder().setAllowedTlds(ImmutableSet.of("newtld"))); + + // Make sure the cache hasn't expired during the test and "newtld" is still not in the cached + // TLDs + assertThat(Registries.getTlds()).doesNotContain("newtld"); + } + @Test public void testLoadByClientIdCached_isTransactionless() { ofy() diff --git a/javatests/google/registry/tools/SetupOteCommandTest.java b/javatests/google/registry/tools/SetupOteCommandTest.java index dc403172b..0b6b92bc3 100644 --- a/javatests/google/registry/tools/SetupOteCommandTest.java +++ b/javatests/google/registry/tools/SetupOteCommandTest.java @@ -34,6 +34,7 @@ import google.registry.model.registrar.RegistrarContact; import google.registry.model.registry.Registry; import google.registry.model.registry.Registry.TldState; import google.registry.testing.DeterministicStringGenerator; +import google.registry.testing.FakeClock; import google.registry.util.CidrAddressBlock; import java.security.cert.CertificateParsingException; import org.joda.money.CurrencyUnit; @@ -46,23 +47,16 @@ import org.junit.Test; /** Unit tests for {@link SetupOteCommand}. */ public class SetupOteCommandTest extends CommandTestCase { - ImmutableList passwords = - ImmutableList.of( - "abcdefghijklmnop", - "qrstuvwxyzabcdef", - "ghijklmnopqrstuv", - "wxyzabcdefghijkl", - "mnopqrstuvwxyzab"); + static final String PASSWORD = "abcdefghijklmnop"; + DeterministicStringGenerator passwordGenerator = new DeterministicStringGenerator("abcdefghijklmnopqrstuvwxyz"); @Before public void init() { - SetupOteCommand.interactive = false; - command.validDnsWriterNames = ImmutableSet.of("FooDnsWriter", "BarDnsWriter", "VoidDnsWriter"); command.passwordGenerator = passwordGenerator; + command.clock = new FakeClock(DateTime.parse("2018-07-07TZ")); persistPremiumList("default_sandbox_list", "sandbox,USD 1000"); - persistPremiumList("alternate_list", "rich,USD 3000"); } /** Verify TLD creation. */ @@ -70,8 +64,6 @@ public class SetupOteCommandTest extends CommandTestCase { String tldName, String roidSuffix, TldState tldState, - String dnsWriter, - String premiumList, Duration addGracePeriodLength, Duration redemptionGracePeriodLength, Duration pendingDeleteLength, @@ -80,9 +72,9 @@ public class SetupOteCommandTest extends CommandTestCase { assertThat(registry).isNotNull(); assertThat(registry.getRoidSuffix()).isEqualTo(roidSuffix); assertThat(registry.getTldState(DateTime.now(UTC))).isEqualTo(tldState); - assertThat(registry.getDnsWriters()).containsExactly(dnsWriter); + assertThat(registry.getDnsWriters()).containsExactly("VoidDnsWriter"); assertThat(registry.getPremiumList()).isNotNull(); - assertThat(registry.getPremiumList().getName()).isEqualTo(premiumList); + assertThat(registry.getPremiumList().getName()).isEqualTo("default_sandbox_list"); assertThat(registry.getAddGracePeriodLength()).isEqualTo(addGracePeriodLength); assertThat(registry.getRedemptionGracePeriodLength()).isEqualTo(redemptionGracePeriodLength); assertThat(registry.getPendingDeleteLength()).isEqualTo(pendingDeleteLength); @@ -98,20 +90,18 @@ public class SetupOteCommandTest extends CommandTestCase { Money.of(CurrencyUnit.USD, 0), DateTime.parse("2018-03-01T00:00:00Z"), Money.of(CurrencyUnit.USD, 100), - DateTime.parse("2022-03-01T00:00:00Z"), + DateTime.parse("2030-03-01T00:00:00Z"), Money.of(CurrencyUnit.USD, 0))); } } /** Verify TLD creation with registry default durations. */ private void verifyTldCreation( - String tldName, String roidSuffix, TldState tldState, String dnsWriter, String premiumList) { + String tldName, String roidSuffix, TldState tldState) { verifyTldCreation( tldName, roidSuffix, tldState, - dnsWriter, - premiumList, Registry.DEFAULT_ADD_GRACE_PERIOD, Registry.DEFAULT_REDEMPTION_GRACE_PERIOD, Registry.DEFAULT_PENDING_DELETE_LENGTH, @@ -162,23 +152,14 @@ public class SetupOteCommandTest extends CommandTestCase { "--ip_whitelist=1.1.1.1", "--registrar=blobio", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--certfile=" + getCertFilename()); - verifyTldCreation( - "blobio-sunrise", - "BLOBIOS0", - TldState.START_DATE_SUNRISE, - "VoidDnsWriter", - "default_sandbox_list"); - verifyTldCreation( - "blobio-landrush", "BLOBIOL1", TldState.LANDRUSH, "VoidDnsWriter", "default_sandbox_list"); + verifyTldCreation("blobio-sunrise", "BLOBIOS0", TldState.START_DATE_SUNRISE); + verifyTldCreation("blobio-landrush", "BLOBIOL1", TldState.LANDRUSH); verifyTldCreation( "blobio-ga", "BLOBIOG2", TldState.GENERAL_AVAILABILITY, - "VoidDnsWriter", - "default_sandbox_list", Duration.standardMinutes(60), Duration.standardMinutes(10), Duration.standardMinutes(5), @@ -187,8 +168,6 @@ public class SetupOteCommandTest extends CommandTestCase { "blobio-eap", "BLOBIOE3", TldState.GENERAL_AVAILABILITY, - "VoidDnsWriter", - "default_sandbox_list", Duration.standardMinutes(60), Duration.standardMinutes(10), Duration.standardMinutes(5), @@ -197,11 +176,11 @@ public class SetupOteCommandTest extends CommandTestCase { ImmutableList ipAddress = ImmutableList.of( CidrAddressBlock.create("1.1.1.1")); - verifyRegistrarCreation("blobio-1", "blobio-sunrise", passwords.get(0), ipAddress); - verifyRegistrarCreation("blobio-2", "blobio-landrush", passwords.get(1), ipAddress); - verifyRegistrarCreation("blobio-3", "blobio-ga", passwords.get(2), ipAddress); - verifyRegistrarCreation("blobio-4", "blobio-ga", passwords.get(3), ipAddress); - verifyRegistrarCreation("blobio-5", "blobio-eap", passwords.get(4), ipAddress); + verifyRegistrarCreation("blobio-1", "blobio-sunrise", PASSWORD, ipAddress); + verifyRegistrarCreation("blobio-2", "blobio-landrush", PASSWORD, ipAddress); + verifyRegistrarCreation("blobio-3", "blobio-ga", PASSWORD, ipAddress); + verifyRegistrarCreation("blobio-4", "blobio-ga", PASSWORD, ipAddress); + verifyRegistrarCreation("blobio-5", "blobio-eap", PASSWORD, ipAddress); verifyRegistrarContactCreation("blobio-1", "contact@email.com"); verifyRegistrarContactCreation("blobio-2", "contact@email.com"); @@ -216,23 +195,14 @@ public class SetupOteCommandTest extends CommandTestCase { "--ip_whitelist=1.1.1.1", "--registrar=abc", "--email=abc@email.com", - "--dns_writers=VoidDnsWriter", "--certfile=" + getCertFilename()); - verifyTldCreation( - "abc-sunrise", - "ABCSUNR0", - TldState.START_DATE_SUNRISE, - "VoidDnsWriter", - "default_sandbox_list"); - verifyTldCreation( - "abc-landrush", "ABCLAND1", TldState.LANDRUSH, "VoidDnsWriter", "default_sandbox_list"); + verifyTldCreation("abc-sunrise", "ABCSUNR0", TldState.START_DATE_SUNRISE); + verifyTldCreation("abc-landrush", "ABCLAND1", TldState.LANDRUSH); verifyTldCreation( "abc-ga", "ABCGA2", TldState.GENERAL_AVAILABILITY, - "VoidDnsWriter", - "default_sandbox_list", Duration.standardMinutes(60), Duration.standardMinutes(10), Duration.standardMinutes(5), @@ -241,8 +211,6 @@ public class SetupOteCommandTest extends CommandTestCase { "abc-eap", "ABCEAP3", TldState.GENERAL_AVAILABILITY, - "VoidDnsWriter", - "default_sandbox_list", Duration.standardMinutes(60), Duration.standardMinutes(10), Duration.standardMinutes(5), @@ -251,11 +219,11 @@ public class SetupOteCommandTest extends CommandTestCase { ImmutableList ipAddress = ImmutableList.of(CidrAddressBlock.create("1.1.1.1")); - verifyRegistrarCreation("abc-1", "abc-sunrise", passwords.get(0), ipAddress); - verifyRegistrarCreation("abc-2", "abc-landrush", passwords.get(1), ipAddress); - verifyRegistrarCreation("abc-3", "abc-ga", passwords.get(2), ipAddress); - verifyRegistrarCreation("abc-4", "abc-ga", passwords.get(3), ipAddress); - verifyRegistrarCreation("abc-5", "abc-eap", passwords.get(4), ipAddress); + verifyRegistrarCreation("abc-1", "abc-sunrise", PASSWORD, ipAddress); + verifyRegistrarCreation("abc-2", "abc-landrush", PASSWORD, ipAddress); + verifyRegistrarCreation("abc-3", "abc-ga", PASSWORD, ipAddress); + verifyRegistrarCreation("abc-4", "abc-ga", PASSWORD, ipAddress); + verifyRegistrarCreation("abc-5", "abc-eap", PASSWORD, ipAddress); verifyRegistrarContactCreation("abc-1", "abc@email.com"); verifyRegistrarContactCreation("abc-2", "abc@email.com"); @@ -267,19 +235,15 @@ public class SetupOteCommandTest extends CommandTestCase { @Test public void testSuccess_certificateHash() throws Exception { runCommandForced( - "--eap_only", "--ip_whitelist=1.1.1.1", "--registrar=blobio", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--certhash=" + SAMPLE_CERT_HASH); verifyTldCreation( "blobio-eap", "BLOBIOE3", TldState.GENERAL_AVAILABILITY, - "VoidDnsWriter", - "default_sandbox_list", Duration.standardMinutes(60), Duration.standardMinutes(10), Duration.standardMinutes(5), @@ -288,36 +252,7 @@ public class SetupOteCommandTest extends CommandTestCase { ImmutableList ipAddress = ImmutableList.of(CidrAddressBlock.create("1.1.1.1")); - verifyRegistrarCreation("blobio-5", "blobio-eap", passwords.get(0), ipAddress, true); - - verifyRegistrarContactCreation("blobio-5", "contact@email.com"); - } - - @Test - public void testSuccess_eapOnly() throws Exception { - runCommandForced( - "--eap_only", - "--ip_whitelist=1.1.1.1", - "--registrar=blobio", - "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", - "--certfile=" + getCertFilename()); - - verifyTldCreation( - "blobio-eap", - "BLOBIOE3", - TldState.GENERAL_AVAILABILITY, - "VoidDnsWriter", - "default_sandbox_list", - Duration.standardMinutes(60), - Duration.standardMinutes(10), - Duration.standardMinutes(5), - true); - - ImmutableList ipAddress = ImmutableList.of( - CidrAddressBlock.create("1.1.1.1")); - - verifyRegistrarCreation("blobio-5", "blobio-eap", passwords.get(0), ipAddress); + verifyRegistrarCreation("blobio-5", "blobio-eap", PASSWORD, ipAddress, true); verifyRegistrarContactCreation("blobio-5", "contact@email.com"); } @@ -328,23 +263,14 @@ public class SetupOteCommandTest extends CommandTestCase { "--ip_whitelist=1.1.1.1,2.2.2.2", "--registrar=blobio", "--email=contact@email.com", - "--dns_writers=FooDnsWriter", "--certfile=" + getCertFilename()); - verifyTldCreation( - "blobio-sunrise", - "BLOBIOS0", - TldState.START_DATE_SUNRISE, - "FooDnsWriter", - "default_sandbox_list"); - verifyTldCreation( - "blobio-landrush", "BLOBIOL1", TldState.LANDRUSH, "FooDnsWriter", "default_sandbox_list"); + verifyTldCreation("blobio-sunrise", "BLOBIOS0", TldState.START_DATE_SUNRISE); + verifyTldCreation("blobio-landrush", "BLOBIOL1", TldState.LANDRUSH); verifyTldCreation( "blobio-ga", "BLOBIOG2", TldState.GENERAL_AVAILABILITY, - "FooDnsWriter", - "default_sandbox_list", Duration.standardMinutes(60), Duration.standardMinutes(10), Duration.standardMinutes(5), @@ -353,8 +279,6 @@ public class SetupOteCommandTest extends CommandTestCase { "blobio-eap", "BLOBIOE3", TldState.GENERAL_AVAILABILITY, - "FooDnsWriter", - "default_sandbox_list", Duration.standardMinutes(60), Duration.standardMinutes(10), Duration.standardMinutes(5), @@ -364,66 +288,11 @@ public class SetupOteCommandTest extends CommandTestCase { CidrAddressBlock.create("1.1.1.1"), CidrAddressBlock.create("2.2.2.2")); - verifyRegistrarCreation("blobio-1", "blobio-sunrise", passwords.get(0), ipAddresses); - verifyRegistrarCreation("blobio-2", "blobio-landrush", passwords.get(1), ipAddresses); - verifyRegistrarCreation("blobio-3", "blobio-ga", passwords.get(2), ipAddresses); - verifyRegistrarCreation("blobio-4", "blobio-ga", passwords.get(3), ipAddresses); - verifyRegistrarCreation("blobio-5", "blobio-eap", passwords.get(4), ipAddresses); - - verifyRegistrarContactCreation("blobio-1", "contact@email.com"); - verifyRegistrarContactCreation("blobio-2", "contact@email.com"); - verifyRegistrarContactCreation("blobio-3", "contact@email.com"); - verifyRegistrarContactCreation("blobio-4", "contact@email.com"); - verifyRegistrarContactCreation("blobio-5", "contact@email.com"); - } - - @Test - public void testSuccess_alternatePremiumList() throws Exception { - runCommandForced( - "--ip_whitelist=1.1.1.1", - "--registrar=blobio", - "--email=contact@email.com", - "--certfile=" + getCertFilename(), - "--dns_writers=BarDnsWriter", - "--premium_list=alternate_list"); - - verifyTldCreation( - "blobio-sunrise", - "BLOBIOS0", - TldState.START_DATE_SUNRISE, - "BarDnsWriter", - "alternate_list"); - verifyTldCreation( - "blobio-landrush", "BLOBIOL1", TldState.LANDRUSH, "BarDnsWriter", "alternate_list"); - verifyTldCreation( - "blobio-ga", - "BLOBIOG2", - TldState.GENERAL_AVAILABILITY, - "BarDnsWriter", - "alternate_list", - Duration.standardMinutes(60), - Duration.standardMinutes(10), - Duration.standardMinutes(5), - false); - verifyTldCreation( - "blobio-eap", - "BLOBIOE3", - TldState.GENERAL_AVAILABILITY, - "BarDnsWriter", - "alternate_list", - Duration.standardMinutes(60), - Duration.standardMinutes(10), - Duration.standardMinutes(5), - true); - - ImmutableList ipAddress = ImmutableList.of( - CidrAddressBlock.create("1.1.1.1")); - - verifyRegistrarCreation("blobio-1", "blobio-sunrise", passwords.get(0), ipAddress); - verifyRegistrarCreation("blobio-2", "blobio-landrush", passwords.get(1), ipAddress); - verifyRegistrarCreation("blobio-3", "blobio-ga", passwords.get(2), ipAddress); - verifyRegistrarCreation("blobio-4", "blobio-ga", passwords.get(3), ipAddress); - verifyRegistrarCreation("blobio-5", "blobio-eap", passwords.get(4), ipAddress); + verifyRegistrarCreation("blobio-1", "blobio-sunrise", PASSWORD, ipAddresses); + verifyRegistrarCreation("blobio-2", "blobio-landrush", PASSWORD, ipAddresses); + verifyRegistrarCreation("blobio-3", "blobio-ga", PASSWORD, ipAddresses); + verifyRegistrarCreation("blobio-4", "blobio-ga", PASSWORD, ipAddresses); + verifyRegistrarCreation("blobio-5", "blobio-eap", PASSWORD, ipAddresses); verifyRegistrarContactCreation("blobio-1", "contact@email.com"); verifyRegistrarContactCreation("blobio-2", "contact@email.com"); @@ -441,7 +310,6 @@ public class SetupOteCommandTest extends CommandTestCase { runCommandForced( "--registrar=blobio", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--certfile=" + getCertFilename())); assertThat(thrown).hasMessageThat().contains("option is required: -w, --ip_whitelist"); } @@ -455,7 +323,6 @@ public class SetupOteCommandTest extends CommandTestCase { runCommandForced( "--ip_whitelist=1.1.1.1", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--certfile=" + getCertFilename())); assertThat(thrown).hasMessageThat().contains("option is required: -r, --registrar"); } @@ -469,7 +336,6 @@ public class SetupOteCommandTest extends CommandTestCase { runCommandForced( "--ip_whitelist=1.1.1.1", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--registrar=blobio")); assertThat(thrown) .hasMessageThat() @@ -486,7 +352,6 @@ public class SetupOteCommandTest extends CommandTestCase { runCommandForced( "--ip_whitelist=1.1.1.1", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--registrar=blobio", "--certfile=" + getCertFilename(), "--certhash=" + SAMPLE_CERT_HASH)); @@ -496,20 +361,6 @@ public class SetupOteCommandTest extends CommandTestCase { "Must specify exactly one of client certificate file or client certificate hash."); } - @Test - public void testFailure_missingDnsWriter() { - ParameterException thrown = - assertThrows( - ParameterException.class, - () -> - runCommandForced( - "--ip_whitelist=1.1.1.1", - "--email=contact@email.com", - "--certfile=" + getCertFilename(), - "--registrar=blobio")); - assertThat(thrown).hasMessageThat().contains("option is required: --dns_writers"); - } - @Test public void testFailure_missingEmail() { ParameterException thrown = @@ -518,7 +369,6 @@ public class SetupOteCommandTest extends CommandTestCase { () -> runCommandForced( "--ip_whitelist=1.1.1.1", - "--dns_writers=VoidDnsWriter", "--certfile=" + getCertFilename(), "--registrar=blobio")); assertThat(thrown).hasMessageThat().contains("option is required: --email"); @@ -534,7 +384,6 @@ public class SetupOteCommandTest extends CommandTestCase { "--ip_whitelist=1.1.1.1", "--registrar=blobio", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--certfile=/dev/null")); assertThat(thrown).hasMessageThat().contains("No X509Certificate found"); } @@ -549,26 +398,8 @@ public class SetupOteCommandTest extends CommandTestCase { "--ip_whitelist=1.1.1.1", "--registrar=3blobio", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--certfile=" + getCertFilename())); - assertThat(thrown).hasMessageThat().contains("Registrar name is invalid"); - } - - @Test - public void testFailure_invalidDnsWriter() { - IllegalArgumentException thrown = - assertThrows( - IllegalArgumentException.class, - () -> - runCommandForced( - "--ip_whitelist=1.1.1.1", - "--registrar=blobio", - "--email=contact@email.com", - "--dns_writers=InvalidDnsWriter", - "--certfile=" + getCertFilename())); - assertThat(thrown) - .hasMessageThat() - .contains("Invalid DNS writer name(s) specified: [InvalidDnsWriter]"); + assertThat(thrown).hasMessageThat().contains("Invalid registrar name: 3blobio"); } @Test @@ -581,9 +412,8 @@ public class SetupOteCommandTest extends CommandTestCase { "--ip_whitelist=1.1.1.1", "--registrar=bl", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--certfile=" + getCertFilename())); - assertThat(thrown).hasMessageThat().contains("Registrar name is invalid"); + assertThat(thrown).hasMessageThat().contains("Invalid registrar name: bl"); } @Test @@ -596,9 +426,8 @@ public class SetupOteCommandTest extends CommandTestCase { "--ip_whitelist=1.1.1.1", "--registrar=blobiotoooolong", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--certfile=" + getCertFilename())); - assertThat(thrown).hasMessageThat().contains("Registrar name is invalid"); + assertThat(thrown).hasMessageThat().contains("Invalid registrar name: blobiotoooolong"); } @Test @@ -611,25 +440,8 @@ public class SetupOteCommandTest extends CommandTestCase { "--ip_whitelist=1.1.1.1", "--registrar=blo#bio", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--certfile=" + getCertFilename())); - assertThat(thrown).hasMessageThat().contains("Registrar name is invalid"); - } - - @Test - public void testFailure_invalidPremiumList() { - IllegalArgumentException thrown = - assertThrows( - IllegalArgumentException.class, - () -> - runCommandForced( - "--ip_whitelist=1.1.1.1", - "--registrar=blobio", - "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", - "--certfile=" + getCertFilename(), - "--premium_list=foo")); - assertThat(thrown).hasMessageThat().contains("The premium list 'foo' doesn't exist"); + assertThat(thrown).hasMessageThat().contains("Invalid registrar name: blo#bio"); } @Test @@ -643,9 +455,23 @@ public class SetupOteCommandTest extends CommandTestCase { "--ip_whitelist=1.1.1.1", "--registrar=blobio", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--certfile=" + getCertFilename())); - assertThat(thrown).hasMessageThat().contains("TLD 'blobio-sunrise' already exists"); + assertThat(thrown).hasMessageThat().contains("Registry(\"blobio-sunrise\")"); + } + + @Test + public void testSuccess_tldExists_replaceExisting() throws Exception { + createTld("blobio-sunrise"); + + runCommandForced( + "--overwrite", + "--ip_whitelist=1.1.1.1", + "--registrar=blobio", + "--email=contact@email.com", + "--certfile=" + getCertFilename()); + + verifyTldCreation("blobio-sunrise", "BLOBIOS0", TldState.START_DATE_SUNRISE); + verifyTldCreation("blobio-landrush", "BLOBIOL1", TldState.LANDRUSH); } @Test @@ -663,8 +489,29 @@ public class SetupOteCommandTest extends CommandTestCase { "--ip_whitelist=1.1.1.1", "--registrar=blobio", "--email=contact@email.com", - "--dns_writers=VoidDnsWriter", "--certfile=" + getCertFilename())); - assertThat(thrown).hasMessageThat().contains("Registrar blobio-1 already exists"); + assertThat(thrown).hasMessageThat().contains("Registrar(\"blobio-1\")"); + } + + @Test + public void testSuccess_registrarExists_replaceExisting() throws Exception { + Registrar registrar = loadRegistrar("TheRegistrar").asBuilder() + .setClientId("blobio-1") + .setRegistrarName("blobio-1") + .build(); + persistResource(registrar); + + runCommandForced( + "--overwrite", + "--ip_whitelist=1.1.1.1", + "--registrar=blobio", + "--email=contact@email.com", + "--certfile=" + getCertFilename()); + + ImmutableList ipAddress = ImmutableList.of( + CidrAddressBlock.create("1.1.1.1")); + + verifyRegistrarCreation("blobio-1", "blobio-sunrise", PASSWORD, ipAddress); + verifyRegistrarCreation("blobio-2", "blobio-landrush", PASSWORD, ipAddress); } } From 214fb4909124da41aca8aea21862eb9727c826bb Mon Sep 17 00:00:00 2001 From: mmuller Date: Mon, 17 Dec 2018 10:59:27 -0800 Subject: [PATCH 102/134] Add "showAllOutput" project property Added a property to show all test output, useful for debugging. Usage: ./gradlew -PshowAllOutput=true build Tested: Verified that the property does the right thing for true and false values and verified that the build doesn't show test output if it's omitted. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=225854375 --- gradle/build.gradle | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/gradle/build.gradle b/gradle/build.gradle index 1a39540b9..742374d65 100644 --- a/gradle/build.gradle +++ b/gradle/build.gradle @@ -20,6 +20,13 @@ plugins { id 'com.bmuschko.docker-java-application' version '4.0.4' apply false } +// Provide defaults for all of the project properties. + +// showAllOutput: boolean. If true, dump all test output during the build. +if (!project.hasProperty('showAllOutput')) { + ext.showAllOutput = 'false' +} + // Only do linting if the build is successful. gradleLint.autoLintAfterFailure = false @@ -245,6 +252,10 @@ subprojects { } } + test { + testLogging.showStandardStreams = Boolean.parseBoolean(showAllOutput) + } + if (project.name == 'core') return ext.relativePath = "google/registry/${project.name}" From 40b05ffb3cc765ef44d5ef06b59a11fb6a326dd2 Mon Sep 17 00:00:00 2001 From: shicong Date: Mon, 17 Dec 2018 12:43:49 -0800 Subject: [PATCH 103/134] Add a script to generate dependency metadata ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=225872347 --- gradle/build.gradle | 10 +++++-- gradle/generate_dependency_metadata.sh | 38 ++++++++++++++++++++++++++ gradle/update_dependency.sh | 37 +++++++++++++++++++------ 3 files changed, 74 insertions(+), 11 deletions(-) create mode 100755 gradle/generate_dependency_metadata.sh diff --git a/gradle/build.gradle b/gradle/build.gradle index 742374d65..dcc577c48 100644 --- a/gradle/build.gradle +++ b/gradle/build.gradle @@ -120,7 +120,7 @@ subprojects { publishing { repositories { maven { - url = project.findProperty('gcsRepo') + url = project.findProperty('repositoryUrl') } } } @@ -133,7 +133,8 @@ subprojects { return } it.resolvedConfiguration.resolvedArtifacts.each { resolvedArtifact -> - if (resolvedArtifact.id.componentIdentifier.displayName in ['project :util', 'project :third_party']) { + if (resolvedArtifact.id.componentIdentifier.displayName in + ['project :core', 'project :proxy', 'project :util', 'project :third_party']) { return } distinctResolvedArtifacts[resolvedArtifact.id.toString()] = resolvedArtifact @@ -203,7 +204,10 @@ subprojects { def repositoryUri = URI.create(mavenRepository.url.toString()) def artifactUri = repositoryUri.resolve(defaultLayout.getPath(artifact)) if (project.ext.urlExists(artifactUri.toURL())) { - project.ext.writeMetadata(resolvedArtifact, artifactUri.toURL(), project.findProperty('privateRepository') + "/${project.name}") + project.ext.writeMetadata( + resolvedArtifact, + artifactUri.toURL(), + project.findProperty('privateRepository') + "/${project.name}") break } } diff --git a/gradle/generate_dependency_metadata.sh b/gradle/generate_dependency_metadata.sh new file mode 100755 index 000000000..394b63271 --- /dev/null +++ b/gradle/generate_dependency_metadata.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2018 The Nomulus Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script runs a workflow to clone a git repository to local, generate +# a metadata file for each dependency artifact and check in the file to remote +# repository. + +set -e + +ALL_SUBPROJECTS="core proxy util" + +USAGE="Usage: ${0} REPO_URL" +REPO_URL=${1:?${USAGE}} + +REPO_DIR="$(mktemp -d)" + +git clone ${REPO_URL} ${REPO_DIR} +for PROJECT in ${ALL_SUBPROJECTS}; do + $(dirname $0)/gradlew -PprivateRepository="${REPO_DIR}" \ + ":${PROJECT}:generateDependencyMetadata" +done +cd "${REPO_DIR}" +git add -A +git diff-index --quiet HEAD \ + || git commit -m "Update dependency metadata file" && git push +rm -rf "${REPO_DIR}" diff --git a/gradle/update_dependency.sh b/gradle/update_dependency.sh index 8276f8cf2..b7cf3f254 100755 --- a/gradle/update_dependency.sh +++ b/gradle/update_dependency.sh @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - # # This script runs a workflow to generate dependency lock file, run a build against # the generated lock file, save the lock file and upload dependency JARs to a private @@ -22,16 +21,38 @@ set -e ALL_SUBPROJECTS="core proxy util" SUBPROJECTS= +REPOSITORY_URL= -if [[ -z "$@" ]]; then +while [[ $# -gt 0 ]]; do + KEY="$1" + case ${KEY} in + --repositoryUrl) + shift + REPOSITORY_URL="$1" + ;; + *) + SUBPROJECTS="${SUBPROJECTS} ${KEY}" + ;; + esac + shift +done + +if [[ -z ${SUBPROJECTS} ]]; then SUBPROJECTS="${ALL_SUBPROJECTS}" -else - SUBPROJECTS="$@" fi +if [[ -z ${REPOSITORY_URL} ]]; then + echo "--repositoryUrl must be specified" + exit 1 +fi + +WORKING_DIR=$(dirname $0) + for PROJECT in ${SUBPROJECTS}; do - ./gradlew ":${PROJECT}:generateLock" - ./gradlew -PdependencyLock.useGeneratedLock=true ":${PROJECT}:build" - ./gradlew ":${PROJECT}:saveLock" - ./gradlew ":${PROJECT}:publish" + ${WORKING_DIR}/gradlew ":${PROJECT}:generateLock" + ${WORKING_DIR}/gradlew -PdependencyLock.useGeneratedLock=true \ + ":${PROJECT}:build" + ${WORKING_DIR}/gradlew ":${PROJECT}:saveLock" + ${WORKING_DIR}/gradlew -PrepositoryUrl="${REPOSITORY_URL}" \ + ":${PROJECT}:publish" done From 27b623105346822da0142e303ec1d1d22c289889 Mon Sep 17 00:00:00 2001 From: jianglai Date: Tue, 18 Dec 2018 09:25:06 -0800 Subject: [PATCH 104/134] Add the ability to provide credential JSON file to the nomulus tool This allows us to run nomulus tool programmatically on environments that do not allow the 3-legged OAuth authentication flow. The provided JSON file corresponds to a service account, which must have GAE admin permission and whose client ID must be whitelisted in the config file. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226008337 --- java/google/registry/tools/AuthModule.java | 47 +++++++++++++------ java/google/registry/tools/RegistryCli.java | 12 ++++- .../registry/tools/RegistryToolComponent.java | 12 ++++- .../registry/tools/RequestFactoryModule.java | 4 +- .../google/registry/tools/AuthModuleTest.java | 21 ++++++++- 5 files changed, 76 insertions(+), 20 deletions(-) diff --git a/java/google/registry/tools/AuthModule.java b/java/google/registry/tools/AuthModule.java index 16f550cff..61286a187 100644 --- a/java/google/registry/tools/AuthModule.java +++ b/java/google/registry/tools/AuthModule.java @@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; import com.google.gson.Gson; import dagger.Binds; +import dagger.Lazy; import dagger.Module; import dagger.Provides; import google.registry.config.CredentialModule.DefaultCredential; @@ -45,12 +46,14 @@ import java.io.IOException; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.nio.file.Files; +import java.nio.file.Paths; +import javax.annotation.Nullable; +import javax.inject.Named; import javax.inject.Qualifier; import javax.inject.Singleton; -/** - * Module providing the dependency graph for authorization credentials. - */ +/** Module providing the dependency graph for authorization credentials. */ @Module public class AuthModule { @@ -84,9 +87,15 @@ public class AuthModule { @Provides @LocalCredential public static GoogleCredential provideLocalCredential( - @LocalCredentialJson String credentialJson) { + @LocalCredentialJson String credentialJson, + @Config("localCredentialOauthScopes") ImmutableList scopes) { try { - return GoogleCredential.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8))); + GoogleCredential credential = + GoogleCredential.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8))); + if (credential.createScopedRequired()) { + credential = credential.createScoped(scopes); + } + return credential; } catch (IOException e) { throw new RuntimeException(e); } @@ -133,15 +142,25 @@ public class AuthModule { @Provides @LocalCredentialJson public static String provideLocalCredentialJson( - GoogleClientSecrets clientSecrets, @StoredCredential Credential credential) { - return new Gson() - .toJson( - ImmutableMap.builder() - .put("type", "authorized_user") - .put("client_id", clientSecrets.getDetails().getClientId()) - .put("client_secret", clientSecrets.getDetails().getClientSecret()) - .put("refresh_token", credential.getRefreshToken()) - .build()); + Lazy clientSecrets, + @StoredCredential Lazy credential, + @Nullable @Named("credentialFileName") String credentialFilename) { + try { + if (credentialFilename != null) { + return new String(Files.readAllBytes(Paths.get(credentialFilename)), UTF_8); + } else { + return new Gson() + .toJson( + ImmutableMap.builder() + .put("type", "authorized_user") + .put("client_id", clientSecrets.get().getDetails().getClientId()) + .put("client_secret", clientSecrets.get().getDetails().getClientSecret()) + .put("refresh_token", credential.get().getRefreshToken()) + .build()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } } @Provides diff --git a/java/google/registry/tools/RegistryCli.java b/java/google/registry/tools/RegistryCli.java index 412d0f472..7baa09e1e 100644 --- a/java/google/registry/tools/RegistryCli.java +++ b/java/google/registry/tools/RegistryCli.java @@ -58,6 +58,13 @@ final class RegistryCli implements AutoCloseable, CommandRunner { description = "Returns all command names.") private boolean showAllCommands; + @Parameter( + names = {"--credential"}, + description = + "Name of a JSON file containing credential information used by the tool. " + + "If not set, credentials saved by running `nomulus login' will be used.") + private String credentialJson = null; + // Do not make this final - compile-time constant inlining may interfere with JCommander. @ParametersDelegate private LoggingParameters loggingParams = new LoggingParameters(); @@ -81,8 +88,6 @@ final class RegistryCli implements AutoCloseable, CommandRunner { this.commands = commands; Security.addProvider(new BouncyCastleProvider()); - - component = DaggerRegistryToolComponent.create(); } // The > wildcard looks a little funny, but is needed so that @@ -146,6 +151,9 @@ final class RegistryCli implements AutoCloseable, CommandRunner { checkState(RegistryToolEnvironment.get() == environment, "RegistryToolEnvironment argument pre-processing kludge failed."); + component = + DaggerRegistryToolComponent.builder().credentialFilename(credentialJson).build(); + // JCommander stores sub-commands as nested JCommander objects containing a list of user objects // to be populated. Extract the subcommand by getting the JCommander wrapper and then // retrieving the first (and, by virtue of our usage, only) object from it. diff --git a/java/google/registry/tools/RegistryToolComponent.java b/java/google/registry/tools/RegistryToolComponent.java index f892a3be6..606cd36b9 100644 --- a/java/google/registry/tools/RegistryToolComponent.java +++ b/java/google/registry/tools/RegistryToolComponent.java @@ -14,6 +14,7 @@ package google.registry.tools; +import dagger.BindsInstance; import dagger.Component; import google.registry.bigquery.BigqueryModule; import google.registry.config.CredentialModule.LocalCredentialJson; @@ -36,6 +37,8 @@ import google.registry.util.AppEngineServiceUtilsImpl.AppEngineServiceUtilsModul import google.registry.util.SystemClock.SystemClockModule; import google.registry.util.SystemSleeper.SystemSleeperModule; import google.registry.whois.WhoisModule; +import javax.annotation.Nullable; +import javax.inject.Named; import javax.inject.Singleton; /** @@ -113,5 +116,12 @@ interface RegistryToolComponent { @LocalCredentialJson String googleCredentialJson(); -} + @Component.Builder + interface Builder { + @BindsInstance + Builder credentialFilename(@Nullable @Named("credentialFileName") String credentialFilename); + + RegistryToolComponent build(); + } +} diff --git a/java/google/registry/tools/RequestFactoryModule.java b/java/google/registry/tools/RequestFactoryModule.java index ac555050c..f338e4f7f 100644 --- a/java/google/registry/tools/RequestFactoryModule.java +++ b/java/google/registry/tools/RequestFactoryModule.java @@ -19,7 +19,7 @@ import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.javanet.NetHttpTransport; import dagger.Module; import dagger.Provides; -import google.registry.config.CredentialModule.LocalCredential; +import google.registry.config.CredentialModule.DefaultCredential; import google.registry.config.RegistryConfig; /** @@ -35,7 +35,7 @@ class RequestFactoryModule { @Provides static HttpRequestFactory provideHttpRequestFactory( - @LocalCredential GoogleCredential credential) { + @DefaultCredential GoogleCredential credential) { if (RegistryConfig.areServersLocal()) { return new NetHttpTransport() .createRequestFactory( diff --git a/javatests/google/registry/tools/AuthModuleTest.java b/javatests/google/registry/tools/AuthModuleTest.java index b58763500..290c1a7a5 100644 --- a/javatests/google/registry/tools/AuthModuleTest.java +++ b/javatests/google/registry/tools/AuthModuleTest.java @@ -16,6 +16,7 @@ package google.registry.tools; import static com.google.common.truth.Truth.assertThat; import static google.registry.testing.JUnitBackports.assertThrows; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -32,11 +33,15 @@ import com.google.api.client.util.store.DataStore; import com.google.common.collect.ImmutableList; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import java.io.File; import java.io.IOException; import java.io.Serializable; +import java.nio.file.Files; import java.util.Map; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -49,6 +54,9 @@ public class AuthModuleTest { private static final String ACCESS_TOKEN = "FakeAccessToken"; private static final String REFRESH_TOKEN = "FakeReFreshToken"; + @Rule + public final TemporaryFolder folder = new TemporaryFolder(); + private final Credential fakeCredential = new Credential.Builder( new Credential.AccessMethod() { @@ -154,7 +162,8 @@ public class AuthModuleTest { @Test public void test_provideLocalCredentialJson() { - String credentialJson = AuthModule.provideLocalCredentialJson(getSecrets(), getCredential()); + String credentialJson = + AuthModule.provideLocalCredentialJson(this::getSecrets, this::getCredential, null); Map jsonMap = new Gson().fromJson(credentialJson, new TypeToken>() {}.getType()); assertThat(jsonMap.get("type")).isEqualTo("authorized_user"); @@ -163,6 +172,16 @@ public class AuthModuleTest { assertThat(jsonMap.get("refresh_token")).isEqualTo(REFRESH_TOKEN); } + @Test + public void test_provideExternalCredentialJson() throws Exception { + File credentialFile = folder.newFile("credential.json"); + Files.write(credentialFile.toPath(), "{some_field: some_value}".getBytes(UTF_8)); + String credentialJson = + AuthModule.provideLocalCredentialJson( + this::getSecrets, this::getCredential, credentialFile.getCanonicalPath()); + assertThat(credentialJson).isEqualTo("{some_field: some_value}"); + } + @Test public void test_provideCredential() { Credential cred = getCredential(); From 7c9b2172fd76db3b5fb7a11b1c5f50346de5bf71 Mon Sep 17 00:00:00 2001 From: guyben Date: Tue, 18 Dec 2018 14:17:27 -0800 Subject: [PATCH 105/134] Set a "nicer" margin value for textareas Currently there's a margin on the top, making the textarea be unaligned with the text naming it. This is annoying on the eye, and will be more annoying in the OT&E cl that will be added soon. - So why not just do this change in that CL? - Because the changes in the Screenshot tests here are irrelevant to that CL and I found make it harder to actually review the actual screenshots we're adding there. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226057985 --- java/google/registry/ui/css/registry.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/google/registry/ui/css/registry.css b/java/google/registry/ui/css/registry.css index ad8e4d45d..3607b22cb 100644 --- a/java/google/registry/ui/css/registry.css +++ b/java/google/registry/ui/css/registry.css @@ -48,7 +48,7 @@ input[readonly], textarea[readonly] { } textarea { - margin: 1em 0; + margin-bottom: 1em; font-family: monospace; font-size: 13px; border: solid 1px #ddd; From da5a8796b8fc295951a6afb3b30168a937c4c79f Mon Sep 17 00:00:00 2001 From: guyben Date: Wed, 19 Dec 2018 08:35:40 -0800 Subject: [PATCH 106/134] Allow XSRF to be sent as POST parameter in addition to HTML header HTML headers can only be sent via JS, we need this change to allow secure POST form submission. The form itself will have a hidden "input" tag with the XSRF token in it. This is how other framework do it as well - see https://en.wikipedia.org/wiki/Cross-site_request_forgery#Synchronizer_token_pattern This is in preparation for the OT&E setup page, which will be a simple form with a "submit" button, so using JS for it is overkill. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226178070 --- .../auth/LegacyAuthenticationMechanism.java | 28 ++- .../registry/security/XsrfTokenManager.java | 3 + .../LegacyAuthenticationMechanismTest.java | 173 ++++++++++++++++++ 3 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 javatests/google/registry/request/auth/LegacyAuthenticationMechanismTest.java diff --git a/java/google/registry/request/auth/LegacyAuthenticationMechanism.java b/java/google/registry/request/auth/LegacyAuthenticationMechanism.java index 6b1d3f50d..a33018af8 100644 --- a/java/google/registry/request/auth/LegacyAuthenticationMechanism.java +++ b/java/google/registry/request/auth/LegacyAuthenticationMechanism.java @@ -14,9 +14,11 @@ package google.registry.request.auth; +import static com.google.common.base.Strings.emptyToNull; import static com.google.common.base.Strings.nullToEmpty; import static google.registry.request.auth.AuthLevel.NONE; import static google.registry.request.auth.AuthLevel.USER; +import static google.registry.security.XsrfTokenManager.P_CSRF_TOKEN; import static google.registry.security.XsrfTokenManager.X_CSRF_TOKEN; import com.google.appengine.api.users.UserService; @@ -52,8 +54,7 @@ public class LegacyAuthenticationMechanism implements AuthenticationMechanism { return AuthResult.create(NONE); } - if (!SAFE_METHODS.contains(request.getMethod()) - && !xsrfTokenManager.validateToken(nullToEmpty(request.getHeader(X_CSRF_TOKEN)))) { + if (!SAFE_METHODS.contains(request.getMethod()) && !validateXsrf(request)) { return AuthResult.create(NONE); } @@ -61,4 +62,27 @@ public class LegacyAuthenticationMechanism implements AuthenticationMechanism { USER, UserAuthInfo.create(userService.getCurrentUser(), userService.isUserAdmin())); } + + private boolean validateXsrf(HttpServletRequest request) { + String headerToken = emptyToNull(request.getHeader(X_CSRF_TOKEN)); + if (headerToken != null) { + return xsrfTokenManager.validateToken(headerToken); + } + // If we got here - the header didn't have the token. + // It might be in the POST data - however even checking whether the POST data has this entry + // could break the Action! + // + // Reason: if we do request.getParameter, any Action that injects @Payload or @JsonPayload + // would break since it uses request.getReader - and it's an error to call both getReader and + // getParameter! + // + // However, in this case it's acceptable since if we got here - the POST request didn't even + // have the XSRF header meaning if it doesn't have POST data - it's not from a valid source at + // all (a valid but outdated source would have a bad header value, but getting here means we had + // no value at all) + // + // TODO(b/120201577): Once we know from the @Action whether we can use getParameter or not - + // only check getParameter if that's how this @Action uses getParameters. + return xsrfTokenManager.validateToken(nullToEmpty(request.getParameter(P_CSRF_TOKEN))); + } } diff --git a/java/google/registry/security/XsrfTokenManager.java b/java/google/registry/security/XsrfTokenManager.java index 5681ddeb2..21318fcab 100644 --- a/java/google/registry/security/XsrfTokenManager.java +++ b/java/google/registry/security/XsrfTokenManager.java @@ -37,6 +37,9 @@ public final class XsrfTokenManager { /** HTTP header used for transmitting XSRF tokens. */ public static final String X_CSRF_TOKEN = "X-CSRF-Token"; + /** POST parameter used for transmitting XSRF tokens. */ + public static final String P_CSRF_TOKEN = "xsrfToken"; + /** Maximum age of an acceptable XSRF token. */ private static final Duration XSRF_VALIDITY = Duration.standardDays(1); diff --git a/javatests/google/registry/request/auth/LegacyAuthenticationMechanismTest.java b/javatests/google/registry/request/auth/LegacyAuthenticationMechanismTest.java new file mode 100644 index 000000000..738bf08e3 --- /dev/null +++ b/javatests/google/registry/request/auth/LegacyAuthenticationMechanismTest.java @@ -0,0 +1,173 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.request.auth; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.appengine.api.users.User; +import com.google.appengine.api.users.UserService; +import google.registry.security.XsrfTokenManager; +import google.registry.testing.AppEngineRule; +import google.registry.testing.FakeClock; +import google.registry.testing.MockitoJUnitRule; +import javax.servlet.http.HttpServletRequest; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; + +@RunWith(JUnit4.class) +public final class LegacyAuthenticationMechanismTest { + + + @Rule + public final AppEngineRule appEngine = AppEngineRule.builder() + .withDatastore() + .build(); + + @Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create(); + + @Mock private UserService userService; + @Mock private HttpServletRequest req; + + private final FakeClock clock = new FakeClock(); + private XsrfTokenManager xsrfTokenManager; + private LegacyAuthenticationMechanism legacyAuthenticationMechanism; + private String goodToken; + + @Before + public void setUp() { + xsrfTokenManager = new XsrfTokenManager(clock, userService); + legacyAuthenticationMechanism = + new LegacyAuthenticationMechanism(userService, xsrfTokenManager); + when(userService.getCurrentUser()).thenReturn(new User("email@example.com", "example.com")); + when(userService.isUserAdmin()).thenReturn(false); + goodToken = xsrfTokenManager.generateToken("email@example.com"); + } + + @After + public void tearDown() { + // Make sure we didn't use getParameter or getInputStream or any of the other "with side + // effects" getters unexpectedly. But allow "no side effect" getters. + // + // Unfortunately HttpServletRequest doesn't document well which getters "have side effects". It + // does explicitly state getReader and getInputStream, and that getParameter can also interfere + // with them, but it doesn't say anything about getParameterNames, getParameterValues, + // getParameterMap - even though I'm pretty sure they are similar to getParameter in that + // effect. + // + // Feel free to add other "no side effect" functions with atLeast(0) to exempt them from the + // verifyNoMoreInteractions + verify(req, atLeast(0)).getMethod(); + verify(req, atLeast(0)).getHeader(any()); + verifyNoMoreInteractions(req); + } + + @Test + public void testAuthenticate_notLoggedIn() { + when(userService.isUserLoggedIn()).thenReturn(false); + assertThat(legacyAuthenticationMechanism.authenticate(req).authLevel()) + .isEqualTo(AuthLevel.NONE); + } + + @Test + public void testAuthenticate_loggedInSafeMethod_get() { + when(userService.isUserLoggedIn()).thenReturn(true); + when(req.getMethod()).thenReturn("GET"); + assertThat(legacyAuthenticationMechanism.authenticate(req).authLevel()) + .isEqualTo(AuthLevel.USER); + } + + @Test + public void testAuthenticate_loggedInSafeMethod_head() { + when(userService.isUserLoggedIn()).thenReturn(true); + when(req.getMethod()).thenReturn("HEAD"); + assertThat(legacyAuthenticationMechanism.authenticate(req).authLevel()) + .isEqualTo(AuthLevel.USER); + } + + @Test + public void testAuthenticate_loggedInUnsafeMethod_post_noXsrfToken() { + when(userService.isUserLoggedIn()).thenReturn(true); + when(req.getMethod()).thenReturn("POST"); + assertThat(legacyAuthenticationMechanism.authenticate(req).authLevel()) + .isEqualTo(AuthLevel.NONE); + + // Make sure we looked for the token in all relevant places before giving up + verify(req).getHeader("X-CSRF-Token"); + verify(req).getParameter("xsrfToken"); + } + + @Test + public void testAuthenticate_loggedInUnsafeMethod_post_goodTokenInHeader() { + when(userService.isUserLoggedIn()).thenReturn(true); + when(req.getMethod()).thenReturn("POST"); + when(req.getHeader("X-CSRF-Token")).thenReturn(goodToken); + assertThat(legacyAuthenticationMechanism.authenticate(req).authLevel()) + .isEqualTo(AuthLevel.USER); + + // Make sure we didn't call getParameter (we already verify it in the @After, but we're doing it + // here explicitly as well for clarity, since this is important in this test) + verify(req, times(0)).getParameter(any()); + } + + @Test + public void testAuthenticate_loggedInUnsafeMethod_post_badTokenInHeader() { + when(userService.isUserLoggedIn()).thenReturn(true); + when(req.getMethod()).thenReturn("POST"); + when(req.getHeader("X-CSRF-Token")).thenReturn("bad"); + assertThat(legacyAuthenticationMechanism.authenticate(req).authLevel()) + .isEqualTo(AuthLevel.NONE); + + // Make sure we didn't call getParameter (we already verify it in the @After, but we're doing it + // here explicitly as well for clarity, since this is important in this test) + verify(req, times(0)).getParameter(any()); + } + + @Test + public void testAuthenticate_loggedInUnsafeMethod_post_goodTokenInParam() { + when(userService.isUserLoggedIn()).thenReturn(true); + when(req.getMethod()).thenReturn("POST"); + when(req.getParameter("xsrfToken")).thenReturn(goodToken); + assertThat(legacyAuthenticationMechanism.authenticate(req).authLevel()) + .isEqualTo(AuthLevel.USER); + + // we allow getParameter to be called in this case (we verify it so it's not caught in the + // @After's verifyNoMoreInteractions) + verify(req).getParameter("xsrfToken"); + } + + @Test + public void testAuthenticate_loggedInUnsafeMethod_post_badTokenInParam() { + when(userService.isUserLoggedIn()).thenReturn(true); + when(req.getMethod()).thenReturn("POST"); + when(req.getParameter("xsrfToken")).thenReturn("bad"); + assertThat(legacyAuthenticationMechanism.authenticate(req).authLevel()) + .isEqualTo(AuthLevel.NONE); + + // we allow getParameter to be called in this case (we verify it so it's not caught in the + // @After's verifyNoMoreInteractions) + verify(req).getParameter("xsrfToken"); + } +} From 4a4989e2a5325e3284879f75f1b934f519c84cc6 Mon Sep 17 00:00:00 2001 From: guyben Date: Wed, 19 Dec 2018 08:44:17 -0800 Subject: [PATCH 107/134] Import the Servlets instead of using full path ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226179113 --- .../registry/server/RegistryTestServer.java | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/javatests/google/registry/server/RegistryTestServer.java b/javatests/google/registry/server/RegistryTestServer.java index 078bfd009..3835ed235 100644 --- a/javatests/google/registry/server/RegistryTestServer.java +++ b/javatests/google/registry/server/RegistryTestServer.java @@ -21,6 +21,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.net.HostAndPort; import com.googlecode.objectify.ObjectifyFilter; import google.registry.model.ofy.OfyFilter; +import google.registry.module.backend.BackendServlet; +import google.registry.module.frontend.FrontendServlet; import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; @@ -53,41 +55,37 @@ public final class RegistryTestServer { private static final ImmutableList ROUTES = ImmutableList.of( // Frontend Services - route("/whois/*", google.registry.module.frontend.FrontendServlet.class), - route("/rdap/*", google.registry.module.frontend.FrontendServlet.class), - route("/registrar-xhr", google.registry.module.frontend.FrontendServlet.class), - route("/check", google.registry.module.frontend.FrontendServlet.class), + route("/whois/*", FrontendServlet.class), + route("/rdap/*", FrontendServlet.class), + route("/registrar-xhr", FrontendServlet.class), + route("/check", FrontendServlet.class), // Proxy Services - route("/_dr/epp", google.registry.module.frontend.FrontendServlet.class), - route("/_dr/whois", google.registry.module.frontend.FrontendServlet.class), + route("/_dr/epp", FrontendServlet.class), + route("/_dr/whois", FrontendServlet.class), // Registry Data Escrow (RDE) - route("/_dr/cron/rdeCreate", google.registry.module.backend.BackendServlet.class), - route("/_dr/task/rdeStaging", google.registry.module.backend.BackendServlet.class), - route("/_dr/task/rdeUpload", google.registry.module.backend.BackendServlet.class), - route("/_dr/task/rdeReport", google.registry.module.backend.BackendServlet.class), - route("/_dr/task/brdaCopy", google.registry.module.backend.BackendServlet.class), + route("/_dr/cron/rdeCreate", BackendServlet.class), + route("/_dr/task/rdeStaging", BackendServlet.class), + route("/_dr/task/rdeUpload", BackendServlet.class), + route("/_dr/task/rdeReport", BackendServlet.class), + route("/_dr/task/brdaCopy", BackendServlet.class), // Trademark Clearinghouse (TMCH) - route("/_dr/cron/tmchDnl", google.registry.module.backend.BackendServlet.class), - route("/_dr/task/tmchSmdrl", google.registry.module.backend.BackendServlet.class), - route("/_dr/task/tmchCrl", google.registry.module.backend.BackendServlet.class), + route("/_dr/cron/tmchDnl", BackendServlet.class), + route("/_dr/task/tmchSmdrl", BackendServlet.class), + route("/_dr/task/tmchCrl", BackendServlet.class), // Notification of Registered Domain Names (NORDN) - route("/_dr/task/nordnUpload", - google.registry.module.backend.BackendServlet.class), - route("/_dr/task/nordnVerify", - google.registry.module.backend.BackendServlet.class), + route("/_dr/task/nordnUpload", BackendServlet.class), + route("/_dr/task/nordnVerify", BackendServlet.class), // Process DNS pull queue - route("/_dr/cron/readDnsQueue", - google.registry.module.backend.BackendServlet.class), + route("/_dr/cron/readDnsQueue", BackendServlet.class), // Registrar Console - route("/registrar", google.registry.module.frontend.FrontendServlet.class), - route("/registrar-settings", - google.registry.module.frontend.FrontendServlet.class)); + route("/registrar", FrontendServlet.class), + route("/registrar-settings", FrontendServlet.class)); private static final ImmutableList> FILTERS = ImmutableList.of( ObjectifyFilter.class, From 56b61ad5a2089e44c634081212b58d26f9a7562b Mon Sep 17 00:00:00 2001 From: mcilwain Date: Wed, 19 Dec 2018 08:52:23 -0800 Subject: [PATCH 108/134] Invalidate premium list cache on update This will only affect the tools service, the primary use case being (1) I go to create a domain through nomulus tool, realize it's premium, (2) update the premium list to not include that domain, (3) kill the tools service instance to wipe out the cached premium value, then (4) create the domain at standard. This commit eliminates step 3. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226180160 --- .../model/registry/label/PremiumList.java | 26 ++++++++++++------- .../registry/label/PremiumListUtils.java | 6 +++++ .../registry/label/PremiumListUtilsTest.java | 19 +++++++++++--- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/java/google/registry/model/registry/label/PremiumList.java b/java/google/registry/model/registry/label/PremiumList.java index c4eb49d52..40b9b62b0 100644 --- a/java/google/registry/model/registry/label/PremiumList.java +++ b/java/google/registry/model/registry/label/PremiumList.java @@ -132,16 +132,22 @@ public final class PremiumList extends BaseDomainLabelListThis is cached for a shorter duration because we need to periodically reload this entity to * check if a new revision has been published, and if so, then use that. */ - static final LoadingCache cachePremiumLists = - CacheBuilder.newBuilder() - .expireAfterWrite(getDomainLabelListCacheDuration().getMillis(), MILLISECONDS) - .build( - new CacheLoader() { - @Override - public PremiumList load(final String name) { - return ofy().doTransactionless(() -> loadPremiumList(name)); - } - }); + @NonFinalForTesting + static LoadingCache cachePremiumLists = + createCachePremiumLists(getDomainLabelListCacheDuration()); + + @VisibleForTesting + static LoadingCache createCachePremiumLists(Duration cachePersistDuration) { + return CacheBuilder.newBuilder() + .expireAfterWrite(cachePersistDuration.getMillis(), MILLISECONDS) + .build( + new CacheLoader() { + @Override + public PremiumList load(final String name) { + return ofy().doTransactionless(() -> loadPremiumList(name)); + } + }); + } private static PremiumList loadPremiumList(String name) { return ofy().load().type(PremiumList.class).parent(getCrossTldKey()).id(name).now(); diff --git a/java/google/registry/model/registry/label/PremiumListUtils.java b/java/google/registry/model/registry/label/PremiumListUtils.java index e9c1515d7..d8df27851 100644 --- a/java/google/registry/model/registry/label/PremiumListUtils.java +++ b/java/google/registry/model/registry/label/PremiumListUtils.java @@ -174,6 +174,12 @@ public final class PremiumListUtils { ofy().save().entities(newList, newRevision); return newList; }); + + // Invalidate the cache on this premium list so the change will take effect instantly. This only + // clears the cache on the same instance that the update was run on, which will typically be the + // only tools instance. + PremiumList.cachePremiumLists.invalidate(premiumList.getName()); + // TODO(b/79888775): Enqueue the oldPremiumList for deletion after at least // RegistryConfig.getDomainLabelListCacheDuration() has elapsed. return updated; diff --git a/javatests/google/registry/model/registry/label/PremiumListUtilsTest.java b/javatests/google/registry/model/registry/label/PremiumListUtilsTest.java index 0feb5e9b4..59c2fc9aa 100644 --- a/javatests/google/registry/model/registry/label/PremiumListUtilsTest.java +++ b/javatests/google/registry/model/registry/label/PremiumListUtilsTest.java @@ -26,6 +26,7 @@ import static google.registry.model.registry.label.DomainLabelMetrics.PremiumLis import static google.registry.model.registry.label.DomainLabelMetrics.PremiumListCheckOutcome.UNCACHED_POSITIVE; import static google.registry.model.registry.label.DomainLabelMetrics.premiumListChecks; import static google.registry.model.registry.label.DomainLabelMetrics.premiumListProcessingTime; +import static google.registry.model.registry.label.PremiumList.createCachePremiumLists; import static google.registry.model.registry.label.PremiumListUtils.deletePremiumList; import static google.registry.model.registry.label.PremiumListUtils.doesPremiumListExist; import static google.registry.model.registry.label.PremiumListUtils.getPremiumPrice; @@ -35,7 +36,7 @@ import static google.registry.testing.DatastoreHelper.loadPremiumListEntries; import static google.registry.testing.DatastoreHelper.persistPremiumList; import static google.registry.testing.DatastoreHelper.persistResource; import static google.registry.testing.JUnitBackports.assertThrows; -import static org.joda.time.Duration.standardMinutes; +import static org.joda.time.Duration.standardDays; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -63,9 +64,11 @@ public class PremiumListUtilsTest { @Before public void before() { - // createTld() overwrites the premium list, so call it first. + // Set long persist times on caches so they can be tested (cache times default to 0 in tests). PremiumList.cachePremiumListEntries = - PremiumList.createCachePremiumListEntries(standardMinutes(1)); + PremiumList.createCachePremiumListEntries(standardDays(1)); + PremiumList.cachePremiumLists = createCachePremiumLists(standardDays(1)); + // createTld() overwrites the premium list, so call it first. createTld("tld"); PremiumList pl = persistPremiumList( @@ -309,6 +312,16 @@ public class PremiumListUtilsTest { assertThat(entriesReloaded.get("test").parent).isEqualTo(resaved.getRevisionKey()); } + @Test + public void test_savePremiumListAndEntries_clearsCache() { + assertThat(PremiumList.cachePremiumLists.getIfPresent("tld")).isNull(); + PremiumList pl = PremiumList.getCached("tld").get(); + assertThat(PremiumList.cachePremiumLists.getIfPresent("tld")).isEqualTo(pl); + savePremiumListAndEntries( + new PremiumList.Builder().setName("tld").build(), ImmutableList.of("test,USD 1")); + assertThat(PremiumList.cachePremiumLists.getIfPresent("tld")).isNull(); + } + @Test public void testDelete() { persistPremiumList("gtld1", "trombone,USD 10"); From 51f22a15ed2fffb26187b10973ec4c245dd6a535 Mon Sep 17 00:00:00 2001 From: guyben Date: Wed, 19 Dec 2018 09:42:22 -0800 Subject: [PATCH 109/134] Move SendEmailUtils to the /ui/server directory SendEmailUtils is a general utility of the web console, and not specifically "only" to the Registrar console. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226187094 --- .../registry/config/RegistryConfig.java | 6 +-- java/google/registry/ui/server/BUILD | 4 ++ .../{registrar => }/SendEmailUtils.java | 40 +++++++++++++----- .../registrar/RegistrarSettingsAction.java | 7 +--- javatests/google/registry/ui/server/BUILD | 3 ++ .../{registrar => }/SendEmailUtilsTest.java | 41 ++++++++++++++----- .../google/registry/ui/server/registrar/BUILD | 1 + .../RegistrarSettingsActionTestCase.java | 8 ++-- 8 files changed, 78 insertions(+), 32 deletions(-) rename java/google/registry/ui/server/{registrar => }/SendEmailUtils.java (65%) rename javatests/google/registry/ui/server/{registrar => }/SendEmailUtilsTest.java (77%) diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index b153eff2c..9cb2f77f2 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -509,7 +509,7 @@ public final class RegistryConfig { /** * The email address that outgoing emails from the app are sent from. * - * @see google.registry.ui.server.registrar.SendEmailUtils + * @see google.registry.ui.server.SendEmailUtils */ @Provides @Config("gSuiteOutgoingEmailAddress") @@ -520,10 +520,10 @@ public final class RegistryConfig { /** * The display name that is used on outgoing emails sent by Nomulus. * - * @see google.registry.ui.server.registrar.SendEmailUtils + * @see google.registry.ui.server.SendEmailUtils */ @Provides - @Config("gSuiteOutoingEmailDisplayName") + @Config("gSuiteOutgoingEmailDisplayName") public static String provideGSuiteOutgoingEmailDisplayName(RegistryConfigSettings config) { return config.gSuite.outgoingEmailDisplayName; } diff --git a/java/google/registry/ui/server/BUILD b/java/google/registry/ui/server/BUILD index af1bc6881..e05f76536 100644 --- a/java/google/registry/ui/server/BUILD +++ b/java/google/registry/ui/server/BUILD @@ -13,12 +13,16 @@ java_library( "//java/google/registry/ui/css:registrar_dbg.css.js", ], deps = [ + "//java/google/registry/config", "//java/google/registry/model", "//java/google/registry/ui", "//java/google/registry/ui/forms", "//java/google/registry/util", "@com_google_appengine_api_1_0_sdk", "@com_google_code_findbugs_jsr305", + "@com_google_dagger", + "@com_google_flogger", + "@com_google_flogger_system_backend", "@com_google_guava", "@com_google_re2j", "@io_bazel_rules_closure//closure/templates", diff --git a/java/google/registry/ui/server/registrar/SendEmailUtils.java b/java/google/registry/ui/server/SendEmailUtils.java similarity index 65% rename from java/google/registry/ui/server/registrar/SendEmailUtils.java rename to java/google/registry/ui/server/SendEmailUtils.java index 90fcc5469..278ab0f4f 100644 --- a/java/google/registry/ui/server/registrar/SendEmailUtils.java +++ b/java/google/registry/ui/server/SendEmailUtils.java @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package google.registry.ui.server.registrar; +package google.registry.ui.server; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.toArray; import com.google.common.base.Joiner; -import com.google.common.collect.Streams; +import com.google.common.collect.ImmutableList; import com.google.common.flogger.FluentLogger; import google.registry.config.RegistryConfig.Config; import google.registry.util.SendEmailService; @@ -37,30 +37,40 @@ public class SendEmailUtils { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private final String gSuiteOutgoingEmailAddress; - private final String gSuiteOutoingEmailDisplayName; + private final String gSuiteOutgoingEmailDisplayName; private final SendEmailService emailService; + private final ImmutableList registrarChangesNotificationEmailAddresses; @Inject public SendEmailUtils( @Config("gSuiteOutgoingEmailAddress") String gSuiteOutgoingEmailAddress, - @Config("gSuiteOutoingEmailDisplayName") String gSuiteOutoingEmailDisplayName, + @Config("gSuiteOutgoingEmailDisplayName") String gSuiteOutgoingEmailDisplayName, + @Config("registrarChangesNotificationEmailAddresses") + ImmutableList registrarChangesNotificationEmailAddresses, SendEmailService emailService) { this.gSuiteOutgoingEmailAddress = gSuiteOutgoingEmailAddress; - this.gSuiteOutoingEmailDisplayName = gSuiteOutoingEmailDisplayName; + this.gSuiteOutgoingEmailDisplayName = gSuiteOutgoingEmailDisplayName; this.emailService = emailService; + this.registrarChangesNotificationEmailAddresses = registrarChangesNotificationEmailAddresses; } /** - * Sends an email from Nomulus to the specified recipient(s). Returns true iff sending was - * successful. + * Sends an email from Nomulus to the registrarChangesNotificationEmailAddresses. Returns true iff + * sending to at least 1 address was successful. + * + *

This means that if there are no recepients ({@link #hasRecepients} returns false), this will + * return false even thought no error happened. + * + *

This also means that if there are multiple recepients, it will return true even if some (but + * not all) of the recepients had an error. */ - public boolean sendEmail(Iterable addresses, final String subject, String body) { + public boolean sendEmail(final String subject, String body) { try { Message msg = emailService.createMessage(); msg.setFrom( - new InternetAddress(gSuiteOutgoingEmailAddress, gSuiteOutoingEmailDisplayName)); + new InternetAddress(gSuiteOutgoingEmailAddress, gSuiteOutgoingEmailDisplayName)); List emails = - Streams.stream(addresses) + registrarChangesNotificationEmailAddresses.stream() .map( emailAddress -> { try { @@ -85,9 +95,17 @@ public class SendEmailUtils { } catch (Throwable t) { logger.atSevere().withCause(t).log( "Could not email to addresses %s with subject '%s'.", - Joiner.on(", ").join(addresses), subject); + Joiner.on(", ").join(registrarChangesNotificationEmailAddresses), subject); return false; } return true; } + + /** + * Returns whether there are any recepients set up. {@link #sendEmail} will always return false if + * there are no recepients. + */ + public boolean hasRecepients() { + return !registrarChangesNotificationEmailAddresses.isEmpty(); + } } diff --git a/java/google/registry/ui/server/registrar/RegistrarSettingsAction.java b/java/google/registry/ui/server/registrar/RegistrarSettingsAction.java index 7da38302b..1d3f9eaa4 100644 --- a/java/google/registry/ui/server/registrar/RegistrarSettingsAction.java +++ b/java/google/registry/ui/server/registrar/RegistrarSettingsAction.java @@ -33,7 +33,6 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import com.google.common.collect.Streams; import com.google.common.flogger.FluentLogger; -import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryEnvironment; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.RegistrarContact; @@ -51,6 +50,7 @@ import google.registry.security.JsonResponseHelper; import google.registry.ui.forms.FormException; import google.registry.ui.forms.FormFieldException; import google.registry.ui.server.RegistrarFormFields; +import google.registry.ui.server.SendEmailUtils; import google.registry.util.AppEngineServiceUtils; import google.registry.util.CollectionUtils; import google.registry.util.DiffUtils; @@ -92,8 +92,6 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA @Inject AuthResult authResult; @Inject RegistryEnvironment registryEnvironment; - @Inject @Config("registrarChangesNotificationEmailAddresses") ImmutableList - registrarChangesNotificationEmailAddresses; @Inject RegistrarSettingsAction() {} private static final Predicate HAS_PHONE = @@ -478,7 +476,7 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA ImmutableSet existingContacts, Registrar updatedRegistrar, ImmutableSet updatedContacts) { - if (registrarChangesNotificationEmailAddresses.isEmpty()) { + if (!sendEmailUtils.hasRecepients()) { return; } @@ -495,7 +493,6 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA enqueueRegistrarSheetSync(appEngineServiceUtils.getCurrentVersionHostname("backend")); String environment = Ascii.toLowerCase(String.valueOf(registryEnvironment)); sendEmailUtils.sendEmail( - registrarChangesNotificationEmailAddresses, String.format( "Registrar %s (%s) updated in %s", existingRegistrar.getRegistrarName(), diff --git a/javatests/google/registry/ui/server/BUILD b/javatests/google/registry/ui/server/BUILD index a9779cfb7..e5f1ea307 100644 --- a/javatests/google/registry/ui/server/BUILD +++ b/javatests/google/registry/ui/server/BUILD @@ -13,12 +13,15 @@ java_library( deps = [ "//java/google/registry/ui/forms", "//java/google/registry/ui/server", + "//java/google/registry/util", "//javatests/google/registry/testing", + "@com_google_appengine_api_1_0_sdk", "@com_google_guava", "@com_google_truth", "@com_google_truth_extensions_truth_java8_extension", "@junit", "@org_hamcrest_library", + "@org_mockito_all", ], ) diff --git a/javatests/google/registry/ui/server/registrar/SendEmailUtilsTest.java b/javatests/google/registry/ui/server/SendEmailUtilsTest.java similarity index 77% rename from javatests/google/registry/ui/server/registrar/SendEmailUtilsTest.java rename to javatests/google/registry/ui/server/SendEmailUtilsTest.java index 119554421..6ad50e5a1 100644 --- a/javatests/google/registry/ui/server/registrar/SendEmailUtilsTest.java +++ b/javatests/google/registry/ui/server/SendEmailUtilsTest.java @@ -12,11 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package google.registry.ui.server.registrar; +package google.registry.ui.server; import static com.google.common.truth.Truth.assertThat; -import static google.registry.config.RegistryConfig.getGSuiteOutgoingEmailAddress; -import static google.registry.config.RegistryConfig.getGSuiteOutgoingEmailDisplayName; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -51,16 +49,23 @@ public class SendEmailUtilsTest { public void init() { message = new MimeMessage(Session.getDefaultInstance(new Properties(), null)); when(emailService.createMessage()).thenReturn(message); + } + + private void setRecepients(ImmutableList recepients) { sendEmailUtils = new SendEmailUtils( - getGSuiteOutgoingEmailAddress(), getGSuiteOutgoingEmailDisplayName(), emailService); + "outgoing@registry.example", + "outgoing display name", + recepients, + emailService); } @Test public void testSuccess_sendToOneAddress() throws Exception { + setRecepients(ImmutableList.of("johnny@fakesite.tld")); + assertThat(sendEmailUtils.hasRecepients()).isTrue(); assertThat( sendEmailUtils.sendEmail( - ImmutableList.of("johnny@fakesite.tld"), "Welcome to the Internet", "It is a dark and scary place.")) .isTrue(); @@ -73,9 +78,10 @@ public class SendEmailUtilsTest { @Test public void testSuccess_sendToMultipleAddresses() throws Exception { + setRecepients(ImmutableList.of("foo@example.com", "bar@example.com")); + assertThat(sendEmailUtils.hasRecepients()).isTrue(); assertThat( sendEmailUtils.sendEmail( - ImmutableList.of("foo@example.com", "bar@example.com"), "Welcome to the Internet", "It is a dark and scary place.")) .isTrue(); @@ -87,9 +93,10 @@ public class SendEmailUtilsTest { @Test public void testSuccess_ignoresMalformedEmailAddress() throws Exception { + setRecepients(ImmutableList.of("foo@example.com", "1iñvalidemail")); + assertThat(sendEmailUtils.hasRecepients()).isTrue(); assertThat( sendEmailUtils.sendEmail( - ImmutableList.of("foo@example.com", "1iñvalidemail"), "Welcome to the Internet", "It is a dark and scary place.")) .isTrue(); @@ -99,10 +106,23 @@ public class SendEmailUtilsTest { } @Test - public void testFailure_onlyGivenMalformedAddress() throws Exception { + public void testFailure_noAddresses() throws Exception { + setRecepients(ImmutableList.of()); + assertThat(sendEmailUtils.hasRecepients()).isFalse(); + assertThat( + sendEmailUtils.sendEmail( + "Welcome to the Internet", + "It is a dark and scary place.")) + .isFalse(); + verify(emailService, never()).sendMessage(any(Message.class)); + } + + @Test + public void testFailure_onlyGivenMalformedAddress() throws Exception { + setRecepients(ImmutableList.of("1iñvalidemail")); + assertThat(sendEmailUtils.hasRecepients()).isTrue(); assertThat( sendEmailUtils.sendEmail( - ImmutableList.of("1iñvalidemail"), "Welcome to the Internet", "It is a dark and scary place.")) .isFalse(); @@ -111,10 +131,11 @@ public class SendEmailUtilsTest { @Test public void testFailure_exceptionThrownDuringSend() throws Exception { + setRecepients(ImmutableList.of("foo@example.com")); + assertThat(sendEmailUtils.hasRecepients()).isTrue(); doThrow(new MessagingException()).when(emailService).sendMessage(any(Message.class)); assertThat( sendEmailUtils.sendEmail( - ImmutableList.of("foo@example.com"), "Welcome to the Internet", "It is a dark and scary place.")) .isFalse(); diff --git a/javatests/google/registry/ui/server/registrar/BUILD b/javatests/google/registry/ui/server/registrar/BUILD index 7e2aa43eb..fd60ec5c6 100644 --- a/javatests/google/registry/ui/server/registrar/BUILD +++ b/javatests/google/registry/ui/server/registrar/BUILD @@ -19,6 +19,7 @@ java_library( "//java/google/registry/request", "//java/google/registry/request/auth", "//java/google/registry/security", + "//java/google/registry/ui/server", "//java/google/registry/ui/server/registrar", "//java/google/registry/util", "//javatests/google/registry/security", diff --git a/javatests/google/registry/ui/server/registrar/RegistrarSettingsActionTestCase.java b/javatests/google/registry/ui/server/registrar/RegistrarSettingsActionTestCase.java index 6764d04aa..0740ebc17 100644 --- a/javatests/google/registry/ui/server/registrar/RegistrarSettingsActionTestCase.java +++ b/javatests/google/registry/ui/server/registrar/RegistrarSettingsActionTestCase.java @@ -41,6 +41,7 @@ import google.registry.testing.AppEngineRule; import google.registry.testing.FakeClock; import google.registry.testing.InjectRule; import google.registry.testing.MockitoJUnitRule; +import google.registry.ui.server.SendEmailUtils; import google.registry.util.AppEngineServiceUtils; import google.registry.util.SendEmailService; import java.io.PrintWriter; @@ -97,11 +98,12 @@ public class RegistrarSettingsActionTestCase { when(appEngineServiceUtils.getCurrentVersionHostname("backend")).thenReturn("backend.hostname"); action.jsonActionRunner = new JsonActionRunner( ImmutableMap.of(), new JsonResponse(new ResponseImpl(rsp))); - action.registrarChangesNotificationEmailAddresses = ImmutableList.of( - "notification@test.example", "notification2@test.example"); action.sendEmailUtils = new SendEmailUtils( - getGSuiteOutgoingEmailAddress(), getGSuiteOutgoingEmailDisplayName(), emailService); + getGSuiteOutgoingEmailAddress(), + getGSuiteOutgoingEmailDisplayName(), + ImmutableList.of("notification@test.example", "notification2@test.example"), + emailService); action.registryEnvironment = RegistryEnvironment.get(); action.registrarConsoleMetrics = new RegistrarConsoleMetrics(); action.authResult = From 1975218f45f8acc4f62d2a0943d548a623f93092 Mon Sep 17 00:00:00 2001 From: guyben Date: Wed, 19 Dec 2018 09:43:26 -0800 Subject: [PATCH 110/134] Mark nullable parameters as nullable A few nullable parameters were not marked as nullable, which causes exceptions to be thrown in debug mode. This had no effect in the deployed web server, because these assert sanity checks aren't performed - but on our local test server this failed. Note that all these fields are checked for "nullness" in the code itself. It's just an oversight in the declaration. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226187227 --- java/google/registry/ui/soy/registrar/SecuritySettings.soy | 2 +- java/google/registry/ui/soy/registrar/WhoisSettings.soy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/java/google/registry/ui/soy/registrar/SecuritySettings.soy b/java/google/registry/ui/soy/registrar/SecuritySettings.soy index bbaa873fa..8923ccf3c 100644 --- a/java/google/registry/ui/soy/registrar/SecuritySettings.soy +++ b/java/google/registry/ui/soy/registrar/SecuritySettings.soy @@ -18,7 +18,7 @@ /** Registrar security settings page for view and edit. */ {template .settings} {@param ipAddressWhitelist: list} - {@param phonePasscode: string} + {@param? phonePasscode: string} {@param? clientCertificate: string} {@param? clientCertificateHash: string} {@param? failoverClientCertificate: string} diff --git a/java/google/registry/ui/soy/registrar/WhoisSettings.soy b/java/google/registry/ui/soy/registrar/WhoisSettings.soy index 3fed68098..1c5a1fec9 100644 --- a/java/google/registry/ui/soy/registrar/WhoisSettings.soy +++ b/java/google/registry/ui/soy/registrar/WhoisSettings.soy @@ -19,8 +19,8 @@ /** Registrar whois settings page for view and edit. */ {template .settings} {@param clientIdentifier: string} - {@param ianaIdentifier: int} - {@param icannReferralEmail: string} + {@param? ianaIdentifier: int} + {@param? icannReferralEmail: string} {@param readonly: bool} {@param? whoisServer: string} {@param? url: string} From 2cc4a9fb2f160ad239f8882a463cec9689ad6f32 Mon Sep 17 00:00:00 2001 From: mmuller Date: Thu, 13 Dec 2018 12:43:27 -0800 Subject: [PATCH 111/134] Add MOE equivalence[s] for 2018-12-13 sync ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=225418342 --- java/google/registry/config/BUILD | 1 + java/google/registry/tools/BUILD | 1 + 2 files changed, 2 insertions(+) diff --git a/java/google/registry/config/BUILD b/java/google/registry/config/BUILD index c0c22deac..fd8c1870f 100644 --- a/java/google/registry/config/BUILD +++ b/java/google/registry/config/BUILD @@ -9,6 +9,7 @@ java_library( srcs = glob(["*.java"]), resources = glob(["files/*.yaml"]), deps = [ + "//java/com/google/api/client/extensions/appengine/auth/oauth2", "//java/google/registry/keyring/api", "//java/google/registry/util", "@com_google_api_client", diff --git a/java/google/registry/tools/BUILD b/java/google/registry/tools/BUILD index b1f1f9d45..3a7653cb0 100644 --- a/java/google/registry/tools/BUILD +++ b/java/google/registry/tools/BUILD @@ -35,6 +35,7 @@ java_library( "//java/google/registry/module/tools", ], deps = [ + "//java/com/google/api/client/extensions/appengine/auth/oauth2", "//java/google/registry/backup", "//java/google/registry/beam/invoicing", "//java/google/registry/beam/spec11", From 7ade0f0adbb1997ea303ddec58ea3c88638bef06 Mon Sep 17 00:00:00 2001 From: guyben Date: Wed, 19 Dec 2018 14:26:35 -0800 Subject: [PATCH 112/134] Fix the monospace font so the screendiff tests work correctly see b/34094769 for context The webdriver tests don't choose a correct font when we specify "monospace". As a result, we don't render correctly pages that use monospace. Here we instead explicitly reference a monospace font we know exists in the webdriver: Courier New. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226233831 --- java/google/registry/ui/css/registry.css | 4 ++-- .../registry/ui/soy/registrar/SecuritySettings.soy | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/google/registry/ui/css/registry.css b/java/google/registry/ui/css/registry.css index 3607b22cb..51d668adc 100644 --- a/java/google/registry/ui/css/registry.css +++ b/java/google/registry/ui/css/registry.css @@ -22,7 +22,7 @@ h1 { } pre { - font-family: monospace; + font-family: "Courier New", Courier, monospace; white-space: pre-wrap; } @@ -49,7 +49,7 @@ input[readonly], textarea[readonly] { textarea { margin-bottom: 1em; - font-family: monospace; + font-family: "Courier New", Courier, monospace; font-size: 13px; border: solid 1px #ddd; } diff --git a/java/google/registry/ui/soy/registrar/SecuritySettings.soy b/java/google/registry/ui/soy/registrar/SecuritySettings.soy index 8923ccf3c..da798bf62 100644 --- a/java/google/registry/ui/soy/registrar/SecuritySettings.soy +++ b/java/google/registry/ui/soy/registrar/SecuritySettings.soy @@ -86,7 +86,7 @@ {elseif isNonnull($clientCertificateHash)} {if $readonly} @@ -100,7 +100,7 @@ Existing certificate SHA256 base64 hash: {$clientCertificateHash} {else} - + {/if} {else} {if $readonly} @@ -109,10 +109,10 @@

No certificate on file. Please enter your certificate below. + rows="16" cols="68">

Example format: - From a153f8ec771ba62e5c221969cdb8b8a9cabe4f0b Mon Sep 17 00:00:00 2001 From: emcmanus Date: Thu, 20 Dec 2018 08:07:58 -0800 Subject: [PATCH 113/134] Switch remaining App Engine dependencies to built-at-head targets Most dependencies on the Old SDK were switched in [] This is just catching up on some OAuth dependencies that remained and some remaining uses of Old build rules. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226337284 --- java/google/registry/config/BUILD | 1 - java/google/registry/tools/BUILD | 1 - 2 files changed, 2 deletions(-) diff --git a/java/google/registry/config/BUILD b/java/google/registry/config/BUILD index fd8c1870f..c0c22deac 100644 --- a/java/google/registry/config/BUILD +++ b/java/google/registry/config/BUILD @@ -9,7 +9,6 @@ java_library( srcs = glob(["*.java"]), resources = glob(["files/*.yaml"]), deps = [ - "//java/com/google/api/client/extensions/appengine/auth/oauth2", "//java/google/registry/keyring/api", "//java/google/registry/util", "@com_google_api_client", diff --git a/java/google/registry/tools/BUILD b/java/google/registry/tools/BUILD index 3a7653cb0..b1f1f9d45 100644 --- a/java/google/registry/tools/BUILD +++ b/java/google/registry/tools/BUILD @@ -35,7 +35,6 @@ java_library( "//java/google/registry/module/tools", ], deps = [ - "//java/com/google/api/client/extensions/appengine/auth/oauth2", "//java/google/registry/backup", "//java/google/registry/beam/invoicing", "//java/google/registry/beam/spec11", From f83e96c448cfea6659e1733c8a6a6851bda3a5e1 Mon Sep 17 00:00:00 2001 From: shicong Date: Thu, 20 Dec 2018 14:04:41 -0800 Subject: [PATCH 114/134] Exit with error code when user is not logged in ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226390329 --- java/google/registry/tools/RegistryCli.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/google/registry/tools/RegistryCli.java b/java/google/registry/tools/RegistryCli.java index 7baa09e1e..2d02cb9bf 100644 --- a/java/google/registry/tools/RegistryCli.java +++ b/java/google/registry/tools/RegistryCli.java @@ -170,6 +170,7 @@ final class RegistryCli implements AutoCloseable, CommandRunner { System.err.println("==================================================================="); System.err.println("You must login using 'nomulus login' prior to running this command."); System.err.println("==================================================================="); + System.exit(1); } else { throw ex; } From 9e155f14c0f667a43bdce96968c83b35c0a63754 Mon Sep 17 00:00:00 2001 From: jianglai Date: Fri, 21 Dec 2018 08:31:01 -0800 Subject: [PATCH 115/134] Add support to bazel 0.21.0 Also fixes the build. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226493872 --- WORKSPACE | 1 + docs/install.md | 6 +++--- java/google/registry/repositories.bzl | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 8109bbdd6..3e2d8b42c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -16,6 +16,7 @@ load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories") closure_repositories( omit_com_google_auto_factory = True, + omit_com_google_protobuf = True, omit_com_google_code_findbugs_jsr305 = True, omit_com_google_guava = True, omit_com_ibm_icu_icu4j = True, diff --git a/docs/install.md b/docs/install.md index 6f6aaec0f..ab6ea914e 100644 --- a/docs/install.md +++ b/docs/install.md @@ -7,8 +7,8 @@ This document covers the steps necessary to download, build, and deploy Nomulus. You will need the following programs installed on your local machine: * A recent version of the [Java 8 JDK][java-jdk8]. -* [Bazel build system](http://bazel.io/) (version [0.20.0][bazel-version] - works as of 2018-12-03). +* [Bazel build system](http://bazel.io/) (version [0.21.0][bazel-version] + works as of 2018-12-20). * [Google App Engine SDK for Java][app-engine-sdk], and configure aliases to to the `gcloud` and `appcfg.sh` utilities (you'll use them a lot). * [Git](https://git-scm.com/) version control system. @@ -181,4 +181,4 @@ See the [first steps tutorial](./first-steps-tutorial.md) for more information. [app-engine-sdk]: https://cloud.google.com/appengine/docs/java/download [java-jdk8]: http://www.oracle.com/technetwork/java/javase/downloads -[bazel-version]: https://github.com/bazelbuild/bazel/releases/download/0.20.0/bazel-0.20.0-installer-linux-x86_64.sh +[bazel-version]: https://github.com/bazelbuild/bazel/releases/download/0.21.0/bazel-0.21.0-installer-linux-x86_64.sh diff --git a/java/google/registry/repositories.bzl b/java/google/registry/repositories.bzl index be7b0347f..30e8d930f 100644 --- a/java/google/registry/repositories.bzl +++ b/java/google/registry/repositories.bzl @@ -15,6 +15,7 @@ """External dependencies for Nomulus.""" +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_import_external") def domain_registry_bazel_check(): @@ -86,6 +87,7 @@ def domain_registry_repositories( omit_com_google_oauth_client_java6 = False, omit_com_google_oauth_client_jetty = False, omit_com_google_oauth_client_servlet = False, + omit_com_google_protobuf = False, omit_com_google_protobuf_java = False, omit_com_google_re2j = False, omit_com_google_template_soy = False, @@ -297,6 +299,8 @@ def domain_registry_repositories( com_google_oauth_client_jetty() if not omit_com_google_oauth_client_servlet: com_google_oauth_client_servlet() + if not omit_com_google_protobuf: + com_google_protobuf() if not omit_com_google_protobuf_java: com_google_protobuf_java() if not omit_com_google_re2j: @@ -1525,6 +1529,17 @@ def com_google_oauth_client_servlet(): ], ) +def com_google_protobuf(): + http_archive( + name = "com_google_protobuf", + strip_prefix = "protobuf-3.6.1.3", + sha256 = "73fdad358857e120fd0fa19e071a96e15c0f23bb25f85d3f7009abfd4f264a2a", + urls = [ + "https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.1.3.tar.gz", + "https://github.com/protocolbuffers/protobuf/archive/v3.6.1.3.tar.gz", + ], + ) + def com_google_protobuf_java(): java_import_external( name = "com_google_protobuf_java", From 3d3c0eb9ae4e590570f2ac709abcdcf6be4cae39 Mon Sep 17 00:00:00 2001 From: mmuller Date: Fri, 21 Dec 2018 09:23:15 -0800 Subject: [PATCH 116/134] Add a set of newly broken tests to the "outcasts" group Recent changes cause a bunch of new tests to have runtime conflicts. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226499557 --- gradle/core/build.gradle | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/gradle/core/build.gradle b/gradle/core/build.gradle index a61894bbf..356b4d1f6 100644 --- a/gradle/core/build.gradle +++ b/gradle/core/build.gradle @@ -13,15 +13,34 @@ def generatedDir = "${project.buildDir}/generated/source/custom/main" def outcastTestPatterns = [ "google/registry/batch/DeleteContactsAndHostsActionTest.*", "google/registry/batch/RefreshDnsOnHostRenameActionTest.*", + "google.registry.export.ExportDomainListsActionTest.*", + "google/registry/export/ExportPremiumTermsActionTest.*", + "google/registry/export/ExportReservedTermsActionTest.*", + "google/registry/export/ExportUtilsTest.*", + "google/registry/export/sheet/SyncRegistrarsSheetTest.*", "google/registry/flows/CheckApiActionTest.*", - "google/registry/flows/EppLifecycleDomainTest.*", - "google/registry/flows/EppLifecycleHostTest.*", "google/registry/flows/domain/DomainAllocateFlowTest.*", "google/registry/flows/domain/DomainApplicationCreateFlowTest.*", "google/registry/flows/domain/DomainApplicationUpdateFlowTest.*", "google/registry/flows/domain/DomainCreateFlowTest.*", "google/registry/flows/domain/DomainUpdateFlowTest.*", + "google/registry/flows/EppLifecycleDomainTest.*", + "google/registry/flows/EppLifecycleHostTest.*", + "google/registry/model/poll/PollMessageExternalKeyConverterTest.*", + "google/registry/model/poll/PollMessageTest.*", + "google/registry/rdap/RdapDomainActionTest.*", + "google/registry/rdap/RdapDomainSearchActionTest.*", + "google/registry/rdap/RdapEntityActionTest.*", + "google/registry/rdap/RdapEntitySearchActionTest.*", + "google/registry/rdap/RdapNameserverSearchActionTest.*", + "google/registry/reporting/icann/IcannHttpReporterTest.*", + "google/registry/tmch/LordnTaskUtilsTest.*", + "google/registry/tmch/NordnUploadActionTest.*", "google/registry/tools/server/CreatePremiumListActionTest.*", + "google/registry/ui/server/registrar/ContactSettingsTest.*", + "google/registry/ui/server/registrar/RegistrarSettingsActionTest.*", + "google/registry/ui/server/registrar/SecuritySettingsTest.*", + // Conflicts with WhoisActionTest "google/registry/whois/WhoisHttpActionTest.*", ] From a09067c51dbe967dcc04f4b38da6a9e0ccce2fb0 Mon Sep 17 00:00:00 2001 From: jianglai Date: Fri, 21 Dec 2018 11:19:00 -0800 Subject: [PATCH 117/134] Make Cloud Build produce artifact ready for Spinnaker to consume The artifacts for each service will be packaged inside a tar file that, when untared, is ready for Spinnaker to deploy to GAE. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226516223 --- cloudbuild-nomulus.yaml | 41 +++++++++++++++++++++-------------------- gradle/build.gradle | 4 +++- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/cloudbuild-nomulus.yaml b/cloudbuild-nomulus.yaml index 3b1e179a8..9f8ac19be 100644 --- a/cloudbuild-nomulus.yaml +++ b/cloudbuild-nomulus.yaml @@ -22,31 +22,32 @@ steps: args: ['source', 'repos', 'clone', 'nomulus-config'] - name: 'alpine' args: ['sh', '-c', 'cp -r nomulus-config/* .'] -# Build the war files. -- name: 'openjdk:8-slim' +# Build the deployment files. +- name: 'google/cloud-sdk' args: ['./gradlew', 'stage', '-x', 'autoLintGradle'] dir: 'gradle' -- name: 'openjdk:8-slim' - args: ['jar', '-cvf', '../../../default-${TAG_NAME}.war', '.'] - dir: 'gradle/services/default/build/exploded-default' -- name: 'openjdk:8-slim' - args: ['jar', '-cvf', '../../../pubapi-${TAG_NAME}.war', '.'] - dir: 'gradle/services/pubapi/build/exploded-pubapi' -- name: 'openjdk:8-slim' - args: ['jar', '-cvf', '../../../backend-${TAG_NAME}.war', '.'] - dir: 'gradle/services/backend/build/exploded-backend' -- name: 'openjdk:8-slim' - args: ['jar', '-cvf', '../../../tools-${TAG_NAME}.war', '.'] - dir: 'gradle/services/tools/build/exploded-tools' -# WAR files to upload to GCS. +# Tar the deployment files as we cannot upload directories to GCS. +- name: 'alpine' + args: ['tar', 'cvf', '../../../default.tar', '.'] + dir: 'gradle/services/default/build/staged-app' +- name: 'alpine' + args: ['tar', 'cvf', '../../../pubapi.tar', '.'] + dir: 'gradle/services/pubapi/build/staged-app' +- name: 'alpine' + args: ['tar', 'cvf', '../../../backend.tar', '.'] + dir: 'gradle/services/backend/build/staged-app' +- name: 'alpine' + args: ['tar', 'cvf', '../../../tools.tar', '.'] + dir: 'gradle/services/tools/build/staged-app' +# Tar files to upload to GCS. artifacts: objects: - location: 'gs://${PROJECT_ID}-war' + location: 'gs://${PROJECT_ID}-deploy/${TAG_NAME}' paths: - - 'gradle/services/default-${TAG_NAME}.war' - - 'gradle/services/pubapi-${TAG_NAME}.war' - - 'gradle/services/backend-${TAG_NAME}.war' - - 'gradle/services/tools-${TAG_NAME}.war' + - 'gradle/services/default.tar' + - 'gradle/services/pubapi.tar' + - 'gradle/services/backend.tar' + - 'gradle/services/tools.tar' timeout: 3600s options: machineType: 'N1_HIGHCPU_8' diff --git a/gradle/build.gradle b/gradle/build.gradle index dcc577c48..68bc6796e 100644 --- a/gradle/build.gradle +++ b/gradle/build.gradle @@ -91,7 +91,7 @@ subprojects { } rootProject.deploy.dependsOn appengineDeploy - rootProject.stage.dependsOn explodeWar + rootProject.stage.dependsOn appengineStage // Return early, do not apply the settings below. return @@ -107,6 +107,8 @@ subprojects { sourceCompatibility = '1.8' targetCompatibility = '1.8' + compileJava {options.encoding = "UTF-8"} + gradleLint.rules = [ // Checks if Gradle wrapper is up-to-date 'archaic-wrapper', From 218a6e58f17b3f1e8c6ea1daf49d1d40b30aaf4f Mon Sep 17 00:00:00 2001 From: jianglai Date: Fri, 21 Dec 2018 13:03:59 -0800 Subject: [PATCH 118/134] Point travis build badge to the right URL We are still using the legacy travis-ci.org server. Migration is not possible at the moment. See https://docs.travis-ci.com/user/open-source-on-travis-ci-com/#existing-open-source-repositories-on-travis-ciorg. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226531134 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1fb3124fd..8192c30a4 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ | Bazel | Gradle | |-------|--------| -|![Build Status](https://storage.googleapis.com/domain-registry-github-build-status/github-ci-status.png)|[![Build Status](https://travis-ci.com/google/nomulus.svg?branch=master)](https://travis-ci.com/google/nomulus)| +|![Build Status](https://storage.googleapis.com/domain-registry-github-build-status/github-ci-status.png)|[![Build Status](https://travis-ci.org/google/nomulus.svg?branch=master)](https://travis-ci.com/google/nomulus)| ![Nomulus logo](./nomulus-logo.png) From 040319a95da87ca2c637ee9c88393460cc93c235 Mon Sep 17 00:00:00 2001 From: mcilwain Date: Fri, 21 Dec 2018 18:11:03 -0800 Subject: [PATCH 119/134] Reduce # of manual instances in production and sandbox Steady state QPS for prod is <30 on both default and pubapi, so number of instances can easily be brought safely down to 15 & 10 instances. Sandbox has negligible steady state QPS (especially on pubapi), so it is brought even lower. Note that, if we have any issues with these levels, we can increase them instantly using the nomulus set_num_instances command, without having to do another release or a rollback; this is therefore low risk. Note that we'll want to go back to 100 instances for the first day of .dev sunrise as well as the entire week of EAP and first day of GA. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226566822 --- .../registry/env/production/default/WEB-INF/appengine-web.xml | 2 +- .../registry/env/production/pubapi/WEB-INF/appengine-web.xml | 2 +- .../registry/env/sandbox/default/WEB-INF/appengine-web.xml | 2 +- .../registry/env/sandbox/pubapi/WEB-INF/appengine-web.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/java/google/registry/env/production/default/WEB-INF/appengine-web.xml b/java/google/registry/env/production/default/WEB-INF/appengine-web.xml index 4e444a23f..7449a161c 100644 --- a/java/google/registry/env/production/default/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/production/default/WEB-INF/appengine-web.xml @@ -7,7 +7,7 @@ true B4_1G - 30 + 15 diff --git a/java/google/registry/env/production/pubapi/WEB-INF/appengine-web.xml b/java/google/registry/env/production/pubapi/WEB-INF/appengine-web.xml index d99ecf96a..de2869151 100644 --- a/java/google/registry/env/production/pubapi/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/production/pubapi/WEB-INF/appengine-web.xml @@ -7,7 +7,7 @@ true B4_1G - 20 + 10 diff --git a/java/google/registry/env/sandbox/default/WEB-INF/appengine-web.xml b/java/google/registry/env/sandbox/default/WEB-INF/appengine-web.xml index 5a8f92fb9..2cd7af386 100644 --- a/java/google/registry/env/sandbox/default/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/sandbox/default/WEB-INF/appengine-web.xml @@ -7,7 +7,7 @@ true B4_1G - 30 + 10 diff --git a/java/google/registry/env/sandbox/pubapi/WEB-INF/appengine-web.xml b/java/google/registry/env/sandbox/pubapi/WEB-INF/appengine-web.xml index e3881c7ce..e82dd364c 100644 --- a/java/google/registry/env/sandbox/pubapi/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/sandbox/pubapi/WEB-INF/appengine-web.xml @@ -7,7 +7,7 @@ true B4_1G - 20 + 5 From 2777018d6aa1970241caebcddc71820a2666b58c Mon Sep 17 00:00:00 2001 From: guyben Date: Fri, 21 Dec 2018 19:09:36 -0800 Subject: [PATCH 120/134] Add the ability to setup OT&E from the web console We create a new endpoint with a simple form that will let admins (including support) setup OT&E for registrars. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=226570568 --- .../env/common/default/WEB-INF/web.xml | 6 + .../model/common/GaeUserIdConverter.java | 7 +- java/google/registry/module/frontend/BUILD | 1 + .../frontend/FrontendRequestComponent.java | 4 +- .../auth/AuthenticatedRegistrarAccessor.java | 73 ++++-- java/google/registry/ui/server/otesetup/BUILD | 43 ++++ .../otesetup/ConsoleOteSetupAction.java | 243 ++++++++++++++++++ .../registrar/RegistrarConsoleModule.java | 12 + java/google/registry/ui/soy/Forms.soy | 15 ++ java/google/registry/ui/soy/otesetup/BUILD | 21 ++ .../registry/ui/soy/otesetup/Console.soy | 169 ++++++++++++ .../ui/soy/registrar/AdminSettings.soy | 8 +- .../frontend/testdata/frontend_routing.txt | 11 +- javatests/google/registry/server/Fixture.java | 4 + .../registry/server/RegistryTestServer.java | 3 +- .../google/registry/ui/server/otesetup/BUILD | 45 ++++ .../otesetup/ConsoleOteSetupActionTest.java | 173 +++++++++++++ 17 files changed, 804 insertions(+), 34 deletions(-) create mode 100644 java/google/registry/ui/server/otesetup/BUILD create mode 100644 java/google/registry/ui/server/otesetup/ConsoleOteSetupAction.java create mode 100644 java/google/registry/ui/soy/otesetup/BUILD create mode 100644 java/google/registry/ui/soy/otesetup/Console.soy create mode 100644 javatests/google/registry/ui/server/otesetup/BUILD create mode 100644 javatests/google/registry/ui/server/otesetup/ConsoleOteSetupActionTest.java diff --git a/java/google/registry/env/common/default/WEB-INF/web.xml b/java/google/registry/env/common/default/WEB-INF/web.xml index ad42c0f06..bcbc77703 100644 --- a/java/google/registry/env/common/default/WEB-INF/web.xml +++ b/java/google/registry/env/common/default/WEB-INF/web.xml @@ -37,6 +37,12 @@ /registrar-settings + + + frontend-servlet + /registrar-ote-setup + + diff --git a/java/google/registry/model/common/GaeUserIdConverter.java b/java/google/registry/model/common/GaeUserIdConverter.java index a4fc98305..57d08e156 100644 --- a/java/google/registry/model/common/GaeUserIdConverter.java +++ b/java/google/registry/model/common/GaeUserIdConverter.java @@ -14,6 +14,7 @@ package google.registry.model.common; +import static com.google.common.base.Preconditions.checkState; import static google.registry.model.ofy.ObjectifyService.allocateId; import static google.registry.model.ofy.ObjectifyService.ofy; @@ -24,6 +25,7 @@ import com.googlecode.objectify.annotation.Id; import google.registry.model.ImmutableObject; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; +import java.util.List; /** * A helper class to convert email addresses to GAE user ids. It does so by persisting a User @@ -46,8 +48,9 @@ public class GaeUserIdConverter extends ImmutableObject { public static String convertEmailAddressToGaeUserId(String emailAddress) { final GaeUserIdConverter gaeUserIdConverter = new GaeUserIdConverter(); gaeUserIdConverter.id = allocateId(); - gaeUserIdConverter.user = - new User(emailAddress, Splitter.on('@').splitToList(emailAddress).get(1)); + List emailParts = Splitter.on('@').splitToList(emailAddress); + checkState(emailParts.size() == 2, "'%s' is not a valid email address", emailAddress); + gaeUserIdConverter.user = new User(emailAddress, emailParts.get(1)); try { // Perform these operations in a transactionless context to avoid enlisting in some outer diff --git a/java/google/registry/module/frontend/BUILD b/java/google/registry/module/frontend/BUILD index 796bee607..fc2d681fc 100644 --- a/java/google/registry/module/frontend/BUILD +++ b/java/google/registry/module/frontend/BUILD @@ -21,6 +21,7 @@ java_library( "//java/google/registry/request:modules", "//java/google/registry/request/auth", "//java/google/registry/ui", + "//java/google/registry/ui/server/otesetup", "//java/google/registry/ui/server/registrar", "//java/google/registry/util", "@com_google_appengine_api_1_0_sdk", diff --git a/java/google/registry/module/frontend/FrontendRequestComponent.java b/java/google/registry/module/frontend/FrontendRequestComponent.java index 19c22964b..3869425a1 100644 --- a/java/google/registry/module/frontend/FrontendRequestComponent.java +++ b/java/google/registry/module/frontend/FrontendRequestComponent.java @@ -25,6 +25,7 @@ import google.registry.monitoring.whitebox.WhiteboxModule; import google.registry.request.RequestComponentBuilder; import google.registry.request.RequestModule; import google.registry.request.RequestScope; +import google.registry.ui.server.otesetup.ConsoleOteSetupAction; import google.registry.ui.server.registrar.ConsoleUiAction; import google.registry.ui.server.registrar.RegistrarConsoleModule; import google.registry.ui.server.registrar.RegistrarSettingsAction; @@ -33,13 +34,14 @@ import google.registry.ui.server.registrar.RegistrarSettingsAction; @RequestScope @Subcomponent( modules = { - RegistrarConsoleModule.class, DnsModule.class, EppTlsModule.class, + RegistrarConsoleModule.class, RequestModule.class, WhiteboxModule.class, }) interface FrontendRequestComponent { + ConsoleOteSetupAction consoleOteSetupAction(); ConsoleUiAction consoleUiAction(); EppConsoleAction eppConsoleAction(); EppTlsAction eppTlsAction(); diff --git a/java/google/registry/request/auth/AuthenticatedRegistrarAccessor.java b/java/google/registry/request/auth/AuthenticatedRegistrarAccessor.java index b04475a65..c1d080730 100644 --- a/java/google/registry/request/auth/AuthenticatedRegistrarAccessor.java +++ b/java/google/registry/request/auth/AuthenticatedRegistrarAccessor.java @@ -68,6 +68,12 @@ public class AuthenticatedRegistrarAccessor { private final String userIdForLogging; + /** + * Whether this user is an Admin, meaning either a GAE-admin or a member of the Support G Suite + * group. + */ + private final boolean isAdmin; + /** * Gives all roles a user has for a given clientId. * @@ -103,33 +109,41 @@ public class AuthenticatedRegistrarAccessor { @Config("registryAdminClientId") String registryAdminClientId, @Config("gSuiteSupportGroupEmailAddress") Optional gSuiteSupportGroupEmailAddress, Lazy lazyGroupsConnection) { - this( - authResult.userIdForLogging(), - createRoleMap( - authResult, - registryAdminClientId, - lazyGroupsConnection, - gSuiteSupportGroupEmailAddress)); + this.isAdmin = userIsAdmin(authResult, gSuiteSupportGroupEmailAddress, lazyGroupsConnection); - logger.atInfo().log( - "%s has the following roles: %s", authResult.userIdForLogging(), roleMap); + this.userIdForLogging = authResult.userIdForLogging(); + this.roleMap = createRoleMap(authResult, this.isAdmin, registryAdminClientId); + + logger.atInfo().log("%s has the following roles: %s", userIdForLogging(), roleMap); } private AuthenticatedRegistrarAccessor( - String userIdForLogging, ImmutableSetMultimap roleMap) { + String userIdForLogging, boolean isAdmin, ImmutableSetMultimap roleMap) { this.userIdForLogging = checkNotNull(userIdForLogging); this.roleMap = checkNotNull(roleMap); + this.isAdmin = isAdmin; } /** * Creates a "logged-in user" accessor with a given role map, used for tests. * + *

The user will be allowed to create Registrars (and hence do OT&E setup) iff they have + * the role of ADMIN for at least one clientId. + * *

The user's "name" in logs and exception messages is "TestUserId". */ @VisibleForTesting public static AuthenticatedRegistrarAccessor createForTesting( ImmutableSetMultimap roleMap) { - return new AuthenticatedRegistrarAccessor("TestUserId", roleMap); + boolean isAdmin = roleMap.values().contains(Role.ADMIN); + return new AuthenticatedRegistrarAccessor("TestUserId", isAdmin, roleMap); + } + + /** + * Returns whether this user is allowed to create new Registrars and TLDs. + */ + public boolean isAdmin() { + return isAdmin; } /** @@ -261,11 +275,32 @@ public class AuthenticatedRegistrarAccessor { } } + private static boolean userIsAdmin( + AuthResult authResult, + Optional gSuiteSupportGroupEmailAddress, + Lazy lazyGroupsConnection) { + + if (!authResult.userAuthInfo().isPresent()) { + return false; + } + + UserAuthInfo userAuthInfo = authResult.userAuthInfo().get(); + + User user = userAuthInfo.user(); + + // both GAE project admin and members of the gSuiteSupportGroupEmailAddress are considered + // admins for the RegistrarConsole. + return bypassAdminCheck + ? false + : userAuthInfo.isUserAdmin() + || checkIsSupport( + lazyGroupsConnection, user.getEmail(), gSuiteSupportGroupEmailAddress); + } + private static ImmutableSetMultimap createRoleMap( AuthResult authResult, - String registryAdminClientId, - Lazy lazyGroupsConnection, - Optional gSuiteSupportGroupEmailAddress) { + boolean isAdmin, + String registryAdminClientId) { if (!authResult.userAuthInfo().isPresent()) { return ImmutableSetMultimap.of(); @@ -274,17 +309,11 @@ public class AuthenticatedRegistrarAccessor { UserAuthInfo userAuthInfo = authResult.userAuthInfo().get(); User user = userAuthInfo.user(); - // both GAE project admin and members of the gSuiteSupportGroupEmailAddress are considered - // admins for the RegistrarConsole. - boolean isAdmin = - bypassAdminCheck - ? false - : userAuthInfo.isUserAdmin() - || checkIsSupport( - lazyGroupsConnection, user.getEmail(), gSuiteSupportGroupEmailAddress); ImmutableSetMultimap.Builder builder = new ImmutableSetMultimap.Builder<>(); + logger.atInfo().log("Checking registrar contacts for user ID %s", user.getUserId()); + ofy() .load() .type(RegistrarContact.class) diff --git a/java/google/registry/ui/server/otesetup/BUILD b/java/google/registry/ui/server/otesetup/BUILD new file mode 100644 index 000000000..4ce526598 --- /dev/null +++ b/java/google/registry/ui/server/otesetup/BUILD @@ -0,0 +1,43 @@ +package( + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) # Apache 2.0 + +java_library( + name = "otesetup", + srcs = glob(["*.java"]), + resources = [ + "//java/google/registry/ui/css:registrar_bin.css.js", + "//java/google/registry/ui/css:registrar_dbg.css.js", + ], + deps = [ + "//java/google/registry/config", + "//java/google/registry/export/sheet", + "//java/google/registry/flows", + "//java/google/registry/model", + "//java/google/registry/request", + "//java/google/registry/request/auth", + "//java/google/registry/security", + "//java/google/registry/ui/forms", + "//java/google/registry/ui/server", + "//java/google/registry/ui/soy:soy_java_wrappers", + "//java/google/registry/ui/soy/otesetup:soy_java_wrappers", + "//java/google/registry/util", + "//third_party/objectify:objectify-v4_1", + "@com_google_appengine_api_1_0_sdk", + "@com_google_auto_value", + "@com_google_code_findbugs_jsr305", + "@com_google_dagger", + "@com_google_flogger", + "@com_google_flogger_system_backend", + "@com_google_guava", + "@com_google_monitoring_client_metrics", + "@com_google_re2j", + "@io_bazel_rules_closure//closure/templates", + "@javax_inject", + "@javax_servlet_api", + "@joda_time", + "@org_joda_money", + ], +) diff --git a/java/google/registry/ui/server/otesetup/ConsoleOteSetupAction.java b/java/google/registry/ui/server/otesetup/ConsoleOteSetupAction.java new file mode 100644 index 000000000..6b9ada0a5 --- /dev/null +++ b/java/google/registry/ui/server/otesetup/ConsoleOteSetupAction.java @@ -0,0 +1,243 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.ui.server.otesetup; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.net.HttpHeaders.LOCATION; +import static com.google.common.net.HttpHeaders.X_FRAME_OPTIONS; +import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; +import static javax.servlet.http.HttpServletResponse.SC_MOVED_TEMPORARILY; + +import com.google.appengine.api.users.User; +import com.google.appengine.api.users.UserService; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ascii; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableMap; +import com.google.common.flogger.FluentLogger; +import com.google.common.io.Resources; +import com.google.common.net.MediaType; +import com.google.template.soy.shared.SoyCssRenamingMap; +import com.google.template.soy.tofu.SoyTofu; +import google.registry.config.RegistryConfig.Config; +import google.registry.config.RegistryEnvironment; +import google.registry.model.OteAccountBuilder; +import google.registry.request.Action; +import google.registry.request.Action.Method; +import google.registry.request.Parameter; +import google.registry.request.RequestMethod; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import google.registry.request.auth.AuthResult; +import google.registry.request.auth.AuthenticatedRegistrarAccessor; +import google.registry.security.XsrfTokenManager; +import google.registry.ui.server.SendEmailUtils; +import google.registry.ui.server.SoyTemplateUtils; +import google.registry.ui.soy.otesetup.ConsoleSoyInfo; +import google.registry.util.StringGenerator; +import java.util.HashMap; +import java.util.Optional; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; + +/** + * Action that serves OT&E setup web page. + * + *

This Action does 2 things: - for GET, just returns the form that asks for the clientId and + * email. - for POST, receives the clientId and email and generates the OTE entities. + * + *

TODO(b/120201577): once we can have 2 different Actions with the same path (different + * Methods), separate this class to 2 Actions. + */ +@Action( + path = ConsoleOteSetupAction.PATH, + method = {Method.POST, Method.GET}, + auth = Auth.AUTH_PUBLIC) +public final class ConsoleOteSetupAction implements Runnable { + + private static final int PASSWORD_LENGTH = 16; + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + public static final String PATH = "/registrar-ote-setup"; + + private static final Supplier TOFU_SUPPLIER = + SoyTemplateUtils.createTofuSupplier( + google.registry.ui.soy.ConsoleSoyInfo.getInstance(), + google.registry.ui.soy.FormsSoyInfo.getInstance(), + google.registry.ui.soy.otesetup.ConsoleSoyInfo.getInstance()); + + @VisibleForTesting // webdriver and screenshot tests need this + public static final Supplier CSS_RENAMING_MAP_SUPPLIER = + SoyTemplateUtils.createCssRenamingMapSupplier( + Resources.getResource("google/registry/ui/css/registrar_bin.css.js"), + Resources.getResource("google/registry/ui/css/registrar_dbg.css.js")); + + @Inject HttpServletRequest req; + @Inject @RequestMethod Method method; + @Inject Response response; + @Inject AuthenticatedRegistrarAccessor registrarAccessor; + @Inject UserService userService; + @Inject XsrfTokenManager xsrfTokenManager; + @Inject AuthResult authResult; + @Inject RegistryEnvironment registryEnvironment; + @Inject SendEmailUtils sendEmailUtils; + @Inject @Config("logoFilename") String logoFilename; + @Inject @Config("productName") String productName; + @Inject @Config("base64StringGenerator") StringGenerator passwordGenerator; + @Inject @Parameter("clientId") Optional clientId; + @Inject @Parameter("email") Optional email; + + @Inject ConsoleOteSetupAction() {} + + @Override + public void run() { + response.setHeader(X_FRAME_OPTIONS, "SAMEORIGIN"); // Disallow iframing. + response.setHeader("X-Ui-Compatible", "IE=edge"); // Ask IE not to be silly. + + logger.atInfo().log( + "User %s is accessing the OT&E setup page. Method= %s", + registrarAccessor.userIdForLogging(), method); + checkState(registryEnvironment != RegistryEnvironment.PRODUCTION, "Can't create OT&E in prod"); + if (!authResult.userAuthInfo().isPresent()) { + response.setStatus(SC_MOVED_TEMPORARILY); + String location; + try { + location = userService.createLoginURL(req.getRequestURI()); + } catch (IllegalArgumentException e) { + // UserServiceImpl.createLoginURL() throws IllegalArgumentException if underlying API call + // returns an error code of NOT_ALLOWED. createLoginURL() assumes that the error is caused + // by an invalid URL. But in fact, the error can also occur if UserService doesn't have any + // user information, which happens when the request has been authenticated as internal. In + // this case, we want to avoid dying before we can send the redirect, so just redirect to + // the root path. + location = "/"; + } + response.setHeader(LOCATION, location); + return; + } + User user = authResult.userAuthInfo().get().user(); + + // Using HashMap to allow null values + HashMap data = new HashMap<>(); + data.put("logoFilename", logoFilename); + data.put("productName", productName); + data.put("username", user.getNickname()); + data.put("logoutUrl", userService.createLogoutURL(PATH)); + data.put("xsrfToken", xsrfTokenManager.generateToken(user.getEmail())); + response.setContentType(MediaType.HTML_UTF_8); + + if (!registrarAccessor.isAdmin()) { + response.setStatus(SC_FORBIDDEN); + response.setPayload( + TOFU_SUPPLIER + .get() + .newRenderer(ConsoleSoyInfo.WHOAREYOU) + .setCssRenamingMap(CSS_RENAMING_MAP_SUPPLIER.get()) + .setData(data) + .render()); + return; + } + switch (method) { + case POST: + runPost(data); + return; + case GET: + runGet(data); + return; + default: + return; + } + } + + private void runPost(HashMap data) { + // This is intentionally outside of the "try/catch", since not having these fields means someone + // tried to "manually" POST to this page. No need to send out a "pretty" result in that case. + checkState(clientId.isPresent() && email.isPresent(), "Must supply clientId and email"); + + data.put("baseClientId", clientId.get()); + data.put("contactEmail", email.get()); + + try { + String password = passwordGenerator.createString(PASSWORD_LENGTH); + ImmutableMap clientIdToTld = + OteAccountBuilder.forClientId(clientId.get()) + .addContact(email.get()) + .setPassword(password) + .buildAndPersist(); + + sendExternalUpdates(clientIdToTld); + + data.put("clientIdToTld", clientIdToTld); + data.put("password", password); + + response.setPayload( + TOFU_SUPPLIER + .get() + .newRenderer(ConsoleSoyInfo.RESULT_SUCCESS) + .setCssRenamingMap(CSS_RENAMING_MAP_SUPPLIER.get()) + .setData(data) + .render()); + } catch (Throwable e) { + logger.atWarning().withCause(e).log( + "Failed to setup OT&E. clientId: %s, email: %s", clientId.get(), email.get()); + data.put("errorMessage", e.getMessage()); + response.setPayload( + TOFU_SUPPLIER + .get() + .newRenderer(ConsoleSoyInfo.FORM_PAGE) + .setCssRenamingMap(CSS_RENAMING_MAP_SUPPLIER.get()) + .setData(data) + .render()); + } + } + + private void runGet(HashMap data) { + // set the values to pre-fill, if given + data.put("baseClientId", clientId.orElse(null)); + data.put("contactEmail", email.orElse(null)); + + response.setPayload( + TOFU_SUPPLIER + .get() + .newRenderer(ConsoleSoyInfo.FORM_PAGE) + .setCssRenamingMap(CSS_RENAMING_MAP_SUPPLIER.get()) + .setData(data) + .render()); + } + + + private void sendExternalUpdates(ImmutableMap clientIdToTld) { + if (!sendEmailUtils.hasRecepients()) { + return; + } + String environment = Ascii.toLowerCase(String.valueOf(registryEnvironment)); + StringBuilder builder = new StringBuilder(); + builder.append( + String.format( + "The following entities were created in %s by %s:\n", + environment, registrarAccessor.userIdForLogging())); + clientIdToTld.forEach( + (clientId, tld) -> + builder.append( + String.format(" Registrar %s with access to TLD %s\n", clientId, tld))); + builder.append(String.format("Gave user %s web access to these Registrars\n", email.get())); + sendEmailUtils.sendEmail( + String.format( + "OT&E for registrar %s created in %s", + clientId.get(), + environment), + builder.toString()); + } +} diff --git a/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java b/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java index 6d6262dbe..d14e89b8b 100644 --- a/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java +++ b/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java @@ -41,4 +41,16 @@ public final class RegistrarConsoleModule { static String provideClientId(HttpServletRequest req) { return extractRequiredParameter(req, PARAM_CLIENT_ID); } + + @Provides + @Parameter("email") + static Optional provideOptionalEmail(HttpServletRequest req) { + return extractOptionalParameter(req, "email"); + } + + @Provides + @Parameter("email") + static String provideEmail(HttpServletRequest req) { + return extractRequiredParameter(req, "email"); + } } diff --git a/java/google/registry/ui/soy/Forms.soy b/java/google/registry/ui/soy/Forms.soy index ea3a0accd..ae1199545 100644 --- a/java/google/registry/ui/soy/Forms.soy +++ b/java/google/registry/ui/soy/Forms.soy @@ -295,6 +295,21 @@ {/template} +/** Submit button. */ +{template .submitRow} + {@param label: string} + + + + + + +{/template} + /** Drop-down select button widget. */ {template .menuButton} diff --git a/java/google/registry/ui/soy/otesetup/BUILD b/java/google/registry/ui/soy/otesetup/BUILD new file mode 100644 index 000000000..f19b6651c --- /dev/null +++ b/java/google/registry/ui/soy/otesetup/BUILD @@ -0,0 +1,21 @@ +package( + default_visibility = ["//java/google/registry:registry_project"], +) + +licenses(["notice"]) # Apache 2.0 + +load("@io_bazel_rules_closure//closure:defs.bzl", "closure_java_template_library", "closure_js_template_library") + +closure_js_template_library( + name = "otesetup", + srcs = glob(["*.soy"]), + data = ["//java/google/registry/ui/css:registrar_raw"], + globals = "//java/google/registry/ui:globals.txt", + deps = ["//java/google/registry/ui/soy"], +) + +closure_java_template_library( + name = "soy_java_wrappers", + srcs = glob(["*.soy"]), + java_package = "google.registry.ui.soy.otesetup", +) diff --git a/java/google/registry/ui/soy/otesetup/Console.soy b/java/google/registry/ui/soy/otesetup/Console.soy new file mode 100644 index 000000000..e96adf4a1 --- /dev/null +++ b/java/google/registry/ui/soy/otesetup/Console.soy @@ -0,0 +1,169 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{namespace registry.soy.registrar.console} + + +/** + * Main page for the OT&E creation. Holds a form with the required data. + */ +{template .formPage} + {@param xsrfToken: string} /** Security token. */ + {@param username: string} /** Arbitrary username to display. */ + {@param logoutUrl: string} /** Generated URL for logging out of Google. */ + {@param productName: string} /** Name to display for this software product. */ + {@param logoFilename: string} + {@param? errorMessage: string} /** If set - display the error message above the form. */ + {@param? baseClientId: string} /** If set - an initial value to fill for the base client ID. */ + {@param? contactEmail: string} /** If set - an initial value to fill for the email. */ + + {call registry.soy.console.header} + {param app: 'registrar' /} + {param subtitle: 'OT&E setup' /} + {/call} + {call registry.soy.console.googlebar data="all" /} +

+
+

OT&E Setup Page

+ {if isNonnull($errorMessage)} +

Failed: {$errorMessage}

+ {/if} + {call .form_ data="all" /} +
+ {call registry.soy.console.footer /} +
+{/template} + + +/** + * Result page for a successful OT&E setup. + */ +{template .resultSuccess} + {@param baseClientId: string} /** The base clientId used for the OT&E setup. */ + {@param contactEmail: string} /** The contact's email added to the registrars. */ + {@param clientIdToTld: map} /** The created registrars->TLD mapping. */ + {@param password: string} /** The password given for the created registrars. */ + {@param username: string} /** Arbitrary username to display. */ + {@param logoutUrl: string} /** Generated URL for logging out of Google. */ + {@param productName: string} /** Name to display for this software product. */ + {@param logoFilename: string} + + {call registry.soy.console.header} + {param app: 'registrar' /} + {param subtitle: 'OT&E setup' /} + {/call} + {call registry.soy.console.googlebar data="all" /} +
+
+

OT&E successfully created for registrar {$baseClientId}!

+ + + + +
+ + Copy and paste this into an email to the registrars + + +
+ Gave web-console access to these registrars. +

Don't forget to set the and for these Registrars!

+ Links to the security page for your convenience:
+ {for $clientId in mapKeys($clientIdToTld)} + {$clientId}
+ {/for} +
+ {call registry.soy.console.footer /} +
+{/template} + + +/** Form for getting registrar info. */ +{template .form_ visibility="private"} + {@param xsrfToken: string} /** Security token. */ + {@param? baseClientId: string} /** If set - an initial value to fill for the base client ID. */ + {@param? contactEmail: string} /** If set - an initial value to fill for the email. */ +
+ + {call registry.soy.forms.inputFieldRowWithValue} + {param label: 'Base clientId' /} + {param name: 'clientId' /} + {param value: $baseClientId /} + {param placeholder: 'registrar\'s ID' /} + {param description kind="text"} + must 1) consist of only lowercase letters, numbers, or hyphens, + 2) start with a letter, and 3) be between 3 and 14 characters (inclusive). + We require 1 and 2 since the registrar name will be used to create TLDs, + and we require 3 since we append \"-[1234]\" to the name to create client + IDs which are required by the EPP XML schema to be between 3-16 chars. + {/param} + {param readonly: false /} + {/call} + {call registry.soy.forms.inputFieldRowWithValue} + {param label: 'contact email' /} + {param name: 'email' /} + {param value: $contactEmail /} + {param placeholder: 'registrar\'s assigned email' /} + {param description kind="text"} + Will be granted web-console access to the OTE registrars. + {/param} + {param readonly: false /} + {/call} + {call registry.soy.forms.submitRow} + {param label: 'create' /} + {/call} +
+ +
+{/template} + + +/** + * Who goes thar?! + */ +{template .whoareyou} + {@param username: string} /** Arbitrary username to display. */ + {@param logoutUrl: string} /** Generated URL for logging out of Google. */ + {@param logoFilename: string} + {@param productName: string} + {call registry.soy.console.header} + {param app: 'registrar' /} + {param subtitle: 'Not Authorized' /} + {/call} +
+ + {$productName} + +

You need permission

+

+ Only {$productName} Admins have access to this page. + You are signed in as {$username}. +

+
+{/template} + + diff --git a/java/google/registry/ui/soy/registrar/AdminSettings.soy b/java/google/registry/ui/soy/registrar/AdminSettings.soy index 960cb2ac2..d9e18993e 100644 --- a/java/google/registry/ui/soy/registrar/AdminSettings.soy +++ b/java/google/registry/ui/soy/registrar/AdminSettings.soy @@ -27,7 +27,6 @@ set or remove TLDs this client is allowed access to. -
@@ -44,8 +43,11 @@ class="{css('kd-button')} {css('btn-add')}">Add
- - + + + + +

Generate new OT&E accounts here {/template} diff --git a/javatests/google/registry/module/frontend/testdata/frontend_routing.txt b/javatests/google/registry/module/frontend/testdata/frontend_routing.txt index 6b91c3704..f60c5ccf9 100644 --- a/javatests/google/registry/module/frontend/testdata/frontend_routing.txt +++ b/javatests/google/registry/module/frontend/testdata/frontend_routing.txt @@ -1,5 +1,6 @@ -PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY -/_dr/epp EppTlsAction POST n INTERNAL,API APP PUBLIC -/registrar ConsoleUiAction GET n INTERNAL,API,LEGACY NONE PUBLIC -/registrar-settings RegistrarSettingsAction POST n API,LEGACY USER PUBLIC -/registrar-xhr EppConsoleAction POST n API,LEGACY USER PUBLIC +PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY +/_dr/epp EppTlsAction POST n INTERNAL,API APP PUBLIC +/registrar ConsoleUiAction GET n INTERNAL,API,LEGACY NONE PUBLIC +/registrar-ote-setup ConsoleOteSetupAction POST,GET n INTERNAL,API,LEGACY NONE PUBLIC +/registrar-settings RegistrarSettingsAction POST n API,LEGACY USER PUBLIC +/registrar-xhr EppConsoleAction POST n API,LEGACY USER PUBLIC diff --git a/javatests/google/registry/server/Fixture.java b/javatests/google/registry/server/Fixture.java index c6d3935ac..afe875d8e 100644 --- a/javatests/google/registry/server/Fixture.java +++ b/javatests/google/registry/server/Fixture.java @@ -22,6 +22,7 @@ import static google.registry.testing.DatastoreHelper.loadRegistrar; import static google.registry.testing.DatastoreHelper.newContactResource; import static google.registry.testing.DatastoreHelper.newDomainResource; import static google.registry.testing.DatastoreHelper.persistActiveHost; +import static google.registry.testing.DatastoreHelper.persistPremiumList; import static google.registry.testing.DatastoreHelper.persistResource; import com.google.common.collect.ImmutableList; @@ -58,6 +59,9 @@ public enum Fixture { public void load() { createTlds("xn--q9jyb4c", "example"); + // Used for OT&E TLDs + persistPremiumList("default_sandbox_list"); + ContactResource google = persistResource(newContactResource("google") .asBuilder() .setLocalizedPostalInfo(new PostalInfo.Builder() diff --git a/javatests/google/registry/server/RegistryTestServer.java b/javatests/google/registry/server/RegistryTestServer.java index 3835ed235..d48f24f73 100644 --- a/javatests/google/registry/server/RegistryTestServer.java +++ b/javatests/google/registry/server/RegistryTestServer.java @@ -85,7 +85,8 @@ public final class RegistryTestServer { // Registrar Console route("/registrar", FrontendServlet.class), - route("/registrar-settings", FrontendServlet.class)); + route("/registrar-settings", FrontendServlet.class), + route("/registrar-ote-setup", FrontendServlet.class)); private static final ImmutableList> FILTERS = ImmutableList.of( ObjectifyFilter.class, diff --git a/javatests/google/registry/ui/server/otesetup/BUILD b/javatests/google/registry/ui/server/otesetup/BUILD new file mode 100644 index 000000000..25d855581 --- /dev/null +++ b/javatests/google/registry/ui/server/otesetup/BUILD @@ -0,0 +1,45 @@ +package( + default_testonly = 1, + default_visibility = ["//java/google/registry:registry_project"], +) + +licenses(["notice"]) # Apache 2.0 + +load("//java/com/google/testing/builddefs:GenTestRules.bzl", "GenTestRules") + +java_library( + name = "otesetup", + srcs = glob(["*.java"]), + deps = [ + "//java/google/registry/config", + "//java/google/registry/model", + "//java/google/registry/request", + "//java/google/registry/request/auth", + "//java/google/registry/security", + "//java/google/registry/ui/server", + "//java/google/registry/ui/server/otesetup", + "//java/google/registry/util", + "//javatests/google/registry/testing", + "//third_party/objectify:objectify-v4_1", + "@com_google_appengine_api_1_0_sdk", + "@com_google_flogger", + "@com_google_flogger_system_backend", + "@com_google_guava", + "@com_google_guava_testlib", + "@com_google_monitoring_client_contrib", + "@com_google_truth", + "@com_google_truth_extensions_truth_java8_extension", + "@com_googlecode_json_simple", + "@javax_servlet_api", + "@joda_time", + "@junit", + "@org_joda_money", + "@org_mockito_all", + ], +) + +GenTestRules( + name = "GeneratedTestRules", + test_files = glob(["*Test.java"]), + deps = [":otesetup"], +) diff --git a/javatests/google/registry/ui/server/otesetup/ConsoleOteSetupActionTest.java b/javatests/google/registry/ui/server/otesetup/ConsoleOteSetupActionTest.java new file mode 100644 index 000000000..79e0f79d2 --- /dev/null +++ b/javatests/google/registry/ui/server/otesetup/ConsoleOteSetupActionTest.java @@ -0,0 +1,173 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.ui.server.otesetup; + +import static com.google.common.net.HttpHeaders.LOCATION; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static google.registry.model.registrar.Registrar.loadByClientId; +import static google.registry.testing.DatastoreHelper.persistPremiumList; +import static google.registry.testing.JUnitBackports.assertThrows; +import static javax.servlet.http.HttpServletResponse.SC_MOVED_TEMPORARILY; +import static org.mockito.Mockito.when; + +import com.google.appengine.api.users.User; +import com.google.appengine.api.users.UserServiceFactory; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSetMultimap; +import google.registry.config.RegistryEnvironment; +import google.registry.model.registry.Registry; +import google.registry.request.Action.Method; +import google.registry.request.auth.AuthLevel; +import google.registry.request.auth.AuthResult; +import google.registry.request.auth.AuthenticatedRegistrarAccessor; +import google.registry.request.auth.UserAuthInfo; +import google.registry.security.XsrfTokenManager; +import google.registry.testing.AppEngineRule; +import google.registry.testing.DeterministicStringGenerator; +import google.registry.testing.FakeClock; +import google.registry.testing.FakeResponse; +import google.registry.testing.MockitoJUnitRule; +import google.registry.ui.server.SendEmailUtils; +import google.registry.util.SendEmailService; +import java.util.Optional; +import java.util.Properties; +import javax.mail.Message; +import javax.mail.Session; +import javax.mail.internet.MimeMessage; +import javax.servlet.http.HttpServletRequest; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; + +@RunWith(JUnit4.class) +public final class ConsoleOteSetupActionTest { + + @Rule + public final AppEngineRule appEngineRule = AppEngineRule.builder() + .withDatastore() + .build(); + + @Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create(); + + private final FakeResponse response = new FakeResponse(); + private final ConsoleOteSetupAction action = new ConsoleOteSetupAction(); + private final User user = new User("marla.singer@example.com", "gmail.com", "12345"); + + @Mock HttpServletRequest request; + @Mock SendEmailService emailService; + Message message; + + @Before + public void setUp() { + persistPremiumList("default_sandbox_list", "sandbox,USD 1000"); + + action.req = request; + action.method = Method.GET; + action.response = response; + action.registrarAccessor = + AuthenticatedRegistrarAccessor.createForTesting( + ImmutableSetMultimap.of("unused", AuthenticatedRegistrarAccessor.Role.ADMIN)); + action.userService = UserServiceFactory.getUserService(); + action.xsrfTokenManager = new XsrfTokenManager(new FakeClock(), action.userService); + action.authResult = AuthResult.create(AuthLevel.USER, UserAuthInfo.create(user, false)); + action.registryEnvironment = RegistryEnvironment.UNITTEST; + action.sendEmailUtils = + new SendEmailUtils( + "outgoing@registry.example", + "UnitTest Registry", + ImmutableList.of("notification@test.example", "notification2@test.example"), + emailService); + action.logoFilename = "logo.png"; + action.productName = "Nomulus"; + action.clientId = Optional.empty(); + action.email = Optional.empty(); + action.passwordGenerator = new DeterministicStringGenerator("abcdefghijklmnopqrstuvwxyz"); + + message = new MimeMessage(Session.getDefaultInstance(new Properties(), null)); + when(emailService.createMessage()).thenReturn(message); + } + + @Test + public void testNoUser_redirect() { + when(request.getRequestURI()).thenReturn("/test"); + action.authResult = AuthResult.NOT_AUTHENTICATED; + action.run(); + assertThat(response.getStatus()).isEqualTo(SC_MOVED_TEMPORARILY); + assertThat(response.getHeaders().get(LOCATION)).isEqualTo("/_ah/login?continue=%2Ftest"); + } + + @Test + public void testGet_authorized() { + action.run(); + assertThat(response.getPayload()).contains("

OT&E Setup Page

"); + } + + @Test + public void testGet_authorized_onProduction() { + action.registryEnvironment = RegistryEnvironment.PRODUCTION; + assertThrows(IllegalStateException.class, action::run); + } + + @Test + public void testGet_unauthorized() { + action.registrarAccessor = + AuthenticatedRegistrarAccessor.createForTesting(ImmutableSetMultimap.of()); + action.run(); + assertThat(response.getPayload()).contains("

You need permission

"); + } + + @Test + public void testPost_authorized() throws Exception { + action.clientId = Optional.of("myclientid"); + action.email = Optional.of("contact@registry.example"); + action.method = Method.POST; + action.run(); + + // We just check some samples to make sure OteAccountBuilder was called successfully. We aren't + // checking that all the entities are there or that they have the correct values. + assertThat(loadByClientId("myclientid-3")).isPresent(); + assertThat(Registry.get("myclientid-ga")).isNotNull(); + assertThat(loadByClientId("myclientid-5").get().getContacts().asList().get(0).getEmailAddress()) + .isEqualTo("contact@registry.example"); + assertThat(response.getPayload()) + .contains("

OT&E successfully created for registrar myclientid!

"); + assertThat(message.getSubject()).isEqualTo("OT&E for registrar myclientid created in unittest"); + assertThat(message.getContent()) + .isEqualTo( + "" + + "The following entities were created in unittest by TestUserId:\n" + + " Registrar myclientid-1 with access to TLD myclientid-sunrise\n" + + " Registrar myclientid-2 with access to TLD myclientid-landrush\n" + + " Registrar myclientid-3 with access to TLD myclientid-ga\n" + + " Registrar myclientid-4 with access to TLD myclientid-ga\n" + + " Registrar myclientid-5 with access to TLD myclientid-eap\n" + + "Gave user contact@registry.example web access to these Registrars\n"); + } + + @Test + public void testPost_unauthorized() { + action.registrarAccessor = + AuthenticatedRegistrarAccessor.createForTesting(ImmutableSetMultimap.of()); + action.clientId = Optional.of("myclientid"); + action.email = Optional.of("contact@registry.example"); + action.method = Method.POST; + action.run(); + assertThat(response.getPayload()).contains("

You need permission

"); + } +} From 2e7d71b238fc86e9e8948fa0c15d19600b0a4639 Mon Sep 17 00:00:00 2001 From: jianglai Date: Fri, 28 Dec 2018 12:28:58 -0800 Subject: [PATCH 121/134] Refactor most of OT&E verification code to exist in utils class This does not change existing functionality but will allow us to use this common code in the yet-to-be-created web console action as well. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=227159346 --- .../registry/model/OteAccountBuilder.java | 2 +- java/google/registry/model/OteStats.java | 277 ++++++++++++++++ .../tools/server/VerifyOteAction.java | 296 ++---------------- .../google/registry/model/OteStatsTest.java | 147 +++++++++ .../registry/model/OteStatsTestHelper.java | 159 ++++++++++ ...main_create_anchor_tenant_fee_standard.xml | 29 ++ .../testdata/domain_create_claim_notice.xml | 0 .../testdata/domain_create_dsdata.xml | 0 .../testdata/domain_create_idn.xml | 0 .../testdata/domain_create_sunrise.xml | 0 .../registry/model/testdata/domain_delete.xml | 17 + .../testdata/domain_restore.xml | 0 .../testdata/domain_transfer_approve.xml | 0 .../testdata/domain_transfer_cancel.xml | 0 .../testdata/domain_transfer_reject.xml | 0 .../testdata/domain_transfer_request.xml | 0 .../testdata/domain_update_with_secdns.xml | 0 .../model/testdata/host_create_complete.xml | 13 + .../registry/model/testdata/host_delete.xml | 17 + .../server => model}/testdata/host_update.xml | 0 javatests/google/registry/tools/server/BUILD | 1 + .../tools/server/VerifyOteActionTest.java | 266 +++++----------- 22 files changed, 754 insertions(+), 470 deletions(-) create mode 100644 java/google/registry/model/OteStats.java create mode 100644 javatests/google/registry/model/OteStatsTest.java create mode 100644 javatests/google/registry/model/OteStatsTestHelper.java create mode 100644 javatests/google/registry/model/testdata/domain_create_anchor_tenant_fee_standard.xml rename javatests/google/registry/{tools/server => model}/testdata/domain_create_claim_notice.xml (100%) rename javatests/google/registry/{tools/server => model}/testdata/domain_create_dsdata.xml (100%) rename javatests/google/registry/{tools/server => model}/testdata/domain_create_idn.xml (100%) rename javatests/google/registry/{tools/server => model}/testdata/domain_create_sunrise.xml (100%) create mode 100644 javatests/google/registry/model/testdata/domain_delete.xml rename javatests/google/registry/{tools/server => model}/testdata/domain_restore.xml (100%) rename javatests/google/registry/{tools/server => model}/testdata/domain_transfer_approve.xml (100%) rename javatests/google/registry/{tools/server => model}/testdata/domain_transfer_cancel.xml (100%) rename javatests/google/registry/{tools/server => model}/testdata/domain_transfer_reject.xml (100%) rename javatests/google/registry/{tools/server => model}/testdata/domain_transfer_request.xml (100%) rename javatests/google/registry/{tools/server => model}/testdata/domain_update_with_secdns.xml (100%) create mode 100644 javatests/google/registry/model/testdata/host_create_complete.xml create mode 100644 javatests/google/registry/model/testdata/host_delete.xml rename javatests/google/registry/{tools/server => model}/testdata/host_update.xml (100%) diff --git a/java/google/registry/model/OteAccountBuilder.java b/java/google/registry/model/OteAccountBuilder.java index 03506889b..e3c795022 100644 --- a/java/google/registry/model/OteAccountBuilder.java +++ b/java/google/registry/model/OteAccountBuilder.java @@ -361,7 +361,7 @@ public final class OteAccountBuilder { } /** Returns the ClientIds of the OT&E, with the TLDs each has access to. */ - private static ImmutableMap createClientIdToTldMap(String baseClientId) { + static ImmutableMap createClientIdToTldMap(String baseClientId) { checkArgument( REGISTRAR_PATTERN.matcher(baseClientId).matches(), "Invalid registrar name: %s", diff --git a/java/google/registry/model/OteStats.java b/java/google/registry/model/OteStats.java new file mode 100644 index 000000000..393d45bf0 --- /dev/null +++ b/java/google/registry/model/OteStats.java @@ -0,0 +1,277 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model; + +import static com.google.common.base.Predicates.equalTo; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static google.registry.model.eppcommon.EppXmlTransformer.unmarshal; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.util.CollectionUtils.isNullOrEmpty; +import static google.registry.util.DomainNameUtils.ACE_PREFIX; + +import com.google.common.base.Ascii; +import com.google.common.base.Predicates; +import com.google.common.collect.HashMultiset; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Multiset; +import com.googlecode.objectify.Key; +import com.googlecode.objectify.cmd.Query; +import google.registry.model.domain.DomainCommand; +import google.registry.model.domain.fee.FeeCreateCommandExtension; +import google.registry.model.domain.launch.LaunchCreateExtension; +import google.registry.model.domain.secdns.SecDnsCreateExtension; +import google.registry.model.domain.secdns.SecDnsUpdateExtension; +import google.registry.model.eppinput.EppInput; +import google.registry.model.eppinput.EppInput.ResourceCommandWrapper; +import google.registry.model.host.HostCommand; +import google.registry.model.reporting.HistoryEntry; +import google.registry.model.reporting.HistoryEntry.Type; +import google.registry.xml.XmlException; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** Represents stats derived from HistoryEntry objects on actions taken by registrars. */ +public class OteStats { + + /** + * Returns the statistics about the OT&E actions that have been taken by a particular registrar. + */ + public static OteStats getFromRegistrar(String registrarName) { + return new OteStats().recordRegistrarHistory(registrarName); + } + + private OteStats() {} + + private static final Predicate HAS_CLAIMS_NOTICE = + eppInput -> { + Optional launchCreate = + eppInput.getSingleExtension(LaunchCreateExtension.class); + return launchCreate.isPresent() && launchCreate.get().getNotice() != null; + }; + + private static final Predicate HAS_SEC_DNS = + eppInput -> + eppInput.getSingleExtension(SecDnsCreateExtension.class).isPresent() + || eppInput.getSingleExtension(SecDnsUpdateExtension.class).isPresent(); + + private static final Predicate IS_SUNRISE = + eppInput -> { + Optional launchCreate = + eppInput.getSingleExtension(LaunchCreateExtension.class); + return launchCreate.isPresent() && !isNullOrEmpty(launchCreate.get().getSignedMarks()); + }; + + private static final Predicate IS_IDN = + eppInput -> + ((DomainCommand.Create) + ((ResourceCommandWrapper) eppInput.getCommandWrapper().getCommand()) + .getResourceCommand()) + .getFullyQualifiedDomainName() + .startsWith(ACE_PREFIX); + + private static final Predicate IS_SUBORDINATE = + eppInput -> + !isNullOrEmpty( + ((HostCommand.Create) + ((ResourceCommandWrapper) eppInput.getCommandWrapper().getCommand()) + .getResourceCommand()) + .getInetAddresses()); + + /** Enum defining the distinct statistics (types of registrar actions) to record. */ + public enum StatType { + CONTACT_CREATES(0, equalTo(Type.CONTACT_CREATE)), + CONTACT_DELETES(0, equalTo(Type.CONTACT_DELETE)), + CONTACT_TRANSFER_APPROVES(0, equalTo(Type.CONTACT_TRANSFER_APPROVE)), + CONTACT_TRANSFER_CANCELS(0, equalTo(Type.CONTACT_TRANSFER_CANCEL)), + CONTACT_TRANSFER_REJECTS(0, equalTo(Type.CONTACT_TRANSFER_REJECT)), + CONTACT_TRANSFER_REQUESTS(0, equalTo(Type.CONTACT_TRANSFER_REQUEST)), + CONTACT_UPDATES(0, equalTo(Type.CONTACT_UPDATE)), + DOMAIN_APPLICATION_CREATES(0, equalTo(Type.DOMAIN_APPLICATION_CREATE)), + DOMAIN_APPLICATION_CREATES_LANDRUSH( + 0, equalTo(Type.DOMAIN_APPLICATION_CREATE), IS_SUNRISE.negate()), + DOMAIN_APPLICATION_CREATES_SUNRISE(0, equalTo(Type.DOMAIN_APPLICATION_CREATE), IS_SUNRISE), + DOMAIN_APPLICATION_DELETES(0, equalTo(Type.DOMAIN_APPLICATION_DELETE)), + DOMAIN_APPLICATION_UPDATES(0, equalTo(Type.DOMAIN_APPLICATION_UPDATE)), + DOMAIN_AUTORENEWS(0, equalTo(Type.DOMAIN_AUTORENEW)), + DOMAIN_CREATES(0, equalTo(Type.DOMAIN_CREATE)), + DOMAIN_CREATES_ASCII(1, equalTo(Type.DOMAIN_CREATE), IS_IDN.negate()), + DOMAIN_CREATES_IDN(1, equalTo(Type.DOMAIN_CREATE), IS_IDN), + DOMAIN_CREATES_START_DATE_SUNRISE(1, equalTo(Type.DOMAIN_CREATE), IS_SUNRISE), + DOMAIN_CREATES_WITH_CLAIMS_NOTICE(1, equalTo(Type.DOMAIN_CREATE), HAS_CLAIMS_NOTICE), + DOMAIN_CREATES_WITH_FEE( + 1, + equalTo(Type.DOMAIN_CREATE), + eppInput -> eppInput.getSingleExtension(FeeCreateCommandExtension.class).isPresent()), + DOMAIN_CREATES_WITH_SEC_DNS(1, equalTo(Type.DOMAIN_CREATE), HAS_SEC_DNS), + DOMAIN_CREATES_WITHOUT_SEC_DNS(0, equalTo(Type.DOMAIN_CREATE), HAS_SEC_DNS.negate()), + DOMAIN_DELETES(2, equalTo(Type.DOMAIN_DELETE)), + DOMAIN_RENEWS(0, equalTo(Type.DOMAIN_RENEW)), + DOMAIN_RESTORES(1, equalTo(Type.DOMAIN_RESTORE)), + DOMAIN_TRANSFER_APPROVES(1, equalTo(Type.DOMAIN_TRANSFER_APPROVE)), + DOMAIN_TRANSFER_CANCELS(1, equalTo(Type.DOMAIN_TRANSFER_CANCEL)), + DOMAIN_TRANSFER_REJECTS(1, equalTo(Type.DOMAIN_TRANSFER_REJECT)), + DOMAIN_TRANSFER_REQUESTS(1, equalTo(Type.DOMAIN_TRANSFER_REQUEST)), + DOMAIN_UPDATES(0, equalTo(Type.DOMAIN_UPDATE)), + DOMAIN_UPDATES_WITH_SEC_DNS(1, equalTo(Type.DOMAIN_UPDATE), HAS_SEC_DNS), + DOMAIN_UPDATES_WITHOUT_SEC_DNS(0, equalTo(Type.DOMAIN_UPDATE), HAS_SEC_DNS.negate()), + HOST_CREATES(0, equalTo(Type.HOST_CREATE)), + HOST_CREATES_EXTERNAL(0, equalTo(Type.HOST_CREATE), IS_SUBORDINATE.negate()), + HOST_CREATES_SUBORDINATE(1, equalTo(Type.HOST_CREATE), IS_SUBORDINATE), + HOST_DELETES(1, equalTo(Type.HOST_DELETE)), + HOST_UPDATES(1, equalTo(Type.HOST_UPDATE)), + UNCLASSIFIED_FLOWS(0, Predicates.alwaysFalse()); + + /** StatTypes with a non-zero requirement */ + public static final ImmutableList REQUIRED_STAT_TYPES = + Arrays.stream(values()) + .filter(statType -> statType.requirement > 0) + .collect(toImmutableList()); + + /** Required number of times registrars must complete this action. */ + private final int requirement; + + /** Filter to check the HistoryEntry Type */ + @SuppressWarnings("ImmutableEnumChecker") // Predicates are immutable. + private final Predicate typeFilter; + + /** Optional filter on the EppInput. */ + @SuppressWarnings("ImmutableEnumChecker") // Predicates are immutable. + private final Optional> eppInputFilter; + + StatType(int requirement, Predicate typeFilter) { + this(requirement, typeFilter, null); + } + + StatType( + int requirement, + Predicate typeFilter, + Predicate eppInputFilter) { + this.requirement = requirement; + this.typeFilter = typeFilter; + if (eppInputFilter == null) { + this.eppInputFilter = Optional.empty(); + } else { + this.eppInputFilter = Optional.of(eppInputFilter); + } + } + + /** Returns the number of times this StatType must be performed. */ + public int getRequirement() { + return requirement; + } + + /** Returns a more human-readable translation of the enum constant. */ + private String description() { + return Ascii.toLowerCase(this.name().replace('_', ' ')); + } + + /** + * Check if the {@link HistoryEntry} type matches as well as the {@link EppInput} if supplied. + */ + private boolean matches(HistoryEntry.Type historyType, Optional eppInput) { + if (eppInputFilter.isPresent() && eppInput.isPresent()) { + return typeFilter.test(historyType) && eppInputFilter.get().test(eppInput.get()); + } else { + return typeFilter.test(historyType); + } + } + } + + /** Stores counts of how many times each action type was performed. */ + private final Multiset statCounts = HashMultiset.create(); + + /** + * Records data on what actions have been performed by the four numbered OT&E variants of the + * registrar name. + * + *

Stops when it notices that all tests have passed. + */ + private OteStats recordRegistrarHistory(String registrarName) { + ImmutableCollection clientIds = + OteAccountBuilder.createClientIdToTldMap(registrarName).keySet(); + + Query query = + ofy() + .load() + .type(HistoryEntry.class) + .filter("clientId in", clientIds) + .order("modificationTime"); + for (HistoryEntry historyEntry : query) { + try { + record(historyEntry); + } catch (XmlException e) { + throw new RuntimeException("Couldn't parse history entry " + Key.create(historyEntry), e); + } + // Break out early if all tests were passed. + if (wereAllTestsPassed()) { + break; + } + } + return this; + } + + /** Interprets the data in the provided HistoryEntry and increments counters. */ + private void record(final HistoryEntry historyEntry) throws XmlException { + byte[] xmlBytes = historyEntry.getXmlBytes(); + // xmlBytes can be null on contact create and update for safe-harbor compliance. + final Optional eppInput = + (xmlBytes == null) ? Optional.empty() : Optional.of(unmarshal(EppInput.class, xmlBytes)); + if (!statCounts.addAll( + EnumSet.allOf(StatType.class).stream() + .filter(statType -> statType.matches(historyEntry.getType(), eppInput)) + .collect(toImmutableList()))) { + statCounts.add(StatType.UNCLASSIFIED_FLOWS); + } + } + + private boolean wereAllTestsPassed() { + return Arrays.stream(StatType.values()).allMatch(s -> statCounts.count(s) >= s.requirement); + } + + /** Returns the total number of actions taken */ + public int getSize() { + return statCounts.size(); + } + + /** Returns the number of times that a particular StatType was seen */ + public int getCount(StatType statType) { + return statCounts.count(statType); + } + + /** + * Returns a list of failures, any cases where the passed stats fail to meet the required + * thresholds, or the empty list if all requirements are met. + */ + public ImmutableList getFailures() { + return StatType.REQUIRED_STAT_TYPES.stream() + .filter(statType -> statCounts.count(statType) < statType.requirement) + .collect(toImmutableList()); + } + + /** Returns a string showing all possible actions and how many times each was performed. */ + @Override + public String toString() { + return String.format( + "%s\nTOTAL: %d", + EnumSet.allOf(StatType.class).stream() + .map(stat -> String.format("%s: %d", stat.description(), statCounts.count(stat))) + .collect(Collectors.joining("\n")), + statCounts.size()); + } +} diff --git a/java/google/registry/tools/server/VerifyOteAction.java b/java/google/registry/tools/server/VerifyOteAction.java index 814b1903b..94b48e413 100644 --- a/java/google/registry/tools/server/VerifyOteAction.java +++ b/java/google/registry/tools/server/VerifyOteAction.java @@ -14,47 +14,19 @@ package google.registry.tools.server; -import static com.google.common.base.Predicates.equalTo; -import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Maps.toMap; -import static google.registry.model.eppcommon.EppXmlTransformer.unmarshal; -import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.util.CollectionUtils.isNullOrEmpty; -import static google.registry.util.DomainNameUtils.ACE_PREFIX; -import com.google.common.base.Ascii; import com.google.common.base.Joiner; -import com.google.common.base.Predicates; -import com.google.common.collect.HashMultiset; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Multiset; -import com.googlecode.objectify.Key; -import com.googlecode.objectify.cmd.Query; -import google.registry.model.domain.DomainCommand; -import google.registry.model.domain.fee.FeeCreateCommandExtension; -import google.registry.model.domain.launch.LaunchCreateExtension; -import google.registry.model.domain.secdns.SecDnsCreateExtension; -import google.registry.model.domain.secdns.SecDnsUpdateExtension; -import google.registry.model.eppinput.EppInput; -import google.registry.model.eppinput.EppInput.ResourceCommandWrapper; -import google.registry.model.host.HostCommand; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.reporting.HistoryEntry.Type; +import com.google.common.collect.Maps; +import google.registry.model.OteStats; +import google.registry.model.OteStats.StatType; import google.registry.request.Action; import google.registry.request.JsonActionRunner; import google.registry.request.JsonActionRunner.JsonAction; import google.registry.request.auth.Auth; -import google.registry.xml.XmlException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.EnumSet; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; import javax.inject.Inject; /** @@ -62,10 +34,9 @@ import javax.inject.Inject; * OT&E commands that have been run just previously to verification may not be picked up yet. */ @Action( - path = VerifyOteAction.PATH, - method = Action.Method.POST, - auth = Auth.AUTH_INTERNAL_OR_ADMIN -) + path = VerifyOteAction.PATH, + method = Action.Method.POST, + auth = Auth.AUTH_INTERNAL_OR_ADMIN) public class VerifyOteAction implements Runnable, JsonAction { public static final String PATH = "/_dr/admin/verifyOte"; @@ -84,252 +55,31 @@ public class VerifyOteAction implements Runnable, JsonAction { @SuppressWarnings("unchecked") public Map handleJsonRequest(Map json) { final boolean summarize = Boolean.parseBoolean((String) json.get("summarize")); - return toMap( - (List) json.get("registrars"), registrar -> checkRegistrar(registrar, summarize)); + + Map registrarResults = + toMap((List) json.get("registrars"), OteStats::getFromRegistrar); + return Maps.transformValues(registrarResults, stats -> transformOteStats(stats, summarize)); } - /** Checks whether the provided registrar has passed OT&E and returns relevant information. */ - private String checkRegistrar(String registrarName, boolean summarize) { - HistoryEntryStats historyEntryStats = - new HistoryEntryStats().recordRegistrarHistory(registrarName); - List failureMessages = historyEntryStats.findFailures(); - int testsPassed = StatType.NUM_REQUIREMENTS - failureMessages.size(); - String status = failureMessages.isEmpty() ? "PASS" : "FAIL"; + private String transformOteStats(OteStats stats, boolean summarize) { + List failures = stats.getFailures(); + int numRequiredTests = StatType.REQUIRED_STAT_TYPES.size(); + int testsPassed = numRequiredTests - failures.size(); + + String status = failures.isEmpty() ? "PASS" : "FAIL"; return summarize ? String.format( "# actions: %4d - Reqs: [%s] %2d/%2d - Overall: %s", - historyEntryStats.statCounts.size(), - historyEntryStats.toSummary(), - testsPassed, - StatType.NUM_REQUIREMENTS, - status) + stats.getSize(), getSummary(stats), testsPassed, numRequiredTests, status) : String.format( "%s\n%s\nRequirements passed: %2d/%2d\nOverall OT&E status: %s\n", - historyEntryStats, - Joiner.on('\n').join(failureMessages), - testsPassed, - StatType.NUM_REQUIREMENTS, - status); + stats, Joiner.on('\n').join(failures), testsPassed, numRequiredTests, status); } - private static final Predicate HAS_CLAIMS_NOTICE = - eppInput -> { - Optional launchCreate = - eppInput.getSingleExtension(LaunchCreateExtension.class); - return launchCreate.isPresent() && launchCreate.get().getNotice() != null; - }; - - private static final Predicate HAS_SEC_DNS = - eppInput -> - (eppInput.getSingleExtension(SecDnsCreateExtension.class).isPresent()) - || (eppInput.getSingleExtension(SecDnsUpdateExtension.class).isPresent()); - private static final Predicate IS_SUNRISE = - eppInput -> { - Optional launchCreate = - eppInput.getSingleExtension(LaunchCreateExtension.class); - return launchCreate.isPresent() && !isNullOrEmpty(launchCreate.get().getSignedMarks()); - }; - - private static final Predicate IS_IDN = - eppInput -> - ((DomainCommand.Create) - ((ResourceCommandWrapper) eppInput.getCommandWrapper().getCommand()) - .getResourceCommand()) - .getFullyQualifiedDomainName() - .startsWith(ACE_PREFIX); - private static final Predicate IS_SUBORDINATE = - eppInput -> - !isNullOrEmpty( - ((HostCommand.Create) - ((ResourceCommandWrapper) eppInput.getCommandWrapper().getCommand()) - .getResourceCommand()) - .getInetAddresses()); - /** Enum defining the distinct statistics (types of registrar actions) to record. */ - public enum StatType { - CONTACT_CREATES(0, equalTo(Type.CONTACT_CREATE)), - CONTACT_DELETES(0, equalTo(Type.CONTACT_DELETE)), - CONTACT_TRANSFER_APPROVES(0, equalTo(Type.CONTACT_TRANSFER_APPROVE)), - CONTACT_TRANSFER_CANCELS(0, equalTo(Type.CONTACT_TRANSFER_CANCEL)), - CONTACT_TRANSFER_REJECTS(0, equalTo(Type.CONTACT_TRANSFER_REJECT)), - CONTACT_TRANSFER_REQUESTS(0, equalTo(Type.CONTACT_TRANSFER_REQUEST)), - CONTACT_UPDATES(0, equalTo(Type.CONTACT_UPDATE)), - DOMAIN_APPLICATION_CREATES(0, equalTo(Type.DOMAIN_APPLICATION_CREATE)), - DOMAIN_APPLICATION_CREATES_LANDRUSH( - 0, equalTo(Type.DOMAIN_APPLICATION_CREATE), IS_SUNRISE.negate()), - DOMAIN_APPLICATION_CREATES_SUNRISE(0, equalTo(Type.DOMAIN_APPLICATION_CREATE), IS_SUNRISE), - DOMAIN_APPLICATION_DELETES(0, equalTo(Type.DOMAIN_APPLICATION_DELETE)), - DOMAIN_APPLICATION_UPDATES(0, equalTo(Type.DOMAIN_APPLICATION_UPDATE)), - DOMAIN_AUTORENEWS(0, equalTo(Type.DOMAIN_AUTORENEW)), - DOMAIN_CREATES(0, equalTo(Type.DOMAIN_CREATE)), - DOMAIN_CREATES_ASCII(1, equalTo(Type.DOMAIN_CREATE), IS_IDN.negate()), - DOMAIN_CREATES_IDN(1, equalTo(Type.DOMAIN_CREATE), IS_IDN), - DOMAIN_CREATES_START_DATE_SUNRISE(1, equalTo(Type.DOMAIN_CREATE), IS_SUNRISE), - DOMAIN_CREATES_WITH_CLAIMS_NOTICE(1, equalTo(Type.DOMAIN_CREATE), HAS_CLAIMS_NOTICE), - DOMAIN_CREATES_WITH_FEE( - 1, - equalTo(Type.DOMAIN_CREATE), - eppInput -> eppInput.getSingleExtension(FeeCreateCommandExtension.class).isPresent()), - DOMAIN_CREATES_WITH_SEC_DNS(1, equalTo(Type.DOMAIN_CREATE), HAS_SEC_DNS), - DOMAIN_CREATES_WITHOUT_SEC_DNS(0, equalTo(Type.DOMAIN_CREATE), HAS_SEC_DNS.negate()), - DOMAIN_DELETES(2, equalTo(Type.DOMAIN_DELETE)), - DOMAIN_RENEWS(0, equalTo(Type.DOMAIN_RENEW)), - DOMAIN_RESTORES(1, equalTo(Type.DOMAIN_RESTORE)), - DOMAIN_TRANSFER_APPROVES(1, equalTo(Type.DOMAIN_TRANSFER_APPROVE)), - DOMAIN_TRANSFER_CANCELS(1, equalTo(Type.DOMAIN_TRANSFER_CANCEL)), - DOMAIN_TRANSFER_REJECTS(1, equalTo(Type.DOMAIN_TRANSFER_REJECT)), - DOMAIN_TRANSFER_REQUESTS(1, equalTo(Type.DOMAIN_TRANSFER_REQUEST)), - DOMAIN_UPDATES(0, equalTo(Type.DOMAIN_UPDATE)), - DOMAIN_UPDATES_WITH_SEC_DNS(1, equalTo(Type.DOMAIN_UPDATE), HAS_SEC_DNS), - DOMAIN_UPDATES_WITHOUT_SEC_DNS(0, equalTo(Type.DOMAIN_UPDATE), HAS_SEC_DNS.negate()), - HOST_CREATES(0, equalTo(Type.HOST_CREATE)), - HOST_CREATES_EXTERNAL(0, equalTo(Type.HOST_CREATE), IS_SUBORDINATE.negate()), - HOST_CREATES_SUBORDINATE(1, equalTo(Type.HOST_CREATE), IS_SUBORDINATE), - HOST_DELETES(1, equalTo(Type.HOST_DELETE)), - HOST_UPDATES(1, equalTo(Type.HOST_UPDATE)), - UNCLASSIFIED_FLOWS(0, Predicates.alwaysFalse()); - - /** The number of StatTypes with a non-zero requirement. */ - private static final int NUM_REQUIREMENTS = - (int) Stream.of(values()).filter(statType -> statType.requirement > 0).count(); - - /** Required number of times registrars must complete this action. */ - final int requirement; - - /** Filter to check the HistoryEntry Type */ - @SuppressWarnings("ImmutableEnumChecker") // Predicates are immutable. - private final Predicate typeFilter; - - /** Optional filter on the EppInput. */ - @SuppressWarnings("ImmutableEnumChecker") // Predicates are immutable. - private final Optional> eppInputFilter; - - StatType(int requirement, Predicate typeFilter) { - this(requirement, typeFilter, null); - } - - StatType( - int requirement, - Predicate typeFilter, - Predicate eppInputFilter) { - this.requirement = requirement; - this.typeFilter = typeFilter; - if (eppInputFilter == null) { - this.eppInputFilter = Optional.empty(); - } else { - this.eppInputFilter = Optional.of(eppInputFilter); - } - } - - /** Returns a more human-readable translation of the enum constant. */ - String description() { - return Ascii.toLowerCase(this.name().replace('_', ' ')); - } - - /** - * Check if the {@link HistoryEntry} type matches as well as the {@link EppInput} if supplied. - */ - boolean matches(HistoryEntry.Type historyType, Optional eppInput) { - if (eppInputFilter.isPresent() && eppInput.isPresent()) { - return typeFilter.test(historyType) && eppInputFilter.get().test(eppInput.get()); - } else { - return typeFilter.test(historyType); - } - } - } - - /** Class to represent stats derived from HistoryEntry objects on actions taken by registrars. */ - static class HistoryEntryStats { - - /** Stores counts of how many times each action type was performed. */ - Multiset statCounts = HashMultiset.create(); - - /** - * Records data in the passed historyEntryStats object on what actions have been performed by - * the four numbered OT&E variants of the registrar name. - * - *

Stops when it notices that all tests have passed. - */ - HistoryEntryStats recordRegistrarHistory(String registrarName) { - ImmutableList clientIds = - IntStream.rangeClosed(1, 4) - .mapToObj(i -> String.format("%s-%d", registrarName, i)) - .collect(toImmutableList()); - - Query query = - ofy() - .load() - .type(HistoryEntry.class) - .filter("clientId in", clientIds) - .order("modificationTime"); - for (HistoryEntry historyEntry : query) { - try { - record(historyEntry); - } catch (XmlException e) { - throw new RuntimeException("Couldn't parse history entry " + Key.create(historyEntry), e); - } - // Break out early if all tests were passed. - if (wereAllTestsPassed()) { - break; - } - } - return this; - } - - /** Interprets the data in the provided HistoryEntry and increments counters. */ - void record(final HistoryEntry historyEntry) throws XmlException { - byte[] xmlBytes = historyEntry.getXmlBytes(); - // xmlBytes can be null on contact create and update for safe-harbor compliance. - final Optional eppInput = - (xmlBytes == null) ? Optional.empty() : Optional.of(unmarshal(EppInput.class, xmlBytes)); - if (!statCounts.addAll( - EnumSet.allOf(StatType.class).stream() - .filter(statType -> statType.matches(historyEntry.getType(), eppInput)) - .collect(toImmutableList()))) { - statCounts.add(StatType.UNCLASSIFIED_FLOWS); - } - } - - boolean wereAllTestsPassed() { - return Arrays.stream(StatType.values()).allMatch(s -> statCounts.count(s) >= s.requirement); - } - - /** - * Returns a list of failure messages describing any cases where the passed stats fail to meet - * the required thresholds, or the empty list if all requirements are met. - */ - List findFailures() { - List messages = new ArrayList<>(); - for (StatType statType : StatType.values()) { - if (statCounts.count(statType) < statType.requirement) { - messages.add( - String.format( - "Failure: %s %s found.", - (statType.requirement == 1 ? "No" : "Not enough"), statType.description())); - } - } - return messages; - } - - /** Returns a string showing all possible actions and how many times each was performed. */ - @Override - public String toString() { - return String.format( - "%s\nTOTAL: %d", - EnumSet.allOf(StatType.class) - .stream() - .map(stat -> String.format("%s: %d", stat.description(), statCounts.count(stat))) - .collect(Collectors.joining("\n")), - statCounts.size()); - } - - /** Returns a string showing the results of each test, one character per test. */ - String toSummary() { - return EnumSet.allOf(StatType.class) - .stream() - .filter(statType -> statType.requirement > 0) - .sorted() - .map(statType -> (statCounts.count(statType) < statType.requirement) ? "." : "-") - .collect(Collectors.joining("")); - } + private String getSummary(OteStats stats) { + return StatType.REQUIRED_STAT_TYPES.stream() + .sorted() + .map(statType -> (stats.getCount(statType) < statType.getRequirement()) ? "." : "-") + .collect(Collectors.joining("")); } } diff --git a/javatests/google/registry/model/OteStatsTest.java b/javatests/google/registry/model/OteStatsTest.java new file mode 100644 index 000000000..cf0a8724e --- /dev/null +++ b/javatests/google/registry/model/OteStatsTest.java @@ -0,0 +1,147 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model; + +import static com.google.common.truth.Truth.assertThat; + +import google.registry.model.OteStats.StatType; +import google.registry.testing.AppEngineRule; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class OteStatsTest { + + @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build(); + + @Before + public void init() throws Exception { + OteStatsTestHelper.setupHistoryEntries(); + } + + @Test + public void testSuccess_allPass() { + OteStats stats = OteStats.getFromRegistrar("blobio"); + assertThat(stats.getFailures()).isEmpty(); + assertThat(stats.getSize()).isEqualTo(31); + } + + @Test + public void testSuccess_someFailures() { + OteStatsTestHelper.deleteHostDeleteHistoryEntry(); + OteStatsTestHelper.deleteDomainCreateHistoryEntry(); + OteStatsTestHelper.deleteDomainRestoreHistoryEntry(); + OteStats stats = OteStats.getFromRegistrar("blobio"); + assertThat(stats.getFailures()) + .containsExactly( + StatType.DOMAIN_CREATES_IDN, StatType.DOMAIN_RESTORES, StatType.HOST_DELETES) + .inOrder(); + assertThat(stats.getSize()).isEqualTo(35); + } + + @Test + public void testSuccess_toString() { + OteStats stats = OteStats.getFromRegistrar("blobio"); + String expected = + "contact creates: 0\n" + + "contact deletes: 0\n" + + "contact transfer approves: 0\n" + + "contact transfer cancels: 0\n" + + "contact transfer rejects: 0\n" + + "contact transfer requests: 0\n" + + "contact updates: 0\n" + + "domain application creates: 0\n" + + "domain application creates landrush: 0\n" + + "domain application creates sunrise: 0\n" + + "domain application deletes: 0\n" + + "domain application updates: 0\n" + + "domain autorenews: 0\n" + + "domain creates: 5\n" + + "domain creates ascii: 4\n" + + "domain creates idn: 1\n" + + "domain creates start date sunrise: 1\n" + + "domain creates with claims notice: 1\n" + + "domain creates with fee: 1\n" + + "domain creates with sec dns: 1\n" + + "domain creates without sec dns: 4\n" + + "domain deletes: 2\n" + + "domain renews: 0\n" + + "domain restores: 1\n" + + "domain transfer approves: 1\n" + + "domain transfer cancels: 1\n" + + "domain transfer rejects: 1\n" + + "domain transfer requests: 1\n" + + "domain updates: 1\n" + + "domain updates with sec dns: 1\n" + + "domain updates without sec dns: 0\n" + + "host creates: 1\n" + + "host creates external: 0\n" + + "host creates subordinate: 1\n" + + "host deletes: 1\n" + + "host updates: 1\n" + + "unclassified flows: 0\n" + + "TOTAL: 31"; + assertThat(stats.toString()).isEqualTo(expected); + } + + @Test + public void testMissingHostDeletes_toString() { + OteStatsTestHelper.deleteHostDeleteHistoryEntry(); + OteStats stats = OteStats.getFromRegistrar("blobio"); + String expected = + "contact creates: 0\n" + + "contact deletes: 0\n" + + "contact transfer approves: 0\n" + + "contact transfer cancels: 0\n" + + "contact transfer rejects: 0\n" + + "contact transfer requests: 0\n" + + "contact updates: 0\n" + + "domain application creates: 0\n" + + "domain application creates landrush: 0\n" + + "domain application creates sunrise: 0\n" + + "domain application deletes: 0\n" + + "domain application updates: 0\n" + + "domain autorenews: 0\n" + + "domain creates: 5\n" + + "domain creates ascii: 4\n" + + "domain creates idn: 1\n" + + "domain creates start date sunrise: 1\n" + + "domain creates with claims notice: 1\n" + + "domain creates with fee: 1\n" + + "domain creates with sec dns: 1\n" + + "domain creates without sec dns: 4\n" + + "domain deletes: 2\n" + + "domain renews: 0\n" + + "domain restores: 1\n" + + "domain transfer approves: 1\n" + + "domain transfer cancels: 1\n" + + "domain transfer rejects: 1\n" + + "domain transfer requests: 1\n" + + "domain updates: 1\n" + + "domain updates with sec dns: 1\n" + + "domain updates without sec dns: 0\n" + + "host creates: 1\n" + + "host creates external: 0\n" + + "host creates subordinate: 1\n" + + "host deletes: 0\n" + + "host updates: 10\n" + + "unclassified flows: 0\n" + + "TOTAL: 39"; + assertThat(stats.toString()).isEqualTo(expected); + } +} diff --git a/javatests/google/registry/model/OteStatsTestHelper.java b/javatests/google/registry/model/OteStatsTestHelper.java new file mode 100644 index 000000000..410e6c111 --- /dev/null +++ b/javatests/google/registry/model/OteStatsTestHelper.java @@ -0,0 +1,159 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model; + +import static google.registry.testing.DatastoreHelper.deleteResource; +import static google.registry.testing.DatastoreHelper.persistPremiumList; +import static google.registry.testing.DatastoreHelper.persistResource; +import static google.registry.testing.TestDataHelper.loadBytes; +import static google.registry.util.DateTimeUtils.END_OF_TIME; + +import google.registry.model.eppcommon.Trid; +import google.registry.model.reporting.HistoryEntry; +import google.registry.model.reporting.HistoryEntry.Type; +import java.io.IOException; + +public final class OteStatsTestHelper { + + private static HistoryEntry hostDeleteHistoryEntry; + private static HistoryEntry domainCreateHistoryEntry; + private static HistoryEntry domainRestoreHistoryEntry; + + public static void setupHistoryEntries() throws IOException { + persistPremiumList("default_sandbox_list", "sandbox,USD 1000"); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_CREATE) + .setXmlBytes(getBytes("domain_create_sunrise.xml")) + .build()); + domainCreateHistoryEntry = + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_CREATE) + .setXmlBytes(getBytes("domain_create_idn.xml")) + .build()); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_CREATE) + .setXmlBytes(getBytes("domain_create_claim_notice.xml")) + .build()); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_CREATE) + .setXmlBytes(getBytes("domain_create_anchor_tenant_fee_standard.xml")) + .build()); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_CREATE) + .setXmlBytes(getBytes("domain_create_dsdata.xml")) + .build()); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_DELETE) + .setXmlBytes(getBytes("domain_delete.xml")) + .build()); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-2") + .setType(Type.DOMAIN_DELETE) + .setXmlBytes(getBytes("domain_delete.xml")) + .build()); + domainRestoreHistoryEntry = + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_RESTORE) + .setXmlBytes(getBytes("domain_restore.xml")) + .build()); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_TRANSFER_APPROVE) + .setXmlBytes(getBytes("domain_transfer_approve.xml")) + .build()); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_TRANSFER_CANCEL) + .setXmlBytes(getBytes("domain_transfer_cancel.xml")) + .build()); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_TRANSFER_REJECT) + .setXmlBytes(getBytes("domain_transfer_reject.xml")) + .build()); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_TRANSFER_REQUEST) + .setXmlBytes(getBytes("domain_transfer_request.xml")) + .build()); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.DOMAIN_UPDATE) + .setXmlBytes(getBytes("domain_update_with_secdns.xml")) + .build()); + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.HOST_CREATE) + .setXmlBytes(getBytes("host_create_complete.xml")) + .build()); + hostDeleteHistoryEntry = + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.HOST_DELETE) + .setXmlBytes(getBytes("host_delete.xml")) + .build()); + // Persist 10 host updates for a total of 25 history entries. Since these also sort last by + // modification time, when these cause all tests to pass, only the first will be recorded and + // the rest will be skipped. + for (int i = 0; i < 10; i++) { + persistResource( + new HistoryEntry.Builder() + .setClientId("blobio-1") + .setType(Type.HOST_UPDATE) + .setXmlBytes(getBytes("host_update.xml")) + .setTrid(Trid.create(null, String.format("blahtrid-%d", i))) + .setModificationTime(END_OF_TIME) + .build()); + } + } + + public static void deleteHostDeleteHistoryEntry() { + deleteResource(hostDeleteHistoryEntry); + } + + public static void deleteDomainCreateHistoryEntry() { + deleteResource(domainCreateHistoryEntry); + } + + public static void deleteDomainRestoreHistoryEntry() { + deleteResource(domainRestoreHistoryEntry); + } + + private static byte[] getBytes(String filename) throws IOException { + return loadBytes(OteStatsTestHelper.class, filename).read(); + } +} diff --git a/javatests/google/registry/model/testdata/domain_create_anchor_tenant_fee_standard.xml b/javatests/google/registry/model/testdata/domain_create_anchor_tenant_fee_standard.xml new file mode 100644 index 000000000..08030ce7f --- /dev/null +++ b/javatests/google/registry/model/testdata/domain_create_anchor_tenant_fee_standard.xml @@ -0,0 +1,29 @@ + + + + + example.tld + 2 + jd1234 + jd1234 + jd1234 + + abcdefghijklmnop + + + + + + anchor-tenant-test + false + true + + + USD + 26.00 + + + RegistryTool + + diff --git a/javatests/google/registry/tools/server/testdata/domain_create_claim_notice.xml b/javatests/google/registry/model/testdata/domain_create_claim_notice.xml similarity index 100% rename from javatests/google/registry/tools/server/testdata/domain_create_claim_notice.xml rename to javatests/google/registry/model/testdata/domain_create_claim_notice.xml diff --git a/javatests/google/registry/tools/server/testdata/domain_create_dsdata.xml b/javatests/google/registry/model/testdata/domain_create_dsdata.xml similarity index 100% rename from javatests/google/registry/tools/server/testdata/domain_create_dsdata.xml rename to javatests/google/registry/model/testdata/domain_create_dsdata.xml diff --git a/javatests/google/registry/tools/server/testdata/domain_create_idn.xml b/javatests/google/registry/model/testdata/domain_create_idn.xml similarity index 100% rename from javatests/google/registry/tools/server/testdata/domain_create_idn.xml rename to javatests/google/registry/model/testdata/domain_create_idn.xml diff --git a/javatests/google/registry/tools/server/testdata/domain_create_sunrise.xml b/javatests/google/registry/model/testdata/domain_create_sunrise.xml similarity index 100% rename from javatests/google/registry/tools/server/testdata/domain_create_sunrise.xml rename to javatests/google/registry/model/testdata/domain_create_sunrise.xml diff --git a/javatests/google/registry/model/testdata/domain_delete.xml b/javatests/google/registry/model/testdata/domain_delete.xml new file mode 100644 index 000000000..b16cfec36 --- /dev/null +++ b/javatests/google/registry/model/testdata/domain_delete.xml @@ -0,0 +1,17 @@ + + + + + example.tld + + + + + Deleted by registry administrator: Test + false + + + RegistryTool + + diff --git a/javatests/google/registry/tools/server/testdata/domain_restore.xml b/javatests/google/registry/model/testdata/domain_restore.xml similarity index 100% rename from javatests/google/registry/tools/server/testdata/domain_restore.xml rename to javatests/google/registry/model/testdata/domain_restore.xml diff --git a/javatests/google/registry/tools/server/testdata/domain_transfer_approve.xml b/javatests/google/registry/model/testdata/domain_transfer_approve.xml similarity index 100% rename from javatests/google/registry/tools/server/testdata/domain_transfer_approve.xml rename to javatests/google/registry/model/testdata/domain_transfer_approve.xml diff --git a/javatests/google/registry/tools/server/testdata/domain_transfer_cancel.xml b/javatests/google/registry/model/testdata/domain_transfer_cancel.xml similarity index 100% rename from javatests/google/registry/tools/server/testdata/domain_transfer_cancel.xml rename to javatests/google/registry/model/testdata/domain_transfer_cancel.xml diff --git a/javatests/google/registry/tools/server/testdata/domain_transfer_reject.xml b/javatests/google/registry/model/testdata/domain_transfer_reject.xml similarity index 100% rename from javatests/google/registry/tools/server/testdata/domain_transfer_reject.xml rename to javatests/google/registry/model/testdata/domain_transfer_reject.xml diff --git a/javatests/google/registry/tools/server/testdata/domain_transfer_request.xml b/javatests/google/registry/model/testdata/domain_transfer_request.xml similarity index 100% rename from javatests/google/registry/tools/server/testdata/domain_transfer_request.xml rename to javatests/google/registry/model/testdata/domain_transfer_request.xml diff --git a/javatests/google/registry/tools/server/testdata/domain_update_with_secdns.xml b/javatests/google/registry/model/testdata/domain_update_with_secdns.xml similarity index 100% rename from javatests/google/registry/tools/server/testdata/domain_update_with_secdns.xml rename to javatests/google/registry/model/testdata/domain_update_with_secdns.xml diff --git a/javatests/google/registry/model/testdata/host_create_complete.xml b/javatests/google/registry/model/testdata/host_create_complete.xml new file mode 100644 index 000000000..b9bc7afed --- /dev/null +++ b/javatests/google/registry/model/testdata/host_create_complete.xml @@ -0,0 +1,13 @@ + + + + + example.tld + 162.100.102.99 + 4.5.6.7 + 2001:0db8:85a3:0000:0000:8a2e:0370:7334 + + + RegistryTool + + diff --git a/javatests/google/registry/model/testdata/host_delete.xml b/javatests/google/registry/model/testdata/host_delete.xml new file mode 100644 index 000000000..b56877485 --- /dev/null +++ b/javatests/google/registry/model/testdata/host_delete.xml @@ -0,0 +1,17 @@ + + + + + ns1.example.tld + + + + + Deleted by registry administrator: Test + false + + + RegistryTool + + diff --git a/javatests/google/registry/tools/server/testdata/host_update.xml b/javatests/google/registry/model/testdata/host_update.xml similarity index 100% rename from javatests/google/registry/tools/server/testdata/host_update.xml rename to javatests/google/registry/model/testdata/host_update.xml diff --git a/javatests/google/registry/tools/server/BUILD b/javatests/google/registry/tools/server/BUILD index 4141c7ebb..55f428ada 100644 --- a/javatests/google/registry/tools/server/BUILD +++ b/javatests/google/registry/tools/server/BUILD @@ -18,6 +18,7 @@ java_library( "//java/google/registry/request", "//java/google/registry/tools/server", "//java/google/registry/util", + "//javatests/google/registry/model", "//javatests/google/registry/testing", "//javatests/google/registry/testing/mapreduce", "//third_party/objectify:objectify-v4_1", diff --git a/javatests/google/registry/tools/server/VerifyOteActionTest.java b/javatests/google/registry/tools/server/VerifyOteActionTest.java index ab905bd39..450328967 100644 --- a/javatests/google/registry/tools/server/VerifyOteActionTest.java +++ b/javatests/google/registry/tools/server/VerifyOteActionTest.java @@ -15,18 +15,12 @@ package google.registry.tools.server; import static com.google.common.truth.Truth.assertThat; -import static google.registry.testing.DatastoreHelper.deleteResource; -import static google.registry.testing.DatastoreHelper.persistResource; -import static google.registry.util.DateTimeUtils.END_OF_TIME; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import google.registry.model.eppcommon.Trid; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.reporting.HistoryEntry.Type; +import google.registry.model.OteStatsTestHelper; import google.registry.testing.AppEngineRule; import java.util.Map; -import java.util.Map.Entry; import java.util.regex.Pattern; import org.junit.Before; import org.junit.Rule; @@ -42,214 +36,94 @@ public class VerifyOteActionTest { private final VerifyOteAction action = new VerifyOteAction(); - private HistoryEntry hostDeleteHistoryEntry; - private HistoryEntry domainCreateHistoryEntry; - private HistoryEntry domainRestoreHistoryEntry; - @Before public void init() throws Exception { - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_CREATE) - .setXmlBytes(ToolsTestData.loadBytes("domain_create_sunrise.xml").read()) - .build()); - domainCreateHistoryEntry = persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_CREATE) - .setXmlBytes(ToolsTestData.loadBytes("domain_create_idn.xml").read()) - .build()); - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_CREATE) - .setXmlBytes(ToolsTestData.loadBytes("domain_create_claim_notice.xml").read()) - .build()); - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_CREATE) - .setXmlBytes(ToolsTestData.loadBytes("domain_create_anchor_tenant_fee_standard.xml").read()) - .build()); - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_CREATE) - .setXmlBytes(ToolsTestData.loadBytes("domain_create_dsdata.xml").read()) - .build()); - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_DELETE) - .setXmlBytes(ToolsTestData.loadBytes("domain_delete.xml").read()) - .build()); - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-2") - .setType(Type.DOMAIN_DELETE) - .setXmlBytes(ToolsTestData.loadBytes("domain_delete.xml").read()) - .build()); - domainRestoreHistoryEntry = persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_RESTORE) - .setXmlBytes(ToolsTestData.loadBytes("domain_restore.xml").read()) - .build()); - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_TRANSFER_APPROVE) - .setXmlBytes(ToolsTestData.loadBytes("domain_transfer_approve.xml").read()) - .build()); - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_TRANSFER_CANCEL) - .setXmlBytes(ToolsTestData.loadBytes("domain_transfer_cancel.xml").read()) - .build()); - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_TRANSFER_REJECT) - .setXmlBytes(ToolsTestData.loadBytes("domain_transfer_reject.xml").read()) - .build()); - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_TRANSFER_REQUEST) - .setXmlBytes(ToolsTestData.loadBytes("domain_transfer_request.xml").read()) - .build()); - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.DOMAIN_UPDATE) - .setXmlBytes(ToolsTestData.loadBytes("domain_update_with_secdns.xml").read()) - .build()); - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.HOST_CREATE) - .setXmlBytes(ToolsTestData.loadBytes("host_create_complete.xml").read()) - .build()); - hostDeleteHistoryEntry = - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.HOST_DELETE) - .setXmlBytes(ToolsTestData.loadBytes("host_delete.xml").read()) - .build()); - // Persist 10 host updates for a total of 25 history entries. Since these also sort last by - // modification time, when these cause all tests to pass, only the first will be recorded and - // the rest will be skipped. - for (int i = 0; i < 10; i++) { - persistResource( - new HistoryEntry.Builder() - .setClientId("blobio-1") - .setType(Type.HOST_UPDATE) - .setXmlBytes(ToolsTestData.loadBytes("host_update.xml").read()) - .setTrid(Trid.create(null, String.format("blahtrid-%d", i))) - .setModificationTime(END_OF_TIME) - .build()); - } + OteStatsTestHelper.setupHistoryEntries(); } @Test public void testSuccess_summarize_allPass() { - Map response = - action.handleJsonRequest( - ImmutableMap.of("summarize", "true", "registrars", ImmutableList.of("blobio"))); - assertThat(response) - .containsExactly( - "blobio", "# actions: 31 - Reqs: [----------------] 16/16 - Overall: PASS"); + assertThat(getResponse(true)) + .isEqualTo("# actions: 31 - Reqs: [----------------] 16/16 - Overall: PASS"); } @Test - public void testSuccess_summarize_someFailures() { - deleteResource(hostDeleteHistoryEntry); - deleteResource(domainCreateHistoryEntry); - deleteResource(domainRestoreHistoryEntry); - Map response = - action.handleJsonRequest( - ImmutableMap.of("summarize", "true", "registrars", ImmutableList.of("blobio"))); - assertThat(response) - .containsExactly( - "blobio", "# actions: 35 - Reqs: [-.-----.------.-] 13/16 - Overall: FAIL"); + public void testFailure_summarize_someFailures() { + OteStatsTestHelper.deleteDomainCreateHistoryEntry(); + OteStatsTestHelper.deleteDomainRestoreHistoryEntry(); + OteStatsTestHelper.deleteHostDeleteHistoryEntry(); + assertThat(getResponse(true)) + .isEqualTo("# actions: 35 - Reqs: [-.-----.------.-] 13/16 - Overall: FAIL"); } @Test public void testSuccess_passNotSummarized() { - Map response = - action.handleJsonRequest( - ImmutableMap.of("summarize", "false", "registrars", ImmutableList.of("blobio"))); - - for (Entry registrar : response.entrySet()) { - assertThat(registrar.getKey()).matches("blobio"); - String expectedOteStatus = - "domain creates idn: 1\n" - + "domain creates start date sunrise: 1\n" - + "domain creates with claims notice: 1\n" - + "domain creates with fee: 1\n" - + "domain creates with sec dns: 1\n" - + ".*" - + "domain deletes: 2\n" - + ".*" - + "domain restores: 1\n" - + "domain transfer approves: 1\n" - + "domain transfer cancels: 1\n" - + "domain transfer rejects: 1\n" - + "domain transfer requests: 1\n" - + ".*" - + "domain updates with sec dns: 1\n" - + ".*" - + "host creates subordinate: 1\n" - + "host deletes: 1\n" - + "host updates: 1\n" - + ".*" - + "Requirements passed: 16/16\n" - + "Overall OT&E status: PASS\n"; - Pattern expectedOteStatusPattern = Pattern.compile(expectedOteStatus, Pattern.DOTALL); - assertThat(registrar.getValue().toString()).containsMatch(expectedOteStatusPattern); - } + String expectedOteStatus = + "domain creates idn: 1\n" + + "domain creates start date sunrise: 1\n" + + "domain creates with claims notice: 1\n" + + "domain creates with fee: 1\n" + + "domain creates with sec dns: 1\n" + + ".*" + + "domain deletes: 2\n" + + ".*" + + "domain restores: 1\n" + + "domain transfer approves: 1\n" + + "domain transfer cancels: 1\n" + + "domain transfer rejects: 1\n" + + "domain transfer requests: 1\n" + + ".*" + + "domain updates with sec dns: 1\n" + + ".*" + + "host creates subordinate: 1\n" + + "host deletes: 1\n" + + "host updates: 1\n" + + ".*" + + "Requirements passed: 16/16\n" + + "Overall OT&E status: PASS\n"; + Pattern expectedOteStatusPattern = Pattern.compile(expectedOteStatus, Pattern.DOTALL); + assertThat(getResponse(false)).containsMatch(expectedOteStatusPattern); } @Test public void testFailure_missingHostDelete() { - deleteResource(hostDeleteHistoryEntry); + OteStatsTestHelper.deleteHostDeleteHistoryEntry(); + String expectedOteStatus = + "domain creates idn: 1\n" + + "domain creates start date sunrise: 1\n" + + "domain creates with claims notice: 1\n" + + "domain creates with fee: 1\n" + + "domain creates with sec dns: 1\n" + + ".*" + + "domain deletes: 2\n" + + ".*" + + "domain restores: 1\n" + + "domain transfer approves: 1\n" + + "domain transfer cancels: 1\n" + + "domain transfer rejects: 1\n" + + "domain transfer requests: 1\n" + + ".*" + + "domain updates with sec dns: 1\n" + + ".*" + + "host creates subordinate: 1\n" + + "host deletes: 0\n" + + "host updates: 10\n" + + ".*" + + "Requirements passed: 15/16\n" + + "Overall OT&E status: FAIL\n"; + Pattern expectedOteStatusPattern = Pattern.compile(expectedOteStatus, Pattern.DOTALL); + assertThat(getResponse(false)).containsMatch(expectedOteStatusPattern); + } + private String getResponse(boolean summarize) { Map response = action.handleJsonRequest( - ImmutableMap.of("summarize", "false", "registrars", ImmutableList.of("blobio"))); - - for (Entry registrar : response.entrySet()) { - assertThat(registrar.getKey()).matches("blobio"); - String oteStatus = registrar.getValue().toString(); - - String expectedOteStatus = - "domain creates idn: 1\n" - + "domain creates start date sunrise: 1\n" - + "domain creates with claims notice: 1\n" - + "domain creates with fee: 1\n" - + "domain creates with sec dns: 1\n" - + ".*" - + "domain deletes: 2\n" - + ".*" - + "domain restores: 1\n" - + "domain transfer approves: 1\n" - + "domain transfer cancels: 1\n" - + "domain transfer rejects: 1\n" - + "domain transfer requests: 1\n" - + ".*" - + "domain updates with sec dns: 1\n" - + ".*" - + "host creates subordinate: 1\n" - + "host deletes: 0\n" - + "host updates: 10\n" - + ".*" - + "Requirements passed: 15/16\n" - + "Overall OT&E status: FAIL\n"; - Pattern expectedOteStatusPattern = Pattern.compile(expectedOteStatus, Pattern.DOTALL); - assertThat(oteStatus).containsMatch(expectedOteStatusPattern); - } + ImmutableMap.of( + "summarize", + Boolean.toString(summarize), + "registrars", + ImmutableList.of("blobio"))); + assertThat(response).containsKey("blobio"); + return response.get("blobio").toString(); } } From 566f60d49508d4ef2c4b95e77fbd67e72a3f035c Mon Sep 17 00:00:00 2001 From: guyben Date: Wed, 2 Jan 2019 11:19:52 -0800 Subject: [PATCH 122/134] Allow using empty string to remove all whitelisted IPs Currently, you have to set "--ip_whitelist=null", which is unintuitive. This adds the option to just give an empty string: "--ip_whitelist=" ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=227550896 --- .../tools/CreateOrUpdateRegistrarCommand.java | 6 ++++-- .../tools/UpdateRegistrarCommandTest.java | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java b/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java index 4b2bc3028..d955ff4f0 100644 --- a/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java +++ b/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java @@ -154,7 +154,7 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand { @Parameter( names = "--ip_whitelist", - description = "Comma-delimited list of IP ranges") + description = "Comma-delimited list of IP ranges. An empty string clears the whitelist.") List ipWhitelist = new ArrayList<>(); @Nullable @@ -332,7 +332,9 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand { ImmutableList.Builder ipWhitelistBuilder = new ImmutableList.Builder<>(); if (!(ipWhitelist.size() == 1 && ipWhitelist.get(0).contains("null"))) { for (String ipRange : ipWhitelist) { - ipWhitelistBuilder.add(CidrAddressBlock.create(ipRange)); + if (!ipRange.isEmpty()) { + ipWhitelistBuilder.add(CidrAddressBlock.create(ipRange)); + } } } builder.setIpAddressWhitelist(ipWhitelistBuilder.build()); diff --git a/javatests/google/registry/tools/UpdateRegistrarCommandTest.java b/javatests/google/registry/tools/UpdateRegistrarCommandTest.java index 44458fa1f..a911156c5 100644 --- a/javatests/google/registry/tools/UpdateRegistrarCommandTest.java +++ b/javatests/google/registry/tools/UpdateRegistrarCommandTest.java @@ -134,7 +134,7 @@ public class UpdateRegistrarCommandTest extends CommandTestCase Date: Wed, 2 Jan 2019 12:18:00 -0800 Subject: [PATCH 123/134] Change pre-epoch times in RdeFixtures ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=227560807 --- .../google/registry/rde/RdeFixtures.java | 20 +++++++++---------- ...pReduce_withDomain_producesExpectedXml.xml | 10 +++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/javatests/google/registry/rde/RdeFixtures.java b/javatests/google/registry/rde/RdeFixtures.java index eb8bbd8fc..3eb79d95a 100644 --- a/javatests/google/registry/rde/RdeFixtures.java +++ b/javatests/google/registry/rde/RdeFixtures.java @@ -75,8 +75,8 @@ final class RdeFixtures { .setClientId("TheRegistrar") .setCost(Money.of(USD, 26)) .setPeriodYears(2) - .setEventTime(DateTime.parse("1910-01-01T00:00:00Z")) - .setBillingTime(DateTime.parse("1910-01-01T00:00:00Z")) + .setEventTime(DateTime.parse("1990-01-01T00:00:00Z")) + .setBillingTime(DateTime.parse("1990-01-01T00:00:00Z")) .setParent(historyEntry) .build()); domain = domain.asBuilder() @@ -94,7 +94,7 @@ final class RdeFixtures { .setDsData(ImmutableSet.of(DelegationSignerData.create( 123, 200, 230, base16().decode("1234567890")))) .setFullyQualifiedDomainName(Idn.toASCII("love." + tld)) - .setLastTransferTime(DateTime.parse("1910-01-01T00:00:00Z")) + .setLastTransferTime(DateTime.parse("1990-01-01T00:00:00Z")) .setLastEppUpdateClientId("IntoTheTempest") .setLastEppUpdateTime(clock.nowUtc()) .setIdnTableName("extended_latin") @@ -104,7 +104,7 @@ final class RdeFixtures { Key.create( makeHostResource( clock, "ns2.cat.みんな", "bad:f00d:cafe::15:beef")))) - .setRegistrationExpirationTime(DateTime.parse("1930-01-01T00:00:00Z")) + .setRegistrationExpirationTime(DateTime.parse("1994-01-01T00:00:00Z")) .setGracePeriods(ImmutableSet.of( GracePeriod.forBillingEvent(GracePeriodStatus.RENEW, persistResource( @@ -114,12 +114,12 @@ final class RdeFixtures { .setClientId("TheRegistrar") .setCost(Money.of(USD, 456)) .setPeriodYears(2) - .setEventTime(DateTime.parse("1920-01-01T00:00:00Z")) - .setBillingTime(DateTime.parse("1920-01-01T00:00:00Z")) + .setEventTime(DateTime.parse("1992-01-01T00:00:00Z")) + .setBillingTime(DateTime.parse("1992-01-01T00:00:00Z")) .setParent(historyEntry) .build())), GracePeriod.create( - GracePeriodStatus.TRANSFER, DateTime.parse("1920-01-01T00:00:00Z"), "foo", null))) + GracePeriodStatus.TRANSFER, DateTime.parse("1992-01-01T00:00:00Z"), "foo", null))) .setSubordinateHosts(ImmutableSet.of("home.by.horror.haunted")) .setStatusValues(ImmutableSet.of( StatusValue.CLIENT_DELETE_PROHIBITED, @@ -150,7 +150,7 @@ final class RdeFixtures { .setTransferData(new TransferData.Builder() .setGainingClientId("gaining") .setLosingClientId("losing") - .setPendingTransferExpirationTime(DateTime.parse("1925-04-20T00:00:00Z")) + .setPendingTransferExpirationTime(DateTime.parse("1993-04-20T00:00:00Z")) .setServerApproveBillingEvent(Key.create(billingEvent)) .setServerApproveAutorenewEvent( Key.create(persistResource( @@ -174,7 +174,7 @@ final class RdeFixtures { .build()))) .setServerApproveEntities(ImmutableSet.of( Key.create(billingEvent))) - .setTransferRequestTime(DateTime.parse("1919-01-01T00:00:00Z")) + .setTransferRequestTime(DateTime.parse("1991-01-01T00:00:00Z")) .setTransferStatus(TransferStatus.PENDING) .setTransferRequestTrid(Trid.create("client-trid", "server-trid")) .build()) @@ -228,7 +228,7 @@ final class RdeFixtures { .setPersistedCurrentSponsorClientId("BusinessCat") .setFullyQualifiedHostName(Idn.toASCII(fqhn)) .setInetAddresses(ImmutableSet.of(InetAddresses.forString(ip))) - .setLastTransferTime(DateTime.parse("1910-01-01T00:00:00Z")) + .setLastTransferTime(DateTime.parse("1990-01-01T00:00:00Z")) .setLastEppUpdateClientId("CeilingCat") .setLastEppUpdateTime(clock.nowUtc()) .setStatusValues(ImmutableSet.of(StatusValue.OK)) diff --git a/javatests/google/registry/rde/testdata/testMapReduce_withDomain_producesExpectedXml.xml b/javatests/google/registry/rde/testdata/testMapReduce_withDomain_producesExpectedXml.xml index ee2aa6249..bed30e277 100644 --- a/javatests/google/registry/rde/testdata/testMapReduce_withDomain_producesExpectedXml.xml +++ b/javatests/google/registry/rde/testdata/testMapReduce_withDomain_producesExpectedXml.xml @@ -113,7 +113,7 @@ 1999-12-31T00:00:00Z CeilingCat 1999-12-31T00:00:00Z - 1910-01-01T00:00:00Z + 1990-01-01T00:00:00Z @@ -126,7 +126,7 @@ 1999-12-31T00:00:00Z CeilingCat 1999-12-31T00:00:00Z - 1910-01-01T00:00:00Z + 1990-01-01T00:00:00Z @@ -160,13 +160,13 @@ 1234567890 - 1925-04-20T00:00:00Z + 1993-04-20T00:00:00Z serverApproved gaining - 1919-01-01T00:00:00Z + 1991-01-01T00:00:00Z losing - 1925-04-20T00:00:00Z + 1993-04-20T00:00:00Z 2001-01-01T00:00:00Z From 2af24c945cc07d018cbe87780adb5c0f0f40d991 Mon Sep 17 00:00:00 2001 From: guyben Date: Thu, 3 Jan 2019 13:05:21 -0800 Subject: [PATCH 124/134] Tweak the registrar-ote-setup web console ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=227736173 --- java/google/registry/ui/css/forms.css | 5 ++ .../otesetup/ConsoleOteSetupAction.java | 3 +- .../registrar/RegistrarConsoleModule.java | 6 ++ .../registry/ui/soy/otesetup/Console.soy | 57 +++++++++++++------ .../otesetup/ConsoleOteSetupActionTest.java | 19 +++++++ 5 files changed, 71 insertions(+), 19 deletions(-) diff --git a/java/google/registry/ui/css/forms.css b/java/google/registry/ui/css/forms.css index f1844a4fa..29c5ca559 100644 --- a/java/google/registry/ui/css/forms.css +++ b/java/google/registry/ui/css/forms.css @@ -54,6 +54,11 @@ tr.subsection h3::first-letter { width: 170px; } +.description ol { + list-style-type: decimal; + margin-left: 2em; +} + /* Setting groups and labels. */ diff --git a/java/google/registry/ui/server/otesetup/ConsoleOteSetupAction.java b/java/google/registry/ui/server/otesetup/ConsoleOteSetupAction.java index 6b9ada0a5..6af97abf8 100644 --- a/java/google/registry/ui/server/otesetup/ConsoleOteSetupAction.java +++ b/java/google/registry/ui/server/otesetup/ConsoleOteSetupAction.java @@ -98,6 +98,7 @@ public final class ConsoleOteSetupAction implements Runnable { @Inject @Config("base64StringGenerator") StringGenerator passwordGenerator; @Inject @Parameter("clientId") Optional clientId; @Inject @Parameter("email") Optional email; + @Inject @Parameter("password") Optional optionalPassword; @Inject ConsoleOteSetupAction() {} @@ -170,7 +171,7 @@ public final class ConsoleOteSetupAction implements Runnable { data.put("contactEmail", email.get()); try { - String password = passwordGenerator.createString(PASSWORD_LENGTH); + String password = optionalPassword.orElse(passwordGenerator.createString(PASSWORD_LENGTH)); ImmutableMap clientIdToTld = OteAccountBuilder.forClientId(clientId.get()) .addContact(email.get()) diff --git a/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java b/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java index d14e89b8b..f4b7d55b1 100644 --- a/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java +++ b/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java @@ -53,4 +53,10 @@ public final class RegistrarConsoleModule { static String provideEmail(HttpServletRequest req) { return extractRequiredParameter(req, "email"); } + + @Provides + @Parameter("password") + static Optional provideOptionalPassword(HttpServletRequest req) { + return extractOptionalParameter(req, "password"); + } } diff --git a/java/google/registry/ui/soy/otesetup/Console.soy b/java/google/registry/ui/soy/otesetup/Console.soy index e96adf4a1..10a477e6d 100644 --- a/java/google/registry/ui/soy/otesetup/Console.soy +++ b/java/google/registry/ui/soy/otesetup/Console.soy @@ -76,9 +76,9 @@ @@ -102,22 +102,34 @@ {@param? contactEmail: string} /** If set - an initial value to fill for the email. */

+ + + + {call registry.soy.forms.inputFieldRowWithValue} - {param label: 'Base clientId' /} - {param name: 'clientId' /} - {param value: $baseClientId /} - {param placeholder: 'registrar\'s ID' /} - {param description kind="text"} - must 1) consist of only lowercase letters, numbers, or hyphens, - 2) start with a letter, and 3) be between 3 and 14 characters (inclusive). - We require 1 and 2 since the registrar name will be used to create TLDs, - and we require 3 since we append \"-[1234]\" to the name to create client - IDs which are required by the EPP XML schema to be between 3-16 chars. - {/param} - {param readonly: false /} - {/call} - {call registry.soy.forms.inputFieldRowWithValue} - {param label: 'contact email' /} + {param label: 'Contact email' /} {param name: 'email' /} {param value: $contactEmail /} {param placeholder: 'registrar\'s assigned email' /} @@ -126,6 +138,15 @@ {/param} {param readonly: false /} {/call} + {call registry.soy.forms.inputFieldRowWithValue} + {param label: 'Password (optional)' /} + {param name: 'password' /} + {param placeholder: 'Optional password' /} + {param description kind="text"} + The password used to for EPP login. Leave blank to auto-generate a password. + {/param} + {param readonly: false /} + {/call} {call registry.soy.forms.submitRow} {param label: 'create' /} {/call} diff --git a/javatests/google/registry/ui/server/otesetup/ConsoleOteSetupActionTest.java b/javatests/google/registry/ui/server/otesetup/ConsoleOteSetupActionTest.java index 79e0f79d2..da34d9d0e 100644 --- a/javatests/google/registry/ui/server/otesetup/ConsoleOteSetupActionTest.java +++ b/javatests/google/registry/ui/server/otesetup/ConsoleOteSetupActionTest.java @@ -97,6 +97,7 @@ public final class ConsoleOteSetupActionTest { action.productName = "Nomulus"; action.clientId = Optional.empty(); action.email = Optional.empty(); + action.optionalPassword = Optional.empty(); action.passwordGenerator = new DeterministicStringGenerator("abcdefghijklmnopqrstuvwxyz"); message = new MimeMessage(Session.getDefaultInstance(new Properties(), null)); @@ -160,6 +161,24 @@ public final class ConsoleOteSetupActionTest { + "Gave user contact@registry.example web access to these Registrars\n"); } + @Test + public void testPost_authorized_setPassword() throws Exception { + action.clientId = Optional.of("myclientid"); + action.email = Optional.of("contact@registry.example"); + action.optionalPassword = Optional.of("SomePassword"); + action.method = Method.POST; + action.run(); + + // We just check some samples to make sure OteAccountBuilder was called successfully. We aren't + // checking that all the entities are there or that they have the correct values. + assertThat(loadByClientId("myclientid-4").get().testPassword("SomePassword")) + .isTrue(); + assertThat(response.getPayload()) + .contains("

OT&E successfully created for registrar myclientid!

"); + assertThat(response.getPayload()) + .contains("SomePassword"); + } + @Test public void testPost_unauthorized() { action.registrarAccessor = From 577c6f6bc9cf87fe9d85a700a2b0c1066b862e41 Mon Sep 17 00:00:00 2001 From: mcilwain Date: Thu, 3 Jan 2019 13:20:39 -0800 Subject: [PATCH 125/134] Add a mapreduce to delete all domain applications This also deletes associated entities including indexes and history entries. This needs to run as a prerequisite to [] which deletes all domain application code entirely. The entities themselves need to be deleted first so that loading DomainBases in the future doesn't accidentally get applications which the code no longer knows how to handle. This deletion is safe to perform because the only remaining applications in our system are historical and we no longer refer to them. Backups will be retained in BigQuery. This mapreduce will be deleted at the same time that the DomainApplication code is. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=227738528 --- .../registry/env/common/tools/WEB-INF/web.xml | 6 ++ .../module/tools/ToolsRequestComponent.java | 2 + .../KillAllDomainApplicationsAction.java | 97 +++++++++++++++++++ .../module/tools/testdata/tools_routing.txt | 43 ++++---- .../KillAllDomainApplicationsActionTest.java | 94 ++++++++++++++++++ 5 files changed, 221 insertions(+), 21 deletions(-) create mode 100644 java/google/registry/tools/server/KillAllDomainApplicationsAction.java create mode 100644 javatests/google/registry/tools/server/KillAllDomainApplicationsActionTest.java diff --git a/java/google/registry/env/common/tools/WEB-INF/web.xml b/java/google/registry/env/common/tools/WEB-INF/web.xml index eb7dde515..0bd9341d1 100644 --- a/java/google/registry/env/common/tools/WEB-INF/web.xml +++ b/java/google/registry/env/common/tools/WEB-INF/web.xml @@ -72,6 +72,12 @@ /_dr/task/resaveAllHistoryEntries + + + tools-servlet + /_dr/task/killAllDomainApplications + + tools-servlet diff --git a/java/google/registry/module/tools/ToolsRequestComponent.java b/java/google/registry/module/tools/ToolsRequestComponent.java index 633a38709..e8e6879c5 100644 --- a/java/google/registry/module/tools/ToolsRequestComponent.java +++ b/java/google/registry/module/tools/ToolsRequestComponent.java @@ -34,6 +34,7 @@ import google.registry.tools.server.CreatePremiumListAction; import google.registry.tools.server.DeleteEntityAction; import google.registry.tools.server.GenerateZoneFilesAction; import google.registry.tools.server.KillAllCommitLogsAction; +import google.registry.tools.server.KillAllDomainApplicationsAction; import google.registry.tools.server.KillAllEppResourcesAction; import google.registry.tools.server.ListDomainsAction; import google.registry.tools.server.ListHostsAction; @@ -69,6 +70,7 @@ interface ToolsRequestComponent { FlowComponent.Builder flowComponentBuilder(); GenerateZoneFilesAction generateZoneFilesAction(); KillAllCommitLogsAction killAllCommitLogsAction(); + KillAllDomainApplicationsAction killAllDomainApplicationsAction(); KillAllEppResourcesAction killAllEppResourcesAction(); ListDomainsAction listDomainsAction(); ListHostsAction listHostsAction(); diff --git a/java/google/registry/tools/server/KillAllDomainApplicationsAction.java b/java/google/registry/tools/server/KillAllDomainApplicationsAction.java new file mode 100644 index 000000000..8b639ba09 --- /dev/null +++ b/java/google/registry/tools/server/KillAllDomainApplicationsAction.java @@ -0,0 +1,97 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.server; + +import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.request.Action.Method.POST; +import static google.registry.util.PipelineUtils.createJobPath; + +import com.google.appengine.tools.mapreduce.Mapper; +import com.google.common.collect.ImmutableList; +import com.google.common.flogger.FluentLogger; +import com.googlecode.objectify.Key; +import google.registry.mapreduce.MapreduceRunner; +import google.registry.model.domain.DomainApplication; +import google.registry.model.index.DomainApplicationIndex; +import google.registry.model.index.EppResourceIndex; +import google.registry.model.reporting.HistoryEntry; +import google.registry.request.Action; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import javax.inject.Inject; + +/** + * Deletes all {@link DomainApplication} entities in Datastore. + * + *

This also deletes the corresponding {@link DomainApplicationIndex}, {@link EppResourceIndex}, + * and descendent {@link HistoryEntry}s. + */ +@Action(path = "/_dr/task/killAllDomainApplications", method = POST, auth = Auth.AUTH_INTERNAL_ONLY) +public class KillAllDomainApplicationsAction implements Runnable { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + @Inject MapreduceRunner mrRunner; + @Inject Response response; + + @Inject + KillAllDomainApplicationsAction() {} + + @Override + public void run() { + response.sendJavaScriptRedirect( + createJobPath( + mrRunner + .setJobName("Delete all domain applications and associated entities") + .setModuleName("tools") + .runMapOnly( + new KillAllDomainApplicationsMapper(), + ImmutableList.of(createEntityInput(DomainApplication.class))))); + } + + static class KillAllDomainApplicationsMapper extends Mapper { + + private static final long serialVersionUID = 2862967335000340688L; + + @Override + public void map(final DomainApplication application) { + ofy() + .transact( + () -> { + if (ofy().load().entity(application).now() == null) { + getContext().incrementCounter("applications already deleted"); + return; + } + Key applicationKey = Key.create(application); + DomainApplicationIndex dai = + ofy().load().key(DomainApplicationIndex.createKey(application)).now(); + EppResourceIndex eri = + ofy().load().entity(EppResourceIndex.create(applicationKey)).now(); + if (dai == null || eri == null) { + logger.atSevere().log( + "Missing index(es) for application %s; skipping.", applicationKey); + getContext().incrementCounter("missing indexes"); + return; + } + // Delete the application, its descendents, and the indexes. + ofy().delete().keys(ofy().load().ancestor(application).keys()); + ofy().delete().entities(dai, eri); + logger.atInfo().log("Deleted domain application %s.", applicationKey); + getContext().incrementCounter("applications deleted"); + }); + } + } +} diff --git a/javatests/google/registry/module/tools/testdata/tools_routing.txt b/javatests/google/registry/module/tools/testdata/tools_routing.txt index 40756e6f4..2d953941c 100644 --- a/javatests/google/registry/module/tools/testdata/tools_routing.txt +++ b/javatests/google/registry/module/tools/testdata/tools_routing.txt @@ -1,21 +1,22 @@ -PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY -/_dr/admin/createGroups CreateGroupsAction POST n INTERNAL,API APP ADMIN -/_dr/admin/createPremiumList CreatePremiumListAction POST n INTERNAL,API APP ADMIN -/_dr/admin/deleteEntity DeleteEntityAction GET n INTERNAL,API APP ADMIN -/_dr/admin/list/domains ListDomainsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/list/hosts ListHostsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/list/premiumLists ListPremiumListsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/list/registrars ListRegistrarsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/list/reservedLists ListReservedListsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/list/tlds ListTldsAction GET,POST n INTERNAL,API APP ADMIN -/_dr/admin/updatePremiumList UpdatePremiumListAction POST n INTERNAL,API APP ADMIN -/_dr/admin/verifyOte VerifyOteAction POST n INTERNAL,API APP ADMIN -/_dr/epptool EppToolAction POST n INTERNAL,API APP ADMIN -/_dr/loadtest LoadTestAction POST y INTERNAL,API APP ADMIN -/_dr/task/generateZoneFiles GenerateZoneFilesAction POST n INTERNAL,API APP ADMIN -/_dr/task/killAllCommitLogs KillAllCommitLogsAction POST n INTERNAL APP IGNORED -/_dr/task/killAllEppResources KillAllEppResourcesAction POST n INTERNAL APP IGNORED -/_dr/task/pollMapreduce PollMapreduceAction POST n INTERNAL APP IGNORED -/_dr/task/refreshDnsForAllDomains RefreshDnsForAllDomainsAction GET n INTERNAL,API APP ADMIN -/_dr/task/resaveAllHistoryEntries ResaveAllHistoryEntriesAction GET n INTERNAL,API APP ADMIN -/_dr/task/restoreCommitLogs RestoreCommitLogsAction POST y INTERNAL,API APP ADMIN +PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY +/_dr/admin/createGroups CreateGroupsAction POST n INTERNAL,API APP ADMIN +/_dr/admin/createPremiumList CreatePremiumListAction POST n INTERNAL,API APP ADMIN +/_dr/admin/deleteEntity DeleteEntityAction GET n INTERNAL,API APP ADMIN +/_dr/admin/list/domains ListDomainsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/list/hosts ListHostsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/list/premiumLists ListPremiumListsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/list/registrars ListRegistrarsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/list/reservedLists ListReservedListsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/list/tlds ListTldsAction GET,POST n INTERNAL,API APP ADMIN +/_dr/admin/updatePremiumList UpdatePremiumListAction POST n INTERNAL,API APP ADMIN +/_dr/admin/verifyOte VerifyOteAction POST n INTERNAL,API APP ADMIN +/_dr/epptool EppToolAction POST n INTERNAL,API APP ADMIN +/_dr/loadtest LoadTestAction POST y INTERNAL,API APP ADMIN +/_dr/task/generateZoneFiles GenerateZoneFilesAction POST n INTERNAL,API APP ADMIN +/_dr/task/killAllCommitLogs KillAllCommitLogsAction POST n INTERNAL APP IGNORED +/_dr/task/killAllDomainApplications KillAllDomainApplicationsAction POST n INTERNAL APP IGNORED +/_dr/task/killAllEppResources KillAllEppResourcesAction POST n INTERNAL APP IGNORED +/_dr/task/pollMapreduce PollMapreduceAction POST n INTERNAL APP IGNORED +/_dr/task/refreshDnsForAllDomains RefreshDnsForAllDomainsAction GET n INTERNAL,API APP ADMIN +/_dr/task/resaveAllHistoryEntries ResaveAllHistoryEntriesAction GET n INTERNAL,API APP ADMIN +/_dr/task/restoreCommitLogs RestoreCommitLogsAction POST y INTERNAL,API APP ADMIN diff --git a/javatests/google/registry/tools/server/KillAllDomainApplicationsActionTest.java b/javatests/google/registry/tools/server/KillAllDomainApplicationsActionTest.java new file mode 100644 index 000000000..102df8adf --- /dev/null +++ b/javatests/google/registry/tools/server/KillAllDomainApplicationsActionTest.java @@ -0,0 +1,94 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.server; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.testing.DatastoreHelper.createTlds; +import static google.registry.testing.DatastoreHelper.persistActiveContact; +import static google.registry.testing.DatastoreHelper.persistActiveDomain; +import static google.registry.testing.DatastoreHelper.persistActiveDomainApplication; +import static google.registry.testing.DatastoreHelper.persistActiveHost; +import static google.registry.testing.DatastoreHelper.persistResource; + +import com.googlecode.objectify.Key; +import google.registry.model.contact.ContactResource; +import google.registry.model.domain.DomainApplication; +import google.registry.model.domain.DomainResource; +import google.registry.model.host.HostResource; +import google.registry.model.index.DomainApplicationIndex; +import google.registry.model.index.EppResourceIndex; +import google.registry.model.index.ForeignKeyIndex; +import google.registry.model.reporting.HistoryEntry; +import google.registry.testing.FakeResponse; +import google.registry.testing.mapreduce.MapreduceTestCase; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link KillAllDomainApplicationsAction}. */ +@RunWith(JUnit4.class) +public class KillAllDomainApplicationsActionTest + extends MapreduceTestCase { + + private void runMapreduce() throws Exception { + action = new KillAllDomainApplicationsAction(); + action.mrRunner = makeDefaultRunner(); + action.response = new FakeResponse(); + action.run(); + executeTasksUntilEmpty("mapreduce"); + } + + @Test + public void test_deletesOnlyApplicationsAndAssociatedEntities() throws Exception { + createTlds("tld1", "tld2"); + + DomainResource domain = persistActiveDomain("foo1.tld1"); + EppResourceIndex domainEri = + ofy().load().entity(EppResourceIndex.create(Key.create(domain))).now(); + ForeignKeyIndex domainFki = + ofy().load().key(ForeignKeyIndex.createKey(domain)).now(); + HistoryEntry domainHistoryEntry = + persistResource(new HistoryEntry.Builder().setParent(domain).build()); + + DomainApplication application = persistActiveDomainApplication("foo2.tld1"); + EppResourceIndex applicationEri = + ofy().load().entity(EppResourceIndex.create(Key.create(application))).now(); + DomainApplicationIndex applicationDai = + ofy().load().key(DomainApplicationIndex.createKey(application)).now(); + HistoryEntry applicationHistoryEntry = + persistResource(new HistoryEntry.Builder().setParent(application).build()); + + ContactResource contact = persistActiveContact("foo"); + HostResource host = persistActiveHost("ns.foo.tld1"); + + runMapreduce(); + ofy().clearSessionCache(); + + // Check that none of the domain, contact, and host entities were deleted. + assertThat( + ofy() + .load() + .entities(domain, domainEri, domainFki, domainHistoryEntry, contact, host) + .values()) + .containsExactly(domain, domainEri, domainFki, domainHistoryEntry, contact, host); + // Check that all of the domain application entities were deleted. + assertThat( + ofy() + .load() + .entities(application, applicationEri, applicationDai, applicationHistoryEntry)) + .isEmpty(); + } +} From a81d45fe5d7c8bdbc75bcaae334fd468c9d5662d Mon Sep 17 00:00:00 2001 From: mcilwain Date: Thu, 3 Jan 2019 16:21:39 -0800 Subject: [PATCH 126/134] Fix stdout of DeleteAllocationTokensCommand It was saying it was deleting tokens it wasn't, because it was outputting the raw input list of tokens rather than the list that filtered out redeemed or domain-specific tokens. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=227769266 --- .../registry/tools/DeleteAllocationTokensCommand.java | 6 +++++- .../registry/tools/DeleteAllocationTokensCommandTest.java | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/java/google/registry/tools/DeleteAllocationTokensCommand.java b/java/google/registry/tools/DeleteAllocationTokensCommand.java index aaf02f7f1..f252616f6 100644 --- a/java/google/registry/tools/DeleteAllocationTokensCommand.java +++ b/java/google/registry/tools/DeleteAllocationTokensCommand.java @@ -101,7 +101,11 @@ final class DeleteAllocationTokensCommand extends ConfirmingCommand System.out.printf( "%s tokens: %s\n", dryRun ? "Would delete" : "Deleted", - JOINER.join(batch.stream().map(Key::getName).collect(toImmutableSet()))); + JOINER.join( + tokensToDelete.stream() + .map(AllocationToken::getToken) + .sorted() + .collect(toImmutableSet()))); return tokensToDelete.size(); } } diff --git a/javatests/google/registry/tools/DeleteAllocationTokensCommandTest.java b/javatests/google/registry/tools/DeleteAllocationTokensCommandTest.java index 0ce485f23..ed2a71a14 100644 --- a/javatests/google/registry/tools/DeleteAllocationTokensCommandTest.java +++ b/javatests/google/registry/tools/DeleteAllocationTokensCommandTest.java @@ -54,6 +54,7 @@ public class DeleteAllocationTokensCommandTest runCommandForced("--prefix", ""); assertThat(reloadTokens(preNot1, preNot2, othrNot)).isEmpty(); assertThat(reloadTokens(preRed1, preRed2, othrRed)).containsExactly(preRed1, preRed2, othrRed); + assertInStdout("Deleted tokens: asdgfho7HASDS, prefix2978204, prefix8ZZZhs8"); } @Test @@ -84,6 +85,7 @@ public class DeleteAllocationTokensCommandTest runCommandForced("--prefix", "", "--dry_run"); assertThat(reloadTokens(preRed1, preRed2, preNot1, preNot2, othrRed, othrNot)) .containsExactly(preRed1, preRed2, preNot1, preNot2, othrRed, othrNot); + assertInStdout("Would delete tokens: asdgfho7HASDS, prefix2978204, prefix8ZZZhs8"); } @Test From 9eaeab9cfe1ff9886644aa20502a0775eabdaced Mon Sep 17 00:00:00 2001 From: guyben Date: Fri, 4 Jan 2019 12:44:28 -0800 Subject: [PATCH 127/134] Allow query parameters in the connection's endpoint ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=227898246 --- .../registry/tools/AppEngineConnection.java | 3 +- .../tools/AppEngineConnectionTest.java | 143 ++++++++++++++++++ javatests/google/registry/tools/BUILD | 1 + 3 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 javatests/google/registry/tools/AppEngineConnectionTest.java diff --git a/java/google/registry/tools/AppEngineConnection.java b/java/google/registry/tools/AppEngineConnection.java index a98e847cb..2a71272fe 100644 --- a/java/google/registry/tools/AppEngineConnection.java +++ b/java/google/registry/tools/AppEngineConnection.java @@ -102,8 +102,7 @@ class AppEngineConnection { private String internalSend( String endpoint, Map params, MediaType contentType, @Nullable byte[] payload) throws IOException { - GenericUrl url = new GenericUrl(getServer()); - url.setRawPath(endpoint); + GenericUrl url = new GenericUrl(String.format("%s%s", getServer(), endpoint)); url.putAll(params); HttpRequest request = (payload != null) diff --git a/javatests/google/registry/tools/AppEngineConnectionTest.java b/javatests/google/registry/tools/AppEngineConnectionTest.java new file mode 100644 index 000000000..f8228223f --- /dev/null +++ b/javatests/google/registry/tools/AppEngineConnectionTest.java @@ -0,0 +1,143 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.security.JsonHttp.JSON_SAFETY_PREFIX; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.Mockito.when; + +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.common.collect.ImmutableMap; +import com.google.common.net.MediaType; +import google.registry.testing.MockitoJUnitRule; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.HashMap; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; + +@RunWith(JUnit4.class) +public final class AppEngineConnectionTest { + + @Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create(); + + AppEngineConnection connection; + TestHttpTransport httpTransport; + TestLowLevelHttpRequest lowLevelHttpRequest; + @Mock LowLevelHttpResponse lowLevelHttpResponse; + + private final class TestHttpTransport extends HttpTransport { + String method = null; + String url = null; + + @Override + protected LowLevelHttpRequest buildRequest(String method, String url) { + // Make sure we only visit once + assertThat(this.method).isNull(); + this.method = method; + this.url = url; + return lowLevelHttpRequest; + } + } + + private final class TestLowLevelHttpRequest extends LowLevelHttpRequest { + final HashMap headers = new HashMap<>(); + + @Override + public void addHeader(String name, String value) { + headers.put(name, value); + } + + @Override + public LowLevelHttpResponse execute() { + return lowLevelHttpResponse; + } + + String getContentString() throws Exception { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + getStreamingContent().writeTo(output); + output.close(); + return new String(output.toByteArray(), UTF_8); + } + } + + @Before + public void setUp() throws Exception { + lowLevelHttpRequest = new TestLowLevelHttpRequest(); + when(lowLevelHttpResponse.getContent()) + .thenReturn(new ByteArrayInputStream("MyContent".getBytes(UTF_8))); + when(lowLevelHttpResponse.getStatusCode()).thenReturn(200); + + connection = new AppEngineConnection(); + httpTransport = new TestHttpTransport(); + connection.requestFactory = httpTransport.createRequestFactory(); + } + + @Test + public void testSendGetRequest() throws Exception { + assertThat( + connection.sendGetRequest( + "/my/path?query", ImmutableMap.of("key1", "value1", "key2", "value2"))) + .isEqualTo("MyContent"); + assertThat(httpTransport.method).isEqualTo("GET"); + assertThat(httpTransport.url) + .isEqualTo("https://localhost/my/path?query&key1=value1&key2=value2"); + assertThat(lowLevelHttpRequest.headers).containsEntry("Cache-Control", "no-cache"); + assertThat(lowLevelHttpRequest.headers).containsEntry("x-requested-with", "RegistryTool"); + } + + @Test + public void testSendPostRequest() throws Exception { + assertThat( + connection.sendPostRequest( + "/my/path?query", + ImmutableMap.of("key1", "value1", "key2", "value2"), + MediaType.PLAIN_TEXT_UTF_8, + "some data".getBytes(UTF_8))) + .isEqualTo("MyContent"); + assertThat(httpTransport.method).isEqualTo("POST"); + assertThat(httpTransport.url) + .isEqualTo("https://localhost/my/path?query&key1=value1&key2=value2"); + assertThat(lowLevelHttpRequest.getContentType()).isEqualTo("text/plain; charset=utf-8"); + assertThat(lowLevelHttpRequest.getContentString()).isEqualTo("some data"); + assertThat(lowLevelHttpRequest.headers).containsEntry("Cache-Control", "no-cache"); + assertThat(lowLevelHttpRequest.headers).containsEntry("x-requested-with", "RegistryTool"); + } + + @Test + public void testSendJsonRequest() throws Exception { + when(lowLevelHttpResponse.getContent()) + .thenReturn( + new ByteArrayInputStream((JSON_SAFETY_PREFIX + "{\"key\":\"value\"}").getBytes(UTF_8))); + assertThat( + connection.sendJson( + "/my/path?query", ImmutableMap.of("string", "value1", "bool", true))) + .containsExactly("key", "value"); + assertThat(httpTransport.method).isEqualTo("POST"); + assertThat(httpTransport.url).isEqualTo("https://localhost/my/path?query"); + assertThat(lowLevelHttpRequest.getContentType()).isEqualTo("application/json; charset=utf-8"); + assertThat(lowLevelHttpRequest.getContentString()) + .isEqualTo("{\"string\":\"value1\",\"bool\":true}"); + assertThat(lowLevelHttpRequest.headers).containsEntry("Cache-Control", "no-cache"); + assertThat(lowLevelHttpRequest.headers).containsEntry("x-requested-with", "RegistryTool"); + } +} diff --git a/javatests/google/registry/tools/BUILD b/javatests/google/registry/tools/BUILD index 07954200e..c80ce057d 100644 --- a/javatests/google/registry/tools/BUILD +++ b/javatests/google/registry/tools/BUILD @@ -24,6 +24,7 @@ java_library( "//java/google/registry/model", "//java/google/registry/rde", "//java/google/registry/request", + "//java/google/registry/security", "//java/google/registry/tmch", "//java/google/registry/tools", "//java/google/registry/tools/params", From 6022353e572e5364672ed27c5912db8e16a1d570 Mon Sep 17 00:00:00 2001 From: weiminyu Date: Fri, 4 Jan 2019 13:49:35 -0800 Subject: [PATCH 128/134] Enable daily backup of Datastore in sandbox and production This uses the new backup implementation, and starts after the old exportSnapshot task has completed. The old task will be removed later. Daily backups in alpha has been running successfully. Manually triggered exports in both environments also completed successfully. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=227908077 --- .../env/alpha/default/WEB-INF/cron.xml | 7 +++++-- .../env/production/default/WEB-INF/cron.xml | 18 ++++++++++++++++++ .../env/sandbox/default/WEB-INF/cron.xml | 18 ++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/java/google/registry/env/alpha/default/WEB-INF/cron.xml b/java/google/registry/env/alpha/default/WEB-INF/cron.xml index 21535c5b2..845822129 100644 --- a/java/google/registry/env/alpha/default/WEB-INF/cron.xml +++ b/java/google/registry/env/alpha/default/WEB-INF/cron.xml @@ -152,8 +152,11 @@ - every day 07:00 + + To facilitate troubleshooting, run after the exportSnapshot task below (typically finishes in + 90 minutes in production) has completed. + TODO(b/122271637): move to 06:00 (UTC) when the exportSnapshot task is removed. --> + every day 09:00 backend diff --git a/java/google/registry/env/production/default/WEB-INF/cron.xml b/java/google/registry/env/production/default/WEB-INF/cron.xml index c1ba7d62d..cf6090023 100644 --- a/java/google/registry/env/production/default/WEB-INF/cron.xml +++ b/java/google/registry/env/production/default/WEB-INF/cron.xml @@ -174,6 +174,24 @@ backend + + + + This job fires off a Datastore managed-export job that generates snapshot files in GCS. + It also enqueues a new task to wait on the completion of that job and then load the resulting + snapshot into bigquery. + + + every day 09:00 + backend + + diff --git a/java/google/registry/env/sandbox/default/WEB-INF/cron.xml b/java/google/registry/env/sandbox/default/WEB-INF/cron.xml index 9ed16a801..a02d57f7c 100644 --- a/java/google/registry/env/sandbox/default/WEB-INF/cron.xml +++ b/java/google/registry/env/sandbox/default/WEB-INF/cron.xml @@ -151,6 +151,24 @@ backend + + + + This job fires off a Datastore managed-export job that generates snapshot files in GCS. + It also enqueues a new task to wait on the completion of that job and then load the resulting + snapshot into bigquery. + + + every day 09:00 + backend + + From 25f3d8544eb02bac8aa036441d199a815312055a Mon Sep 17 00:00:00 2001 From: weiminyu Date: Fri, 4 Jan 2019 13:57:33 -0800 Subject: [PATCH 129/134] Update Nomulus open source install guide appcfg fails if version is not specified. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=227909394 --- docs/install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/install.md b/docs/install.md index ab6ea914e..949f78610 100644 --- a/docs/install.md +++ b/docs/install.md @@ -135,11 +135,11 @@ $ ls /path/to/app-dir/acme-registry-alpha backend default META-INF tools ``` -Now deploy the code to App Engine. +Now deploy the code to App Engine. We must provide a version string, e.g., live. ```shell $ appcfg.sh -A acme-registry-alpha --enable_jar_splitting \ - update /path/to/app-dir/acme-registry-alpha + -V live update /path/to/app-dir/acme-registry-alpha Reading application configuration data... Processing module default Oct 05, 2016 12:16:59 PM com.google.apphosting.utils.config.IndexesXmlReader readConfigXml From b6a293a9ff27d25702dd794f792698b1f9da43b0 Mon Sep 17 00:00:00 2001 From: jianglai Date: Mon, 7 Jan 2019 14:53:24 -0800 Subject: [PATCH 130/134] Fix travis build badge link [] fixed the link to the image, but the hyperlink to the travis site itself was left out. This CL fixed that. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=228241047 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8192c30a4..d69b9c31b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ | Bazel | Gradle | |-------|--------| -|![Build Status](https://storage.googleapis.com/domain-registry-github-build-status/github-ci-status.png)|[![Build Status](https://travis-ci.org/google/nomulus.svg?branch=master)](https://travis-ci.com/google/nomulus)| +|![Build Status](https://storage.googleapis.com/domain-registry-github-build-status/github-ci-status.png)|[![Build Status](https://travis-ci.org/google/nomulus.svg?branch=master)](https://travis-ci.org/google/nomulus)| ![Nomulus logo](./nomulus-logo.png) From c3de49130a0d51e603932227ba99b118e9c47949 Mon Sep 17 00:00:00 2001 From: jianglai Date: Mon, 7 Jan 2019 16:16:25 -0800 Subject: [PATCH 131/134] Fix Gradle build Added SOY templates for OTE console need to be complied. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=228255528 --- gradle/core/build.gradle | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gradle/core/build.gradle b/gradle/core/build.gradle index 356b4d1f6..2049a27a6 100644 --- a/gradle/core/build.gradle +++ b/gradle/core/build.gradle @@ -361,6 +361,12 @@ task soyToJava { dir: "${javaDir}/google/registry/ui/soy/registrar", include: ['**/*.soy'])) + soyToJava('google.registry.ui.soy.otesetup', + "${generatedDir}/google/registry/ui/soy/otesetup", + fileTree( + dir: "${javaDir}/google/registry/ui/soy/otesetup", + include: ['**/*.soy'])) + soyToJava('google.registry.ui.soy', "${generatedDir}/google/registry/ui/soy", files { From a79e254e4901fbae1cfeab673a2706a4c09ae8e7 Mon Sep 17 00:00:00 2001 From: Shicong Huang Date: Tue, 8 Jan 2019 08:18:30 -0800 Subject: [PATCH 132/134] Add an action to check the status of OT&E verification ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=228333195 --- .../registry/model/OteAccountBuilder.java | 15 +- java/google/registry/model/OteStats.java | 4 +- .../frontend/FrontendRequestComponent.java | 2 + .../ui/server/registrar/OteStatusAction.java | 121 ++++++++++++++ .../registry/model/OteAccountBuilderTest.java | 49 ++++++ .../frontend/testdata/frontend_routing.txt | 13 +- .../registry/testing/DatastoreHelper.java | 2 +- .../google/registry/ui/server/registrar/BUILD | 1 + .../server/registrar/OteStatusActionTest.java | 149 ++++++++++++++++++ 9 files changed, 346 insertions(+), 10 deletions(-) create mode 100644 java/google/registry/ui/server/registrar/OteStatusAction.java create mode 100644 javatests/google/registry/ui/server/registrar/OteStatusActionTest.java diff --git a/java/google/registry/model/OteAccountBuilder.java b/java/google/registry/model/OteAccountBuilder.java index e3c795022..52997c497 100644 --- a/java/google/registry/model/OteAccountBuilder.java +++ b/java/google/registry/model/OteAccountBuilder.java @@ -361,7 +361,7 @@ public final class OteAccountBuilder { } /** Returns the ClientIds of the OT&E, with the TLDs each has access to. */ - static ImmutableMap createClientIdToTldMap(String baseClientId) { + public static ImmutableMap createClientIdToTldMap(String baseClientId) { checkArgument( REGISTRAR_PATTERN.matcher(baseClientId).matches(), "Invalid registrar name: %s", @@ -374,4 +374,17 @@ public final class OteAccountBuilder { .put(baseClientId + "-5", baseClientId + "-eap") .build(); } + + /** Returns the base client ID that correspond to a given OT&E client ID. */ + public static String getBaseClientId(String oteClientId) { + int index = oteClientId.lastIndexOf('-'); + checkArgument(index > 0, "Invalid OT&E client ID: %s", oteClientId); + String baseClientId = oteClientId.substring(0, index); + checkArgument( + createClientIdToTldMap(baseClientId).containsKey(oteClientId), + "ID %s is not one of the OT&E client IDs for base %s", + oteClientId, + baseClientId); + return baseClientId; + } } diff --git a/java/google/registry/model/OteStats.java b/java/google/registry/model/OteStats.java index 393d45bf0..55301624b 100644 --- a/java/google/registry/model/OteStats.java +++ b/java/google/registry/model/OteStats.java @@ -177,7 +177,7 @@ public class OteStats { } /** Returns a more human-readable translation of the enum constant. */ - private String description() { + public String getDescription() { return Ascii.toLowerCase(this.name().replace('_', ' ')); } @@ -270,7 +270,7 @@ public class OteStats { return String.format( "%s\nTOTAL: %d", EnumSet.allOf(StatType.class).stream() - .map(stat -> String.format("%s: %d", stat.description(), statCounts.count(stat))) + .map(stat -> String.format("%s: %d", stat.getDescription(), statCounts.count(stat))) .collect(Collectors.joining("\n")), statCounts.size()); } diff --git a/java/google/registry/module/frontend/FrontendRequestComponent.java b/java/google/registry/module/frontend/FrontendRequestComponent.java index 3869425a1..277f2d745 100644 --- a/java/google/registry/module/frontend/FrontendRequestComponent.java +++ b/java/google/registry/module/frontend/FrontendRequestComponent.java @@ -27,6 +27,7 @@ import google.registry.request.RequestModule; import google.registry.request.RequestScope; import google.registry.ui.server.otesetup.ConsoleOteSetupAction; import google.registry.ui.server.registrar.ConsoleUiAction; +import google.registry.ui.server.registrar.OteStatusAction; import google.registry.ui.server.registrar.RegistrarConsoleModule; import google.registry.ui.server.registrar.RegistrarSettingsAction; @@ -46,6 +47,7 @@ interface FrontendRequestComponent { EppConsoleAction eppConsoleAction(); EppTlsAction eppTlsAction(); FlowComponent.Builder flowComponentBuilder(); + OteStatusAction oteStatusAction(); RegistrarSettingsAction registrarSettingsAction(); @Subcomponent.Builder diff --git a/java/google/registry/ui/server/registrar/OteStatusAction.java b/java/google/registry/ui/server/registrar/OteStatusAction.java new file mode 100644 index 000000000..fcd46cd56 --- /dev/null +++ b/java/google/registry/ui/server/registrar/OteStatusAction.java @@ -0,0 +1,121 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.ui.server.registrar; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static google.registry.security.JsonResponseHelper.Status.ERROR; +import static google.registry.security.JsonResponseHelper.Status.SUCCESS; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.common.flogger.FluentLogger; +import google.registry.model.OteAccountBuilder; +import google.registry.model.OteStats; +import google.registry.model.OteStats.StatType; +import google.registry.model.registrar.Registrar; +import google.registry.model.registrar.Registrar.Type; +import google.registry.request.Action; +import google.registry.request.JsonActionRunner; +import google.registry.request.auth.Auth; +import google.registry.request.auth.AuthenticatedRegistrarAccessor; +import google.registry.request.auth.AuthenticatedRegistrarAccessor.RegistrarAccessDeniedException; +import google.registry.security.JsonResponseHelper; +import java.util.Map; +import java.util.Optional; +import javax.inject.Inject; + +/** + * Admin servlet that allows creating or updating a registrar. Deletes are not allowed so as to + * preserve history. + */ +@Action(path = OteStatusAction.PATH, method = Action.Method.POST, auth = Auth.AUTH_PUBLIC_LOGGED_IN) +public final class OteStatusAction implements Runnable, JsonActionRunner.JsonAction { + + public static final String PATH = "/registrar-ote-status"; + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private static final String CLIENT_ID_PARAM = "clientId"; + private static final String COMPLETED_PARAM = "completed"; + private static final String DETAILS_PARAM = "details"; + private static final String STAT_TYPE_DESCRIPTION_PARAM = "description"; + private static final String STAT_TYPE_REQUIREMENT_PARAM = "requirement"; + private static final String STAT_TYPE_TIMES_PERFORMED_PARAM = "timesPerformed"; + + @Inject AuthenticatedRegistrarAccessor registrarAccessor; + @Inject JsonActionRunner jsonActionRunner; + + @Inject + OteStatusAction() {} + + @Override + public void run() { + jsonActionRunner.run(this); + } + + @Override + public Map handleJsonRequest(Map input) { + try { + checkArgument(input != null, "Malformed JSON"); + + String oteClientId = (String) input.get(CLIENT_ID_PARAM); + checkArgument( + !Strings.isNullOrEmpty(oteClientId), "Missing key for OT&E client: %s", CLIENT_ID_PARAM); + + String baseClientId = OteAccountBuilder.getBaseClientId(oteClientId); + Registrar oteRegistrar = registrarAccessor.getRegistrar(oteClientId); + verifyOteAccess(baseClientId); + checkArgument( + Type.OTE.equals(oteRegistrar.getType()), + "Registrar with ID %s is not an OT&E registrar", + oteClientId); + + OteStats oteStats = OteStats.getFromRegistrar(baseClientId); + return JsonResponseHelper.create( + SUCCESS, "OT&E check completed successfully", convertOteStats(baseClientId, oteStats)); + } catch (Throwable e) { + logger.atWarning().withCause(e).log( + "Failed to verify OT&E status for registrar with input %s", input); + return JsonResponseHelper.create( + ERROR, Optional.ofNullable(e.getMessage()).orElse("Unspecified error")); + } + } + + private void verifyOteAccess(String baseClientId) throws RegistrarAccessDeniedException { + for (String oteClientId : OteAccountBuilder.createClientIdToTldMap(baseClientId).keySet()) { + registrarAccessor.verifyAccess(oteClientId); + } + } + + private Map convertOteStats(String baseClientId, OteStats oteStats) { + return ImmutableMap.of( + CLIENT_ID_PARAM, baseClientId, + COMPLETED_PARAM, oteStats.getFailures().isEmpty(), + DETAILS_PARAM, + StatType.REQUIRED_STAT_TYPES.stream() + .map(statType -> convertSingleRequirement(statType, oteStats.getCount(statType))) + .collect(toImmutableList())); + } + + private Map convertSingleRequirement(StatType statType, int count) { + int requirement = statType.getRequirement(); + return ImmutableMap.of( + STAT_TYPE_DESCRIPTION_PARAM, statType.getDescription(), + STAT_TYPE_REQUIREMENT_PARAM, requirement, + STAT_TYPE_TIMES_PERFORMED_PARAM, count, + COMPLETED_PARAM, count >= requirement); + } +} diff --git a/javatests/google/registry/model/OteAccountBuilderTest.java b/javatests/google/registry/model/OteAccountBuilderTest.java index 792f8cd84..cf8ff12b4 100644 --- a/javatests/google/registry/model/OteAccountBuilderTest.java +++ b/javatests/google/registry/model/OteAccountBuilderTest.java @@ -299,4 +299,53 @@ public final class OteAccountBuilderTest { assertContactExists("myclientid-3", "other@example.com"); assertContactExists("myclientid-3", "email@example.com"); } + + @Test + public void testCreateClientIdToTldMap_validEntries() { + assertThat(OteAccountBuilder.createClientIdToTldMap("myclientid")) + .containsExactly( + "myclientid-1", "myclientid-sunrise", + "myclientid-2", "myclientid-landrush", + "myclientid-3", "myclientid-ga", + "myclientid-4", "myclientid-ga", + "myclientid-5", "myclientid-eap"); + } + + @Test + public void testCreateClientIdToTldMap_invalidId() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, () -> OteAccountBuilder.createClientIdToTldMap("a")); + assertThat(exception).hasMessageThat().isEqualTo("Invalid registrar name: a"); + } + + @Test + public void testGetBaseClientId_validOteId() { + assertThat(OteAccountBuilder.getBaseClientId("myclientid-4")).isEqualTo("myclientid"); + } + + @Test + public void testGetBaseClientId_multipleDashesValid() { + assertThat(OteAccountBuilder.getBaseClientId("two-dashes-3")).isEqualTo("two-dashes"); + } + + @Test + public void testGetBaseClientId_invalidInput_malformed() { + assertThat( + assertThrows( + IllegalArgumentException.class, + () -> OteAccountBuilder.getBaseClientId("myclientid"))) + .hasMessageThat() + .isEqualTo("Invalid OT&E client ID: myclientid"); + } + + @Test + public void testGetBaseClientId_invalidInput_wrongForBase() { + assertThat( + assertThrows( + IllegalArgumentException.class, + () -> OteAccountBuilder.getBaseClientId("myclientid-7"))) + .hasMessageThat() + .isEqualTo("ID myclientid-7 is not one of the OT&E client IDs for base myclientid"); + } } diff --git a/javatests/google/registry/module/frontend/testdata/frontend_routing.txt b/javatests/google/registry/module/frontend/testdata/frontend_routing.txt index f60c5ccf9..22d10ca49 100644 --- a/javatests/google/registry/module/frontend/testdata/frontend_routing.txt +++ b/javatests/google/registry/module/frontend/testdata/frontend_routing.txt @@ -1,6 +1,7 @@ -PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY -/_dr/epp EppTlsAction POST n INTERNAL,API APP PUBLIC -/registrar ConsoleUiAction GET n INTERNAL,API,LEGACY NONE PUBLIC -/registrar-ote-setup ConsoleOteSetupAction POST,GET n INTERNAL,API,LEGACY NONE PUBLIC -/registrar-settings RegistrarSettingsAction POST n API,LEGACY USER PUBLIC -/registrar-xhr EppConsoleAction POST n API,LEGACY USER PUBLIC +PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY +/_dr/epp EppTlsAction POST n INTERNAL,API APP PUBLIC +/registrar ConsoleUiAction GET n INTERNAL,API,LEGACY NONE PUBLIC +/registrar-ote-setup ConsoleOteSetupAction POST,GET n INTERNAL,API,LEGACY NONE PUBLIC +/registrar-ote-status OteStatusAction POST n API,LEGACY USER PUBLIC +/registrar-settings RegistrarSettingsAction POST n API,LEGACY USER PUBLIC +/registrar-xhr EppConsoleAction POST n API,LEGACY USER PUBLIC diff --git a/javatests/google/registry/testing/DatastoreHelper.java b/javatests/google/registry/testing/DatastoreHelper.java index af66dda89..1145be6e5 100644 --- a/javatests/google/registry/testing/DatastoreHelper.java +++ b/javatests/google/registry/testing/DatastoreHelper.java @@ -701,7 +701,7 @@ public class DatastoreHelper { /** Persists and returns a {@link Registrar} with the specified attributes. */ public static Registrar persistNewRegistrar( - String clientId, String registrarName, Registrar.Type type, long ianaIdentifier) { + String clientId, String registrarName, Registrar.Type type, @Nullable Long ianaIdentifier) { return persistSimpleResource( new Registrar.Builder() .setClientId(clientId) diff --git a/javatests/google/registry/ui/server/registrar/BUILD b/javatests/google/registry/ui/server/registrar/BUILD index fd60ec5c6..16a8bdf8c 100644 --- a/javatests/google/registry/ui/server/registrar/BUILD +++ b/javatests/google/registry/ui/server/registrar/BUILD @@ -22,6 +22,7 @@ java_library( "//java/google/registry/ui/server", "//java/google/registry/ui/server/registrar", "//java/google/registry/util", + "//javatests/google/registry/model", "//javatests/google/registry/security", "//javatests/google/registry/testing", "//third_party/objectify:objectify-v4_1", diff --git a/javatests/google/registry/ui/server/registrar/OteStatusActionTest.java b/javatests/google/registry/ui/server/registrar/OteStatusActionTest.java new file mode 100644 index 000000000..e9d5ff2d1 --- /dev/null +++ b/javatests/google/registry/ui/server/registrar/OteStatusActionTest.java @@ -0,0 +1,149 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.ui.server.registrar; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap; +import static com.google.common.truth.Truth.assertThat; +import static google.registry.testing.DatastoreHelper.persistNewRegistrar; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Iterables; +import google.registry.model.OteAccountBuilder; +import google.registry.model.OteStats.StatType; +import google.registry.model.OteStatsTestHelper; +import google.registry.model.registrar.Registrar.Type; +import google.registry.request.auth.AuthenticatedRegistrarAccessor; +import google.registry.request.auth.AuthenticatedRegistrarAccessor.Role; +import google.registry.testing.AppEngineRule; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link OteStatusAction} */ +@RunWith(JUnit4.class) +public final class OteStatusActionTest { + + private static final String CLIENT_ID = "blobio-1"; + private static final String BASE_CLIENT_ID = "blobio"; + + private final OteStatusAction action = new OteStatusAction(); + + @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build(); + + @Before + public void init() throws Exception { + OteStatsTestHelper.setupHistoryEntries(); + persistNewRegistrar(CLIENT_ID, "SomeRegistrar", Type.OTE, null); + + ImmutableSetMultimap authValues = + OteAccountBuilder.createClientIdToTldMap("blobio").keySet().stream() + .collect(toImmutableSetMultimap(Function.identity(), ignored -> Role.OWNER)); + action.registrarAccessor = AuthenticatedRegistrarAccessor.createForTesting(authValues); + } + + @Test + @SuppressWarnings("unchecked") + public void testSuccess_finishedOte() { + Map actionResult = action.handleJsonRequest(ImmutableMap.of("clientId", CLIENT_ID)); + assertThat(actionResult).containsEntry("status", "SUCCESS"); + assertThat(actionResult).containsEntry("message", "OT&E check completed successfully"); + Map results = + Iterables.getOnlyElement((List>) actionResult.get("results")); + assertThat(results).containsEntry("clientId", BASE_CLIENT_ID); + assertThat(results).containsEntry("completed", true); + assertThat(getFailingResultDetails(results)).isEmpty(); + } + + @Test + @SuppressWarnings("unchecked") + public void testSuccess_unfinishedOte() { + OteStatsTestHelper.deleteHostDeleteHistoryEntry(); + + Map actionResult = action.handleJsonRequest(ImmutableMap.of("clientId", CLIENT_ID)); + assertThat(actionResult).containsEntry("status", "SUCCESS"); + assertThat(actionResult).containsEntry("message", "OT&E check completed successfully"); + Map results = + Iterables.getOnlyElement((List>) actionResult.get("results")); + assertThat(results).containsEntry("clientId", BASE_CLIENT_ID); + assertThat(results).containsEntry("completed", false); + assertThat(getFailingResultDetails(results)) + .containsExactly( + ImmutableMap.of( + "description", StatType.HOST_DELETES.getDescription(), + "requirement", StatType.HOST_DELETES.getRequirement(), + "timesPerformed", 0, + "completed", false)); + } + + @Test + public void testFailure_malformedInput() { + assertThat(action.handleJsonRequest(null)) + .containsExactlyEntriesIn(errorResultWithMessage("Malformed JSON")); + assertThat(action.handleJsonRequest(ImmutableMap.of())) + .containsExactlyEntriesIn(errorResultWithMessage("Missing key for OT&E client: clientId")); + } + + @Test + public void testFailure_noRegistrar() { + assertThat(action.handleJsonRequest(ImmutableMap.of("clientId", "nonexistent-id-2"))) + .containsExactlyEntriesIn( + errorResultWithMessage("TestUserId doesn't have access to registrar nonexistent-id-2")); + } + + @Test + public void testFailure_notAuthorized() { + action.registrarAccessor = + AuthenticatedRegistrarAccessor.createForTesting(ImmutableSetMultimap.of()); + assertThat(action.handleJsonRequest(ImmutableMap.of("clientId", CLIENT_ID))) + .containsExactlyEntriesIn( + errorResultWithMessage("TestUserId doesn't have access to registrar blobio-1")); + } + + @Test + public void testFailure_malformedRegistrarName() { + assertThat(action.handleJsonRequest(ImmutableMap.of("clientId", "bad-client-id"))) + .containsExactlyEntriesIn( + errorResultWithMessage( + "ID bad-client-id is not one of the OT&E client IDs for base bad-client")); + } + + @Test + public void testFailure_nonOteRegistrar() { + persistNewRegistrar(CLIENT_ID, "SomeRegistrar", Type.REAL, 1L); + assertThat(action.handleJsonRequest(ImmutableMap.of("clientId", CLIENT_ID))) + .containsExactlyEntriesIn( + errorResultWithMessage("Registrar with ID blobio-1 is not an OT&E registrar")); + } + + @SuppressWarnings("unchecked") + private List> getFailingResultDetails(Map results) { + return ((List>) results.get("details")) + .stream() + .filter(result -> !Boolean.TRUE.equals(result.get("completed"))) + .collect(toImmutableList()); + } + + private ImmutableMap errorResultWithMessage(String message) { + return ImmutableMap.of("status", "ERROR", "message", message, "results", ImmutableList.of()); + } +} From 04a495bc9995274609fe2f1ab4121356fb33e955 Mon Sep 17 00:00:00 2001 From: shicong Date: Tue, 8 Jan 2019 12:25:37 -0800 Subject: [PATCH 133/134] Fix Gradle build There were two issues in the Gradle build. 1. Missing soyToJava conversion for newly added google.registry.ui.soy.otesetup. 2. testSuccess_setAllowedTldsUncached_newTldNotInCache in RegistrarTest modified the cache duration to 600 seconds which broke other tests as those tests write to the storage in initiation stage and expect the data to be available in the cache immediately, this requires the duration to be 0. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=228377715 --- gradle/core/build.gradle | 31 +++------ .../model/registrar/RegistrarTest.java | 67 ++++++++++--------- 2 files changed, 47 insertions(+), 51 deletions(-) diff --git a/gradle/core/build.gradle b/gradle/core/build.gradle index 2049a27a6..475fe450f 100644 --- a/gradle/core/build.gradle +++ b/gradle/core/build.gradle @@ -13,34 +13,15 @@ def generatedDir = "${project.buildDir}/generated/source/custom/main" def outcastTestPatterns = [ "google/registry/batch/DeleteContactsAndHostsActionTest.*", "google/registry/batch/RefreshDnsOnHostRenameActionTest.*", - "google.registry.export.ExportDomainListsActionTest.*", - "google/registry/export/ExportPremiumTermsActionTest.*", - "google/registry/export/ExportReservedTermsActionTest.*", - "google/registry/export/ExportUtilsTest.*", - "google/registry/export/sheet/SyncRegistrarsSheetTest.*", "google/registry/flows/CheckApiActionTest.*", + "google/registry/flows/EppLifecycleHostTest.*", "google/registry/flows/domain/DomainAllocateFlowTest.*", "google/registry/flows/domain/DomainApplicationCreateFlowTest.*", "google/registry/flows/domain/DomainApplicationUpdateFlowTest.*", "google/registry/flows/domain/DomainCreateFlowTest.*", "google/registry/flows/domain/DomainUpdateFlowTest.*", - "google/registry/flows/EppLifecycleDomainTest.*", - "google/registry/flows/EppLifecycleHostTest.*", - "google/registry/model/poll/PollMessageExternalKeyConverterTest.*", - "google/registry/model/poll/PollMessageTest.*", - "google/registry/rdap/RdapDomainActionTest.*", - "google/registry/rdap/RdapDomainSearchActionTest.*", - "google/registry/rdap/RdapEntityActionTest.*", - "google/registry/rdap/RdapEntitySearchActionTest.*", - "google/registry/rdap/RdapNameserverSearchActionTest.*", - "google/registry/reporting/icann/IcannHttpReporterTest.*", - "google/registry/tmch/LordnTaskUtilsTest.*", - "google/registry/tmch/NordnUploadActionTest.*", + "google/registry/tools/CreateDomainCommandTest.*", "google/registry/tools/server/CreatePremiumListActionTest.*", - "google/registry/ui/server/registrar/ContactSettingsTest.*", - "google/registry/ui/server/registrar/RegistrarSettingsActionTest.*", - "google/registry/ui/server/registrar/SecuritySettingsTest.*", - // Conflicts with WhoisActionTest "google/registry/whois/WhoisHttpActionTest.*", ] @@ -374,6 +355,14 @@ task soyToJava { }.filter { it.name.endsWith(".soy") }) + + soyToJava('google.registry.ui.soy.otesetup', + "${generatedDir}/google/registry/ui/soy/otesetup", + files { + file("${javaDir}/google/registry/ui/soy/otesetup").listFiles() + }.filter { + it.name.endsWith(".soy") + }) } } diff --git a/javatests/google/registry/model/registrar/RegistrarTest.java b/javatests/google/registry/model/registrar/RegistrarTest.java index ed095087f..f9fe18cb7 100644 --- a/javatests/google/registry/model/registrar/RegistrarTest.java +++ b/javatests/google/registry/model/registrar/RegistrarTest.java @@ -454,39 +454,46 @@ public class RegistrarTest extends EntityTestCase { @Test public void testSuccess_setAllowedTldsUncached_newTldNotInCache() { - // Cache duration in tests is 0. To make sure the data isn't in the cache we have to set it to a - // higher value and reset the cache. - RegistryConfig.CONFIG_SETTINGS.get().caching.singletonCacheRefreshSeconds = 600; - Registries.resetCache(); - // Make sure the TLD we want to create doesn't exist yet. - // This is also important because getTlds fills out the cache when used. - assertThat(Registries.getTlds()).doesNotContain("newtld"); - // We can't use createTld here because it failes when the cache is used. - persistResource(newRegistry("newtld", "NEWTLD")); - // Make sure we set up the cache correctly, so the newly created TLD isn't in the cache - assertThat(Registries.getTlds()).doesNotContain("newtld"); + try { + // Cache duration in tests is 0. To make sure the data isn't in the cache we have to set it + // to a higher value and reset the cache. + RegistryConfig.CONFIG_SETTINGS.get().caching.singletonCacheRefreshSeconds = 600; + Registries.resetCache(); + // Make sure the TLD we want to create doesn't exist yet. + // This is also important because getTlds fills out the cache when used. + assertThat(Registries.getTlds()).doesNotContain("newtld"); + // We can't use createTld here because it failes when the cache is used. + persistResource(newRegistry("newtld", "NEWTLD")); + // Make sure we set up the cache correctly, so the newly created TLD isn't in the cache + assertThat(Registries.getTlds()).doesNotContain("newtld"); - // Test that the uncached version works - assertThat( - registrar.asBuilder() - .setAllowedTldsUncached(ImmutableSet.of("newtld")) - .build() - .getAllowedTlds()) - .containsExactly("newtld"); + // Test that the uncached version works + assertThat( + registrar + .asBuilder() + .setAllowedTldsUncached(ImmutableSet.of("newtld")) + .build() + .getAllowedTlds()) + .containsExactly("newtld"); - // Test that the "regular" cached version fails. If this doesn't throw - then we changed how the - // cached version works: - // - either we switched to a different cache type/duration, and we haven't actually set up that - // cache in the test - // - or we stopped using the cache entirely and we should rethink if the Uncached version is - // still needed - assertThrows( - IllegalArgumentException.class, - () -> registrar.asBuilder().setAllowedTlds(ImmutableSet.of("newtld"))); + // Test that the "regular" cached version fails. If this doesn't throw - then we changed how + // the cached version works: + // - either we switched to a different cache type/duration, and we haven't actually set up + // that cache in the test + // - or we stopped using the cache entirely and we should rethink if the Uncached version is + // still needed + assertThrows( + IllegalArgumentException.class, + () -> registrar.asBuilder().setAllowedTlds(ImmutableSet.of("newtld"))); - // Make sure the cache hasn't expired during the test and "newtld" is still not in the cached - // TLDs - assertThat(Registries.getTlds()).doesNotContain("newtld"); + // Make sure the cache hasn't expired during the test and "newtld" is still not in the cached + // TLDs + assertThat(Registries.getTlds()).doesNotContain("newtld"); + } finally { + // Set the cache duration back to 0 to satisfy other tests. + RegistryConfig.CONFIG_SETTINGS.get().caching.singletonCacheRefreshSeconds = 0; + Registries.resetCache(); + } } @Test From 9e0b675275d1d2b2d79c0e04ffb9ec301f66215a Mon Sep 17 00:00:00 2001 From: mmuller Date: Tue, 8 Jan 2019 13:06:24 -0800 Subject: [PATCH 134/134] Don't use Files.copy() when decrypting to stdout Files.copy() attempts to delete the file if it already exists, which obviously won't work very well for /dev/stdout. Instead copy directly from the decoder to standard output. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=228384575 --- gradle/core/build.gradle | 1 + java/google/registry/repositories.bzl | 14 +++++++++++++ java/google/registry/tools/BUILD | 1 + .../registry/tools/GhostrydeCommand.java | 13 +++++++++--- .../registry/tools/GhostrydeCommandTest.java | 21 +++++++++++++++++++ 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/gradle/core/build.gradle b/gradle/core/build.gradle index 475fe450f..0182ad928 100644 --- a/gradle/core/build.gradle +++ b/gradle/core/build.gradle @@ -151,6 +151,7 @@ dependencies { maybeRuntime 'com.sun.activation:javax.activation:1.2.0' maybeRuntime 'com.thoughtworks.paranamer:paranamer:2.7' maybeRuntime 'commons-codec:commons-codec:1.10' + compile group: 'commons-io', name: 'commons-io', version: '2.6' maybeRuntime 'commons-logging:commons-logging:1.2' compile 'dnsjava:dnsjava:2.1.7' maybeRuntime 'io.netty:netty-buffer:4.1.28.Final' diff --git a/java/google/registry/repositories.bzl b/java/google/registry/repositories.bzl index 30e8d930f..cd5a23c68 100644 --- a/java/google/registry/repositories.bzl +++ b/java/google/registry/repositories.bzl @@ -106,6 +106,7 @@ def domain_registry_repositories( omit_com_sun_xml_bind_jaxb_xjc = False, omit_com_thoughtworks_paranamer = False, omit_commons_codec = False, + omit_commons_io = False, omit_commons_logging = False, omit_dnsjava = False, omit_io_netty_buffer = False, @@ -337,6 +338,8 @@ def domain_registry_repositories( com_thoughtworks_paranamer() if not omit_commons_codec: commons_codec() + if not omit_commons_io: + commons_io() if not omit_commons_logging: commons_logging() if not omit_dnsjava: @@ -1799,6 +1802,17 @@ def commons_codec(): ], ) +def commons_io(): + java_import_external( + name = "commons_io", + licenses = ["notice"], # Apache License, Version 2.0 + jar_sha256 = "a10418348d234968600ccb1d988efcbbd08716e1d96936ccc1880e7d22513474", + jar_urls = [ + "http://maven.ibiblio.org/maven2/commons-io/commons-io/2.5/commons-io-2.5.jar", + "http://repo1.maven.org/maven2/commons-io/commons-io/2.5/commons-io-2.5.jar", + ], + ) + def commons_logging(): java_import_external( name = "commons_logging", diff --git a/java/google/registry/tools/BUILD b/java/google/registry/tools/BUILD index b1f1f9d45..23b5a7999 100644 --- a/java/google/registry/tools/BUILD +++ b/java/google/registry/tools/BUILD @@ -65,6 +65,7 @@ java_library( "//java/google/registry/whois", "//java/google/registry/xjc", "//java/google/registry/xml", + "//third_party/java/jakarta_commons_io", "//third_party/jaxb", "//third_party/objectify:objectify-v4_1", "@com_beust_jcommander", diff --git a/java/google/registry/tools/GhostrydeCommand.java b/java/google/registry/tools/GhostrydeCommand.java index efeee8fd3..ee8807592 100644 --- a/java/google/registry/tools/GhostrydeCommand.java +++ b/java/google/registry/tools/GhostrydeCommand.java @@ -31,6 +31,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import javax.inject.Inject; import javax.inject.Provider; +import org.apache.commons.io.IOUtils; import org.bouncycastle.openpgp.PGPException; import org.bouncycastle.openpgp.PGPPrivateKey; import org.bouncycastle.openpgp.PGPPublicKey; @@ -101,9 +102,15 @@ final class GhostrydeCommand implements CommandWithRemoteApi { private void runDecrypt() throws IOException, PGPException { try (InputStream in = Files.newInputStream(input); InputStream ghostDecoder = Ghostryde.decoder(in, rdeStagingDecryptionKey.get())) { - Path outFile = - Files.isDirectory(output) ? output.resolve(input.getFileName() + ".decrypt") : output; - Files.copy(ghostDecoder, outFile, REPLACE_EXISTING); + System.err.println("output = " + output); + if (output.toString().equals("/dev/stdout")) { + System.err.println("doing copy"); + IOUtils.copy(ghostDecoder, System.out); + } else { + Path outFile = + Files.isDirectory(output) ? output.resolve(input.getFileName() + ".decrypt") : output; + Files.copy(ghostDecoder, outFile, REPLACE_EXISTING); + } } } } diff --git a/javatests/google/registry/tools/GhostrydeCommandTest.java b/javatests/google/registry/tools/GhostrydeCommandTest.java index 4236e3919..72464a082 100644 --- a/javatests/google/registry/tools/GhostrydeCommandTest.java +++ b/javatests/google/registry/tools/GhostrydeCommandTest.java @@ -22,9 +22,12 @@ import google.registry.rde.Ghostryde; import google.registry.testing.BouncyCastleProviderRule; import google.registry.testing.FakeKeyringModule; import google.registry.testing.InjectRule; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -58,12 +61,19 @@ public class GhostrydeCommandTest extends CommandTestCase { public final BouncyCastleProviderRule bouncy = new BouncyCastleProviderRule(); private Keyring keyring; + private PrintStream orgStdout; @Before public void before() { keyring = new FakeKeyringModule().get(); command.rdeStagingDecryptionKey = keyring::getRdeStagingDecryptionKey; command.rdeStagingEncryptionKey = keyring::getRdeStagingEncryptionKey; + orgStdout = System.out; + } + + @After + public void after() { + System.setOut(orgStdout); } @Test @@ -112,4 +122,15 @@ public class GhostrydeCommandTest extends CommandTestCase { Path outFile = outDir.resolve("atrain.ghostryde.decrypt"); assertThat(Files.readAllBytes(outFile)).isEqualTo(SONG_BY_CHRISTINA_ROSSETTI); } + + @Test + public void testDecrypt_outputIsStdOut() throws Exception { + Path inFile = Paths.get(tmpDir.newFolder().toString()).resolve("atrain.ghostryde"); + Files.write( + inFile, Ghostryde.encode(SONG_BY_CHRISTINA_ROSSETTI, keyring.getRdeStagingEncryptionKey())); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + runCommand("--decrypt", "--input=" + inFile); + assertThat(out.toByteArray()).isEqualTo(SONG_BY_CHRISTINA_ROSSETTI); + } }

+ {call registry.soy.forms.inputFieldLabel} + {param label: 'Base client ID' /} + {/call} + + {call registry.soy.forms.inputFieldValue } + {param name: 'clientId' /} + {param value: $baseClientId /} + {param placeholder: 'registrar\'s ID' /} + {param readonly: false /} + {/call} + + Must: +
    +
  1. consist of only lowercase letters, numbers, or hyphens,
    2. +
  2. start with a letter, and
    3. +
  3. be between 3 and 14 characters (inclusive).
    4. +
+ We require 1 and 2 since the registrar name will be used to create TLDs, and we + require 3 since we append "-[12345]" to the name to create client IDs which are + required by the EPP XML schema to be between 3-16 chars. + +