diff --git a/core/src/main/java/google/registry/module/backend/BackendComponent.java b/core/src/main/java/google/registry/module/backend/BackendComponent.java index 9b615f0cd..042a4e715 100644 --- a/core/src/main/java/google/registry/module/backend/BackendComponent.java +++ b/core/src/main/java/google/registry/module/backend/BackendComponent.java @@ -43,7 +43,7 @@ import google.registry.rde.JSchModule; import google.registry.request.Modules.DatastoreServiceModule; import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.NetHttpTransportModule; -import google.registry.request.Modules.URLFetchServiceModule; +import google.registry.request.Modules.UrlConnectionServiceModule; import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UserServiceModule; import google.registry.request.auth.AuthModule; @@ -80,7 +80,7 @@ import javax.inject.Singleton; ServerTridProviderModule.class, SheetsServiceModule.class, StackdriverModule.class, - URLFetchServiceModule.class, + UrlConnectionServiceModule.class, UrlFetchTransportModule.class, UserServiceModule.class, VoidDnsWriterModule.class, diff --git a/core/src/main/java/google/registry/module/frontend/FrontendComponent.java b/core/src/main/java/google/registry/module/frontend/FrontendComponent.java index f5d3c2008..3a7c76b7a 100644 --- a/core/src/main/java/google/registry/module/frontend/FrontendComponent.java +++ b/core/src/main/java/google/registry/module/frontend/FrontendComponent.java @@ -34,7 +34,6 @@ import google.registry.monitoring.whitebox.StackdriverModule; import google.registry.privileges.secretmanager.SecretManagerModule; import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.NetHttpTransportModule; -import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UserServiceModule; import google.registry.request.auth.AuthModule; import google.registry.ui.ConsoleDebug.ConsoleConfigModule; @@ -65,7 +64,6 @@ import javax.inject.Singleton; SecretManagerModule.class, ServerTridProviderModule.class, StackdriverModule.class, - UrlFetchTransportModule.class, UserServiceModule.class, UtilsModule.class }) diff --git a/core/src/main/java/google/registry/module/pubapi/PubApiComponent.java b/core/src/main/java/google/registry/module/pubapi/PubApiComponent.java index 479fad057..b04b03a2c 100644 --- a/core/src/main/java/google/registry/module/pubapi/PubApiComponent.java +++ b/core/src/main/java/google/registry/module/pubapi/PubApiComponent.java @@ -34,7 +34,6 @@ import google.registry.persistence.PersistenceModule; import google.registry.privileges.secretmanager.SecretManagerModule; import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.NetHttpTransportModule; -import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UserServiceModule; import google.registry.request.auth.AuthModule; import google.registry.util.UtilsModule; @@ -62,7 +61,6 @@ import javax.inject.Singleton; SecretManagerModule.class, ServerTridProviderModule.class, StackdriverModule.class, - UrlFetchTransportModule.class, UserServiceModule.class, UtilsModule.class }) diff --git a/core/src/main/java/google/registry/module/tools/ToolsComponent.java b/core/src/main/java/google/registry/module/tools/ToolsComponent.java index 2a332be02..0a5243787 100644 --- a/core/src/main/java/google/registry/module/tools/ToolsComponent.java +++ b/core/src/main/java/google/registry/module/tools/ToolsComponent.java @@ -36,7 +36,6 @@ import google.registry.privileges.secretmanager.SecretManagerModule; import google.registry.request.Modules.DatastoreServiceModule; import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.NetHttpTransportModule; -import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UserServiceModule; import google.registry.request.auth.AuthModule; import google.registry.util.UtilsModule; @@ -66,7 +65,6 @@ import javax.inject.Singleton; ServerTridProviderModule.class, StackdriverModule.class, ToolsRequestComponentModule.class, - UrlFetchTransportModule.class, UserServiceModule.class, UtilsModule.class }) diff --git a/core/src/main/java/google/registry/rde/RdeReporter.java b/core/src/main/java/google/registry/rde/RdeReporter.java index f8cfaff37..f4fdd3811 100644 --- a/core/src/main/java/google/registry/rde/RdeReporter.java +++ b/core/src/main/java/google/registry/rde/RdeReporter.java @@ -14,26 +14,24 @@ package google.registry.rde; -import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate; -import static com.google.appengine.api.urlfetch.HTTPMethod.PUT; -import static com.google.common.io.BaseEncoding.base64; -import static com.google.common.net.HttpHeaders.AUTHORIZATION; -import static com.google.common.net.HttpHeaders.CONTENT_TYPE; +import static google.registry.request.UrlConnectionUtils.getResponseBytes; +import static google.registry.request.UrlConnectionUtils.setBasicAuth; +import static google.registry.request.UrlConnectionUtils.setPayload; import static google.registry.util.DomainNameUtils.canonicalizeDomainName; import static java.nio.charset.StandardCharsets.UTF_8; import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; import static javax.servlet.http.HttpServletResponse.SC_OK; -import com.google.appengine.api.urlfetch.HTTPHeader; -import com.google.appengine.api.urlfetch.HTTPRequest; +import com.google.api.client.http.HttpMethods; import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; import com.google.common.flogger.FluentLogger; +import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.keyring.api.KeyModule.Key; import google.registry.request.HttpException.InternalServerErrorException; +import google.registry.request.UrlConnectionService; import google.registry.util.Retrier; -import google.registry.util.UrlFetchException; +import google.registry.util.UrlConnectionException; import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.iirdea.XjcIirdeaResponseElement; import google.registry.xjc.iirdea.XjcIirdeaResult; @@ -41,6 +39,7 @@ import google.registry.xjc.rdeheader.XjcRdeHeader; import google.registry.xjc.rdereport.XjcRdeReportReport; import google.registry.xml.XmlException; import java.io.ByteArrayInputStream; +import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.SocketTimeoutException; import java.net.URL; @@ -55,12 +54,15 @@ public class RdeReporter { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - /** @see - * ICANN Registry Interfaces - Interface details*/ - private static final String REPORT_MIME = "text/xml"; + /** + * @see + * ICANN Registry Interfaces - Interface details + */ + private static final MediaType MEDIA_TYPE = MediaType.XML_UTF_8; @Inject Retrier retrier; - @Inject URLFetchService urlFetchService; + @Inject UrlConnectionService urlConnectionService; + @Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix; @Inject @Key("icannReportingPassword") String password; @Inject RdeReporter() {} @@ -74,29 +76,24 @@ public class RdeReporter { // Send a PUT request to ICANN's HTTPS server. URL url = makeReportUrl(header.getTld(), report.getId()); String username = header.getTld() + "_ry"; - String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8)); - final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d)); - req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME)); - req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token)); - req.setPayload(reportBytes); logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8)); - HTTPResponse rsp = + byte[] responseBytes = retrier.callWithRetry( () -> { - HTTPResponse rsp1 = urlFetchService.fetch(req); - switch (rsp1.getResponseCode()) { - case SC_OK: - case SC_BAD_REQUEST: - break; - default: - throw new UrlFetchException("PUT failed", req, rsp1); + HttpURLConnection connection = urlConnectionService.createConnection(url); + connection.setRequestMethod(HttpMethods.PUT); + setBasicAuth(connection, username, password); + setPayload(connection, reportBytes, MEDIA_TYPE.toString()); + int responseCode = connection.getResponseCode(); + if (responseCode == SC_OK || responseCode == SC_BAD_REQUEST) { + return getResponseBytes(connection); } - return rsp1; + throw new UrlConnectionException("PUT failed", connection); }, SocketTimeoutException.class); // Ensure the XML response is valid. - XjcIirdeaResult result = parseResult(rsp); + XjcIirdeaResult result = parseResult(responseBytes); if (result.getCode().getValue() != 1000) { logger.atWarning().log( "PUT rejected: %d %s\n%s", @@ -108,11 +105,11 @@ public class RdeReporter { /** * Unmarshals IIRDEA XML result object from {@link HTTPResponse} payload. * - * @see + * @see * ICANN Registry Interfaces - IIRDEA Result Object */ - private XjcIirdeaResult parseResult(HTTPResponse rsp) throws XmlException { - byte[] responseBytes = rsp.getContent(); + private XjcIirdeaResult parseResult(byte[] responseBytes) throws XmlException { logger.atInfo().log("Received response:\n%s", new String(responseBytes, UTF_8)); XjcIirdeaResponseElement response = XjcXmlTransformer.unmarshal( XjcIirdeaResponseElement.class, new ByteArrayInputStream(responseBytes)); diff --git a/core/src/main/java/google/registry/request/Modules.java b/core/src/main/java/google/registry/request/Modules.java index af043552a..110f72c85 100644 --- a/core/src/main/java/google/registry/request/Modules.java +++ b/core/src/main/java/google/registry/request/Modules.java @@ -23,12 +23,11 @@ import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson2.JacksonFactory; import com.google.appengine.api.datastore.DatastoreService; -import com.google.appengine.api.urlfetch.URLFetchService; -import com.google.appengine.api.urlfetch.URLFetchServiceFactory; import com.google.appengine.api.users.UserService; import com.google.appengine.api.users.UserServiceFactory; import dagger.Module; import dagger.Provides; +import java.net.HttpURLConnection; import javax.inject.Singleton; /** Dagger modules for App Engine services and other vendor classes. */ @@ -45,14 +44,12 @@ public final class Modules { } } - /** Dagger module for {@link URLFetchService}. */ + /** Dagger module for {@link UrlConnectionService}. */ @Module - public static final class URLFetchServiceModule { - private static final URLFetchService fetchService = URLFetchServiceFactory.getURLFetchService(); - + public static final class UrlConnectionServiceModule { @Provides - static URLFetchService provideURLFetchService() { - return fetchService; + static UrlConnectionService provideUrlConnectionService() { + return url -> (HttpURLConnection) url.openConnection(); } } diff --git a/core/src/main/java/google/registry/request/UrlConnectionService.java b/core/src/main/java/google/registry/request/UrlConnectionService.java new file mode 100644 index 000000000..a3d1ba33d --- /dev/null +++ b/core/src/main/java/google/registry/request/UrlConnectionService.java @@ -0,0 +1,25 @@ +// Copyright 2022 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.request; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; + +/** Functional interface for opening a connection from a URL, injectable for testing. */ +public interface UrlConnectionService { + + HttpURLConnection createConnection(URL url) throws IOException; +} diff --git a/util/src/main/java/google/registry/util/UrlFetchUtils.java b/core/src/main/java/google/registry/request/UrlConnectionUtils.java similarity index 56% rename from util/src/main/java/google/registry/util/UrlFetchUtils.java rename to core/src/main/java/google/registry/request/UrlConnectionUtils.java index 4f6e60105..62759f006 100644 --- a/util/src/main/java/google/registry/util/UrlFetchUtils.java +++ b/core/src/main/java/google/registry/request/UrlConnectionUtils.java @@ -1,4 +1,4 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// Copyright 2022 The Nomulus Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package google.registry.util; +package google.registry.request; import static com.google.common.base.Preconditions.checkState; import static com.google.common.io.BaseEncoding.base64; @@ -22,36 +22,41 @@ import static com.google.common.net.HttpHeaders.CONTENT_LENGTH; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.api.urlfetch.HTTPHeader; -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.common.base.Ascii; import com.google.common.base.Strings; +import com.google.common.io.ByteStreams; import com.google.common.net.MediaType; -import java.util.Optional; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.URLConnection; import java.util.Random; -/** Helper methods for the App Engine URL fetch service. */ -public final class UrlFetchUtils { +/** Utilities for common functionality relating to {@link java.net.URLConnection}s. */ +public class UrlConnectionUtils { - /** Returns value of first header matching {@code name}. */ - public static Optional getHeaderFirst(HTTPResponse rsp, String name) { - return getHeaderFirstInternal(rsp.getHeadersUncombined(), name); + /** Retrieves the response from the given connection as a byte array. */ + public static byte[] getResponseBytes(URLConnection connection) throws IOException { + return ByteStreams.toByteArray(connection.getInputStream()); } - /** Returns value of first header matching {@code name}. */ - public static Optional getHeaderFirst(HTTPRequest req, String name) { - return getHeaderFirstInternal(req.getHeaders(), name); + /** Sets auth on the given connection with the given username/password. */ + public static void setBasicAuth(URLConnection connection, String username, String password) { + setBasicAuth(connection, String.format("%s:%s", username, password)); } - private static Optional getHeaderFirstInternal(Iterable hdrs, String name) { - name = Ascii.toLowerCase(name); - for (HTTPHeader header : hdrs) { - if (Ascii.toLowerCase(header.getName()).equals(name)) { - return Optional.of(header.getValue()); - } + /** Sets auth on the given connection with the given string, formatted "username:password". */ + public static void setBasicAuth(URLConnection connection, String usernameAndPassword) { + String token = base64().encode(usernameAndPassword.getBytes(UTF_8)); + connection.setRequestProperty(AUTHORIZATION, "Basic " + token); + } + + /** Sets the given byte[] payload on the given connection with a particular content type. */ + public static void setPayload(URLConnection connection, byte[] bytes, String contentType) + throws IOException { + connection.setRequestProperty(CONTENT_TYPE, contentType); + connection.setDoOutput(true); + try (DataOutputStream dataStream = new DataOutputStream(connection.getOutputStream())) { + dataStream.write(bytes); } - return Optional.empty(); } /** @@ -62,16 +67,16 @@ public final class UrlFetchUtils { * @see RFC2388 - Returning Values from Forms */ public static void setPayloadMultipart( - HTTPRequest request, + URLConnection connection, String name, String filename, MediaType contentType, String data, - Random random) { + Random random) + throws IOException { String boundary = createMultipartBoundary(random); checkState( - !data.contains(boundary), - "Multipart data contains autogenerated boundary: %s", boundary); + !data.contains(boundary), "Multipart data contains autogenerated boundary: %s", boundary); String multipart = String.format("--%s\r\n", boundary) + String.format( @@ -83,11 +88,9 @@ public final class UrlFetchUtils { + "\r\n" + String.format("--%s--\r\n", boundary); byte[] payload = multipart.getBytes(UTF_8); - request.addHeader( - new HTTPHeader( - CONTENT_TYPE, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary))); - request.addHeader(new HTTPHeader(CONTENT_LENGTH, Integer.toString(payload.length))); - request.setPayload(payload); + connection.setRequestProperty(CONTENT_LENGTH, Integer.toString(payload.length)); + setPayload( + connection, payload, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary)); } private static String createMultipartBoundary(Random random) { @@ -98,12 +101,4 @@ public final class UrlFetchUtils { // See https://tools.ietf.org/html/rfc2046#section-5.1.1 return Strings.repeat("-", 30) + base64().encode(rand); } - - /** Sets the HTTP Basic Authentication header on an {@link HTTPRequest}. */ - public static void setAuthorizationHeader(HTTPRequest req, Optional login) { - if (login.isPresent()) { - String token = base64().encode(login.get().getBytes(UTF_8)); - req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token)); - } - } } diff --git a/core/src/main/java/google/registry/tmch/LordnRequestInitializer.java b/core/src/main/java/google/registry/tmch/LordnRequestInitializer.java index 8bedfce71..815859793 100644 --- a/core/src/main/java/google/registry/tmch/LordnRequestInitializer.java +++ b/core/src/main/java/google/registry/tmch/LordnRequestInitializer.java @@ -15,12 +15,12 @@ package google.registry.tmch; import static com.google.common.base.Verify.verifyNotNull; -import static google.registry.util.UrlFetchUtils.setAuthorizationHeader; -import com.google.appengine.api.urlfetch.HTTPRequest; import com.google.common.flogger.FluentLogger; import google.registry.keyring.api.KeyModule.Key; import google.registry.model.tld.Registry; +import google.registry.request.UrlConnectionUtils; +import java.net.HttpURLConnection; import java.util.Optional; import javax.inject.Inject; @@ -37,8 +37,9 @@ final class LordnRequestInitializer { } /** Initializes a URL fetch request for talking to the MarksDB server. */ - void initialize(HTTPRequest request, String tld) { - setAuthorizationHeader(request, getMarksDbLordnCredentials(tld)); + void initialize(HttpURLConnection connection, String tld) { + getMarksDbLordnCredentials(tld) + .ifPresent(login -> UrlConnectionUtils.setBasicAuth(connection, login)); } /** Returns the username and password for the current TLD to login to the MarksDB server. */ diff --git a/core/src/main/java/google/registry/tmch/Marksdb.java b/core/src/main/java/google/registry/tmch/Marksdb.java index 40109b091..afbbcea10 100644 --- a/core/src/main/java/google/registry/tmch/Marksdb.java +++ b/core/src/main/java/google/registry/tmch/Marksdb.java @@ -14,25 +14,23 @@ package google.registry.tmch; -import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate; -import static com.google.appengine.api.urlfetch.HTTPMethod.GET; import static com.google.common.base.Preconditions.checkArgument; +import static google.registry.request.UrlConnectionUtils.getResponseBytes; +import static google.registry.request.UrlConnectionUtils.setBasicAuth; import static google.registry.util.HexDumper.dumpHex; -import static google.registry.util.UrlFetchUtils.setAuthorizationHeader; import static java.nio.charset.StandardCharsets.US_ASCII; import static javax.servlet.http.HttpServletResponse.SC_OK; -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; import com.google.common.collect.ImmutableList; import com.google.common.flogger.FluentLogger; import com.google.common.io.ByteSource; import google.registry.config.RegistryConfig.Config; import google.registry.keyring.api.KeyModule.Key; -import google.registry.util.UrlFetchException; +import google.registry.request.UrlConnectionService; +import google.registry.util.UrlConnectionException; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.net.HttpURLConnection; import java.net.URL; import java.security.Security; import java.security.SignatureException; @@ -57,7 +55,8 @@ public final class Marksdb { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final int MAX_DNL_LOGGING_LENGTH = 500; - @Inject URLFetchService fetchService; + @Inject UrlConnectionService urlConnectionService; + @Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl; @Inject @Key("marksdbPublicKey") PGPPublicKey marksdbPublicKey; @Inject Marksdb() {} @@ -112,19 +111,16 @@ public final class Marksdb { } byte[] fetch(URL url, Optional loginAndPassword) throws IOException { - HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d)); - setAuthorizationHeader(req, loginAndPassword); - HTTPResponse rsp; + HttpURLConnection connection = urlConnectionService.createConnection(url); + loginAndPassword.ifPresent(auth -> setBasicAuth(connection, auth)); try { - rsp = fetchService.fetch(req); + if (connection.getResponseCode() != SC_OK) { + throw new UrlConnectionException("Failed to fetch from MarksDB", connection); + } + return getResponseBytes(connection); } catch (IOException e) { - throw new IOException( - String.format("Error connecting to MarksDB at URL %s", url), e); + throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e); } - if (rsp.getResponseCode() != SC_OK) { - throw new UrlFetchException("Failed to fetch from MarksDB", req, rsp); - } - return rsp.getContent(); } List fetchSignedCsv(Optional loginAndPassword, String csvPath, String sigPath) diff --git a/core/src/main/java/google/registry/tmch/NordnUploadAction.java b/core/src/main/java/google/registry/tmch/NordnUploadAction.java index b015b3ba9..9b4f3f8d1 100644 --- a/core/src/main/java/google/registry/tmch/NordnUploadAction.java +++ b/core/src/main/java/google/registry/tmch/NordnUploadAction.java @@ -16,28 +16,23 @@ package google.registry.tmch; import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl; -import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate; -import static com.google.appengine.api.urlfetch.HTTPMethod.POST; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.net.HttpHeaders.LOCATION; import static com.google.common.net.MediaType.CSV_UTF_8; +import static google.registry.request.UrlConnectionUtils.getResponseBytes; import static google.registry.tmch.LordnTaskUtils.COLUMNS_CLAIMS; import static google.registry.tmch.LordnTaskUtils.COLUMNS_SUNRISE; -import static google.registry.util.UrlFetchUtils.getHeaderFirst; -import static google.registry.util.UrlFetchUtils.setPayloadMultipart; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED; +import com.google.api.client.http.HttpMethods; import com.google.appengine.api.taskqueue.LeaseOptions; import com.google.appengine.api.taskqueue.Queue; import com.google.appengine.api.taskqueue.TaskHandle; import com.google.appengine.api.taskqueue.TaskOptions; import com.google.appengine.api.taskqueue.TransientFailureException; -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; import com.google.apphosting.api.DeadlineExceededException; import com.google.common.base.Joiner; import com.google.common.base.Strings; @@ -49,16 +44,18 @@ import google.registry.config.RegistryConfig.Config; import google.registry.request.Action; import google.registry.request.Parameter; import google.registry.request.RequestParameters; +import google.registry.request.UrlConnectionService; +import google.registry.request.UrlConnectionUtils; import google.registry.request.auth.Auth; import google.registry.util.Clock; import google.registry.util.Retrier; import google.registry.util.TaskQueueUtils; -import google.registry.util.UrlFetchException; +import google.registry.util.UrlConnectionException; import java.io.IOException; +import java.net.HttpURLConnection; import java.net.URL; import java.security.SecureRandom; import java.util.List; -import java.util.Optional; import java.util.Random; import java.util.concurrent.TimeUnit; import javax.inject.Inject; @@ -97,7 +94,8 @@ public final class NordnUploadAction implements Runnable { @Inject Retrier retrier; @Inject SecureRandom random; @Inject LordnRequestInitializer lordnRequestInitializer; - @Inject URLFetchService fetchService; + @Inject UrlConnectionService urlConnectionService; + @Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl; @Inject @Parameter(LORDN_PHASE_PARAM) String phase; @Inject @Parameter(RequestParameters.PARAM_TLD) String tld; @@ -193,47 +191,48 @@ public final class NordnUploadAction implements Runnable { *

Idempotency: If the exact same LORDN report is uploaded twice, the MarksDB server will * return the same confirmation number. * - * @see - * TMCH functional specifications - LORDN File + * @see TMCH + * functional specifications - LORDN File */ private void uploadCsvToLordn(String urlPath, String csvData) throws IOException { String url = tmchMarksdbUrl + urlPath; logger.atInfo().log( "LORDN upload task %s: Sending to URL: %s ; data: %s", actionLogId, url, csvData); - HTTPRequest req = new HTTPRequest(new URL(url), POST, validateCertificate().setDeadline(60d)); - lordnRequestInitializer.initialize(req, tld); - setPayloadMultipart(req, "file", "claims.csv", CSV_UTF_8, csvData, random); - HTTPResponse rsp; + HttpURLConnection connection = urlConnectionService.createConnection(new URL(url)); + connection.setRequestMethod(HttpMethods.POST); + lordnRequestInitializer.initialize(connection, tld); + UrlConnectionUtils.setPayloadMultipart( + connection, "file", "claims.csv", CSV_UTF_8, csvData, random); try { - rsp = fetchService.fetch(req); + int responseCode = connection.getResponseCode(); + if (logger.atInfo().isEnabled()) { + String responseContent = new String(getResponseBytes(connection), US_ASCII); + if (responseContent.isEmpty()) { + responseContent = "(null)"; + } + logger.atInfo().log( + "LORDN upload task %s response: HTTP response code %d, response data: %s", + actionLogId, responseCode, responseContent); + } + if (responseCode != SC_ACCEPTED) { + throw new UrlConnectionException( + String.format( + "LORDN upload task %s error: Failed to upload LORDN claims to MarksDB", + actionLogId), + connection); + } + String location = connection.getHeaderField(LOCATION); + if (location == null) { + throw new UrlConnectionException( + String.format( + "LORDN upload task %s error: MarksDB failed to provide a Location header", + actionLogId), + connection); + } + getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location))); } catch (IOException e) { - throw new IOException( - String.format("Error connecting to MarksDB at URL %s", url), e); + throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e); } - if (logger.atInfo().isEnabled()) { - String response = - (rsp.getContent() == null) ? "(null)" : new String(rsp.getContent(), US_ASCII); - logger.atInfo().log( - "LORDN upload task %s response: HTTP response code %d, response data: %s", - actionLogId, rsp.getResponseCode(), response); - } - if (rsp.getResponseCode() != SC_ACCEPTED) { - throw new UrlFetchException( - String.format( - "LORDN upload task %s error: Failed to upload LORDN claims to MarksDB", actionLogId), - req, - rsp); - } - Optional location = getHeaderFirst(rsp, LOCATION); - if (!location.isPresent()) { - throw new UrlFetchException( - String.format( - "LORDN upload task %s error: MarksDB failed to provide a Location header", - actionLogId), - req, - rsp); - } - getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location.get()))); } private TaskOptions makeVerifyTask(URL url) { diff --git a/core/src/main/java/google/registry/tmch/NordnVerifyAction.java b/core/src/main/java/google/registry/tmch/NordnVerifyAction.java index da6a5640f..a2f6589f9 100644 --- a/core/src/main/java/google/registry/tmch/NordnVerifyAction.java +++ b/core/src/main/java/google/registry/tmch/NordnVerifyAction.java @@ -14,15 +14,11 @@ package google.registry.tmch; -import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate; -import static com.google.appengine.api.urlfetch.HTTPMethod.GET; +import static google.registry.request.UrlConnectionUtils.getResponseBytes; import static java.nio.charset.StandardCharsets.UTF_8; import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; import static javax.servlet.http.HttpServletResponse.SC_OK; -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; import com.google.common.annotations.VisibleForTesting; import com.google.common.flogger.FluentLogger; import com.google.common.io.ByteSource; @@ -32,9 +28,11 @@ import google.registry.request.HttpException.ConflictException; import google.registry.request.Parameter; import google.registry.request.RequestParameters; import google.registry.request.Response; +import google.registry.request.UrlConnectionService; import google.registry.request.auth.Auth; -import google.registry.util.UrlFetchException; +import google.registry.util.UrlConnectionException; import java.io.IOException; +import java.net.HttpURLConnection; import java.net.URL; import java.util.Map.Entry; import javax.inject.Inject; @@ -68,7 +66,8 @@ public final class NordnVerifyAction implements Runnable { @Inject LordnRequestInitializer lordnRequestInitializer; @Inject Response response; - @Inject URLFetchService fetchService; + @Inject UrlConnectionService urlConnectionService; + @Inject @Header(URL_HEADER) URL url; @Inject @Header(HEADER_ACTION_LOG_ID) String actionLogId; @Inject @Parameter(RequestParameters.PARAM_TLD) String tld; @@ -96,51 +95,49 @@ public final class NordnVerifyAction implements Runnable { @VisibleForTesting LordnLog verify() throws IOException { logger.atInfo().log("LORDN verify task %s: Sending request to URL %s", actionLogId, url); - HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d)); - lordnRequestInitializer.initialize(req, tld); - HTTPResponse rsp; + HttpURLConnection connection = urlConnectionService.createConnection(url); + lordnRequestInitializer.initialize(connection, tld); try { - rsp = fetchService.fetch(req); - } catch (IOException e) { - throw new IOException( - String.format("Error connecting to MarksDB at URL %s", url), e); - } - logger.atInfo().log( - "LORDN verify task %s response: HTTP response code %d, response data: %s", - actionLogId, rsp.getResponseCode(), rsp.getContent()); - if (rsp.getResponseCode() == SC_NO_CONTENT) { - // Send a 400+ status code so App Engine will retry the task. - throw new ConflictException("Not ready"); - } - if (rsp.getResponseCode() != SC_OK) { - throw new UrlFetchException( - String.format("LORDN verify task %s: Failed to verify LORDN upload to MarksDB.", - actionLogId), - req, rsp); - } - LordnLog log = - LordnLog.parse(ByteSource.wrap(rsp.getContent()).asCharSource(UTF_8).readLines()); - if (log.getStatus() == LordnLog.Status.ACCEPTED) { - logger.atInfo().log("LORDN verify task %s: Upload accepted.", actionLogId); - } else { - logger.atSevere().log( - "LORDN verify task %s: Upload rejected with reason: %s", actionLogId, log); - } - for (Entry result : log) { - switch (result.getValue().getOutcome()) { - case OK: - break; - case WARNING: - // fall through - case ERROR: - logger.atWarning().log(result.toString()); - break; - default: - logger.atWarning().log( - "LORDN verify task %s: Unexpected outcome: %s", actionLogId, result); - break; + int responseCode = connection.getResponseCode(); + logger.atInfo().log( + "LORDN verify task %s response: HTTP response code %d", actionLogId, responseCode); + if (responseCode == SC_NO_CONTENT) { + // Send a 400+ status code so App Engine will retry the task. + throw new ConflictException("Not ready"); } + if (responseCode != SC_OK) { + throw new UrlConnectionException( + String.format( + "LORDN verify task %s: Failed to verify LORDN upload to MarksDB.", actionLogId), + connection); + } + LordnLog log = + LordnLog.parse( + ByteSource.wrap(getResponseBytes(connection)).asCharSource(UTF_8).readLines()); + if (log.getStatus() == LordnLog.Status.ACCEPTED) { + logger.atInfo().log("LORDN verify task %s: Upload accepted.", actionLogId); + } else { + logger.atSevere().log( + "LORDN verify task %s: Upload rejected with reason: %s", actionLogId, log); + } + for (Entry result : log) { + switch (result.getValue().getOutcome()) { + case OK: + break; + case WARNING: + // fall through + case ERROR: + logger.atWarning().log(result.toString()); + break; + default: + logger.atWarning().log( + "LORDN verify task %s: Unexpected outcome: %s", actionLogId, result); + break; + } + } + return log; + } catch (IOException e) { + throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e); } - return log; } } diff --git a/core/src/main/java/google/registry/tools/RegistryToolComponent.java b/core/src/main/java/google/registry/tools/RegistryToolComponent.java index e6a4d665e..6429acd73 100644 --- a/core/src/main/java/google/registry/tools/RegistryToolComponent.java +++ b/core/src/main/java/google/registry/tools/RegistryToolComponent.java @@ -38,8 +38,7 @@ import google.registry.privileges.secretmanager.SecretManagerModule; import google.registry.rde.RdeModule; import google.registry.request.Modules.DatastoreServiceModule; import google.registry.request.Modules.Jackson2Module; -import google.registry.request.Modules.URLFetchServiceModule; -import google.registry.request.Modules.UrlFetchTransportModule; +import google.registry.request.Modules.UrlConnectionServiceModule; import google.registry.request.Modules.UserServiceModule; import google.registry.tools.AuthModule.LocalCredentialModule; import google.registry.tools.javascrap.CompareEscrowDepositsCommand; @@ -79,8 +78,7 @@ import javax.inject.Singleton; RegistryToolDataflowModule.class, RequestFactoryModule.class, SecretManagerModule.class, - URLFetchServiceModule.class, - UrlFetchTransportModule.class, + UrlConnectionServiceModule.class, UserServiceModule.class, UtilsModule.class, VoidDnsWriterModule.class, diff --git a/core/src/test/java/google/registry/rde/RdeReportActionTest.java b/core/src/test/java/google/registry/rde/RdeReportActionTest.java index 6890caae0..3d77cad01 100644 --- a/core/src/test/java/google/registry/rde/RdeReportActionTest.java +++ b/core/src/test/java/google/registry/rde/RdeReportActionTest.java @@ -14,7 +14,6 @@ package google.registry.rde; -import static com.google.appengine.api.urlfetch.HTTPMethod.PUT; import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; import static com.google.common.truth.Truth.assertThat; import static google.registry.model.common.Cursor.CursorType.RDE_REPORT; @@ -29,20 +28,13 @@ import static javax.servlet.http.HttpServletResponse.SC_OK; import static org.joda.time.Duration.standardDays; import static org.joda.time.Duration.standardSeconds; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import com.google.appengine.api.urlfetch.HTTPHeader; -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; -import com.google.common.base.Ascii; -import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSource; import google.registry.gcs.GcsUtils; import google.registry.model.common.Cursor; @@ -57,20 +49,21 @@ import google.registry.testing.FakeClock; import google.registry.testing.FakeKeyringModule; import google.registry.testing.FakeResponse; import google.registry.testing.FakeSleeper; +import google.registry.testing.FakeUrlConnectionService; import google.registry.testing.TestOfyAndSql; import google.registry.util.Retrier; import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.rdereport.XjcRdeReportReport; import google.registry.xml.XmlException; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.net.HttpURLConnection; import java.net.SocketTimeoutException; -import java.util.Map; import java.util.Optional; import org.bouncycastle.openpgp.PGPPublicKey; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.ArgumentCaptor; /** Unit tests for {@link RdeReportAction}. */ @DualDatabaseTest @@ -89,20 +82,21 @@ public class RdeReportActionTest { private final FakeResponse response = new FakeResponse(); private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class); - private final URLFetchService urlFetchService = mock(URLFetchService.class); - private final ArgumentCaptor request = ArgumentCaptor.forClass(HTTPRequest.class); - private final HTTPResponse httpResponse = mock(HTTPResponse.class); private final PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey(); private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final BlobId reportFile = BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde"); + private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class); + private final FakeUrlConnectionService urlConnectionService = + new FakeUrlConnectionService(httpUrlConnection); + private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream(); private RdeReportAction createAction() { RdeReporter reporter = new RdeReporter(); reporter.reportUrlPrefix = "https://rde-report.example"; - reporter.urlFetchService = urlFetchService; reporter.password = "foo"; + reporter.urlConnectionService = urlConnectionService; reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); RdeReportAction action = new RdeReportAction(); action.gcsUtils = gcsUtils; @@ -127,6 +121,7 @@ public class RdeReportActionTest { Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test"))); gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0)); + when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream); } @TestOfyAndSql @@ -142,24 +137,22 @@ public class RdeReportActionTest { @TestOfyAndSql void testRunWithLock() throws Exception { - when(httpResponse.getResponseCode()).thenReturn(SC_OK); - when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); - when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); + when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK); + when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream()); createAction().runWithLock(loadRdeReportCursor()); assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8); assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n"); // Verify the HTTP request was correct. - assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT); - assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https"); - assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001"); - Map headers = mapifyHeaders(request.getValue().getHeaders()); - assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml"); - assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28="); + verify(httpUrlConnection).setRequestMethod("PUT"); + assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https"); + assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001"); + verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8"); + verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28="); // Verify the payload XML was the same as what's in testdata/report.xml. - XjcRdeReportReport report = parseReport(request.getValue().getPayload()); + XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray()); assertThat(report.getId()).isEqualTo("20101017001"); assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z")); assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z")); @@ -167,9 +160,8 @@ public class RdeReportActionTest { @TestOfyAndSql void testRunWithLock_withPrefix() throws Exception { - when(httpResponse.getResponseCode()).thenReturn(SC_OK); - when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); - when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); + when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK); + when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream()); RdeReportAction action = createAction(); action.prefix = Optional.of("job-name/"); gcsUtils.delete(reportFile); @@ -182,16 +174,14 @@ public class RdeReportActionTest { assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n"); // Verify the HTTP request was correct. - assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT); - assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https"); - assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001"); - Map headers = mapifyHeaders(request.getValue().getHeaders()); - assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml"); - assertThat(headers) - .containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28="); + verify(httpUrlConnection).setRequestMethod("PUT"); + assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https"); + assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001"); + verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8"); + verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28="); // Verify the payload XML was the same as what's in testdata/report.xml. - XjcRdeReportReport report = parseReport(request.getValue().getPayload()); + XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray()); assertThat(report.getId()).isEqualTo("20101017001"); assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z")); assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z")); @@ -204,9 +194,8 @@ public class RdeReportActionTest { PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey(); gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1)); - when(httpResponse.getResponseCode()).thenReturn(SC_OK); - when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); - when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); + when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK); + when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream()); createAction().runWithLock(loadRdeReportCursor()); assertThat(response.getStatus()).isEqualTo(200); } @@ -239,9 +228,8 @@ public class RdeReportActionTest { @TestOfyAndSql void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception { - when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST); - when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read()); - when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); + when(httpUrlConnection.getResponseCode()).thenReturn(SC_BAD_REQUEST); + when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openStream()); InternalServerErrorException thrown = assertThrows( InternalServerErrorException.class, @@ -252,18 +240,17 @@ public class RdeReportActionTest { @TestOfyAndSql void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception { class ExpectedThrownException extends RuntimeException {} - when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException()); + when(httpUrlConnection.getResponseCode()).thenThrow(new ExpectedThrownException()); assertThrows( ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor())); } @TestOfyAndSql void testRunWithLock_socketTimeout_doesRetry() throws Exception { - when(httpResponse.getResponseCode()).thenReturn(SC_OK); - when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); - when(urlFetchService.fetch(request.capture())) + when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream()); + when(httpUrlConnection.getResponseCode()) .thenThrow(new SocketTimeoutException()) - .thenReturn(httpResponse); + .thenReturn(SC_OK); createAction().runWithLock(loadRdeReportCursor()); assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8); @@ -274,14 +261,6 @@ public class RdeReportActionTest { return loadByKey(Cursor.createVKey(RDE_REPORT, "test")).getCursorTime(); } - private static ImmutableMap mapifyHeaders(Iterable headers) { - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - for (HTTPHeader header : headers) { - builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue()); - } - return builder.build(); - } - private static XjcRdeReportReport parseReport(byte[] data) { try { return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data)); diff --git a/core/src/test/java/google/registry/util/UrlFetchUtilsTest.java b/core/src/test/java/google/registry/request/UrlConnectionUtilsTest.java similarity index 62% rename from core/src/test/java/google/registry/util/UrlFetchUtilsTest.java rename to core/src/test/java/google/registry/request/UrlConnectionUtilsTest.java index d425347c9..620218a7c 100644 --- a/core/src/test/java/google/registry/util/UrlFetchUtilsTest.java +++ b/core/src/test/java/google/registry/request/UrlConnectionUtilsTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// Copyright 2022 The Nomulus Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package google.registry.util; +package google.registry.request; import static com.google.common.net.HttpHeaders.CONTENT_LENGTH; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; import static com.google.common.net.MediaType.CSV_UTF_8; import static com.google.common.truth.Truth.assertThat; -import static google.registry.util.UrlFetchUtils.setPayloadMultipart; +import static google.registry.request.UrlConnectionUtils.setPayloadMultipart; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -27,22 +27,19 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; -import com.google.appengine.api.urlfetch.HTTPHeader; -import com.google.appengine.api.urlfetch.HTTPRequest; -import google.registry.testing.AppEngineExtension; +import java.io.ByteArrayOutputStream; import java.util.Arrays; import java.util.List; import java.util.Random; +import javax.net.ssl.HttpsURLConnection; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.ArgumentCaptor; -/** Unit tests for {@link UrlFetchUtils}. */ -class UrlFetchUtilsTest { - - @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().build(); +/** Tests for {@link UrlConnectionUtils}. */ +public class UrlConnectionUtilsTest { private final Random random = mock(Random.class); @@ -58,25 +55,28 @@ class UrlFetchUtilsTest { } @Test - void testSetPayloadMultipart() { - HTTPRequest request = mock(HTTPRequest.class); + void testSetPayloadMultipart() throws Exception { + HttpsURLConnection connection = mock(HttpsURLConnection.class); + ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream(); + when(connection.getOutputStream()).thenReturn(connectionOutputStream); setPayloadMultipart( - request, + connection, "lol", "cat", CSV_UTF_8, "The nice people at the store say hello. ヘ(◕。◕ヘ)", random); - ArgumentCaptor headerCaptor = ArgumentCaptor.forClass(HTTPHeader.class); - verify(request, times(2)).addHeader(headerCaptor.capture()); - List addedHeaders = headerCaptor.getAllValues(); - assertThat(addedHeaders.get(0).getName()).isEqualTo(CONTENT_TYPE); - assertThat(addedHeaders.get(0).getValue()) - .isEqualTo( - "multipart/form-data; " - + "boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\""); - assertThat(addedHeaders.get(1).getName()).isEqualTo(CONTENT_LENGTH); - assertThat(addedHeaders.get(1).getValue()).isEqualTo("294"); + ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor valueCaptor = ArgumentCaptor.forClass(String.class); + verify(connection, times(2)).setRequestProperty(keyCaptor.capture(), valueCaptor.capture()); + List addedKeys = keyCaptor.getAllValues(); + assertThat(addedKeys).containsExactly(CONTENT_TYPE, CONTENT_LENGTH); + List addedValues = valueCaptor.getAllValues(); + assertThat(addedValues) + .containsExactly( + "multipart/form-data;" + + " boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"", + "294"); String payload = "--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\n" + "Content-Disposition: form-data; name=\"lol\"; filename=\"cat\"\r\n" @@ -84,18 +84,20 @@ class UrlFetchUtilsTest { + "\r\n" + "The nice people at the store say hello. ヘ(◕。◕ヘ)\r\n" + "--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA--\r\n"; - verify(request).setPayload(payload.getBytes(UTF_8)); - verifyNoMoreInteractions(request); + verify(connection).setDoOutput(true); + verify(connection).getOutputStream(); + assertThat(connectionOutputStream.toByteArray()).isEqualTo(payload.getBytes(UTF_8)); + verifyNoMoreInteractions(connection); } @Test void testSetPayloadMultipart_boundaryInPayload() { - HTTPRequest request = mock(HTTPRequest.class); + HttpsURLConnection connection = mock(HttpsURLConnection.class); String payload = "I screamed------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHHH"; IllegalStateException thrown = assertThrows( IllegalStateException.class, - () -> setPayloadMultipart(request, "lol", "cat", CSV_UTF_8, payload, random)); + () -> setPayloadMultipart(connection, "lol", "cat", CSV_UTF_8, payload, random)); assertThat(thrown) .hasMessageThat() .contains( diff --git a/core/src/test/java/google/registry/testing/FakeURLFetchService.java b/core/src/test/java/google/registry/testing/FakeURLFetchService.java deleted file mode 100644 index f080b086a..000000000 --- a/core/src/test/java/google/registry/testing/FakeURLFetchService.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.testing; - -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; -import com.google.common.collect.ImmutableList; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.Map; - -/** - * A fake {@link URLFetchService} that serves constructed {@link HTTPResponse} objects from - * a simple {@link Map} ({@link URL} to {@link HTTPResponse}) lookup. - */ -public class FakeURLFetchService extends ForwardingURLFetchService { - - private Map backingMap; - - public FakeURLFetchService(Map backingMap) { - this.backingMap = backingMap; - } - - @Override - public HTTPResponse fetch(HTTPRequest request) { - URL requestURL = request.getURL(); - if (backingMap.containsKey(requestURL)) { - return backingMap.get(requestURL); - } else { - return new HTTPResponse(HttpURLConnection.HTTP_NOT_FOUND, null, null, ImmutableList.of()); - } - } -} diff --git a/core/src/test/java/google/registry/testing/FakeUrlConnectionService.java b/core/src/test/java/google/registry/testing/FakeUrlConnectionService.java new file mode 100644 index 000000000..de4bd5c81 --- /dev/null +++ b/core/src/test/java/google/registry/testing/FakeUrlConnectionService.java @@ -0,0 +1,46 @@ +// Copyright 2022 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.testing; + +import static org.mockito.Mockito.when; + +import google.registry.request.UrlConnectionService; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +/** A fake {@link UrlConnectionService} with a mocked HTTP connection for testing. */ +public class FakeUrlConnectionService implements UrlConnectionService { + + private final HttpURLConnection mockConnection; + private final List connectedUrls; + + public FakeUrlConnectionService(HttpURLConnection mockConnection) { + this(mockConnection, new ArrayList<>()); + } + + public FakeUrlConnectionService(HttpURLConnection mockConnection, List connectedUrls) { + this.mockConnection = mockConnection; + this.connectedUrls = connectedUrls; + } + + @Override + public HttpURLConnection createConnection(URL url) { + connectedUrls.add(url); + when(mockConnection.getURL()).thenReturn(url); + return mockConnection; + } +} diff --git a/core/src/test/java/google/registry/testing/ForwardingURLFetchService.java b/core/src/test/java/google/registry/testing/ForwardingURLFetchService.java deleted file mode 100644 index 56ab78083..000000000 --- a/core/src/test/java/google/registry/testing/ForwardingURLFetchService.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.testing; - -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; -import com.google.common.util.concurrent.Futures; -import java.io.IOException; -import java.net.URL; -import java.util.concurrent.Future; - -/** - * An implementation of the {@link URLFetchService} interface that forwards all requests through - * a synchronous fetch call. - */ -public abstract class ForwardingURLFetchService implements URLFetchService { - - @Override - public HTTPResponse fetch(URL url) throws IOException { - return fetch(new HTTPRequest(url)); // Defaults to HTTPMethod.GET - } - - @Override - public Future fetchAsync(URL url) { - return fetchAsync(new HTTPRequest(url)); // Defaults to HTTPMethod.GET - } - - @Override - public Future fetchAsync(HTTPRequest request) { - try { - return Futures.immediateFuture(fetch(request)); - } catch (Exception e) { - return Futures.immediateFailedFuture(e); - } - } -} diff --git a/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java b/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java index 86e7be307..6a137da2f 100644 --- a/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java +++ b/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java @@ -19,21 +19,22 @@ import static com.google.common.net.HttpHeaders.CONTENT_TYPE; import static com.google.common.net.HttpHeaders.LOCATION; import static com.google.common.net.MediaType.FORM_DATA; import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth8.assertThat; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.loadRegistrar; import static google.registry.testing.DatabaseHelper.newDomainBase; import static google.registry.testing.DatabaseHelper.persistDomainAndEnqueueLordn; import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; -import static google.registry.util.UrlFetchUtils.getHeaderFirst; -import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.startsWith; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -43,10 +44,6 @@ import com.google.appengine.api.taskqueue.TaskHandle; import com.google.appengine.api.taskqueue.TaskOptions; import com.google.appengine.api.taskqueue.TaskOptions.Method; import com.google.appengine.api.taskqueue.TransientFailureException; -import com.google.appengine.api.urlfetch.HTTPHeader; -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; import com.google.apphosting.api.DeadlineExceededException; import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; @@ -57,11 +54,15 @@ import google.registry.model.tld.Registry; import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeClock; import google.registry.testing.FakeSleeper; +import google.registry.testing.FakeUrlConnectionService; import google.registry.testing.InjectExtension; import google.registry.testing.TaskQueueHelper.TaskMatcher; import google.registry.util.Retrier; import google.registry.util.TaskQueueUtils; -import google.registry.util.UrlFetchException; +import google.registry.util.UrlConnectionException; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.net.HttpURLConnection; import java.net.URL; import java.security.SecureRandom; import java.util.List; @@ -69,17 +70,9 @@ import java.util.Optional; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; /** Unit tests for {@link NordnUploadAction}. */ -@ExtendWith(MockitoExtension.class) class NordnUploadActionTest { private static final String CLAIMS_CSV = @@ -101,29 +94,30 @@ class NordnUploadActionTest { AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build(); @RegisterExtension public final InjectExtension inject = new InjectExtension(); - - @Mock private URLFetchService fetchService; - @Mock private HTTPResponse httpResponse; - @Captor private ArgumentCaptor httpRequestCaptor; - + private final FakeClock clock = new FakeClock(DateTime.parse("2010-05-01T10:11:12Z")); private final LordnRequestInitializer lordnRequestInitializer = new LordnRequestInitializer(Optional.of("attack")); private final NordnUploadAction action = new NordnUploadAction(); + private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class); + private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream(); + private final FakeUrlConnectionService urlConnectionService = + new FakeUrlConnectionService(httpUrlConnection); + @BeforeEach void beforeEach() throws Exception { inject.setStaticField(Ofy.class, "clock", clock); - when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse); - when(httpResponse.getContent()).thenReturn("Success".getBytes(US_ASCII)); - when(httpResponse.getResponseCode()).thenReturn(SC_ACCEPTED); - when(httpResponse.getHeadersUncombined()) - .thenReturn(ImmutableList.of(new HTTPHeader(LOCATION, "http://trololol"))); + when(httpUrlConnection.getInputStream()) + .thenReturn(new ByteArrayInputStream("Success".getBytes(UTF_8))); + when(httpUrlConnection.getResponseCode()).thenReturn(SC_ACCEPTED); + when(httpUrlConnection.getHeaderField(LOCATION)).thenReturn("http://trololol"); + when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream); persistResource(loadRegistrar("TheRegistrar").asBuilder().setIanaIdentifier(99999L).build()); createTld("tld"); persistResource(Registry.get("tld").asBuilder().setLordnUsername("lolcat").build()); action.clock = clock; - action.fetchService = fetchService; + action.urlConnectionService = urlConnectionService; action.lordnRequestInitializer = lordnRequestInitializer; action.phase = "claims"; action.taskQueueUtils = new TaskQueueUtils(new Retrier(new FakeSleeper(clock), 3)); @@ -133,7 +127,6 @@ class NordnUploadActionTest { action.retrier = new Retrier(new FakeSleeper(clock), 3); } - @MockitoSettings(strictness = Strictness.LENIENT) @Test void test_convertTasksToCsv() { List tasks = @@ -145,7 +138,6 @@ class NordnUploadActionTest { .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n"); } - @MockitoSettings(strictness = Strictness.LENIENT) @Test void test_convertTasksToCsv_dedupesDuplicates() { List tasks = @@ -158,14 +150,12 @@ class NordnUploadActionTest { .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n"); } - @MockitoSettings(strictness = Strictness.LENIENT) @Test void test_convertTasksToCsv_doesntFailOnEmptyTasks() { assertThat(NordnUploadAction.convertTasksToCsv(ImmutableList.of(), clock.nowUtc(), "col1,col2")) .isEqualTo("1,2010-05-01T10:11:12.000Z,0\ncol1,col2\n"); } - @MockitoSettings(strictness = Strictness.LENIENT) @Test void test_convertTasksToCsv_throwsNpeOnNullTasks() { assertThrows( @@ -173,7 +163,6 @@ class NordnUploadActionTest { () -> NordnUploadAction.convertTasksToCsv(null, clock.nowUtc(), "header")); } - @MockitoSettings(strictness = Strictness.LENIENT) @SuppressWarnings("unchecked") @Test void test_loadAllTasks_retryLogic_thirdTrysTheCharm() { @@ -186,7 +175,6 @@ class NordnUploadActionTest { assertThat(action.loadAllTasks(queue, "tld")).containsExactly(task); } - @MockitoSettings(strictness = Strictness.LENIENT) @Test void test_loadAllTasks_retryLogic_allFailures() { Queue queue = mock(Queue.class); @@ -201,47 +189,47 @@ class NordnUploadActionTest { void testRun_claimsMode_appendsTldAndClaimsToRequestUrl() throws Exception { persistClaimsModeDomain(); action.run(); - assertThat(getCapturedHttpRequest().getURL()) - .isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims")); + assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims")); } @Test void testRun_sunriseMode_appendsTldAndClaimsToRequestUrl() throws Exception { persistSunriseModeDomain(); action.run(); - assertThat(getCapturedHttpRequest().getURL()) - .isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise")); + assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise")); } @Test void testRun_usesMultipartContentType() throws Exception { persistClaimsModeDomain(); action.run(); - assertThat(getHeaderFirst(getCapturedHttpRequest(), CONTENT_TYPE).get()) - .startsWith("multipart/form-data; boundary="); + verify(httpUrlConnection) + .setRequestProperty(eq(CONTENT_TYPE), startsWith("multipart/form-data; boundary=")); + verify(httpUrlConnection).setRequestMethod("POST"); } @Test - void testRun_hasPassword_setsAuthorizationHeader() throws Exception { + void testRun_hasPassword_setsAuthorizationHeader() { persistClaimsModeDomain(); action.run(); - assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)) - .hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64 + verify(httpUrlConnection) + .setRequestProperty( + AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64 } @Test - void testRun_noPassword_doesntSendAuthorizationHeader() throws Exception { + void testRun_noPassword_doesntSendAuthorizationHeader() { action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty()); persistClaimsModeDomain(); action.run(); - assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty(); + verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString()); } @Test - void testRun_claimsMode_payloadMatchesClaimsCsv() throws Exception { + void testRun_claimsMode_payloadMatchesClaimsCsv() { persistClaimsModeDomain(); action.run(); - assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(CLAIMS_CSV); + assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(CLAIMS_CSV); } @Test @@ -257,19 +245,19 @@ class NordnUploadActionTest { } @Test - void testRun_sunriseMode_payloadMatchesSunriseCsv() throws Exception { + void testRun_sunriseMode_payloadMatchesSunriseCsv() { persistSunriseModeDomain(); action.run(); - assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV); + assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV); } @Test void test_noResponseContent_stillWorksNormally() throws Exception { // Returning null only affects logging. - when(httpResponse.getContent()).thenReturn(null); + when(httpUrlConnection.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[] {})); persistSunriseModeDomain(); action.run(); - assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV); + assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV); } @Test @@ -284,7 +272,6 @@ class NordnUploadActionTest { .header(CONTENT_TYPE, FORM_DATA.toString())); } - @MockitoSettings(strictness = Strictness.LENIENT) @Test void testFailure_nullRegistryUser() { persistClaimsModeDomain(); @@ -293,17 +280,11 @@ class NordnUploadActionTest { assertThat(thrown).hasMessageThat().contains("lordnUsername is not set for tld."); } - @MockitoSettings(strictness = Strictness.LENIENT) @Test - void testFetchFailure() { + void testFetchFailure() throws Exception { persistClaimsModeDomain(); - when(httpResponse.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR); - assertThrows(UrlFetchException.class, action::run); - } - - private HTTPRequest getCapturedHttpRequest() throws Exception { - verify(fetchService).fetch(httpRequestCaptor.capture()); - return httpRequestCaptor.getAllValues().get(0); + when(httpUrlConnection.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR); + assertThrows(UrlConnectionException.class, action::run); } private void persistClaimsModeDomain() { diff --git a/core/src/test/java/google/registry/tmch/NordnVerifyActionTest.java b/core/src/test/java/google/registry/tmch/NordnVerifyActionTest.java index 30bfc5c78..3426cd00b 100644 --- a/core/src/test/java/google/registry/tmch/NordnVerifyActionTest.java +++ b/core/src/test/java/google/registry/tmch/NordnVerifyActionTest.java @@ -16,39 +16,34 @@ package google.registry.tmch; import static com.google.common.net.HttpHeaders.AUTHORIZATION; import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth8.assertThat; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.util.UrlFetchUtils.getHeaderFirst; import static java.nio.charset.StandardCharsets.UTF_8; import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; import static javax.servlet.http.HttpServletResponse.SC_OK; import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; import google.registry.model.tld.Registry; import google.registry.request.HttpException.ConflictException; import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeResponse; +import google.registry.testing.FakeUrlConnectionService; +import java.io.ByteArrayInputStream; +import java.net.HttpURLConnection; import java.net.URL; import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; /** Unit tests for {@link NordnVerifyAction}. */ -@ExtendWith(MockitoExtension.class) class NordnVerifyActionTest { private static final String LOG_ACCEPTED = @@ -81,84 +76,83 @@ class NordnVerifyActionTest { public final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build(); - @Mock private URLFetchService fetchService; - @Mock private HTTPResponse httpResponse; - @Captor private ArgumentCaptor httpRequestCaptor; - private final FakeResponse response = new FakeResponse(); private final LordnRequestInitializer lordnRequestInitializer = new LordnRequestInitializer(Optional.of("attack")); private final NordnVerifyAction action = new NordnVerifyAction(); + private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class); + private final FakeUrlConnectionService urlConnectionService = + new FakeUrlConnectionService(httpUrlConnection); + @BeforeEach void beforeEach() throws Exception { - when(httpResponse.getResponseCode()).thenReturn(SC_OK); - when(httpResponse.getContent()).thenReturn(LOG_ACCEPTED.getBytes(UTF_8)); - when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse); createTld("gtld"); persistResource(Registry.get("gtld").asBuilder().setLordnUsername("lolcat").build()); action.tld = "gtld"; - action.fetchService = fetchService; + action.urlConnectionService = urlConnectionService; + when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK); + when(httpUrlConnection.getInputStream()) + .thenReturn(new ByteArrayInputStream(LOG_ACCEPTED.getBytes(UTF_8))); action.lordnRequestInitializer = lordnRequestInitializer; action.response = response; action.url = new URL("http://127.0.0.1/blobio"); } - private HTTPRequest getCapturedHttpRequest() throws Exception { - verify(fetchService).fetch(httpRequestCaptor.capture()); - return httpRequestCaptor.getAllValues().get(0); - } - @Test void testSuccess_sendHttpRequest_urlIsCorrect() throws Exception { action.run(); - assertThat(getCapturedHttpRequest().getURL()).isEqualTo(new URL("http://127.0.0.1/blobio")); + assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/blobio")); } @Test - void testSuccess_hasLordnPassword_sendsAuthorizationHeader() throws Exception { + void testSuccess_hasLordnPassword_sendsAuthorizationHeader() { action.run(); - assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)) - .hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64 + verify(httpUrlConnection) + .setRequestProperty( + AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64 } @Test - void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() throws Exception { + void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() { action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty()); action.run(); - assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty(); + verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString()); } @Test void successVerifyRejected() throws Exception { - when(httpResponse.getContent()).thenReturn(LOG_REJECTED.getBytes(UTF_8)); + when(httpUrlConnection.getInputStream()) + .thenReturn(new ByteArrayInputStream(LOG_REJECTED.getBytes(UTF_8))); LordnLog lastLog = action.verify(); assertThat(lastLog.getStatus()).isEqualTo(LordnLog.Status.REJECTED); } @Test void successVerifyWarnings() throws Exception { - when(httpResponse.getContent()).thenReturn(LOG_WARNINGS.getBytes(UTF_8)); + when(httpUrlConnection.getInputStream()) + .thenReturn(new ByteArrayInputStream(LOG_WARNINGS.getBytes(UTF_8))); LordnLog lastLog = action.verify(); assertThat(lastLog.hasWarnings()).isTrue(); } @Test void successVerifyErrors() throws Exception { - when(httpResponse.getContent()).thenReturn(LOG_ERRORS.getBytes(UTF_8)); + when(httpUrlConnection.getInputStream()) + .thenReturn(new ByteArrayInputStream(LOG_ERRORS.getBytes(UTF_8))); LordnLog lastLog = action.verify(); assertThat(lastLog.hasWarnings()).isTrue(); } @Test - void failureVerifyUnauthorized() { - when(httpResponse.getResponseCode()).thenReturn(SC_UNAUTHORIZED); + void failureVerifyUnauthorized() throws Exception { + when(httpUrlConnection.getResponseCode()).thenReturn(SC_UNAUTHORIZED); assertThrows(Exception.class, action::run); } @Test - void failureVerifyNotReady() { - when(httpResponse.getResponseCode()).thenReturn(SC_NO_CONTENT); + void failureVerifyNotReady() throws Exception { + when(httpUrlConnection.getResponseCode()).thenReturn(SC_NO_CONTENT); ConflictException thrown = assertThrows(ConflictException.class, action::run); assertThat(thrown).hasMessageThat().contains("Not ready"); } diff --git a/core/src/test/java/google/registry/tmch/TmchActionTestCase.java b/core/src/test/java/google/registry/tmch/TmchActionTestCase.java index ce4688a6e..8eb148c3f 100644 --- a/core/src/test/java/google/registry/tmch/TmchActionTestCase.java +++ b/core/src/test/java/google/registry/tmch/TmchActionTestCase.java @@ -15,21 +15,19 @@ package google.registry.tmch; import static javax.servlet.http.HttpServletResponse.SC_OK; -import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; import google.registry.testing.AppEngineExtension; import google.registry.testing.BouncyCastleProviderExtension; import google.registry.testing.FakeClock; +import google.registry.testing.FakeUrlConnectionService; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; /** Common code for unit tests of classes that extend {@link Marksdb}. */ @@ -46,19 +44,19 @@ abstract class TmchActionTestCase { @RegisterExtension public final BouncyCastleProviderExtension bouncy = new BouncyCastleProviderExtension(); - @Mock URLFetchService fetchService; - @Mock HTTPResponse httpResponse; - @Captor ArgumentCaptor httpRequest; - final FakeClock clock = new FakeClock(); final Marksdb marksdb = new Marksdb(); + protected final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class); + protected final ArrayList connectedUrls = new ArrayList<>(); + protected FakeUrlConnectionService urlConnectionService = + new FakeUrlConnectionService(httpUrlConnection, connectedUrls); + @BeforeEach public void beforeEachTmchActionTestCase() throws Exception { - marksdb.fetchService = fetchService; marksdb.tmchMarksdbUrl = MARKSDB_URL; marksdb.marksdbPublicKey = TmchData.loadPublicKey(TmchTestData.loadBytes("pubkey")); - when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse); - when(httpResponse.getResponseCode()).thenReturn(SC_OK); + marksdb.urlConnectionService = urlConnectionService; + when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK); } } diff --git a/core/src/test/java/google/registry/tmch/TmchCrlActionTest.java b/core/src/test/java/google/registry/tmch/TmchCrlActionTest.java index 3e18ab46d..eeb7861ea 100644 --- a/core/src/test/java/google/registry/tmch/TmchCrlActionTest.java +++ b/core/src/test/java/google/registry/tmch/TmchCrlActionTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import google.registry.config.RegistryConfig.ConfigModule.TmchCaMode; +import java.io.ByteArrayInputStream; import java.net.MalformedURLException; import java.net.URL; import java.security.SignatureException; @@ -44,19 +45,22 @@ class TmchCrlActionTest extends TmchActionTestCase { @Test void testSuccess() throws Exception { clock.setTo(DateTime.parse("2013-07-24TZ")); - when(httpResponse.getContent()).thenReturn( - readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()); + when(httpUrlConnection.getInputStream()) + .thenReturn( + new ByteArrayInputStream( + readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read())); newTmchCrlAction(TmchCaMode.PRODUCTION).run(); - verify(httpResponse).getContent(); - verify(fetchService).fetch(httpRequest.capture()); - assertThat(httpRequest.getValue().getURL().toString()).isEqualTo("http://sloth.lol/tmch.crl"); + verify(httpUrlConnection).getInputStream(); + assertThat(connectedUrls).containsExactly(new URL("http://sloth.lol/tmch.crl")); } @Test void testFailure_crlTooOld() throws Exception { clock.setTo(DateTime.parse("2020-01-01TZ")); - when(httpResponse.getContent()) - .thenReturn(loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read()); + when(httpUrlConnection.getInputStream()) + .thenReturn( + new ByteArrayInputStream( + loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read())); TmchCrlAction action = newTmchCrlAction(TmchCaMode.PILOT); Exception e = assertThrows(Exception.class, action::run); assertThat(e).hasCauseThat().isInstanceOf(CRLException.class); @@ -69,8 +73,10 @@ class TmchCrlActionTest extends TmchActionTestCase { @Test void testFailure_crlNotSignedByRoot() throws Exception { clock.setTo(DateTime.parse("2013-07-24TZ")); - when(httpResponse.getContent()) - .thenReturn(readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()); + when(httpUrlConnection.getInputStream()) + .thenReturn( + new ByteArrayInputStream( + readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read())); Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run); assertThat(e).hasCauseThat().isInstanceOf(SignatureException.class); assertThat(e).hasCauseThat().hasMessageThat().isEqualTo("Signature does not match."); @@ -79,8 +85,10 @@ class TmchCrlActionTest extends TmchActionTestCase { @Test void testFailure_crlNotYetValid() throws Exception { clock.setTo(DateTime.parse("1984-01-01TZ")); - when(httpResponse.getContent()).thenReturn( - readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read()); + when(httpUrlConnection.getInputStream()) + .thenReturn( + new ByteArrayInputStream( + readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read())); Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run); assertThat(e).hasCauseThat().isInstanceOf(CertificateNotYetValidException.class); } diff --git a/core/src/test/java/google/registry/tmch/TmchDnlActionTest.java b/core/src/test/java/google/registry/tmch/TmchDnlActionTest.java index 8018b2d79..0845956e2 100644 --- a/core/src/test/java/google/registry/tmch/TmchDnlActionTest.java +++ b/core/src/test/java/google/registry/tmch/TmchDnlActionTest.java @@ -14,6 +14,7 @@ package google.registry.tmch; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth8.assertThat; import static org.mockito.Mockito.times; @@ -22,6 +23,8 @@ import static org.mockito.Mockito.when; import google.registry.model.tmch.ClaimsList; import google.registry.model.tmch.ClaimsListDao; +import java.io.ByteArrayInputStream; +import java.net.URL; import java.util.Optional; import org.joda.time.DateTime; import org.junit.jupiter.api.Test; @@ -39,15 +42,13 @@ class TmchDnlActionTest extends TmchActionTestCase { @Test void testDnl() throws Exception { assertThat(ClaimsListDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty(); - when(httpResponse.getContent()) - .thenReturn(TmchTestData.loadBytes("dnl-latest.csv").read()) - .thenReturn(TmchTestData.loadBytes("dnl-latest.sig").read()); + when(httpUrlConnection.getInputStream()) + .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.csv").read())) + .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.sig").read())); newTmchDnlAction().run(); - verify(fetchService, times(2)).fetch(httpRequest.capture()); - assertThat(httpRequest.getAllValues().get(0).getURL().toString()) - .isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.csv"); - assertThat(httpRequest.getAllValues().get(1).getURL().toString()) - .isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.sig"); + verify(httpUrlConnection, times(2)).getInputStream(); + assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList())) + .containsExactly(MARKSDB_URL + "/dnl/dnl-latest.csv", MARKSDB_URL + "/dnl/dnl-latest.sig"); // Make sure the contents of testdata/dnl-latest.csv got inserted into the database. ClaimsList claimsList = ClaimsListDao.get(); diff --git a/core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java b/core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java index 4dfd9aa5e..181955381 100644 --- a/core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java +++ b/core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java @@ -14,6 +14,7 @@ package google.registry.tmch; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.truth.Truth.assertThat; import static google.registry.tmch.TmchTestData.loadBytes; import static org.mockito.Mockito.times; @@ -21,6 +22,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import google.registry.model.smd.SignedMarkRevocationList; +import java.io.ByteArrayInputStream; +import java.net.URL; import java.util.Optional; import org.joda.time.DateTime; import org.junit.jupiter.api.Test; @@ -42,15 +45,14 @@ class TmchSmdrlActionTest extends TmchActionTestCase { SignedMarkRevocationList smdrl = SignedMarkRevocationList.get(); assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isFalse(); assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse(); - when(httpResponse.getContent()) - .thenReturn(loadBytes("smdrl-latest.csv").read()) - .thenReturn(loadBytes("smdrl-latest.sig").read()); + when(httpUrlConnection.getInputStream()) + .thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.csv").read())) + .thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.sig").read())); newTmchSmdrlAction().run(); - verify(fetchService, times(2)).fetch(httpRequest.capture()); - assertThat(httpRequest.getAllValues().get(0).getURL().toString()) - .isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.csv"); - assertThat(httpRequest.getAllValues().get(1).getURL().toString()) - .isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.sig"); + verify(httpUrlConnection, times(2)).getInputStream(); + assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList())) + .containsExactly( + MARKSDB_URL + "/smdrl/smdrl-latest.csv", MARKSDB_URL + "/smdrl/smdrl-latest.sig"); smdrl = SignedMarkRevocationList.get(); assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isTrue(); assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse(); diff --git a/util/src/main/java/google/registry/util/UrlConnectionException.java b/util/src/main/java/google/registry/util/UrlConnectionException.java new file mode 100644 index 000000000..019a50eb4 --- /dev/null +++ b/util/src/main/java/google/registry/util/UrlConnectionException.java @@ -0,0 +1,71 @@ +// Copyright 2022 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.util; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.io.ByteStreams; +import java.io.IOException; +import java.net.HttpURLConnection; + +/** + * Used when HTTP requests return a bad response, with troubleshooting info. + * + *

This class displays lots of helpful troubleshooting information. + */ +public class UrlConnectionException extends RuntimeException { + + private final HttpURLConnection connection; + + public UrlConnectionException(String message, HttpURLConnection connection) { + super(message); + this.connection = connection; + } + + @Override + public String getMessage() { + byte[] resultContent; + int responseCode; + try { + resultContent = ByteStreams.toByteArray(connection.getInputStream()); + responseCode = connection.getResponseCode(); + } catch (IOException e) { + resultContent = new byte[] {}; + responseCode = 0; + } + StringBuilder result = + new StringBuilder(2048 + resultContent.length) + .append( + String.format( + "%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\n", + getClass().getSimpleName(), + super.getMessage(), + responseCode, + connection.getURL().toString())); + connection + .getRequestProperties() + .forEach( + (key, value) -> { + result.append(key); + result.append(": "); + result.append(value); + result.append('\n'); + }); + result.append(">>>\n"); + result.append(new String(resultContent, UTF_8)); + result.append("\n<<<"); + return result.toString(); + } +} diff --git a/util/src/main/java/google/registry/util/UrlFetchException.java b/util/src/main/java/google/registry/util/UrlFetchException.java deleted file mode 100644 index 4e2730c3b..000000000 --- a/util/src/main/java/google/registry/util/UrlFetchException.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.util; - -import static com.google.common.base.Preconditions.checkNotNull; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.google.appengine.api.urlfetch.HTTPHeader; -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; - -/** - * Exception for when App Engine HTTP requests return a bad response. - * - *

This class displays lots of helpful troubleshooting information. - */ -public class UrlFetchException extends RuntimeException { - - private final HTTPRequest req; - private final HTTPResponse rsp; - - public UrlFetchException(String message, HTTPRequest req, HTTPResponse rsp) { - super(message); - this.req = checkNotNull(req, "req"); - this.rsp = checkNotNull(rsp, "rsp"); - } - - @Override - public String getMessage() { - StringBuilder res = - new StringBuilder(2048 + rsp.getContent().length) - .append( - String.format( - "%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\nX-Final-URL: %s\n", - getClass().getSimpleName(), - super.getMessage(), - rsp.getResponseCode(), - req.getURL().toString(), - rsp.getFinalUrl())); - for (HTTPHeader header : rsp.getHeadersUncombined()) { - res.append(header.getName()); - res.append(": "); - res.append(header.getValue()); - res.append('\n'); - } - res.append(">>>\n"); - res.append(new String(rsp.getContent(), UTF_8)); - res.append("\n<<<"); - return res.toString(); - } -}