Use built-in Java URL connections instead of UrlFetchService (#1535)

- Use the standard HttpsURLConnection to write/read data
- Rewrite RdeReporter, Nordn*Action, and Marksdb classes and related
  tests to conform to the new format
- Remove FakeURLFetchService and ForwardingUrlFetchService as they weren't used
- Refactor UrlFetchException to UrlConnectionException
- Refactor UrlFetchUtils to UrlConnectionUtils

I will need to test this on Alpha. Fortunately the connections that
don't require auth (e.g. TMDB downloading) should be testable.

core/src/main/java/google/registry/module/backend/BackendComponent.java

@@ -43,7 +43,7 @@ import google.registry.rde.JSchModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.URLFetchServiceModule;
+import google.registry.request.Modules.UrlConnectionServiceModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
@@ -80,7 +80,7 @@ import javax.inject.Singleton;
       ServerTridProviderModule.class,
       SheetsServiceModule.class,
       StackdriverModule.class,
-      URLFetchServiceModule.class,
+      UrlConnectionServiceModule.class,
       UrlFetchTransportModule.class,
       UserServiceModule.class,
       VoidDnsWriterModule.class,

core/src/main/java/google/registry/module/frontend/FrontendComponent.java

@@ -34,7 +34,6 @@ import google.registry.monitoring.whitebox.StackdriverModule;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.ui.ConsoleDebug.ConsoleConfigModule;
@@ -65,7 +64,6 @@ import javax.inject.Singleton;
       SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/module/pubapi/PubApiComponent.java

@@ -34,7 +34,6 @@ import google.registry.persistence.PersistenceModule;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.util.UtilsModule;
@@ -62,7 +61,6 @@ import javax.inject.Singleton;
       SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/module/tools/ToolsComponent.java

@@ -36,7 +36,6 @@ import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.util.UtilsModule;
@@ -66,7 +65,6 @@ import javax.inject.Singleton;
       ServerTridProviderModule.class,
       StackdriverModule.class,
       ToolsRequestComponentModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/rde/RdeReporter.java

@@ -14,26 +14,24 @@
 
 package google.registry.rde;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
-import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
+import static google.registry.request.UrlConnectionUtils.setBasicAuth;
-import static com.google.common.io.BaseEncoding.base64;
+import static google.registry.request.UrlConnectionUtils.setPayload;
-import static com.google.common.net.HttpHeaders.AUTHORIZATION;
-import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
+import com.google.api.client.http.HttpMethods;
-import com.google.appengine.api.urlfetch.HTTPRequest;
 import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.request.HttpException.InternalServerErrorException;
+import google.registry.request.UrlConnectionService;
 import google.registry.util.Retrier;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
 import google.registry.xjc.iirdea.XjcIirdeaResult;
@@ -41,6 +39,7 @@ import google.registry.xjc.rdeheader.XjcRdeHeader;
 import google.registry.xjc.rdereport.XjcRdeReportReport;
 import google.registry.xml.XmlException;
 import java.io.ByteArrayInputStream;
+import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.SocketTimeoutException;
 import java.net.URL;
@@ -55,12 +54,15 @@ public class RdeReporter {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  /** @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
+  /**
-   *     ICANN Registry Interfaces - Interface details</a>*/
+   * @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
-  private static final String REPORT_MIME = "text/xml";
+   *     ICANN Registry Interfaces - Interface details</a>
+   */
+  private static final MediaType MEDIA_TYPE = MediaType.XML_UTF_8;
 
   @Inject Retrier retrier;
-  @Inject URLFetchService urlFetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix;
   @Inject @Key("icannReportingPassword") String password;
   @Inject RdeReporter() {}
@@ -74,29 +76,24 @@ public class RdeReporter {
     // Send a PUT request to ICANN's HTTPS server.
     URL url = makeReportUrl(header.getTld(), report.getId());
     String username = header.getTld() + "_ry";
-    String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8));
-    final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d));
-    req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME));
-    req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
-    req.setPayload(reportBytes);
     logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8));
-    HTTPResponse rsp =
+    byte[] responseBytes =
         retrier.callWithRetry(
             () -> {
-              HTTPResponse rsp1 = urlFetchService.fetch(req);
+              HttpURLConnection connection = urlConnectionService.createConnection(url);
-              switch (rsp1.getResponseCode()) {
+              connection.setRequestMethod(HttpMethods.PUT);
-                case SC_OK:
+              setBasicAuth(connection, username, password);
-                case SC_BAD_REQUEST:
+              setPayload(connection, reportBytes, MEDIA_TYPE.toString());
-                  break;
+              int responseCode = connection.getResponseCode();
-                default:
+              if (responseCode == SC_OK || responseCode == SC_BAD_REQUEST) {
-                  throw new UrlFetchException("PUT failed", req, rsp1);
+                return getResponseBytes(connection);
               }
-              return rsp1;
+              throw new UrlConnectionException("PUT failed", connection);
             },
             SocketTimeoutException.class);
 
     // Ensure the XML response is valid.
-    XjcIirdeaResult result = parseResult(rsp);
+    XjcIirdeaResult result = parseResult(responseBytes);
     if (result.getCode().getValue() != 1000) {
       logger.atWarning().log(
           "PUT rejected: %d %s\n%s",
@@ -108,11 +105,11 @@ public class RdeReporter {
   /**
    * Unmarshals IIRDEA XML result object from {@link HTTPResponse} payload.
    *
-   * @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1">
+   * @see <a
+   *     href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1">
    *     ICANN Registry Interfaces - IIRDEA Result Object</a>
    */
-  private XjcIirdeaResult parseResult(HTTPResponse rsp) throws XmlException {
+  private XjcIirdeaResult parseResult(byte[] responseBytes) throws XmlException {
-    byte[] responseBytes = rsp.getContent();
     logger.atInfo().log("Received response:\n%s", new String(responseBytes, UTF_8));
     XjcIirdeaResponseElement response = XjcXmlTransformer.unmarshal(
         XjcIirdeaResponseElement.class, new ByteArrayInputStream(responseBytes));

core/src/main/java/google/registry/request/Modules.java

@@ -23,12 +23,11 @@ import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.appengine.api.datastore.DatastoreService;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
 import com.google.appengine.api.users.UserService;
 import com.google.appengine.api.users.UserServiceFactory;
 import dagger.Module;
 import dagger.Provides;
+import java.net.HttpURLConnection;
 import javax.inject.Singleton;
 
 /** Dagger modules for App Engine services and other vendor classes. */
@@ -45,14 +44,12 @@ public final class Modules {
     }
   }
 
-  /** Dagger module for {@link URLFetchService}. */
+  /** Dagger module for {@link UrlConnectionService}. */
   @Module
-  public static final class URLFetchServiceModule {
+  public static final class UrlConnectionServiceModule {
-    private static final URLFetchService fetchService = URLFetchServiceFactory.getURLFetchService();
-
     @Provides
-    static URLFetchService provideURLFetchService() {
+    static UrlConnectionService provideUrlConnectionService() {
-      return fetchService;
+      return url -> (HttpURLConnection) url.openConnection();
     }
   }
 

core/src/main/java/google/registry/request/UrlConnectionService.java

@@ -0,0 +1,25 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.request;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/** Functional interface for opening a connection from a URL, injectable for testing. */
+public interface UrlConnectionService {
+
+  HttpURLConnection createConnection(URL url) throws IOException;
+}

core/src/main/java/google/registry/request/UrlConnectionUtils.java

@@ -1,4 +1,4 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.util;
+package google.registry.request;
 
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.io.BaseEncoding.base64;
@@ -22,36 +22,41 @@ import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
 import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.common.base.Ascii;
 import com.google.common.base.Strings;
+import com.google.common.io.ByteStreams;
 import com.google.common.net.MediaType;
-import java.util.Optional;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.net.URLConnection;
 import java.util.Random;
 
-/** Helper methods for the App Engine URL fetch service. */
+/** Utilities for common functionality relating to {@link java.net.URLConnection}s. */
-public final class UrlFetchUtils {
+public class UrlConnectionUtils {
 
-  /** Returns value of first header matching {@code name}. */
+  /** Retrieves the response from the given connection as a byte array. */
-  public static Optional<String> getHeaderFirst(HTTPResponse rsp, String name) {
+  public static byte[] getResponseBytes(URLConnection connection) throws IOException {
-    return getHeaderFirstInternal(rsp.getHeadersUncombined(), name);
+    return ByteStreams.toByteArray(connection.getInputStream());
   }
 
-  /** Returns value of first header matching {@code name}. */
+  /** Sets auth on the given connection with the given username/password. */
-  public static Optional<String> getHeaderFirst(HTTPRequest req, String name) {
+  public static void setBasicAuth(URLConnection connection, String username, String password) {
-    return getHeaderFirstInternal(req.getHeaders(), name);
+    setBasicAuth(connection, String.format("%s:%s", username, password));
   }
 
-  private static Optional<String> getHeaderFirstInternal(Iterable<HTTPHeader> hdrs, String name) {
+  /** Sets auth on the given connection with the given string, formatted "username:password". */
-    name = Ascii.toLowerCase(name);
+  public static void setBasicAuth(URLConnection connection, String usernameAndPassword) {
-    for (HTTPHeader header : hdrs) {
+    String token = base64().encode(usernameAndPassword.getBytes(UTF_8));
-      if (Ascii.toLowerCase(header.getName()).equals(name)) {
+    connection.setRequestProperty(AUTHORIZATION, "Basic " + token);
-        return Optional.of(header.getValue());
   }
+
+  /** Sets the given byte[] payload on the given connection with a particular content type. */
+  public static void setPayload(URLConnection connection, byte[] bytes, String contentType)
+      throws IOException {
+    connection.setRequestProperty(CONTENT_TYPE, contentType);
+    connection.setDoOutput(true);
+    try (DataOutputStream dataStream = new DataOutputStream(connection.getOutputStream())) {
+      dataStream.write(bytes);
     }
-    return Optional.empty();
   }
 
   /**
@@ -62,16 +67,16 @@ public final class UrlFetchUtils {
    * @see <a href="http://www.ietf.org/rfc/rfc2388.txt">RFC2388 - Returning Values from Forms</a>
    */
   public static void setPayloadMultipart(
-      HTTPRequest request,
+      URLConnection connection,
       String name,
       String filename,
       MediaType contentType,
       String data,
-      Random random) {
+      Random random)
+      throws IOException {
     String boundary = createMultipartBoundary(random);
     checkState(
-        !data.contains(boundary),
+        !data.contains(boundary), "Multipart data contains autogenerated boundary: %s", boundary);
-        "Multipart data contains autogenerated boundary: %s", boundary);
     String multipart =
         String.format("--%s\r\n", boundary)
             + String.format(
@@ -83,11 +88,9 @@ public final class UrlFetchUtils {
             + "\r\n"
             + String.format("--%s--\r\n", boundary);
     byte[] payload = multipart.getBytes(UTF_8);
-    request.addHeader(
+    connection.setRequestProperty(CONTENT_LENGTH, Integer.toString(payload.length));
-        new HTTPHeader(
+    setPayload(
-            CONTENT_TYPE, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary)));
+        connection, payload, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary));
-    request.addHeader(new HTTPHeader(CONTENT_LENGTH, Integer.toString(payload.length)));
-    request.setPayload(payload);
   }
 
   private static String createMultipartBoundary(Random random) {
@@ -98,12 +101,4 @@ public final class UrlFetchUtils {
     // See https://tools.ietf.org/html/rfc2046#section-5.1.1
     return Strings.repeat("-", 30) + base64().encode(rand);
   }
-
-  /** Sets the HTTP Basic Authentication header on an {@link HTTPRequest}. */
-  public static void setAuthorizationHeader(HTTPRequest req, Optional<String> login) {
-    if (login.isPresent()) {
-      String token = base64().encode(login.get().getBytes(UTF_8));
-      req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
-    }
-  }
 }

core/src/main/java/google/registry/tmch/LordnRequestInitializer.java

@@ -15,12 +15,12 @@
 package google.registry.tmch;
 
 import static com.google.common.base.Verify.verifyNotNull;
-import static google.registry.util.UrlFetchUtils.setAuthorizationHeader;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
 import com.google.common.flogger.FluentLogger;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.model.tld.Registry;
+import google.registry.request.UrlConnectionUtils;
+import java.net.HttpURLConnection;
 import java.util.Optional;
 import javax.inject.Inject;
 
@@ -37,8 +37,9 @@ final class LordnRequestInitializer {
   }
 
   /** Initializes a URL fetch request for talking to the MarksDB server. */
-  void initialize(HTTPRequest request, String tld) {
+  void initialize(HttpURLConnection connection, String tld) {
-    setAuthorizationHeader(request, getMarksDbLordnCredentials(tld));
+    getMarksDbLordnCredentials(tld)
+        .ifPresent(login -> UrlConnectionUtils.setBasicAuth(connection, login));
   }
 
   /** Returns the username and password for the current TLD to login to the MarksDB server. */

core/src/main/java/google/registry/tmch/Marksdb.java

@@ -14,25 +14,23 @@
 
 package google.registry.tmch;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.GET;
 import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
+import static google.registry.request.UrlConnectionUtils.setBasicAuth;
 import static google.registry.util.HexDumper.dumpHex;
-import static google.registry.util.UrlFetchUtils.setAuthorizationHeader;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteSource;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
-import google.registry.util.UrlFetchException;
+import google.registry.request.UrlConnectionService;
+import google.registry.util.UrlConnectionException;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.Security;
 import java.security.SignatureException;
@@ -57,7 +55,8 @@ public final class Marksdb {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   private static final int MAX_DNL_LOGGING_LENGTH = 500;
 
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl;
   @Inject @Key("marksdbPublicKey") PGPPublicKey marksdbPublicKey;
   @Inject Marksdb() {}
@@ -112,19 +111,16 @@ public final class Marksdb {
   }
 
   byte[] fetch(URL url, Optional<String> loginAndPassword) throws IOException {
-    HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d));
+    HttpURLConnection connection = urlConnectionService.createConnection(url);
-    setAuthorizationHeader(req, loginAndPassword);
+    loginAndPassword.ifPresent(auth -> setBasicAuth(connection, auth));
-    HTTPResponse rsp;
     try {
-      rsp = fetchService.fetch(req);
+      if (connection.getResponseCode() != SC_OK) {
+        throw new UrlConnectionException("Failed to fetch from MarksDB", connection);
+      }
+      return getResponseBytes(connection);
     } catch (IOException e) {
-      throw new IOException(
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
-          String.format("Error connecting to MarksDB at URL %s", url), e);
     }
-    if (rsp.getResponseCode() != SC_OK) {
-      throw new UrlFetchException("Failed to fetch from MarksDB", req, rsp);
-    }
-    return rsp.getContent();
   }
 
   List<String> fetchSignedCsv(Optional<String> loginAndPassword, String csvPath, String sigPath)

core/src/main/java/google/registry/tmch/NordnUploadAction.java

@@ -16,28 +16,23 @@ package google.registry.tmch;
 
 import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.POST;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.net.HttpHeaders.LOCATION;
 import static com.google.common.net.MediaType.CSV_UTF_8;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
 import static google.registry.tmch.LordnTaskUtils.COLUMNS_CLAIMS;
 import static google.registry.tmch.LordnTaskUtils.COLUMNS_SUNRISE;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
-import static google.registry.util.UrlFetchUtils.setPayloadMultipart;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 
+import com.google.api.client.http.HttpMethods;
 import com.google.appengine.api.taskqueue.LeaseOptions;
 import com.google.appengine.api.taskqueue.Queue;
 import com.google.appengine.api.taskqueue.TaskHandle;
 import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TransientFailureException;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.apphosting.api.DeadlineExceededException;
 import com.google.common.base.Joiner;
 import com.google.common.base.Strings;
@@ -49,16 +44,18 @@ import google.registry.config.RegistryConfig.Config;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
+import google.registry.request.UrlConnectionService;
+import google.registry.request.UrlConnectionUtils;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
 import google.registry.util.Retrier;
 import google.registry.util.TaskQueueUtils;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.SecureRandom;
 import java.util.List;
-import java.util.Optional;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
@@ -97,7 +94,8 @@ public final class NordnUploadAction implements Runnable {
   @Inject Retrier retrier;
   @Inject SecureRandom random;
   @Inject LordnRequestInitializer lordnRequestInitializer;
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl;
   @Inject @Parameter(LORDN_PHASE_PARAM) String phase;
   @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@@ -193,47 +191,48 @@ public final class NordnUploadAction implements Runnable {
    * <p>Idempotency: If the exact same LORDN report is uploaded twice, the MarksDB server will
    * return the same confirmation number.
    *
-   * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">
+   * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">TMCH
-   *     TMCH functional specifications - LORDN File</a>
+   *     functional specifications - LORDN File</a>
    */
   private void uploadCsvToLordn(String urlPath, String csvData) throws IOException {
     String url = tmchMarksdbUrl + urlPath;
     logger.atInfo().log(
         "LORDN upload task %s: Sending to URL: %s ; data: %s", actionLogId, url, csvData);
-    HTTPRequest req = new HTTPRequest(new URL(url), POST, validateCertificate().setDeadline(60d));
+    HttpURLConnection connection = urlConnectionService.createConnection(new URL(url));
-    lordnRequestInitializer.initialize(req, tld);
+    connection.setRequestMethod(HttpMethods.POST);
-    setPayloadMultipart(req, "file", "claims.csv", CSV_UTF_8, csvData, random);
+    lordnRequestInitializer.initialize(connection, tld);
-    HTTPResponse rsp;
+    UrlConnectionUtils.setPayloadMultipart(
+        connection, "file", "claims.csv", CSV_UTF_8, csvData, random);
     try {
-      rsp = fetchService.fetch(req);
+      int responseCode = connection.getResponseCode();
-    } catch (IOException e) {
-      throw new IOException(
-          String.format("Error connecting to MarksDB at URL %s", url), e);
-    }
       if (logger.atInfo().isEnabled()) {
-      String response =
+        String responseContent = new String(getResponseBytes(connection), US_ASCII);
-          (rsp.getContent() == null) ? "(null)" : new String(rsp.getContent(), US_ASCII);
+        if (responseContent.isEmpty()) {
+          responseContent = "(null)";
+        }
         logger.atInfo().log(
             "LORDN upload task %s response: HTTP response code %d, response data: %s",
-          actionLogId, rsp.getResponseCode(), response);
+            actionLogId, responseCode, responseContent);
       }
-    if (rsp.getResponseCode() != SC_ACCEPTED) {
+      if (responseCode != SC_ACCEPTED) {
-      throw new UrlFetchException(
+        throw new UrlConnectionException(
             String.format(
-              "LORDN upload task %s error: Failed to upload LORDN claims to MarksDB", actionLogId),
+                "LORDN upload task %s error: Failed to upload LORDN claims to MarksDB",
-          req,
+                actionLogId),
-          rsp);
+            connection);
       }
-    Optional<String> location = getHeaderFirst(rsp, LOCATION);
+      String location = connection.getHeaderField(LOCATION);
-    if (!location.isPresent()) {
+      if (location == null) {
-      throw new UrlFetchException(
+        throw new UrlConnectionException(
             String.format(
                 "LORDN upload task %s error: MarksDB failed to provide a Location header",
                 actionLogId),
-          req,
+            connection);
-          rsp);
+      }
+      getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location)));
+    } catch (IOException e) {
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
     }
-    getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location.get())));
   }
 
   private TaskOptions makeVerifyTask(URL url) {

core/src/main/java/google/registry/tmch/NordnVerifyAction.java

@@ -14,15 +14,11 @@
 
 package google.registry.tmch;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
-import static com.google.appengine.api.urlfetch.HTTPMethod.GET;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteSource;
@@ -32,9 +28,11 @@ import google.registry.request.HttpException.ConflictException;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
 import google.registry.request.Response;
+import google.registry.request.UrlConnectionService;
 import google.registry.request.auth.Auth;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Map.Entry;
 import javax.inject.Inject;
@@ -68,7 +66,8 @@ public final class NordnVerifyAction implements Runnable {
 
   @Inject LordnRequestInitializer lordnRequestInitializer;
   @Inject Response response;
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Header(URL_HEADER) URL url;
   @Inject @Header(HEADER_ACTION_LOG_ID) String actionLogId;
   @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@@ -96,30 +95,25 @@ public final class NordnVerifyAction implements Runnable {
   @VisibleForTesting
   LordnLog verify() throws IOException {
     logger.atInfo().log("LORDN verify task %s: Sending request to URL %s", actionLogId, url);
-    HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d));
+    HttpURLConnection connection = urlConnectionService.createConnection(url);
-    lordnRequestInitializer.initialize(req, tld);
+    lordnRequestInitializer.initialize(connection, tld);
-    HTTPResponse rsp;
     try {
-      rsp = fetchService.fetch(req);
+      int responseCode = connection.getResponseCode();
-    } catch (IOException e) {
-      throw new IOException(
-          String.format("Error connecting to MarksDB at URL %s", url), e);
-    }
       logger.atInfo().log(
-        "LORDN verify task %s response: HTTP response code %d, response data: %s",
+          "LORDN verify task %s response: HTTP response code %d", actionLogId, responseCode);
-        actionLogId, rsp.getResponseCode(), rsp.getContent());
+      if (responseCode == SC_NO_CONTENT) {
-    if (rsp.getResponseCode() == SC_NO_CONTENT) {
         // Send a 400+ status code so App Engine will retry the task.
         throw new ConflictException("Not ready");
       }
-    if (rsp.getResponseCode() != SC_OK) {
+      if (responseCode != SC_OK) {
-      throw new UrlFetchException(
+        throw new UrlConnectionException(
-          String.format("LORDN verify task %s: Failed to verify LORDN upload to MarksDB.",
+            String.format(
-              actionLogId),
+                "LORDN verify task %s: Failed to verify LORDN upload to MarksDB.", actionLogId),
-          req, rsp);
+            connection);
       }
       LordnLog log =
-        LordnLog.parse(ByteSource.wrap(rsp.getContent()).asCharSource(UTF_8).readLines());
+          LordnLog.parse(
+              ByteSource.wrap(getResponseBytes(connection)).asCharSource(UTF_8).readLines());
       if (log.getStatus() == LordnLog.Status.ACCEPTED) {
         logger.atInfo().log("LORDN verify task %s: Upload accepted.", actionLogId);
       } else {
@@ -142,5 +136,8 @@ public final class NordnVerifyAction implements Runnable {
         }
       }
       return log;
+    } catch (IOException e) {
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
+    }
   }
 }

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -38,8 +38,7 @@ import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.rde.RdeModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
-import google.registry.request.Modules.URLFetchServiceModule;
+import google.registry.request.Modules.UrlConnectionServiceModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
 import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
@@ -79,8 +78,7 @@ import javax.inject.Singleton;
       RegistryToolDataflowModule.class,
       RequestFactoryModule.class,
       SecretManagerModule.class,
-      URLFetchServiceModule.class,
+      UrlConnectionServiceModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class,
       VoidDnsWriterModule.class,

core/src/test/java/google/registry/rde/RdeReportActionTest.java

@@ -14,7 +14,6 @@
 
 package google.registry.rde;
 
-import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
@@ -29,20 +28,13 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.joda.time.Duration.standardDays;
 import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.cloud.storage.BlobId;
 import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
-import com.google.common.base.Ascii;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteSource;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.common.Cursor;
@@ -57,20 +49,21 @@ import google.registry.testing.FakeClock;
 import google.registry.testing.FakeKeyringModule;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.FakeSleeper;
+import google.registry.testing.FakeUrlConnectionService;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.util.Retrier;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.rdereport.XjcRdeReportReport;
 import google.registry.xml.XmlException;
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.net.HttpURLConnection;
 import java.net.SocketTimeoutException;
-import java.util.Map;
 import java.util.Optional;
 import org.bouncycastle.openpgp.PGPPublicKey;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
 
 /** Unit tests for {@link RdeReportAction}. */
 @DualDatabaseTest
@@ -89,20 +82,21 @@ public class RdeReportActionTest {
 
   private final FakeResponse response = new FakeResponse();
   private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class);
-  private final URLFetchService urlFetchService = mock(URLFetchService.class);
-  private final ArgumentCaptor<HTTPRequest> request = ArgumentCaptor.forClass(HTTPRequest.class);
-  private final HTTPResponse httpResponse = mock(HTTPResponse.class);
   private final PGPPublicKey encryptKey =
       new FakeKeyringModule().get().getRdeStagingEncryptionKey();
   private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
   private final BlobId reportFile =
       BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde");
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+  private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
 
   private RdeReportAction createAction() {
     RdeReporter reporter = new RdeReporter();
     reporter.reportUrlPrefix = "https://rde-report.example";
-    reporter.urlFetchService = urlFetchService;
     reporter.password = "foo";
+    reporter.urlConnectionService = urlConnectionService;
     reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
     RdeReportAction action = new RdeReportAction();
     action.gcsUtils = gcsUtils;
@@ -127,6 +121,7 @@ public class RdeReportActionTest {
         Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test")));
     gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
+    when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
   }
 
   @TestOfyAndSql
@@ -142,24 +137,22 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
     assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
 
     // Verify the HTTP request was correct.
-    assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
+    verify(httpUrlConnection).setRequestMethod("PUT");
-    assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
+    assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
-    assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
+    assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
-    Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
+    verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
-    assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
+    verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
-    assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
 
     // Verify the payload XML was the same as what's in testdata/report.xml.
-    XjcRdeReportReport report = parseReport(request.getValue().getPayload());
+    XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
     assertThat(report.getId()).isEqualTo("20101017001");
     assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@@ -167,9 +160,8 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock_withPrefix() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
     RdeReportAction action = createAction();
     action.prefix = Optional.of("job-name/");
     gcsUtils.delete(reportFile);
@@ -182,16 +174,14 @@ public class RdeReportActionTest {
     assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
 
     // Verify the HTTP request was correct.
-    assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
+    verify(httpUrlConnection).setRequestMethod("PUT");
-    assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
+    assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
-    assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
+    assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
-    Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
+    verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
-    assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
+    verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
-    assertThat(headers)
-        .containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
 
     // Verify the payload XML was the same as what's in testdata/report.xml.
-    XjcRdeReportReport report = parseReport(request.getValue().getPayload());
+    XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
     assertThat(report.getId()).isEqualTo("20101017001");
     assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@@ -204,9 +194,8 @@ public class RdeReportActionTest {
     PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
     gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
   }
@@ -239,9 +228,8 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_BAD_REQUEST);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read());
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openStream());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
     InternalServerErrorException thrown =
         assertThrows(
             InternalServerErrorException.class,
@@ -252,18 +240,17 @@ public class RdeReportActionTest {
   @TestOfyAndSql
   void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception {
     class ExpectedThrownException extends RuntimeException {}
-    when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException());
+    when(httpUrlConnection.getResponseCode()).thenThrow(new ExpectedThrownException());
     assertThrows(
         ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
   }
 
   @TestOfyAndSql
   void testRunWithLock_socketTimeout_doesRetry() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
+    when(httpUrlConnection.getResponseCode())
-    when(urlFetchService.fetch(request.capture()))
         .thenThrow(new SocketTimeoutException())
-        .thenReturn(httpResponse);
+        .thenReturn(SC_OK);
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
@@ -274,14 +261,6 @@ public class RdeReportActionTest {
     return loadByKey(Cursor.createVKey(RDE_REPORT, "test")).getCursorTime();
   }
 
-  private static ImmutableMap<String, String> mapifyHeaders(Iterable<HTTPHeader> headers) {
-    ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
-    for (HTTPHeader header : headers) {
-      builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue());
-    }
-    return builder.build();
-  }
-
   private static XjcRdeReportReport parseReport(byte[] data) {
     try {
       return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data));

core/src/test/java/google/registry/request/UrlConnectionUtilsTest.java

@@ -1,4 +1,4 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,13 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.util;
+package google.registry.request;
 
 import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
 import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static com.google.common.net.MediaType.CSV_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.util.UrlFetchUtils.setPayloadMultipart;
+import static google.registry.request.UrlConnectionUtils.setPayloadMultipart;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
@@ -27,22 +27,19 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
+import java.io.ByteArrayOutputStream;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import google.registry.testing.AppEngineExtension;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
+import javax.net.ssl.HttpsURLConnection;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
 import org.mockito.ArgumentCaptor;
 
-/** Unit tests for {@link UrlFetchUtils}. */
+/** Tests for {@link UrlConnectionUtils}. */
-class UrlFetchUtilsTest {
+public class UrlConnectionUtilsTest {
-
-  @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().build();
 
   private final Random random = mock(Random.class);
 
@@ -58,25 +55,28 @@ class UrlFetchUtilsTest {
   }
 
   @Test
-  void testSetPayloadMultipart() {
+  void testSetPayloadMultipart() throws Exception {
-    HTTPRequest request = mock(HTTPRequest.class);
+    HttpsURLConnection connection = mock(HttpsURLConnection.class);
+    ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
+    when(connection.getOutputStream()).thenReturn(connectionOutputStream);
     setPayloadMultipart(
-        request,
+        connection,
         "lol",
         "cat",
         CSV_UTF_8,
         "The nice people at the store say hello. ヘ(◕。◕ヘ)",
         random);
-    ArgumentCaptor<HTTPHeader> headerCaptor = ArgumentCaptor.forClass(HTTPHeader.class);
+    ArgumentCaptor<String> keyCaptor = ArgumentCaptor.forClass(String.class);
-    verify(request, times(2)).addHeader(headerCaptor.capture());
+    ArgumentCaptor<String> valueCaptor = ArgumentCaptor.forClass(String.class);
-    List<HTTPHeader> addedHeaders = headerCaptor.getAllValues();
+    verify(connection, times(2)).setRequestProperty(keyCaptor.capture(), valueCaptor.capture());
-    assertThat(addedHeaders.get(0).getName()).isEqualTo(CONTENT_TYPE);
+    List<String> addedKeys = keyCaptor.getAllValues();
-    assertThat(addedHeaders.get(0).getValue())
+    assertThat(addedKeys).containsExactly(CONTENT_TYPE, CONTENT_LENGTH);
-        .isEqualTo(
+    List<String> addedValues = valueCaptor.getAllValues();
-            "multipart/form-data; "
+    assertThat(addedValues)
-                + "boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"");
+        .containsExactly(
-    assertThat(addedHeaders.get(1).getName()).isEqualTo(CONTENT_LENGTH);
+            "multipart/form-data;"
-    assertThat(addedHeaders.get(1).getValue()).isEqualTo("294");
+                + " boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"",
+            "294");
     String payload =
         "--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\n"
             + "Content-Disposition: form-data; name=\"lol\"; filename=\"cat\"\r\n"
@@ -84,18 +84,20 @@ class UrlFetchUtilsTest {
             + "\r\n"
             + "The nice people at the store say hello. ヘ(◕。◕ヘ)\r\n"
             + "--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA--\r\n";
-    verify(request).setPayload(payload.getBytes(UTF_8));
+    verify(connection).setDoOutput(true);
-    verifyNoMoreInteractions(request);
+    verify(connection).getOutputStream();
+    assertThat(connectionOutputStream.toByteArray()).isEqualTo(payload.getBytes(UTF_8));
+    verifyNoMoreInteractions(connection);
   }
 
   @Test
   void testSetPayloadMultipart_boundaryInPayload() {
-    HTTPRequest request = mock(HTTPRequest.class);
+    HttpsURLConnection connection = mock(HttpsURLConnection.class);
     String payload = "I screamed------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHHH";
     IllegalStateException thrown =
         assertThrows(
             IllegalStateException.class,
-            () -> setPayloadMultipart(request, "lol", "cat", CSV_UTF_8, payload, random));
+            () -> setPayloadMultipart(connection, "lol", "cat", CSV_UTF_8, payload, random));
     assertThat(thrown)
         .hasMessageThat()
         .contains(

core/src/test/java/google/registry/testing/FakeURLFetchService.java

@@ -1,46 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.testing;
-
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.common.collect.ImmutableList;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.Map;
-
-/**
- * A fake {@link URLFetchService} that serves constructed {@link HTTPResponse} objects from
- * a simple {@link Map} ({@link URL} to {@link HTTPResponse}) lookup.
- */
-public class FakeURLFetchService extends ForwardingURLFetchService {
-
-  private Map<URL, HTTPResponse> backingMap;
-
-  public FakeURLFetchService(Map<URL, HTTPResponse> backingMap) {
-    this.backingMap = backingMap;
-  }
-
-  @Override
-  public HTTPResponse fetch(HTTPRequest request) {
-    URL requestURL = request.getURL();
-    if (backingMap.containsKey(requestURL)) {
-      return backingMap.get(requestURL);
-    } else {
-      return new HTTPResponse(HttpURLConnection.HTTP_NOT_FOUND, null, null, ImmutableList.of());
-    }
-  }
-}

core/src/test/java/google/registry/testing/FakeUrlConnectionService.java

@@ -0,0 +1,46 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.testing;
+
+import static org.mockito.Mockito.when;
+
+import google.registry.request.UrlConnectionService;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+/** A fake {@link UrlConnectionService} with a mocked HTTP connection for testing. */
+public class FakeUrlConnectionService implements UrlConnectionService {
+
+  private final HttpURLConnection mockConnection;
+  private final List<URL> connectedUrls;
+
+  public FakeUrlConnectionService(HttpURLConnection mockConnection) {
+    this(mockConnection, new ArrayList<>());
+  }
+
+  public FakeUrlConnectionService(HttpURLConnection mockConnection, List<URL> connectedUrls) {
+    this.mockConnection = mockConnection;
+    this.connectedUrls = connectedUrls;
+  }
+
+  @Override
+  public HttpURLConnection createConnection(URL url) {
+    connectedUrls.add(url);
+    when(mockConnection.getURL()).thenReturn(url);
+    return mockConnection;
+  }
+}

core/src/test/java/google/registry/testing/ForwardingURLFetchService.java

@@ -1,49 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.testing;
-
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.common.util.concurrent.Futures;
-import java.io.IOException;
-import java.net.URL;
-import java.util.concurrent.Future;
-
-/**
- * An implementation of the {@link URLFetchService} interface that forwards all requests through
- * a synchronous fetch call.
- */
-public abstract class ForwardingURLFetchService implements URLFetchService {
-
-  @Override
-  public HTTPResponse fetch(URL url) throws IOException {
-    return fetch(new HTTPRequest(url)); // Defaults to HTTPMethod.GET
-  }
-
-  @Override
-  public Future<HTTPResponse> fetchAsync(URL url) {
-    return fetchAsync(new HTTPRequest(url)); // Defaults to HTTPMethod.GET
-  }
-
-  @Override
-  public Future<HTTPResponse> fetchAsync(HTTPRequest request) {
-    try {
-      return Futures.immediateFuture(fetch(request));
-    } catch (Exception e) {
-      return Futures.immediateFailedFuture(e);
-    }
-  }
-}

core/src/test/java/google/registry/tmch/NordnUploadActionTest.java

@@ -19,21 +19,22 @@ import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static com.google.common.net.HttpHeaders.LOCATION;
 import static com.google.common.net.MediaType.FORM_DATA;
 import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.loadRegistrar;
 import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.persistDomainAndEnqueueLordn;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
-import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.startsWith;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -43,10 +44,6 @@ import com.google.appengine.api.taskqueue.TaskHandle;
 import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.appengine.api.taskqueue.TransientFailureException;
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.apphosting.api.DeadlineExceededException;
 import com.google.common.base.VerifyException;
 import com.google.common.collect.ImmutableList;
@@ -57,11 +54,15 @@ import google.registry.model.tld.Registry;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
+import google.registry.testing.FakeUrlConnectionService;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.util.Retrier;
 import google.registry.util.TaskQueueUtils;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.SecureRandom;
 import java.util.List;
@@ -69,17 +70,9 @@ import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
-import org.mockito.junit.jupiter.MockitoSettings;
-import org.mockito.quality.Strictness;
 
 /** Unit tests for {@link NordnUploadAction}. */
-@ExtendWith(MockitoExtension.class)
 class NordnUploadActionTest {
 
   private static final String CLAIMS_CSV =
@@ -102,28 +95,29 @@ class NordnUploadActionTest {
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
   
-  @Mock private URLFetchService fetchService;
-  @Mock private HTTPResponse httpResponse;
-  @Captor private ArgumentCaptor<HTTPRequest> httpRequestCaptor;
-
   private final FakeClock clock = new FakeClock(DateTime.parse("2010-05-01T10:11:12Z"));
   private final LordnRequestInitializer lordnRequestInitializer =
       new LordnRequestInitializer(Optional.of("attack"));
   private final NordnUploadAction action = new NordnUploadAction();
 
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+
   @BeforeEach
   void beforeEach() throws Exception {
     inject.setStaticField(Ofy.class, "clock", clock);
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
+    when(httpUrlConnection.getInputStream())
-    when(httpResponse.getContent()).thenReturn("Success".getBytes(US_ASCII));
+        .thenReturn(new ByteArrayInputStream("Success".getBytes(UTF_8)));
-    when(httpResponse.getResponseCode()).thenReturn(SC_ACCEPTED);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_ACCEPTED);
-    when(httpResponse.getHeadersUncombined())
+    when(httpUrlConnection.getHeaderField(LOCATION)).thenReturn("http://trololol");
-        .thenReturn(ImmutableList.of(new HTTPHeader(LOCATION, "http://trololol")));
+    when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
     persistResource(loadRegistrar("TheRegistrar").asBuilder().setIanaIdentifier(99999L).build());
     createTld("tld");
     persistResource(Registry.get("tld").asBuilder().setLordnUsername("lolcat").build());
     action.clock = clock;
-    action.fetchService = fetchService;
+    action.urlConnectionService = urlConnectionService;
     action.lordnRequestInitializer = lordnRequestInitializer;
     action.phase = "claims";
     action.taskQueueUtils = new TaskQueueUtils(new Retrier(new FakeSleeper(clock), 3));
@@ -133,7 +127,6 @@ class NordnUploadActionTest {
     action.retrier = new Retrier(new FakeSleeper(clock), 3);
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv() {
     List<TaskHandle> tasks =
@@ -145,7 +138,6 @@ class NordnUploadActionTest {
         .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_dedupesDuplicates() {
     List<TaskHandle> tasks =
@@ -158,14 +150,12 @@ class NordnUploadActionTest {
         .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_doesntFailOnEmptyTasks() {
     assertThat(NordnUploadAction.convertTasksToCsv(ImmutableList.of(), clock.nowUtc(), "col1,col2"))
         .isEqualTo("1,2010-05-01T10:11:12.000Z,0\ncol1,col2\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_throwsNpeOnNullTasks() {
     assertThrows(
@@ -173,7 +163,6 @@ class NordnUploadActionTest {
         () -> NordnUploadAction.convertTasksToCsv(null, clock.nowUtc(), "header"));
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @SuppressWarnings("unchecked")
   @Test
   void test_loadAllTasks_retryLogic_thirdTrysTheCharm() {
@@ -186,7 +175,6 @@ class NordnUploadActionTest {
     assertThat(action.loadAllTasks(queue, "tld")).containsExactly(task);
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_loadAllTasks_retryLogic_allFailures() {
     Queue queue = mock(Queue.class);
@@ -201,47 +189,47 @@ class NordnUploadActionTest {
   void testRun_claimsMode_appendsTldAndClaimsToRequestUrl() throws Exception {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getCapturedHttpRequest().getURL())
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims"));
-        .isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims"));
   }
 
   @Test
   void testRun_sunriseMode_appendsTldAndClaimsToRequestUrl() throws Exception {
     persistSunriseModeDomain();
     action.run();
-    assertThat(getCapturedHttpRequest().getURL())
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise"));
-        .isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise"));
   }
 
   @Test
   void testRun_usesMultipartContentType() throws Exception {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), CONTENT_TYPE).get())
+    verify(httpUrlConnection)
-        .startsWith("multipart/form-data; boundary=");
+        .setRequestProperty(eq(CONTENT_TYPE), startsWith("multipart/form-data; boundary="));
+    verify(httpUrlConnection).setRequestMethod("POST");
   }
 
   @Test
-  void testRun_hasPassword_setsAuthorizationHeader() throws Exception {
+  void testRun_hasPassword_setsAuthorizationHeader() {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION))
+    verify(httpUrlConnection)
-        .hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
+        .setRequestProperty(
+            AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
   }
 
   @Test
-  void testRun_noPassword_doesntSendAuthorizationHeader() throws Exception {
+  void testRun_noPassword_doesntSendAuthorizationHeader() {
     action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty());
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty();
+    verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString());
   }
 
   @Test
-  void testRun_claimsMode_payloadMatchesClaimsCsv() throws Exception {
+  void testRun_claimsMode_payloadMatchesClaimsCsv() {
     persistClaimsModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(CLAIMS_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(CLAIMS_CSV);
   }
 
   @Test
@@ -257,19 +245,19 @@ class NordnUploadActionTest {
   }
 
   @Test
-  void testRun_sunriseMode_payloadMatchesSunriseCsv() throws Exception {
+  void testRun_sunriseMode_payloadMatchesSunriseCsv() {
     persistSunriseModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV);
   }
 
   @Test
   void test_noResponseContent_stillWorksNormally() throws Exception {
     // Returning null only affects logging.
-    when(httpResponse.getContent()).thenReturn(null);
+    when(httpUrlConnection.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[] {}));
     persistSunriseModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV);
   }
 
   @Test
@@ -284,7 +272,6 @@ class NordnUploadActionTest {
             .header(CONTENT_TYPE, FORM_DATA.toString()));
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void testFailure_nullRegistryUser() {
     persistClaimsModeDomain();
@@ -293,17 +280,11 @@ class NordnUploadActionTest {
     assertThat(thrown).hasMessageThat().contains("lordnUsername is not set for tld.");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
-  void testFetchFailure() {
+  void testFetchFailure() throws Exception {
     persistClaimsModeDomain();
-    when(httpResponse.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
-    assertThrows(UrlFetchException.class, action::run);
+    assertThrows(UrlConnectionException.class, action::run);
-  }
-
-  private HTTPRequest getCapturedHttpRequest() throws Exception {
-    verify(fetchService).fetch(httpRequestCaptor.capture());
-    return httpRequestCaptor.getAllValues().get(0);
   }
 
   private void persistClaimsModeDomain() {

core/src/test/java/google/registry/tmch/NordnVerifyActionTest.java

@@ -16,39 +16,34 @@ package google.registry.tmch;
 
 import static com.google.common.net.HttpHeaders.AUTHORIZATION;
 import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.persistResource;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import google.registry.model.tld.Registry;
 import google.registry.request.HttpException.ConflictException;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeResponse;
+import google.registry.testing.FakeUrlConnectionService;
+import java.io.ByteArrayInputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Optional;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
 
 /** Unit tests for {@link NordnVerifyAction}. */
-@ExtendWith(MockitoExtension.class)
 class NordnVerifyActionTest {
 
   private static final String LOG_ACCEPTED =
@@ -81,84 +76,83 @@ class NordnVerifyActionTest {
   public final AppEngineExtension appEngine =
       AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
 
-  @Mock private URLFetchService fetchService;
-  @Mock private HTTPResponse httpResponse;
-  @Captor private ArgumentCaptor<HTTPRequest> httpRequestCaptor;
-
   private final FakeResponse response = new FakeResponse();
   private final LordnRequestInitializer lordnRequestInitializer =
       new LordnRequestInitializer(Optional.of("attack"));
   private final NordnVerifyAction action = new NordnVerifyAction();
 
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+
   @BeforeEach
   void beforeEach() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(LOG_ACCEPTED.getBytes(UTF_8));
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
     createTld("gtld");
     persistResource(Registry.get("gtld").asBuilder().setLordnUsername("lolcat").build());
     action.tld = "gtld";
-    action.fetchService = fetchService;
+    action.urlConnectionService = urlConnectionService;
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_ACCEPTED.getBytes(UTF_8)));
     action.lordnRequestInitializer = lordnRequestInitializer;
     action.response = response;
     action.url = new URL("http://127.0.0.1/blobio");
   }
 
-  private HTTPRequest getCapturedHttpRequest() throws Exception {
-    verify(fetchService).fetch(httpRequestCaptor.capture());
-    return httpRequestCaptor.getAllValues().get(0);
-  }
-
   @Test
   void testSuccess_sendHttpRequest_urlIsCorrect() throws Exception {
     action.run();
-    assertThat(getCapturedHttpRequest().getURL()).isEqualTo(new URL("http://127.0.0.1/blobio"));
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/blobio"));
   }
 
   @Test
-  void testSuccess_hasLordnPassword_sendsAuthorizationHeader() throws Exception {
+  void testSuccess_hasLordnPassword_sendsAuthorizationHeader() {
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION))
+    verify(httpUrlConnection)
-        .hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
+        .setRequestProperty(
+            AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
   }
 
   @Test
-  void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() throws Exception {
+  void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() {
     action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty());
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty();
+    verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString());
   }
 
   @Test
   void successVerifyRejected() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_REJECTED.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_REJECTED.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.getStatus()).isEqualTo(LordnLog.Status.REJECTED);
   }
 
   @Test
   void successVerifyWarnings() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_WARNINGS.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_WARNINGS.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.hasWarnings()).isTrue();
   }
 
   @Test
   void successVerifyErrors() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_ERRORS.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_ERRORS.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.hasWarnings()).isTrue();
   }
 
   @Test
-  void failureVerifyUnauthorized() {
+  void failureVerifyUnauthorized() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_UNAUTHORIZED);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_UNAUTHORIZED);
     assertThrows(Exception.class, action::run);
   }
 
   @Test
-  void failureVerifyNotReady() {
+  void failureVerifyNotReady() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_NO_CONTENT);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_NO_CONTENT);
     ConflictException thrown = assertThrows(ConflictException.class, action::run);
     assertThat(thrown).hasMessageThat().contains("Not ready");
   }

core/src/test/java/google/registry/tmch/TmchActionTestCase.java

@@ -15,21 +15,19 @@
 package google.registry.tmch;
 
 import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.BouncyCastleProviderExtension;
 import google.registry.testing.FakeClock;
+import google.registry.testing.FakeUrlConnectionService;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
 /** Common code for unit tests of classes that extend {@link Marksdb}. */
@@ -46,19 +44,19 @@ abstract class TmchActionTestCase {
   @RegisterExtension
   public final BouncyCastleProviderExtension bouncy = new BouncyCastleProviderExtension();
 
-  @Mock URLFetchService fetchService;
-  @Mock HTTPResponse httpResponse;
-  @Captor ArgumentCaptor<HTTPRequest> httpRequest;
-
   final FakeClock clock = new FakeClock();
   final Marksdb marksdb = new Marksdb();
 
+  protected final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  protected final ArrayList<URL> connectedUrls = new ArrayList<>();
+  protected FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection, connectedUrls);
+
   @BeforeEach
   public void beforeEachTmchActionTestCase() throws Exception {
-    marksdb.fetchService = fetchService;
     marksdb.tmchMarksdbUrl = MARKSDB_URL;
     marksdb.marksdbPublicKey = TmchData.loadPublicKey(TmchTestData.loadBytes("pubkey"));
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
+    marksdb.urlConnectionService = urlConnectionService;
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
   }
 }

core/src/test/java/google/registry/tmch/TmchCrlActionTest.java

@@ -22,6 +22,7 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import google.registry.config.RegistryConfig.ConfigModule.TmchCaMode;
+import java.io.ByteArrayInputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.security.SignatureException;
@@ -44,19 +45,22 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testSuccess() throws Exception {
     clock.setTo(DateTime.parse("2013-07-24TZ"));
-    when(httpResponse.getContent()).thenReturn(
+    when(httpUrlConnection.getInputStream())
-        readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read());
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()));
     newTmchCrlAction(TmchCaMode.PRODUCTION).run();
-    verify(httpResponse).getContent();
+    verify(httpUrlConnection).getInputStream();
-    verify(fetchService).fetch(httpRequest.capture());
+    assertThat(connectedUrls).containsExactly(new URL("http://sloth.lol/tmch.crl"));
-    assertThat(httpRequest.getValue().getURL().toString()).isEqualTo("http://sloth.lol/tmch.crl");
   }
 
   @Test
   void testFailure_crlTooOld() throws Exception {
     clock.setTo(DateTime.parse("2020-01-01TZ"));
-    when(httpResponse.getContent())
+    when(httpUrlConnection.getInputStream())
-        .thenReturn(loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read());
+        .thenReturn(
+            new ByteArrayInputStream(
+                loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read()));
     TmchCrlAction action = newTmchCrlAction(TmchCaMode.PILOT);
     Exception e = assertThrows(Exception.class, action::run);
     assertThat(e).hasCauseThat().isInstanceOf(CRLException.class);
@@ -69,8 +73,10 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testFailure_crlNotSignedByRoot() throws Exception {
     clock.setTo(DateTime.parse("2013-07-24TZ"));
-    when(httpResponse.getContent())
+    when(httpUrlConnection.getInputStream())
-        .thenReturn(readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read());
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()));
     Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run);
     assertThat(e).hasCauseThat().isInstanceOf(SignatureException.class);
     assertThat(e).hasCauseThat().hasMessageThat().isEqualTo("Signature does not match.");
@@ -79,8 +85,10 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testFailure_crlNotYetValid() throws Exception {
     clock.setTo(DateTime.parse("1984-01-01TZ"));
-    when(httpResponse.getContent()).thenReturn(
+    when(httpUrlConnection.getInputStream())
-        readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read());
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read()));
     Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run);
     assertThat(e).hasCauseThat().isInstanceOf(CertificateNotYetValidException.class);
   }

core/src/test/java/google/registry/tmch/TmchDnlActionTest.java

@@ -14,6 +14,7 @@
 
 package google.registry.tmch;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
 import static org.mockito.Mockito.times;
@@ -22,6 +23,8 @@ import static org.mockito.Mockito.when;
 
 import google.registry.model.tmch.ClaimsList;
 import google.registry.model.tmch.ClaimsListDao;
+import java.io.ByteArrayInputStream;
+import java.net.URL;
 import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
@@ -39,15 +42,13 @@ class TmchDnlActionTest extends TmchActionTestCase {
   @Test
   void testDnl() throws Exception {
     assertThat(ClaimsListDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty();
-    when(httpResponse.getContent())
+    when(httpUrlConnection.getInputStream())
-        .thenReturn(TmchTestData.loadBytes("dnl-latest.csv").read())
+        .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.csv").read()))
-        .thenReturn(TmchTestData.loadBytes("dnl-latest.sig").read());
+        .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.sig").read()));
     newTmchDnlAction().run();
-    verify(fetchService, times(2)).fetch(httpRequest.capture());
+    verify(httpUrlConnection, times(2)).getInputStream();
-    assertThat(httpRequest.getAllValues().get(0).getURL().toString())
+    assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
-        .isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.csv");
+        .containsExactly(MARKSDB_URL + "/dnl/dnl-latest.csv", MARKSDB_URL + "/dnl/dnl-latest.sig");
-    assertThat(httpRequest.getAllValues().get(1).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.sig");
 
     // Make sure the contents of testdata/dnl-latest.csv got inserted into the database.
     ClaimsList claimsList = ClaimsListDao.get();

core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java

@@ -14,6 +14,7 @@
 
 package google.registry.tmch;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.tmch.TmchTestData.loadBytes;
 import static org.mockito.Mockito.times;
@@ -21,6 +22,8 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import google.registry.model.smd.SignedMarkRevocationList;
+import java.io.ByteArrayInputStream;
+import java.net.URL;
 import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
@@ -42,15 +45,14 @@ class TmchSmdrlActionTest extends TmchActionTestCase {
     SignedMarkRevocationList smdrl = SignedMarkRevocationList.get();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isFalse();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse();
-    when(httpResponse.getContent())
+    when(httpUrlConnection.getInputStream())
-        .thenReturn(loadBytes("smdrl-latest.csv").read())
+        .thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.csv").read()))
-        .thenReturn(loadBytes("smdrl-latest.sig").read());
+        .thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.sig").read()));
     newTmchSmdrlAction().run();
-    verify(fetchService, times(2)).fetch(httpRequest.capture());
+    verify(httpUrlConnection, times(2)).getInputStream();
-    assertThat(httpRequest.getAllValues().get(0).getURL().toString())
+    assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
-        .isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.csv");
+        .containsExactly(
-    assertThat(httpRequest.getAllValues().get(1).getURL().toString())
+            MARKSDB_URL + "/smdrl/smdrl-latest.csv", MARKSDB_URL + "/smdrl/smdrl-latest.sig");
-        .isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.sig");
     smdrl = SignedMarkRevocationList.get();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isTrue();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse();

util/src/main/java/google/registry/util/UrlConnectionException.java

@@ -0,0 +1,71 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.util;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.io.ByteStreams;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+
+/**
+ * Used when HTTP requests return a bad response, with troubleshooting info.
+ *
+ * <p>This class displays lots of helpful troubleshooting information.
+ */
+public class UrlConnectionException extends RuntimeException {
+
+  private final HttpURLConnection connection;
+
+  public UrlConnectionException(String message, HttpURLConnection connection) {
+    super(message);
+    this.connection = connection;
+  }
+
+  @Override
+  public String getMessage() {
+    byte[] resultContent;
+    int responseCode;
+    try {
+      resultContent = ByteStreams.toByteArray(connection.getInputStream());
+      responseCode = connection.getResponseCode();
+    } catch (IOException e) {
+      resultContent = new byte[] {};
+      responseCode = 0;
+    }
+    StringBuilder result =
+        new StringBuilder(2048 + resultContent.length)
+            .append(
+                String.format(
+                    "%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\n",
+                    getClass().getSimpleName(),
+                    super.getMessage(),
+                    responseCode,
+                    connection.getURL().toString()));
+    connection
+        .getRequestProperties()
+        .forEach(
+            (key, value) -> {
+              result.append(key);
+              result.append(": ");
+              result.append(value);
+              result.append('\n');
+            });
+    result.append(">>>\n");
+    result.append(new String(resultContent, UTF_8));
+    result.append("\n<<<");
+    return result.toString();
+  }
+}

util/src/main/java/google/registry/util/UrlFetchException.java

@@ -1,63 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.util;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-
-/**
- * Exception for when App Engine HTTP requests return a bad response.
- *
- * <p>This class displays lots of helpful troubleshooting information.
- */
-public class UrlFetchException extends RuntimeException {
-
-  private final HTTPRequest req;
-  private final HTTPResponse rsp;
-
-  public UrlFetchException(String message, HTTPRequest req, HTTPResponse rsp) {
-    super(message);
-    this.req = checkNotNull(req, "req");
-    this.rsp = checkNotNull(rsp, "rsp");
-  }
-
-  @Override
-  public String getMessage() {
-    StringBuilder res =
-        new StringBuilder(2048 + rsp.getContent().length)
-            .append(
-                String.format(
-                    "%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\nX-Final-URL: %s\n",
-                    getClass().getSimpleName(),
-                    super.getMessage(),
-                    rsp.getResponseCode(),
-                    req.getURL().toString(),
-                    rsp.getFinalUrl()));
-    for (HTTPHeader header : rsp.getHeadersUncombined()) {
-      res.append(header.getName());
-      res.append(": ");
-      res.append(header.getValue());
-      res.append('\n');
-    }
-    res.append(">>>\n");
-    res.append(new String(rsp.getContent(), UTF_8));
-    res.append("\n<<<");
-    return res.toString();
-  }
-}