zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-04-29 19:47:51 +02:00
Code Issues Projects Releases Packages Wiki Activity

Use built-in Java URL connections instead of UrlFetchService (#1535)

Browse source 
- Use the standard HttpsURLConnection to write/read data
- Rewrite RdeReporter, Nordn*Action, and Marksdb classes and related
  tests to conform to the new format
- Remove FakeURLFetchService and ForwardingUrlFetchService as they weren't used
- Refactor UrlFetchException to UrlConnectionException
- Refactor UrlFetchUtils to UrlConnectionUtils

I will need to test this on Alpha. Fortunately the connections that
don't require auth (e.g. TMDB downloading) should be testable.
This commit is contained in:
gbrodman 2022-03-04 14:16:22 -05:00 • committed by GitHub
parent cc4f2c0c52
commit 0d62ac0410
26 changed files with 507 additions and 584 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
core/src/main/java/google/registry/module/backend/BackendComponent.java
View file

@ -43,7 +43,7 @@ import google.registry.rde.JSchModule;
import google.registry.request.Modules.DatastoreServiceModule;
import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.NetHttpTransportModule;
import google.registry.request.Modules.URLFetchServiceModule;
import google.registry.request.Modules.UrlConnectionServiceModule;
import google.registry.request.Modules.UrlFetchTransportModule;
import google.registry.request.Modules.UserServiceModule;
import google.registry.request.auth.AuthModule;
@ -80,7 +80,7 @@ import javax.inject.Singleton;
ServerTridProviderModule.class,
SheetsServiceModule.class,
StackdriverModule.class,
URLFetchServiceModule.class,
UrlConnectionServiceModule.class,
UrlFetchTransportModule.class,
UserServiceModule.class,
VoidDnsWriterModule.class,

2
core/src/main/java/google/registry/module/frontend/FrontendComponent.java
View file

@ -34,7 +34,6 @@ import google.registry.monitoring.whitebox.StackdriverModule;
import google.registry.privileges.secretmanager.SecretManagerModule;
import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.NetHttpTransportModule;
import google.registry.request.Modules.UrlFetchTransportModule;
import google.registry.request.Modules.UserServiceModule;
import google.registry.request.auth.AuthModule;
import google.registry.ui.ConsoleDebug.ConsoleConfigModule;
@ -65,7 +64,6 @@ import javax.inject.Singleton;
SecretManagerModule.class,
ServerTridProviderModule.class,
StackdriverModule.class,
UrlFetchTransportModule.class,
UserServiceModule.class,
UtilsModule.class
})

2
core/src/main/java/google/registry/module/pubapi/PubApiComponent.java
View file

@ -34,7 +34,6 @@ import google.registry.persistence.PersistenceModule;
import google.registry.privileges.secretmanager.SecretManagerModule;
import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.NetHttpTransportModule;
import google.registry.request.Modules.UrlFetchTransportModule;
import google.registry.request.Modules.UserServiceModule;
import google.registry.request.auth.AuthModule;
import google.registry.util.UtilsModule;
@ -62,7 +61,6 @@ import javax.inject.Singleton;
SecretManagerModule.class,
ServerTridProviderModule.class,
StackdriverModule.class,
UrlFetchTransportModule.class,
UserServiceModule.class,
UtilsModule.class
})

2
core/src/main/java/google/registry/module/tools/ToolsComponent.java
View file

@ -36,7 +36,6 @@ import google.registry.privileges.secretmanager.SecretManagerModule;
import google.registry.request.Modules.DatastoreServiceModule;
import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.NetHttpTransportModule;
import google.registry.request.Modules.UrlFetchTransportModule;
import google.registry.request.Modules.UserServiceModule;
import google.registry.request.auth.AuthModule;
import google.registry.util.UtilsModule;
@ -66,7 +65,6 @@ import javax.inject.Singleton;
ServerTridProviderModule.class,
StackdriverModule.class,
ToolsRequestComponentModule.class,
UrlFetchTransportModule.class,
UserServiceModule.class,
UtilsModule.class
})

59
core/src/main/java/google/registry/rde/RdeReporter.java
View file

@ -14,26 +14,24 @@
package google.registry.rde;
import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
import static com.google.common.io.BaseEncoding.base64;
import static com.google.common.net.HttpHeaders.AUTHORIZATION;
import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
import static google.registry.request.UrlConnectionUtils.getResponseBytes;
import static google.registry.request.UrlConnectionUtils.setBasicAuth;
import static google.registry.request.UrlConnectionUtils.setPayload;
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.api.client.http.HttpMethods;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.common.flogger.FluentLogger;
import com.google.common.net.MediaType;
import google.registry.config.RegistryConfig.Config;
import google.registry.keyring.api.KeyModule.Key;
import google.registry.request.HttpException.InternalServerErrorException;
import google.registry.request.UrlConnectionService;
import google.registry.util.Retrier;
import google.registry.util.UrlFetchException;
import google.registry.util.UrlConnectionException;
import google.registry.xjc.XjcXmlTransformer;
import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
import google.registry.xjc.iirdea.XjcIirdeaResult;
@ -41,6 +39,7 @@ import google.registry.xjc.rdeheader.XjcRdeHeader;
import google.registry.xjc.rdereport.XjcRdeReportReport;
import google.registry.xml.XmlException;
import java.io.ByteArrayInputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URL;
@ -55,12 +54,15 @@ public class RdeReporter {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
/** @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
* ICANN Registry Interfaces - Interface details</a>*/
private static final String REPORT_MIME = "text/xml";
/**
* @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
* ICANN Registry Interfaces - Interface details</a>
*/
private static final MediaType MEDIA_TYPE = MediaType.XML_UTF_8;
@Inject Retrier retrier;
@Inject URLFetchService urlFetchService;
@Inject UrlConnectionService urlConnectionService;
@Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix;
@Inject @Key("icannReportingPassword") String password;
@Inject RdeReporter() {}
@ -74,29 +76,24 @@ public class RdeReporter {
// Send a PUT request to ICANN's HTTPS server.
URL url = makeReportUrl(header.getTld(), report.getId());
String username = header.getTld() + "_ry";
String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8));
final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d));
req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME));
req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
req.setPayload(reportBytes);
logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8));
HTTPResponse rsp =
byte[] responseBytes =
retrier.callWithRetry(
() -> {
HTTPResponse rsp1 = urlFetchService.fetch(req);
switch (rsp1.getResponseCode()) {
case SC_OK:
case SC_BAD_REQUEST:
break;
default:
throw new UrlFetchException("PUT failed", req, rsp1);
HttpURLConnection connection = urlConnectionService.createConnection(url);
connection.setRequestMethod(HttpMethods.PUT);
setBasicAuth(connection, username, password);
setPayload(connection, reportBytes, MEDIA_TYPE.toString());
int responseCode = connection.getResponseCode();
if (responseCode == SC_OK || responseCode == SC_BAD_REQUEST) {
return getResponseBytes(connection);
}
return rsp1;
throw new UrlConnectionException("PUT failed", connection);
},
SocketTimeoutException.class);
// Ensure the XML response is valid.
XjcIirdeaResult result = parseResult(rsp);
XjcIirdeaResult result = parseResult(responseBytes);
if (result.getCode().getValue() != 1000) {
logger.atWarning().log(
"PUT rejected: %d %s\n%s",
@ -108,11 +105,11 @@ public class RdeReporter {
/**
* Unmarshals IIRDEA XML result object from {@link HTTPResponse} payload.
*
* @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1">
* @see <a
* href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1">
* ICANN Registry Interfaces - IIRDEA Result Object</a>
*/
private XjcIirdeaResult parseResult(HTTPResponse rsp) throws XmlException {
byte[] responseBytes = rsp.getContent();
private XjcIirdeaResult parseResult(byte[] responseBytes) throws XmlException {
logger.atInfo().log("Received response:\n%s", new String(responseBytes, UTF_8));
XjcIirdeaResponseElement response = XjcXmlTransformer.unmarshal(
XjcIirdeaResponseElement.class, new ByteArrayInputStream(responseBytes));

13
core/src/main/java/google/registry/request/Modules.java
View file

@ -23,12 +23,11 @@ import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
import com.google.appengine.api.users.UserService;
import com.google.appengine.api.users.UserServiceFactory;
import dagger.Module;
import dagger.Provides;
import java.net.HttpURLConnection;
import javax.inject.Singleton;
/** Dagger modules for App Engine services and other vendor classes. */
@ -45,14 +44,12 @@ public final class Modules {
}
}
/** Dagger module for {@link URLFetchService}. */
/** Dagger module for {@link UrlConnectionService}. */
@Module
public static final class URLFetchServiceModule {
private static final URLFetchService fetchService = URLFetchServiceFactory.getURLFetchService();
public static final class UrlConnectionServiceModule {
@Provides
static URLFetchService provideURLFetchService() {
return fetchService;
static UrlConnectionService provideUrlConnectionService() {
return url -> (HttpURLConnection) url.openConnection();
}
}

25
core/src/main/java/google/registry/request/UrlConnectionService.java Normal file
View file

@ -0,0 +1,25 @@
// Copyright 2022 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.request;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
/** Functional interface for opening a connection from a URL, injectable for testing. */
public interface UrlConnectionService {
HttpURLConnection createConnection(URL url) throws IOException;
}

73
util/src/main/java/google/registry/util/UrlFetchUtils.java → core/src/main/java/google/registry/request/UrlConnectionUtils.java
View file

@ -1,4 +1,4 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
// Copyright 2022 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.util;
package google.registry.request;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.io.BaseEncoding.base64;
@ -22,36 +22,41 @@ import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.common.base.Ascii;
import com.google.common.base.Strings;
import com.google.common.io.ByteStreams;
import com.google.common.net.MediaType;
import java.util.Optional;
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.URLConnection;
import java.util.Random;
/** Helper methods for the App Engine URL fetch service. */
public final class UrlFetchUtils {
/** Utilities for common functionality relating to {@link java.net.URLConnection}s. */
public class UrlConnectionUtils {
/** Returns value of first header matching {@code name}. */
public static Optional<String> getHeaderFirst(HTTPResponse rsp, String name) {
return getHeaderFirstInternal(rsp.getHeadersUncombined(), name);
/** Retrieves the response from the given connection as a byte array. */
public static byte[] getResponseBytes(URLConnection connection) throws IOException {
return ByteStreams.toByteArray(connection.getInputStream());
}
/** Returns value of first header matching {@code name}. */
public static Optional<String> getHeaderFirst(HTTPRequest req, String name) {
return getHeaderFirstInternal(req.getHeaders(), name);
/** Sets auth on the given connection with the given username/password. */
public static void setBasicAuth(URLConnection connection, String username, String password) {
setBasicAuth(connection, String.format("%s:%s", username, password));
}
private static Optional<String> getHeaderFirstInternal(Iterable<HTTPHeader> hdrs, String name) {
name = Ascii.toLowerCase(name);
for (HTTPHeader header : hdrs) {
if (Ascii.toLowerCase(header.getName()).equals(name)) {
return Optional.of(header.getValue());
}
/** Sets auth on the given connection with the given string, formatted "username:password". */
public static void setBasicAuth(URLConnection connection, String usernameAndPassword) {
String token = base64().encode(usernameAndPassword.getBytes(UTF_8));
connection.setRequestProperty(AUTHORIZATION, "Basic " + token);
}
/** Sets the given byte[] payload on the given connection with a particular content type. */
public static void setPayload(URLConnection connection, byte[] bytes, String contentType)
throws IOException {
connection.setRequestProperty(CONTENT_TYPE, contentType);
connection.setDoOutput(true);
try (DataOutputStream dataStream = new DataOutputStream(connection.getOutputStream())) {
dataStream.write(bytes);
}
return Optional.empty();
}
/**
@ -62,16 +67,16 @@ public final class UrlFetchUtils {
* @see <a href="http://www.ietf.org/rfc/rfc2388.txt">RFC2388 - Returning Values from Forms</a>
*/
public static void setPayloadMultipart(
HTTPRequest request,
URLConnection connection,
String name,
String filename,
MediaType contentType,
String data,
Random random) {
Random random)
throws IOException {
String boundary = createMultipartBoundary(random);
checkState(
!data.contains(boundary),
"Multipart data contains autogenerated boundary: %s", boundary);
!data.contains(boundary), "Multipart data contains autogenerated boundary: %s", boundary);
String multipart =
String.format("--%s\r\n", boundary)
+ String.format(
@ -83,11 +88,9 @@ public final class UrlFetchUtils {
+ "\r\n"
+ String.format("--%s--\r\n", boundary);
byte[] payload = multipart.getBytes(UTF_8);
request.addHeader(
new HTTPHeader(
CONTENT_TYPE, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary)));
request.addHeader(new HTTPHeader(CONTENT_LENGTH, Integer.toString(payload.length)));
request.setPayload(payload);
connection.setRequestProperty(CONTENT_LENGTH, Integer.toString(payload.length));
setPayload(
connection, payload, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary));
}
private static String createMultipartBoundary(Random random) {
@ -98,12 +101,4 @@ public final class UrlFetchUtils {
// See https://tools.ietf.org/html/rfc2046#section-5.1.1
return Strings.repeat("-", 30) + base64().encode(rand);
}
/** Sets the HTTP Basic Authentication header on an {@link HTTPRequest}. */
public static void setAuthorizationHeader(HTTPRequest req, Optional<String> login) {
if (login.isPresent()) {
String token = base64().encode(login.get().getBytes(UTF_8));
req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
}
}
}

9
core/src/main/java/google/registry/tmch/LordnRequestInitializer.java
View file

@ -15,12 +15,12 @@
package google.registry.tmch;
import static com.google.common.base.Verify.verifyNotNull;
import static google.registry.util.UrlFetchUtils.setAuthorizationHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.common.flogger.FluentLogger;
import google.registry.keyring.api.KeyModule.Key;
import google.registry.model.tld.Registry;
import google.registry.request.UrlConnectionUtils;
import java.net.HttpURLConnection;
import java.util.Optional;
import javax.inject.Inject;
@ -37,8 +37,9 @@ final class LordnRequestInitializer {
}
/** Initializes a URL fetch request for talking to the MarksDB server. */
void initialize(HTTPRequest request, String tld) {
setAuthorizationHeader(request, getMarksDbLordnCredentials(tld));
void initialize(HttpURLConnection connection, String tld) {
getMarksDbLordnCredentials(tld)
.ifPresent(login -> UrlConnectionUtils.setBasicAuth(connection, login));
}
/** Returns the username and password for the current TLD to login to the MarksDB server. */

32
core/src/main/java/google/registry/tmch/Marksdb.java
View file

@ -14,25 +14,23 @@
package google.registry.tmch;
import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
import static com.google.appengine.api.urlfetch.HTTPMethod.GET;
import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.request.UrlConnectionUtils.getResponseBytes;
import static google.registry.request.UrlConnectionUtils.setBasicAuth;
import static google.registry.util.HexDumper.dumpHex;
import static google.registry.util.UrlFetchUtils.setAuthorizationHeader;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.common.collect.ImmutableList;
import com.google.common.flogger.FluentLogger;
import com.google.common.io.ByteSource;
import google.registry.config.RegistryConfig.Config;
import google.registry.keyring.api.KeyModule.Key;
import google.registry.util.UrlFetchException;
import google.registry.request.UrlConnectionService;
import google.registry.util.UrlConnectionException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.Security;
import java.security.SignatureException;
@ -57,7 +55,8 @@ public final class Marksdb {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final int MAX_DNL_LOGGING_LENGTH = 500;
@Inject URLFetchService fetchService;
@Inject UrlConnectionService urlConnectionService;
@Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl;
@Inject @Key("marksdbPublicKey") PGPPublicKey marksdbPublicKey;
@Inject Marksdb() {}
@ -112,19 +111,16 @@ public final class Marksdb {
}
byte[] fetch(URL url, Optional<String> loginAndPassword) throws IOException {
HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d));
setAuthorizationHeader(req, loginAndPassword);
HTTPResponse rsp;
HttpURLConnection connection = urlConnectionService.createConnection(url);
loginAndPassword.ifPresent(auth -> setBasicAuth(connection, auth));
try {
rsp = fetchService.fetch(req);
if (connection.getResponseCode() != SC_OK) {
throw new UrlConnectionException("Failed to fetch from MarksDB", connection);
}
return getResponseBytes(connection);
} catch (IOException e) {
throw new IOException(
String.format("Error connecting to MarksDB at URL %s", url), e);
throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
}
if (rsp.getResponseCode() != SC_OK) {
throw new UrlFetchException("Failed to fetch from MarksDB", req, rsp);
}
return rsp.getContent();
}
List<String> fetchSignedCsv(Optional<String> loginAndPassword, String csvPath, String sigPath)

85
core/src/main/java/google/registry/tmch/NordnUploadAction.java
View file

@ -16,28 +16,23 @@ package google.registry.tmch;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
import static com.google.appengine.api.urlfetch.HTTPMethod.POST;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.net.HttpHeaders.LOCATION;
import static com.google.common.net.MediaType.CSV_UTF_8;
import static google.registry.request.UrlConnectionUtils.getResponseBytes;
import static google.registry.tmch.LordnTaskUtils.COLUMNS_CLAIMS;
import static google.registry.tmch.LordnTaskUtils.COLUMNS_SUNRISE;
import static google.registry.util.UrlFetchUtils.getHeaderFirst;
import static google.registry.util.UrlFetchUtils.setPayloadMultipart;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
import com.google.api.client.http.HttpMethods;
import com.google.appengine.api.taskqueue.LeaseOptions;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TransientFailureException;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.apphosting.api.DeadlineExceededException;
import com.google.common.base.Joiner;
import com.google.common.base.Strings;
@ -49,16 +44,18 @@ import google.registry.config.RegistryConfig.Config;
import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.RequestParameters;
import google.registry.request.UrlConnectionService;
import google.registry.request.UrlConnectionUtils;
import google.registry.request.auth.Auth;
import google.registry.util.Clock;
import google.registry.util.Retrier;
import google.registry.util.TaskQueueUtils;
import google.registry.util.UrlFetchException;
import google.registry.util.UrlConnectionException;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.SecureRandom;
import java.util.List;
import java.util.Optional;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
@ -97,7 +94,8 @@ public final class NordnUploadAction implements Runnable {
@Inject Retrier retrier;
@Inject SecureRandom random;
@Inject LordnRequestInitializer lordnRequestInitializer;
@Inject URLFetchService fetchService;
@Inject UrlConnectionService urlConnectionService;
@Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl;
@Inject @Parameter(LORDN_PHASE_PARAM) String phase;
@Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@ -193,47 +191,48 @@ public final class NordnUploadAction implements Runnable {
* <p>Idempotency: If the exact same LORDN report is uploaded twice, the MarksDB server will
* return the same confirmation number.
*
* @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">
* TMCH functional specifications - LORDN File</a>
* @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">TMCH
* functional specifications - LORDN File</a>
*/
private void uploadCsvToLordn(String urlPath, String csvData) throws IOException {
String url = tmchMarksdbUrl + urlPath;
logger.atInfo().log(
"LORDN upload task %s: Sending to URL: %s ; data: %s", actionLogId, url, csvData);
HTTPRequest req = new HTTPRequest(new URL(url), POST, validateCertificate().setDeadline(60d));
lordnRequestInitializer.initialize(req, tld);
setPayloadMultipart(req, "file", "claims.csv", CSV_UTF_8, csvData, random);
HTTPResponse rsp;
HttpURLConnection connection = urlConnectionService.createConnection(new URL(url));
connection.setRequestMethod(HttpMethods.POST);
lordnRequestInitializer.initialize(connection, tld);
UrlConnectionUtils.setPayloadMultipart(
connection, "file", "claims.csv", CSV_UTF_8, csvData, random);
try {
rsp = fetchService.fetch(req);
int responseCode = connection.getResponseCode();
if (logger.atInfo().isEnabled()) {
String responseContent = new String(getResponseBytes(connection), US_ASCII);
if (responseContent.isEmpty()) {
responseContent = "(null)";
}
logger.atInfo().log(
"LORDN upload task %s response: HTTP response code %d, response data: %s",
actionLogId, responseCode, responseContent);
}
if (responseCode != SC_ACCEPTED) {
throw new UrlConnectionException(
String.format(
"LORDN upload task %s error: Failed to upload LORDN claims to MarksDB",
actionLogId),
connection);
}
String location = connection.getHeaderField(LOCATION);
if (location == null) {
throw new UrlConnectionException(
String.format(
"LORDN upload task %s error: MarksDB failed to provide a Location header",
actionLogId),
connection);
}
getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location)));
} catch (IOException e) {
throw new IOException(
String.format("Error connecting to MarksDB at URL %s", url), e);
throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
}
if (logger.atInfo().isEnabled()) {
String response =
(rsp.getContent() == null) ? "(null)" : new String(rsp.getContent(), US_ASCII);
logger.atInfo().log(
"LORDN upload task %s response: HTTP response code %d, response data: %s",
actionLogId, rsp.getResponseCode(), response);
}
if (rsp.getResponseCode() != SC_ACCEPTED) {
throw new UrlFetchException(
String.format(
"LORDN upload task %s error: Failed to upload LORDN claims to MarksDB", actionLogId),
req,
rsp);
}
Optional<String> location = getHeaderFirst(rsp, LOCATION);
if (!location.isPresent()) {
throw new UrlFetchException(
String.format(
"LORDN upload task %s error: MarksDB failed to provide a Location header",
actionLogId),
req,
rsp);
}
getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location.get())));
}
private TaskOptions makeVerifyTask(URL url) {

97
core/src/main/java/google/registry/tmch/NordnVerifyAction.java
View file

@ -14,15 +14,11 @@
package google.registry.tmch;
import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
import static com.google.appengine.api.urlfetch.HTTPMethod.GET;
import static google.registry.request.UrlConnectionUtils.getResponseBytes;
import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.flogger.FluentLogger;
import com.google.common.io.ByteSource;
@ -32,9 +28,11 @@ import google.registry.request.HttpException.ConflictException;
import google.registry.request.Parameter;
import google.registry.request.RequestParameters;
import google.registry.request.Response;
import google.registry.request.UrlConnectionService;
import google.registry.request.auth.Auth;
import google.registry.util.UrlFetchException;
import google.registry.util.UrlConnectionException;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Map.Entry;
import javax.inject.Inject;
@ -68,7 +66,8 @@ public final class NordnVerifyAction implements Runnable {
@Inject LordnRequestInitializer lordnRequestInitializer;
@Inject Response response;
@Inject URLFetchService fetchService;
@Inject UrlConnectionService urlConnectionService;
@Inject @Header(URL_HEADER) URL url;
@Inject @Header(HEADER_ACTION_LOG_ID) String actionLogId;
@Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@ -96,51 +95,49 @@ public final class NordnVerifyAction implements Runnable {
@VisibleForTesting
LordnLog verify() throws IOException {
logger.atInfo().log("LORDN verify task %s: Sending request to URL %s", actionLogId, url);
HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d));
lordnRequestInitializer.initialize(req, tld);
HTTPResponse rsp;
HttpURLConnection connection = urlConnectionService.createConnection(url);
lordnRequestInitializer.initialize(connection, tld);
try {
rsp = fetchService.fetch(req);
} catch (IOException e) {
throw new IOException(
String.format("Error connecting to MarksDB at URL %s", url), e);
}
logger.atInfo().log(
"LORDN verify task %s response: HTTP response code %d, response data: %s",
actionLogId, rsp.getResponseCode(), rsp.getContent());
if (rsp.getResponseCode() == SC_NO_CONTENT) {
// Send a 400+ status code so App Engine will retry the task.
throw new ConflictException("Not ready");
}
if (rsp.getResponseCode() != SC_OK) {
throw new UrlFetchException(
String.format("LORDN verify task %s: Failed to verify LORDN upload to MarksDB.",
actionLogId),
req, rsp);
}
LordnLog log =
LordnLog.parse(ByteSource.wrap(rsp.getContent()).asCharSource(UTF_8).readLines());
if (log.getStatus() == LordnLog.Status.ACCEPTED) {
logger.atInfo().log("LORDN verify task %s: Upload accepted.", actionLogId);
} else {
logger.atSevere().log(
"LORDN verify task %s: Upload rejected with reason: %s", actionLogId, log);
}
for (Entry<String, LordnLog.Result> result : log) {
switch (result.getValue().getOutcome()) {
case OK:
break;
case WARNING:
// fall through
case ERROR:
logger.atWarning().log(result.toString());
break;
default:
logger.atWarning().log(
"LORDN verify task %s: Unexpected outcome: %s", actionLogId, result);
break;
int responseCode = connection.getResponseCode();
logger.atInfo().log(
"LORDN verify task %s response: HTTP response code %d", actionLogId, responseCode);
if (responseCode == SC_NO_CONTENT) {
// Send a 400+ status code so App Engine will retry the task.
throw new ConflictException("Not ready");
}
if (responseCode != SC_OK) {
throw new UrlConnectionException(
String.format(
"LORDN verify task %s: Failed to verify LORDN upload to MarksDB.", actionLogId),
connection);
}
LordnLog log =
LordnLog.parse(
ByteSource.wrap(getResponseBytes(connection)).asCharSource(UTF_8).readLines());
if (log.getStatus() == LordnLog.Status.ACCEPTED) {
logger.atInfo().log("LORDN verify task %s: Upload accepted.", actionLogId);
} else {
logger.atSevere().log(
"LORDN verify task %s: Upload rejected with reason: %s", actionLogId, log);
}
for (Entry<String, LordnLog.Result> result : log) {
switch (result.getValue().getOutcome()) {
case OK:
break;
case WARNING:
// fall through
case ERROR:
logger.atWarning().log(result.toString());
break;
default:
logger.atWarning().log(
"LORDN verify task %s: Unexpected outcome: %s", actionLogId, result);
break;
}
}
return log;
} catch (IOException e) {
throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
}
return log;
}
}

6
core/src/main/java/google/registry/tools/RegistryToolComponent.java
View file

@ -38,8 +38,7 @@ import google.registry.privileges.secretmanager.SecretManagerModule;
import google.registry.rde.RdeModule;
import google.registry.request.Modules.DatastoreServiceModule;
import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.URLFetchServiceModule;
import google.registry.request.Modules.UrlFetchTransportModule;
import google.registry.request.Modules.UrlConnectionServiceModule;
import google.registry.request.Modules.UserServiceModule;
import google.registry.tools.AuthModule.LocalCredentialModule;
import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
@ -79,8 +78,7 @@ import javax.inject.Singleton;
RegistryToolDataflowModule.class,
RequestFactoryModule.class,
SecretManagerModule.class,
URLFetchServiceModule.class,
UrlFetchTransportModule.class,
UrlConnectionServiceModule.class,
UserServiceModule.class,
UtilsModule.class,
VoidDnsWriterModule.class,

87
core/src/test/java/google/registry/rde/RdeReportActionTest.java
View file

@ -14,7 +14,6 @@
package google.registry.rde;
import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
@ -29,20 +28,13 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
import static org.joda.time.Duration.standardDays;
import static org.joda.time.Duration.standardSeconds;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.ByteSource;
import google.registry.gcs.GcsUtils;
import google.registry.model.common.Cursor;
@ -57,20 +49,21 @@ import google.registry.testing.FakeClock;
import google.registry.testing.FakeKeyringModule;
import google.registry.testing.FakeResponse;
import google.registry.testing.FakeSleeper;
import google.registry.testing.FakeUrlConnectionService;
import google.registry.testing.TestOfyAndSql;
import google.registry.util.Retrier;
import google.registry.xjc.XjcXmlTransformer;
import google.registry.xjc.rdereport.XjcRdeReportReport;
import google.registry.xml.XmlException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.util.Map;
import java.util.Optional;
import org.bouncycastle.openpgp.PGPPublicKey;
import org.joda.time.DateTime;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.ArgumentCaptor;
/** Unit tests for {@link RdeReportAction}. */
@DualDatabaseTest
@ -89,20 +82,21 @@ public class RdeReportActionTest {
private final FakeResponse response = new FakeResponse();
private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class);
private final URLFetchService urlFetchService = mock(URLFetchService.class);
private final ArgumentCaptor<HTTPRequest> request = ArgumentCaptor.forClass(HTTPRequest.class);
private final HTTPResponse httpResponse = mock(HTTPResponse.class);
private final PGPPublicKey encryptKey =
new FakeKeyringModule().get().getRdeStagingEncryptionKey();
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
private final BlobId reportFile =
BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde");
private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
private final FakeUrlConnectionService urlConnectionService =
new FakeUrlConnectionService(httpUrlConnection);
private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
private RdeReportAction createAction() {
RdeReporter reporter = new RdeReporter();
reporter.reportUrlPrefix = "https://rde-report.example";
reporter.urlFetchService = urlFetchService;
reporter.password = "foo";
reporter.urlConnectionService = urlConnectionService;
reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
RdeReportAction action = new RdeReportAction();
action.gcsUtils = gcsUtils;
@ -127,6 +121,7 @@ public class RdeReportActionTest {
Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test")));
gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
}
@TestOfyAndSql
@ -142,24 +137,22 @@ public class RdeReportActionTest {
@TestOfyAndSql
void testRunWithLock() throws Exception {
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
createAction().runWithLock(loadRdeReportCursor());
assertThat(response.getStatus()).isEqualTo(200);
assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
// Verify the HTTP request was correct.
assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
verify(httpUrlConnection).setRequestMethod("PUT");
assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
// Verify the payload XML was the same as what's in testdata/report.xml.
XjcRdeReportReport report = parseReport(request.getValue().getPayload());
XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
assertThat(report.getId()).isEqualTo("20101017001");
assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@ -167,9 +160,8 @@ public class RdeReportActionTest {
@TestOfyAndSql
void testRunWithLock_withPrefix() throws Exception {
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
RdeReportAction action = createAction();
action.prefix = Optional.of("job-name/");
gcsUtils.delete(reportFile);
@ -182,16 +174,14 @@ public class RdeReportActionTest {
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
// Verify the HTTP request was correct.
assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
assertThat(headers)
.containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
verify(httpUrlConnection).setRequestMethod("PUT");
assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
// Verify the payload XML was the same as what's in testdata/report.xml.
XjcRdeReportReport report = parseReport(request.getValue().getPayload());
XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
assertThat(report.getId()).isEqualTo("20101017001");
assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@ -204,9 +194,8 @@ public class RdeReportActionTest {
PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
createAction().runWithLock(loadRdeReportCursor());
assertThat(response.getStatus()).isEqualTo(200);
}
@ -239,9 +228,8 @@ public class RdeReportActionTest {
@TestOfyAndSql
void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception {
when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST);
when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
when(httpUrlConnection.getResponseCode()).thenReturn(SC_BAD_REQUEST);
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openStream());
InternalServerErrorException thrown =
assertThrows(
InternalServerErrorException.class,
@ -252,18 +240,17 @@ public class RdeReportActionTest {
@TestOfyAndSql
void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception {
class ExpectedThrownException extends RuntimeException {}
when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException());
when(httpUrlConnection.getResponseCode()).thenThrow(new ExpectedThrownException());
assertThrows(
ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
}
@TestOfyAndSql
void testRunWithLock_socketTimeout_doesRetry() throws Exception {
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture()))
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
when(httpUrlConnection.getResponseCode())
.thenThrow(new SocketTimeoutException())
.thenReturn(httpResponse);
.thenReturn(SC_OK);
createAction().runWithLock(loadRdeReportCursor());
assertThat(response.getStatus()).isEqualTo(200);
assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
@ -274,14 +261,6 @@ public class RdeReportActionTest {
return loadByKey(Cursor.createVKey(RDE_REPORT, "test")).getCursorTime();
}
private static ImmutableMap<String, String> mapifyHeaders(Iterable<HTTPHeader> headers) {
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
for (HTTPHeader header : headers) {
builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue());
}
return builder.build();
}
private static XjcRdeReportReport parseReport(byte[] data) {
try {
return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data));

58
core/src/test/java/google/registry/util/UrlFetchUtilsTest.java → core/src/test/java/google/registry/request/UrlConnectionUtilsTest.java
View file

@ -1,4 +1,4 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
// Copyright 2022 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,13 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.util;
package google.registry.request;
import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
import static com.google.common.net.MediaType.CSV_UTF_8;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.util.UrlFetchUtils.setPayloadMultipart;
import static google.registry.request.UrlConnectionUtils.setPayloadMultipart;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
@ -27,22 +27,19 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import google.registry.testing.AppEngineExtension;
import java.io.ByteArrayOutputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import javax.net.ssl.HttpsURLConnection;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.ArgumentCaptor;
/** Unit tests for {@link UrlFetchUtils}. */
class UrlFetchUtilsTest {
@RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().build();
/** Tests for {@link UrlConnectionUtils}. */
public class UrlConnectionUtilsTest {
private final Random random = mock(Random.class);
@ -58,25 +55,28 @@ class UrlFetchUtilsTest {
}
@Test
void testSetPayloadMultipart() {
HTTPRequest request = mock(HTTPRequest.class);
void testSetPayloadMultipart() throws Exception {
HttpsURLConnection connection = mock(HttpsURLConnection.class);
ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
when(connection.getOutputStream()).thenReturn(connectionOutputStream);
setPayloadMultipart(
request,
connection,
"lol",
"cat",
CSV_UTF_8,
"The nice people at the store say hello. ヘ(◕。◕ヘ)",
random);
ArgumentCaptor<HTTPHeader> headerCaptor = ArgumentCaptor.forClass(HTTPHeader.class);
verify(request, times(2)).addHeader(headerCaptor.capture());
List<HTTPHeader> addedHeaders = headerCaptor.getAllValues();
assertThat(addedHeaders.get(0).getName()).isEqualTo(CONTENT_TYPE);
assertThat(addedHeaders.get(0).getValue())
.isEqualTo(
"multipart/form-data; "
+ "boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"");
assertThat(addedHeaders.get(1).getName()).isEqualTo(CONTENT_LENGTH);
assertThat(addedHeaders.get(1).getValue()).isEqualTo("294");
ArgumentCaptor<String> keyCaptor = ArgumentCaptor.forClass(String.class);
ArgumentCaptor<String> valueCaptor = ArgumentCaptor.forClass(String.class);
verify(connection, times(2)).setRequestProperty(keyCaptor.capture(), valueCaptor.capture());
List<String> addedKeys = keyCaptor.getAllValues();
assertThat(addedKeys).containsExactly(CONTENT_TYPE, CONTENT_LENGTH);
List<String> addedValues = valueCaptor.getAllValues();
assertThat(addedValues)
.containsExactly(
"multipart/form-data;"
+ " boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"",
"294");
String payload =
"--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\n"
+ "Content-Disposition: form-data; name=\"lol\"; filename=\"cat\"\r\n"
@ -84,18 +84,20 @@ class UrlFetchUtilsTest {
+ "\r\n"
+ "The nice people at the store say hello. ヘ(◕。◕ヘ)\r\n"
+ "--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA--\r\n";
verify(request).setPayload(payload.getBytes(UTF_8));
verifyNoMoreInteractions(request);
verify(connection).setDoOutput(true);
verify(connection).getOutputStream();
assertThat(connectionOutputStream.toByteArray()).isEqualTo(payload.getBytes(UTF_8));
verifyNoMoreInteractions(connection);
}
@Test
void testSetPayloadMultipart_boundaryInPayload() {
HTTPRequest request = mock(HTTPRequest.class);
HttpsURLConnection connection = mock(HttpsURLConnection.class);
String payload = "I screamed------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHHH";
IllegalStateException thrown =
assertThrows(
IllegalStateException.class,
() -> setPayloadMultipart(request, "lol", "cat", CSV_UTF_8, payload, random));
() -> setPayloadMultipart(connection, "lol", "cat", CSV_UTF_8, payload, random));
assertThat(thrown)
.hasMessageThat()
.contains(

46
core/src/test/java/google/registry/testing/FakeURLFetchService.java
View file

@ -1,46 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.testing;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.common.collect.ImmutableList;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Map;
/**
* A fake {@link URLFetchService} that serves constructed {@link HTTPResponse} objects from
* a simple {@link Map} ({@link URL} to {@link HTTPResponse}) lookup.
*/
public class FakeURLFetchService extends ForwardingURLFetchService {
private Map<URL, HTTPResponse> backingMap;
public FakeURLFetchService(Map<URL, HTTPResponse> backingMap) {
this.backingMap = backingMap;
}
@Override
public HTTPResponse fetch(HTTPRequest request) {
URL requestURL = request.getURL();
if (backingMap.containsKey(requestURL)) {
return backingMap.get(requestURL);
} else {
return new HTTPResponse(HttpURLConnection.HTTP_NOT_FOUND, null, null, ImmutableList.of());
}
}
}

46
core/src/test/java/google/registry/testing/FakeUrlConnectionService.java Normal file
View file

@ -0,0 +1,46 @@
// Copyright 2022 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.testing;
import static org.mockito.Mockito.when;
import google.registry.request.UrlConnectionService;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
/** A fake {@link UrlConnectionService} with a mocked HTTP connection for testing. */
public class FakeUrlConnectionService implements UrlConnectionService {
private final HttpURLConnection mockConnection;
private final List<URL> connectedUrls;
public FakeUrlConnectionService(HttpURLConnection mockConnection) {
this(mockConnection, new ArrayList<>());
}
public FakeUrlConnectionService(HttpURLConnection mockConnection, List<URL> connectedUrls) {
this.mockConnection = mockConnection;
this.connectedUrls = connectedUrls;
}
@Override
public HttpURLConnection createConnection(URL url) {
connectedUrls.add(url);
when(mockConnection.getURL()).thenReturn(url);
return mockConnection;
}
}

49
core/src/test/java/google/registry/testing/ForwardingURLFetchService.java
View file

@ -1,49 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.testing;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.common.util.concurrent.Futures;
import java.io.IOException;
import java.net.URL;
import java.util.concurrent.Future;
/**
* An implementation of the {@link URLFetchService} interface that forwards all requests through
* a synchronous fetch call.
*/
public abstract class ForwardingURLFetchService implements URLFetchService {
@Override
public HTTPResponse fetch(URL url) throws IOException {
return fetch(new HTTPRequest(url)); // Defaults to HTTPMethod.GET
}
@Override
public Future<HTTPResponse> fetchAsync(URL url) {
return fetchAsync(new HTTPRequest(url)); // Defaults to HTTPMethod.GET
}
@Override
public Future<HTTPResponse> fetchAsync(HTTPRequest request) {
try {
return Futures.immediateFuture(fetch(request));
} catch (Exception e) {
return Futures.immediateFailedFuture(e);
}
}
}

101
core/src/test/java/google/registry/tmch/NordnUploadActionTest.java
View file

@ -19,21 +19,22 @@ import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
import static com.google.common.net.HttpHeaders.LOCATION;
import static com.google.common.net.MediaType.FORM_DATA;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static google.registry.testing.DatabaseHelper.createTld;
import static google.registry.testing.DatabaseHelper.loadRegistrar;
import static google.registry.testing.DatabaseHelper.newDomainBase;
import static google.registry.testing.DatabaseHelper.persistDomainAndEnqueueLordn;
import static google.registry.testing.DatabaseHelper.persistResource;
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
import static google.registry.util.UrlFetchUtils.getHeaderFirst;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.startsWith;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ -43,10 +44,6 @@ import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.appengine.api.taskqueue.TransientFailureException;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.apphosting.api.DeadlineExceededException;
import com.google.common.base.VerifyException;
import com.google.common.collect.ImmutableList;
@ -57,11 +54,15 @@ import google.registry.model.tld.Registry;
import google.registry.testing.AppEngineExtension;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeSleeper;
import google.registry.testing.FakeUrlConnectionService;
import google.registry.testing.InjectExtension;
import google.registry.testing.TaskQueueHelper.TaskMatcher;
import google.registry.util.Retrier;
import google.registry.util.TaskQueueUtils;
import google.registry.util.UrlFetchException;
import google.registry.util.UrlConnectionException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.SecureRandom;
import java.util.List;
@ -69,17 +70,9 @@ import java.util.Optional;
import org.joda.time.DateTime;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
/** Unit tests for {@link NordnUploadAction}. */
@ExtendWith(MockitoExtension.class)
class NordnUploadActionTest {
private static final String CLAIMS_CSV =
@ -101,29 +94,30 @@ class NordnUploadActionTest {
AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
@RegisterExtension public final InjectExtension inject = new InjectExtension();
@Mock private URLFetchService fetchService;
@Mock private HTTPResponse httpResponse;
@Captor private ArgumentCaptor<HTTPRequest> httpRequestCaptor;
private final FakeClock clock = new FakeClock(DateTime.parse("2010-05-01T10:11:12Z"));
private final LordnRequestInitializer lordnRequestInitializer =
new LordnRequestInitializer(Optional.of("attack"));
private final NordnUploadAction action = new NordnUploadAction();
private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
private final FakeUrlConnectionService urlConnectionService =
new FakeUrlConnectionService(httpUrlConnection);
@BeforeEach
void beforeEach() throws Exception {
inject.setStaticField(Ofy.class, "clock", clock);
when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
when(httpResponse.getContent()).thenReturn("Success".getBytes(US_ASCII));
when(httpResponse.getResponseCode()).thenReturn(SC_ACCEPTED);
when(httpResponse.getHeadersUncombined())
.thenReturn(ImmutableList.of(new HTTPHeader(LOCATION, "http://trololol")));
when(httpUrlConnection.getInputStream())
.thenReturn(new ByteArrayInputStream("Success".getBytes(UTF_8)));
when(httpUrlConnection.getResponseCode()).thenReturn(SC_ACCEPTED);
when(httpUrlConnection.getHeaderField(LOCATION)).thenReturn("http://trololol");
when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
persistResource(loadRegistrar("TheRegistrar").asBuilder().setIanaIdentifier(99999L).build());
createTld("tld");
persistResource(Registry.get("tld").asBuilder().setLordnUsername("lolcat").build());
action.clock = clock;
action.fetchService = fetchService;
action.urlConnectionService = urlConnectionService;
action.lordnRequestInitializer = lordnRequestInitializer;
action.phase = "claims";
action.taskQueueUtils = new TaskQueueUtils(new Retrier(new FakeSleeper(clock), 3));
@ -133,7 +127,6 @@ class NordnUploadActionTest {
action.retrier = new Retrier(new FakeSleeper(clock), 3);
}
@MockitoSettings(strictness = Strictness.LENIENT)
@Test
void test_convertTasksToCsv() {
List<TaskHandle> tasks =
@ -145,7 +138,6 @@ class NordnUploadActionTest {
.isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
}
@MockitoSettings(strictness = Strictness.LENIENT)
@Test
void test_convertTasksToCsv_dedupesDuplicates() {
List<TaskHandle> tasks =
@ -158,14 +150,12 @@ class NordnUploadActionTest {
.isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
}
@MockitoSettings(strictness = Strictness.LENIENT)
@Test
void test_convertTasksToCsv_doesntFailOnEmptyTasks() {
assertThat(NordnUploadAction.convertTasksToCsv(ImmutableList.of(), clock.nowUtc(), "col1,col2"))
.isEqualTo("1,2010-05-01T10:11:12.000Z,0\ncol1,col2\n");
}
@MockitoSettings(strictness = Strictness.LENIENT)
@Test
void test_convertTasksToCsv_throwsNpeOnNullTasks() {
assertThrows(
@ -173,7 +163,6 @@ class NordnUploadActionTest {
() -> NordnUploadAction.convertTasksToCsv(null, clock.nowUtc(), "header"));
}
@MockitoSettings(strictness = Strictness.LENIENT)
@SuppressWarnings("unchecked")
@Test
void test_loadAllTasks_retryLogic_thirdTrysTheCharm() {
@ -186,7 +175,6 @@ class NordnUploadActionTest {
assertThat(action.loadAllTasks(queue, "tld")).containsExactly(task);
}
@MockitoSettings(strictness = Strictness.LENIENT)
@Test
void test_loadAllTasks_retryLogic_allFailures() {
Queue queue = mock(Queue.class);
@ -201,47 +189,47 @@ class NordnUploadActionTest {
void testRun_claimsMode_appendsTldAndClaimsToRequestUrl() throws Exception {
persistClaimsModeDomain();
action.run();
assertThat(getCapturedHttpRequest().getURL())
.isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims"));
assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims"));
}
@Test
void testRun_sunriseMode_appendsTldAndClaimsToRequestUrl() throws Exception {
persistSunriseModeDomain();
action.run();
assertThat(getCapturedHttpRequest().getURL())
.isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise"));
assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise"));
}
@Test
void testRun_usesMultipartContentType() throws Exception {
persistClaimsModeDomain();
action.run();
assertThat(getHeaderFirst(getCapturedHttpRequest(), CONTENT_TYPE).get())
.startsWith("multipart/form-data; boundary=");
verify(httpUrlConnection)
.setRequestProperty(eq(CONTENT_TYPE), startsWith("multipart/form-data; boundary="));
verify(httpUrlConnection).setRequestMethod("POST");
}
@Test
void testRun_hasPassword_setsAuthorizationHeader() throws Exception {
void testRun_hasPassword_setsAuthorizationHeader() {
persistClaimsModeDomain();
action.run();
assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION))
.hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
verify(httpUrlConnection)
.setRequestProperty(
AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
}
@Test
void testRun_noPassword_doesntSendAuthorizationHeader() throws Exception {
void testRun_noPassword_doesntSendAuthorizationHeader() {
action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty());
persistClaimsModeDomain();
action.run();
assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty();
verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString());
}
@Test
void testRun_claimsMode_payloadMatchesClaimsCsv() throws Exception {
void testRun_claimsMode_payloadMatchesClaimsCsv() {
persistClaimsModeDomain();
action.run();
assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(CLAIMS_CSV);
assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(CLAIMS_CSV);
}
@Test
@ -257,19 +245,19 @@ class NordnUploadActionTest {
}
@Test
void testRun_sunriseMode_payloadMatchesSunriseCsv() throws Exception {
void testRun_sunriseMode_payloadMatchesSunriseCsv() {
persistSunriseModeDomain();
action.run();
assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV);
assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV);
}
@Test
void test_noResponseContent_stillWorksNormally() throws Exception {
// Returning null only affects logging.
when(httpResponse.getContent()).thenReturn(null);
when(httpUrlConnection.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[] {}));
persistSunriseModeDomain();
action.run();
assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV);
assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV);
}
@Test
@ -284,7 +272,6 @@ class NordnUploadActionTest {
.header(CONTENT_TYPE, FORM_DATA.toString()));
}
@MockitoSettings(strictness = Strictness.LENIENT)
@Test
void testFailure_nullRegistryUser() {
persistClaimsModeDomain();
@ -293,17 +280,11 @@ class NordnUploadActionTest {
assertThat(thrown).hasMessageThat().contains("lordnUsername is not set for tld.");
}
@MockitoSettings(strictness = Strictness.LENIENT)
@Test
void testFetchFailure() {
void testFetchFailure() throws Exception {
persistClaimsModeDomain();
when(httpResponse.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
assertThrows(UrlFetchException.class, action::run);
}
private HTTPRequest getCapturedHttpRequest() throws Exception {
verify(fetchService).fetch(httpRequestCaptor.capture());
return httpRequestCaptor.getAllValues().get(0);
when(httpUrlConnection.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
assertThrows(UrlConnectionException.class, action::run);
}
private void persistClaimsModeDomain() {

70
core/src/test/java/google/registry/tmch/NordnVerifyActionTest.java
View file

@ -16,39 +16,34 @@ package google.registry.tmch;
import static com.google.common.net.HttpHeaders.AUTHORIZATION;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static google.registry.testing.DatabaseHelper.createTld;
import static google.registry.testing.DatabaseHelper.persistResource;
import static google.registry.util.UrlFetchUtils.getHeaderFirst;
import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import google.registry.model.tld.Registry;
import google.registry.request.HttpException.ConflictException;
import google.registry.testing.AppEngineExtension;
import google.registry.testing.FakeResponse;
import google.registry.testing.FakeUrlConnectionService;
import java.io.ByteArrayInputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
/** Unit tests for {@link NordnVerifyAction}. */
@ExtendWith(MockitoExtension.class)
class NordnVerifyActionTest {
private static final String LOG_ACCEPTED =
@ -81,84 +76,83 @@ class NordnVerifyActionTest {
public final AppEngineExtension appEngine =
AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
@Mock private URLFetchService fetchService;
@Mock private HTTPResponse httpResponse;
@Captor private ArgumentCaptor<HTTPRequest> httpRequestCaptor;
private final FakeResponse response = new FakeResponse();
private final LordnRequestInitializer lordnRequestInitializer =
new LordnRequestInitializer(Optional.of("attack"));
private final NordnVerifyAction action = new NordnVerifyAction();
private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
private final FakeUrlConnectionService urlConnectionService =
new FakeUrlConnectionService(httpUrlConnection);
@BeforeEach
void beforeEach() throws Exception {
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(LOG_ACCEPTED.getBytes(UTF_8));
when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
createTld("gtld");
persistResource(Registry.get("gtld").asBuilder().setLordnUsername("lolcat").build());
action.tld = "gtld";
action.fetchService = fetchService;
action.urlConnectionService = urlConnectionService;
when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
when(httpUrlConnection.getInputStream())
.thenReturn(new ByteArrayInputStream(LOG_ACCEPTED.getBytes(UTF_8)));
action.lordnRequestInitializer = lordnRequestInitializer;
action.response = response;
action.url = new URL("http://127.0.0.1/blobio");
}
private HTTPRequest getCapturedHttpRequest() throws Exception {
verify(fetchService).fetch(httpRequestCaptor.capture());
return httpRequestCaptor.getAllValues().get(0);
}
@Test
void testSuccess_sendHttpRequest_urlIsCorrect() throws Exception {
action.run();
assertThat(getCapturedHttpRequest().getURL()).isEqualTo(new URL("http://127.0.0.1/blobio"));
assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/blobio"));
}
@Test
void testSuccess_hasLordnPassword_sendsAuthorizationHeader() throws Exception {
void testSuccess_hasLordnPassword_sendsAuthorizationHeader() {
action.run();
assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION))
.hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
verify(httpUrlConnection)
.setRequestProperty(
AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
}
@Test
void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() throws Exception {
void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() {
action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty());
action.run();
assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty();
verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString());
}
@Test
void successVerifyRejected() throws Exception {
when(httpResponse.getContent()).thenReturn(LOG_REJECTED.getBytes(UTF_8));
when(httpUrlConnection.getInputStream())
.thenReturn(new ByteArrayInputStream(LOG_REJECTED.getBytes(UTF_8)));
LordnLog lastLog = action.verify();
assertThat(lastLog.getStatus()).isEqualTo(LordnLog.Status.REJECTED);
}
@Test
void successVerifyWarnings() throws Exception {
when(httpResponse.getContent()).thenReturn(LOG_WARNINGS.getBytes(UTF_8));
when(httpUrlConnection.getInputStream())
.thenReturn(new ByteArrayInputStream(LOG_WARNINGS.getBytes(UTF_8)));
LordnLog lastLog = action.verify();
assertThat(lastLog.hasWarnings()).isTrue();
}
@Test
void successVerifyErrors() throws Exception {
when(httpResponse.getContent()).thenReturn(LOG_ERRORS.getBytes(UTF_8));
when(httpUrlConnection.getInputStream())
.thenReturn(new ByteArrayInputStream(LOG_ERRORS.getBytes(UTF_8)));
LordnLog lastLog = action.verify();
assertThat(lastLog.hasWarnings()).isTrue();
}
@Test
void failureVerifyUnauthorized() {
when(httpResponse.getResponseCode()).thenReturn(SC_UNAUTHORIZED);
void failureVerifyUnauthorized() throws Exception {
when(httpUrlConnection.getResponseCode()).thenReturn(SC_UNAUTHORIZED);
assertThrows(Exception.class, action::run);
}
@Test
void failureVerifyNotReady() {
when(httpResponse.getResponseCode()).thenReturn(SC_NO_CONTENT);
void failureVerifyNotReady() throws Exception {
when(httpUrlConnection.getResponseCode()).thenReturn(SC_NO_CONTENT);
ConflictException thrown = assertThrows(ConflictException.class, action::run);
assertThat(thrown).hasMessageThat().contains("Not ready");
}

26
core/src/test/java/google/registry/tmch/TmchActionTestCase.java
View file

@ -15,21 +15,19 @@
package google.registry.tmch;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import google.registry.testing.AppEngineExtension;
import google.registry.testing.BouncyCastleProviderExtension;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeUrlConnectionService;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
/** Common code for unit tests of classes that extend {@link Marksdb}. */
@ -46,19 +44,19 @@ abstract class TmchActionTestCase {
@RegisterExtension
public final BouncyCastleProviderExtension bouncy = new BouncyCastleProviderExtension();
@Mock URLFetchService fetchService;
@Mock HTTPResponse httpResponse;
@Captor ArgumentCaptor<HTTPRequest> httpRequest;
final FakeClock clock = new FakeClock();
final Marksdb marksdb = new Marksdb();
protected final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
protected final ArrayList<URL> connectedUrls = new ArrayList<>();
protected FakeUrlConnectionService urlConnectionService =
new FakeUrlConnectionService(httpUrlConnection, connectedUrls);
@BeforeEach
public void beforeEachTmchActionTestCase() throws Exception {
marksdb.fetchService = fetchService;
marksdb.tmchMarksdbUrl = MARKSDB_URL;
marksdb.marksdbPublicKey = TmchData.loadPublicKey(TmchTestData.loadBytes("pubkey"));
when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
marksdb.urlConnectionService = urlConnectionService;
when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
}
}

30
core/src/test/java/google/registry/tmch/TmchCrlActionTest.java
View file

@ -22,6 +22,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import google.registry.config.RegistryConfig.ConfigModule.TmchCaMode;
import java.io.ByteArrayInputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.SignatureException;
@ -44,19 +45,22 @@ class TmchCrlActionTest extends TmchActionTestCase {
@Test
void testSuccess() throws Exception {
clock.setTo(DateTime.parse("2013-07-24TZ"));
when(httpResponse.getContent()).thenReturn(
readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read());
when(httpUrlConnection.getInputStream())
.thenReturn(
new ByteArrayInputStream(
readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()));
newTmchCrlAction(TmchCaMode.PRODUCTION).run();
verify(httpResponse).getContent();
verify(fetchService).fetch(httpRequest.capture());
assertThat(httpRequest.getValue().getURL().toString()).isEqualTo("http://sloth.lol/tmch.crl");
verify(httpUrlConnection).getInputStream();
assertThat(connectedUrls).containsExactly(new URL("http://sloth.lol/tmch.crl"));
}
@Test
void testFailure_crlTooOld() throws Exception {
clock.setTo(DateTime.parse("2020-01-01TZ"));
when(httpResponse.getContent())
.thenReturn(loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read());
when(httpUrlConnection.getInputStream())
.thenReturn(
new ByteArrayInputStream(
loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read()));
TmchCrlAction action = newTmchCrlAction(TmchCaMode.PILOT);
Exception e = assertThrows(Exception.class, action::run);
assertThat(e).hasCauseThat().isInstanceOf(CRLException.class);
@ -69,8 +73,10 @@ class TmchCrlActionTest extends TmchActionTestCase {
@Test
void testFailure_crlNotSignedByRoot() throws Exception {
clock.setTo(DateTime.parse("2013-07-24TZ"));
when(httpResponse.getContent())
.thenReturn(readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read());
when(httpUrlConnection.getInputStream())
.thenReturn(
new ByteArrayInputStream(
readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()));
Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run);
assertThat(e).hasCauseThat().isInstanceOf(SignatureException.class);
assertThat(e).hasCauseThat().hasMessageThat().isEqualTo("Signature does not match.");
@ -79,8 +85,10 @@ class TmchCrlActionTest extends TmchActionTestCase {
@Test
void testFailure_crlNotYetValid() throws Exception {
clock.setTo(DateTime.parse("1984-01-01TZ"));
when(httpResponse.getContent()).thenReturn(
readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read());
when(httpUrlConnection.getInputStream())
.thenReturn(
new ByteArrayInputStream(
readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read()));
Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run);
assertThat(e).hasCauseThat().isInstanceOf(CertificateNotYetValidException.class);
}

17
core/src/test/java/google/registry/tmch/TmchDnlActionTest.java
View file

@ -14,6 +14,7 @@
package google.registry.tmch;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static org.mockito.Mockito.times;
@ -22,6 +23,8 @@ import static org.mockito.Mockito.when;
import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListDao;
import java.io.ByteArrayInputStream;
import java.net.URL;
import java.util.Optional;
import org.joda.time.DateTime;
import org.junit.jupiter.api.Test;
@ -39,15 +42,13 @@ class TmchDnlActionTest extends TmchActionTestCase {
@Test
void testDnl() throws Exception {
assertThat(ClaimsListDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty();
when(httpResponse.getContent())
.thenReturn(TmchTestData.loadBytes("dnl-latest.csv").read())
.thenReturn(TmchTestData.loadBytes("dnl-latest.sig").read());
when(httpUrlConnection.getInputStream())
.thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.csv").read()))
.thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.sig").read()));
newTmchDnlAction().run();
verify(fetchService, times(2)).fetch(httpRequest.capture());
assertThat(httpRequest.getAllValues().get(0).getURL().toString())
.isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.csv");
assertThat(httpRequest.getAllValues().get(1).getURL().toString())
.isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.sig");
verify(httpUrlConnection, times(2)).getInputStream();
assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
.containsExactly(MARKSDB_URL + "/dnl/dnl-latest.csv", MARKSDB_URL + "/dnl/dnl-latest.sig");
// Make sure the contents of testdata/dnl-latest.csv got inserted into the database.
ClaimsList claimsList = ClaimsListDao.get();

18
core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java
View file

@ -14,6 +14,7 @@
package google.registry.tmch;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.tmch.TmchTestData.loadBytes;
import static org.mockito.Mockito.times;
@ -21,6 +22,8 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import google.registry.model.smd.SignedMarkRevocationList;
import java.io.ByteArrayInputStream;
import java.net.URL;
import java.util.Optional;
import org.joda.time.DateTime;
import org.junit.jupiter.api.Test;
@ -42,15 +45,14 @@ class TmchSmdrlActionTest extends TmchActionTestCase {
SignedMarkRevocationList smdrl = SignedMarkRevocationList.get();
assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isFalse();
assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse();
when(httpResponse.getContent())
.thenReturn(loadBytes("smdrl-latest.csv").read())
.thenReturn(loadBytes("smdrl-latest.sig").read());
when(httpUrlConnection.getInputStream())
.thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.csv").read()))
.thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.sig").read()));
newTmchSmdrlAction().run();
verify(fetchService, times(2)).fetch(httpRequest.capture());
assertThat(httpRequest.getAllValues().get(0).getURL().toString())
.isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.csv");
assertThat(httpRequest.getAllValues().get(1).getURL().toString())
.isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.sig");
verify(httpUrlConnection, times(2)).getInputStream();
assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
.containsExactly(
MARKSDB_URL + "/smdrl/smdrl-latest.csv", MARKSDB_URL + "/smdrl/smdrl-latest.sig");
smdrl = SignedMarkRevocationList.get();
assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isTrue();
assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse();

71
util/src/main/java/google/registry/util/UrlConnectionException.java Normal file
View file

@ -0,0 +1,71 @@
// Copyright 2022 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.util;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.io.ByteStreams;
import java.io.IOException;
import java.net.HttpURLConnection;
/**
* Used when HTTP requests return a bad response, with troubleshooting info.
*
* <p>This class displays lots of helpful troubleshooting information.
*/
public class UrlConnectionException extends RuntimeException {
private final HttpURLConnection connection;
public UrlConnectionException(String message, HttpURLConnection connection) {
super(message);
this.connection = connection;
}
@Override
public String getMessage() {
byte[] resultContent;
int responseCode;
try {
resultContent = ByteStreams.toByteArray(connection.getInputStream());
responseCode = connection.getResponseCode();
} catch (IOException e) {
resultContent = new byte[] {};
responseCode = 0;
}
StringBuilder result =
new StringBuilder(2048 + resultContent.length)
.append(
String.format(
"%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\n",
getClass().getSimpleName(),
super.getMessage(),
responseCode,
connection.getURL().toString()));
connection
.getRequestProperties()
.forEach(
(key, value) -> {
result.append(key);
result.append(": ");
result.append(value);
result.append('\n');
});
result.append(">>>\n");
result.append(new String(resultContent, UTF_8));
result.append("\n<<<");
return result.toString();
}
}

63
util/src/main/java/google/registry/util/UrlFetchException.java
View file

@ -1,63 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.util;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
/**
* Exception for when App Engine HTTP requests return a bad response.
*
* <p>This class displays lots of helpful troubleshooting information.
*/
public class UrlFetchException extends RuntimeException {
private final HTTPRequest req;
private final HTTPResponse rsp;
public UrlFetchException(String message, HTTPRequest req, HTTPResponse rsp) {
super(message);
this.req = checkNotNull(req, "req");
this.rsp = checkNotNull(rsp, "rsp");
}
@Override
public String getMessage() {
StringBuilder res =
new StringBuilder(2048 + rsp.getContent().length)
.append(
String.format(
"%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\nX-Final-URL: %s\n",
getClass().getSimpleName(),
super.getMessage(),
rsp.getResponseCode(),
req.getURL().toString(),
rsp.getFinalUrl()));
for (HTTPHeader header : rsp.getHeadersUncombined()) {
res.append(header.getName());
res.append(": ");
res.append(header.getValue());
res.append('\n');
}
res.append(">>>\n");
res.append(new String(rsp.getContent(), UTF_8));
res.append("\n<<<");
return res.toString();
}
}