Use UrlFetch for RDE and default TLS (1.2) for other URL connections (#1578)

* Use UrlFetch for RDE and default TLS (1.2) for other URL connections

This removes the TLS 1.3-settings in the module providers and,
essentially, reverts the changes in #1535 only to the RdeReporter and
RdeReportActionTest
This commit is contained in:
gbrodman 2022-03-31 14:08:28 -04:00 committed by GitHub
parent f340b735c1
commit 86226d6ffa
5 changed files with 96 additions and 65 deletions

View file

@ -44,6 +44,7 @@ import google.registry.request.Modules.DatastoreServiceModule;
import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.NetHttpTransportModule;
import google.registry.request.Modules.UrlConnectionServiceModule;
import google.registry.request.Modules.UrlFetchServiceModule;
import google.registry.request.Modules.UrlFetchTransportModule;
import google.registry.request.Modules.UserServiceModule;
import google.registry.request.auth.AuthModule;
@ -81,6 +82,7 @@ import javax.inject.Singleton;
SheetsServiceModule.class,
StackdriverModule.class,
UrlConnectionServiceModule.class,
UrlFetchServiceModule.class,
UrlFetchTransportModule.class,
UserServiceModule.class,
VoidDnsWriterModule.class,

View file

@ -14,24 +14,25 @@
package google.registry.rde;
import static google.registry.request.UrlConnectionUtils.getResponseBytes;
import static google.registry.request.UrlConnectionUtils.setBasicAuth;
import static google.registry.request.UrlConnectionUtils.setPayload;
import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
import static com.google.common.io.BaseEncoding.base64;
import static com.google.common.net.HttpHeaders.AUTHORIZATION;
import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.api.client.http.HttpMethods;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.common.flogger.FluentLogger;
import com.google.common.net.MediaType;
import google.registry.config.RegistryConfig.Config;
import google.registry.keyring.api.KeyModule.Key;
import google.registry.request.HttpException.InternalServerErrorException;
import google.registry.request.UrlConnectionService;
import google.registry.util.Retrier;
import google.registry.util.UrlConnectionException;
import google.registry.xjc.XjcXmlTransformer;
import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
import google.registry.xjc.iirdea.XjcIirdeaResult;
@ -39,7 +40,6 @@ import google.registry.xjc.rdeheader.XjcRdeHeader;
import google.registry.xjc.rdereport.XjcRdeReportReport;
import google.registry.xml.XmlException;
import java.io.ByteArrayInputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URL;
@ -58,10 +58,10 @@ public class RdeReporter {
* @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
* ICANN Registry Interfaces - Interface details</a>
*/
private static final MediaType MEDIA_TYPE = MediaType.XML_UTF_8;
private static final String REPORT_MIME = "text/xml";
@Inject Retrier retrier;
@Inject UrlConnectionService urlConnectionService;
@Inject URLFetchService urlFetchService;
@Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix;
@Inject @Key("icannReportingPassword") String password;
@ -76,24 +76,29 @@ public class RdeReporter {
// Send a PUT request to ICANN's HTTPS server.
URL url = makeReportUrl(header.getTld(), report.getId());
String username = header.getTld() + "_ry";
String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8));
final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d));
req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME));
req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
req.setPayload(reportBytes);
logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8));
byte[] responseBytes =
HTTPResponse rsp =
retrier.callWithRetry(
() -> {
HttpURLConnection connection = urlConnectionService.createConnection(url);
connection.setRequestMethod(HttpMethods.PUT);
setBasicAuth(connection, username, password);
setPayload(connection, reportBytes, MEDIA_TYPE.toString());
int responseCode = connection.getResponseCode();
if (responseCode == SC_OK || responseCode == SC_BAD_REQUEST) {
return getResponseBytes(connection);
HTTPResponse rsp1 = urlFetchService.fetch(req);
switch (rsp1.getResponseCode()) {
case SC_OK:
case SC_BAD_REQUEST:
break;
default:
throw new RuntimeException("PUT failed");
}
throw new UrlConnectionException("PUT failed", connection);
return rsp1;
},
SocketTimeoutException.class);
// Ensure the XML response is valid.
XjcIirdeaResult result = parseResult(responseBytes);
XjcIirdeaResult result = parseResult(rsp.getContent());
if (result.getCode().getValue() != 1000) {
logger.atWarning().log(
"PUT rejected: %d %s\n%s",

View file

@ -23,14 +23,14 @@ import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
import com.google.appengine.api.users.UserService;
import com.google.appengine.api.users.UserServiceFactory;
import dagger.Module;
import dagger.Provides;
import java.net.HttpURLConnection;
import javax.inject.Singleton;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
/** Dagger modules for App Engine services and other vendor classes. */
public final class Modules {
@ -51,16 +51,18 @@ public final class Modules {
public static final class UrlConnectionServiceModule {
@Provides
static UrlConnectionService provideUrlConnectionService() {
return url -> {
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
if (connection instanceof HttpsURLConnection) {
HttpsURLConnection httpsConnection = (HttpsURLConnection) connection;
SSLContext tls13Context = SSLContext.getInstance("TLSv1.3");
tls13Context.init(null, null, null);
httpsConnection.setSSLSocketFactory(tls13Context.getSocketFactory());
}
return connection;
};
return url -> (HttpURLConnection) url.openConnection();
}
}
/** Dagger module for {@link URLFetchService}. */
@Module
public static final class UrlFetchServiceModule {
private static final URLFetchService fetchService = URLFetchServiceFactory.getURLFetchService();
@Provides
static URLFetchService provideUrlFetchService() {
return fetchService;
}
}

View file

@ -40,6 +40,7 @@ import google.registry.rde.RdeModule;
import google.registry.request.Modules.DatastoreServiceModule;
import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.UrlConnectionServiceModule;
import google.registry.request.Modules.UrlFetchServiceModule;
import google.registry.request.Modules.UserServiceModule;
import google.registry.tools.AuthModule.LocalCredentialModule;
import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
@ -80,6 +81,7 @@ import javax.inject.Singleton;
RequestFactoryModule.class,
SecretManagerModule.class,
UrlConnectionServiceModule.class,
UrlFetchServiceModule.class,
UserServiceModule.class,
UtilsModule.class,
VoidDnsWriterModule.class,

View file

@ -14,6 +14,7 @@
package google.registry.rde;
import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
@ -28,13 +29,20 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
import static org.joda.time.Duration.standardDays;
import static org.joda.time.Duration.standardSeconds;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.ByteSource;
import google.registry.gcs.GcsUtils;
import google.registry.model.common.Cursor;
@ -49,21 +57,20 @@ import google.registry.testing.FakeClock;
import google.registry.testing.FakeKeyringModule;
import google.registry.testing.FakeResponse;
import google.registry.testing.FakeSleeper;
import google.registry.testing.FakeUrlConnectionService;
import google.registry.testing.TestOfyAndSql;
import google.registry.util.Retrier;
import google.registry.xjc.XjcXmlTransformer;
import google.registry.xjc.rdereport.XjcRdeReportReport;
import google.registry.xml.XmlException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.util.Map;
import java.util.Optional;
import org.bouncycastle.openpgp.PGPPublicKey;
import org.joda.time.DateTime;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.ArgumentCaptor;
/** Unit tests for {@link RdeReportAction}. */
@DualDatabaseTest
@ -82,21 +89,20 @@ public class RdeReportActionTest {
private final FakeResponse response = new FakeResponse();
private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class);
private final URLFetchService urlFetchService = mock(URLFetchService.class);
private final ArgumentCaptor<HTTPRequest> request = ArgumentCaptor.forClass(HTTPRequest.class);
private final HTTPResponse httpResponse = mock(HTTPResponse.class);
private final PGPPublicKey encryptKey =
new FakeKeyringModule().get().getRdeStagingEncryptionKey();
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
private final BlobId reportFile =
BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde");
private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
private final FakeUrlConnectionService urlConnectionService =
new FakeUrlConnectionService(httpUrlConnection);
private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
private RdeReportAction createAction() {
RdeReporter reporter = new RdeReporter();
reporter.reportUrlPrefix = "https://rde-report.example";
reporter.urlFetchService = urlFetchService;
reporter.password = "foo";
reporter.urlConnectionService = urlConnectionService;
reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
RdeReportAction action = new RdeReportAction();
action.gcsUtils = gcsUtils;
@ -121,7 +127,6 @@ public class RdeReportActionTest {
Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test")));
gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
}
@TestOfyAndSql
@ -137,22 +142,24 @@ public class RdeReportActionTest {
@TestOfyAndSql
void testRunWithLock() throws Exception {
when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
createAction().runWithLock(loadRdeReportCursor());
assertThat(response.getStatus()).isEqualTo(200);
assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
// Verify the HTTP request was correct.
verify(httpUrlConnection).setRequestMethod("PUT");
assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
// Verify the payload XML was the same as what's in testdata/report.xml.
XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
XjcRdeReportReport report = parseReport(request.getValue().getPayload());
assertThat(report.getId()).isEqualTo("20101017001");
assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@ -160,8 +167,9 @@ public class RdeReportActionTest {
@TestOfyAndSql
void testRunWithLock_withPrefix() throws Exception {
when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
RdeReportAction action = createAction();
action.prefix = Optional.of("job-name/");
gcsUtils.delete(reportFile);
@ -174,14 +182,15 @@ public class RdeReportActionTest {
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
// Verify the HTTP request was correct.
verify(httpUrlConnection).setRequestMethod("PUT");
assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
// Verify the payload XML was the same as what's in testdata/report.xml.
XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
XjcRdeReportReport report = parseReport(request.getValue().getPayload());
assertThat(report.getId()).isEqualTo("20101017001");
assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@ -194,8 +203,9 @@ public class RdeReportActionTest {
PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
createAction().runWithLock(loadRdeReportCursor());
assertThat(response.getStatus()).isEqualTo(200);
}
@ -228,8 +238,9 @@ public class RdeReportActionTest {
@TestOfyAndSql
void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception {
when(httpUrlConnection.getResponseCode()).thenReturn(SC_BAD_REQUEST);
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openStream());
when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST);
when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
InternalServerErrorException thrown =
assertThrows(
InternalServerErrorException.class,
@ -240,17 +251,18 @@ public class RdeReportActionTest {
@TestOfyAndSql
void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception {
class ExpectedThrownException extends RuntimeException {}
when(httpUrlConnection.getResponseCode()).thenThrow(new ExpectedThrownException());
when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException());
assertThrows(
ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
}
@TestOfyAndSql
void testRunWithLock_socketTimeout_doesRetry() throws Exception {
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
when(httpUrlConnection.getResponseCode())
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture()))
.thenThrow(new SocketTimeoutException())
.thenReturn(SC_OK);
.thenReturn(httpResponse);
createAction().runWithLock(loadRdeReportCursor());
assertThat(response.getStatus()).isEqualTo(200);
assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
@ -261,6 +273,14 @@ public class RdeReportActionTest {
return loadByKey(Cursor.createVKey(RDE_REPORT, "test")).getCursorTime();
}
private static ImmutableMap<String, String> mapifyHeaders(Iterable<HTTPHeader> headers) {
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
for (HTTPHeader header : headers) {
builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue());
}
return builder.build();
}
private static XjcRdeReportReport parseReport(byte[] data) {
try {
return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data));