zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-07-23 19:20:44 +02:00
Code Issues Projects Releases Packages Wiki Activity

Remove URLFetch (#2181)

Browse source 
We previously needed to use URLFetch in some instances where TLS 1.3 is
required (mostly when connecting to ICANN servers),and the JDK-bundled SSL
engine that came with App Engine runtime did not support TLS 1.3.

It appears now that the Java 8 runtime on App Engine supports TLS 1.3
out of the box, which allows us to get rid of URLFetch, which depends on
App Engine APIs.

Also removed some redundant retry and logging logic, now that we know
the HTTP client behaves correctly.

TESTED=modified the CannedScriptExecutionAction and deployed to alpha, used the
new HTTP client to connect to the three URL endpoints that were
problematic before and confirmed that TLS connections can be established. HTTP
sessions were rejected in some cases when authentication failed, but
that was expected.
This commit is contained in:
Lai Jiang 2023-10-19 14:51:51 -04:00 • committed by GitHub
parent bf3bb5d804
commit af303bd26f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 356 additions and 503 deletions
Download patch file Download diff file Expand all files Collapse all files

6
core/src/main/java/google/registry/batch/BatchModule.java
View file

@ -43,6 +43,12 @@ public class BatchModule {
public static final String PARAM_DRY_RUN = "dryRun"; public static final String PARAM_DRY_RUN = "dryRun";
public static final String PARAM_FAST = "fast"; public static final String PARAM_FAST = "fast";
@Provides
@Parameter("url")
static String provideUrl(HttpServletRequest req) {
return extractRequiredParameter(req, "url");
}
@Provides @Provides
@Parameter("jobName") @Parameter("jobName")
static Optional<String> provideJobName(HttpServletRequest req) { static Optional<String> provideJobName(HttpServletRequest req) {

119
core/src/main/java/google/registry/batch/CannedScriptExecutionAction.java
View file

@ -14,26 +14,20 @@
package google.registry.batch; package google.registry.batch;
import static com.google.common.collect.ImmutableList.toImmutableList; import static google.registry.request.Action.Method.GET;
import static google.registry.request.Action.Method.POST; import static google.registry.request.Action.Method.POST;
import static google.registry.util.RegistrarUtils.normalizeRegistrarId; import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Streams;
import com.google.common.flogger.FluentLogger; import com.google.common.flogger.FluentLogger;
import google.registry.config.RegistryConfig.Config;
import google.registry.groups.GmailClient;
import google.registry.groups.GroupsConnection;
import google.registry.model.registrar.Registrar;
import google.registry.model.registrar.RegistrarPoc;
import google.registry.request.Action; import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.UrlConnectionService;
import google.registry.request.UrlConnectionUtils;
import google.registry.request.auth.Auth; import google.registry.request.auth.Auth;
import google.registry.util.EmailMessage; import java.net.URL;
import java.io.IOException;
import java.util.Set;
import javax.inject.Inject; import javax.inject.Inject;
import javax.mail.internet.AddressException; import javax.net.ssl.HttpsURLConnection;
import javax.mail.internet.InternetAddress;
/** /**
* Action that executes a canned script specified by the caller. * Action that executes a canned script specified by the caller.
@ -50,88 +44,45 @@ import javax.mail.internet.InternetAddress;
@Action( @Action(
service = Action.Service.BACKEND, service = Action.Service.BACKEND,
path = "/_dr/task/executeCannedScript", path = "/_dr/task/executeCannedScript",
method = POST, method = {POST, GET},
automaticallyPrintOk = true, automaticallyPrintOk = true,
auth = Auth.AUTH_API_ADMIN) auth = Auth.AUTH_API_ADMIN)
public class CannedScriptExecutionAction implements Runnable { public class CannedScriptExecutionAction implements Runnable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final GroupsConnection groupsConnection; @Inject UrlConnectionService urlConnectionService;
private final GmailClient gmailClient; @Inject Response response;
private final InternetAddress senderAddress;
private final InternetAddress recipientAddress;
private final String gSuiteDomainName;
@Inject @Inject
CannedScriptExecutionAction( @Parameter("url")
GroupsConnection groupsConnection, String url;
GmailClient gmailClient,
@Config("projectId") String projectId, @Inject
@Config("gSuiteDomainName") String gSuiteDomainName, CannedScriptExecutionAction() {}
@Config("newAlertRecipientEmailAddress") InternetAddress recipientAddress) {
this.groupsConnection = groupsConnection;
this.gmailClient = gmailClient;
this.gSuiteDomainName = gSuiteDomainName;
try {
this.senderAddress = new InternetAddress(String.format("%s@%s", projectId, gSuiteDomainName));
} catch (AddressException e) {
throw new RuntimeException(e);
}
this.recipientAddress = recipientAddress;
logger.atInfo().log("Sender:%s; Recipient: %s.", this.senderAddress, this.recipientAddress);
}
@Override @Override
public void run() { public void run() {
Integer responseCode = null;
String responseContent = null;
try { try {
// Invoke canned scripts here. logger.atInfo().log("Connecting to: %s", url);
checkGroupApi(); HttpsURLConnection connection =
EmailMessage message = createEmail(); (HttpsURLConnection) urlConnectionService.createConnection(new URL(url));
this.gmailClient.sendEmail(message); responseCode = connection.getResponseCode();
logger.atInfo().log("Finished running scripts."); logger.atInfo().log("Code: %d", responseCode);
} catch (Throwable t) { logger.atInfo().log("Headers: %s", connection.getHeaderFields());
logger.atWarning().withCause(t).log("Error executing scripts."); responseContent = new String(UrlConnectionUtils.getResponseBytes(connection), UTF_8);
throw new RuntimeException("Execution failed."); logger.atInfo().log("Response: %s", responseContent);
} } catch (Exception e) {
} logger.atWarning().withCause(e).log("Connection to %s failed", url);
throw new RuntimeException(e);
// Checks if Directory and GroupSettings still work after GWorkspace changes. } finally {
void checkGroupApi() { if (responseCode != null) {
ImmutableList<Registrar> registrars = response.setStatus(responseCode);
Streams.stream(Registrar.loadAllCached()) }
.filter(registrar -> registrar.isLive() && registrar.getType() == Registrar.Type.REAL) if (responseContent != null) {
.collect(toImmutableList()); response.setPayload(responseContent);
logger.atInfo().log("Found %s registrars.", registrars.size());
for (Registrar registrar : registrars) {
for (final RegistrarPoc.Type type : RegistrarPoc.Type.values()) {
String groupKey =
String.format(
"%s-%s-contacts@%s",
normalizeRegistrarId(registrar.getRegistrarId()),
type.getDisplayName(),
gSuiteDomainName);
try {
Set<String> currentMembers = groupsConnection.getMembersOfGroup(groupKey);
logger.atInfo().log("%s has %s members.", groupKey, currentMembers.size());
// One success is enough for validation.
return;
} catch (IOException e) {
logger.atWarning().withCause(e).log("Failed to check %s", groupKey);
}
} }
} }
logger.atInfo().log("Finished checking GroupApis.");
}
EmailMessage createEmail() {
return EmailMessage.newBuilder()
.setFrom(senderAddress)
.setSubject("Test: Please ignore<eom>.")
.setRecipients(ImmutableList.of(recipientAddress))
.setBody("Sent from Nomulus through Google Workspace.")
.build();
} }
} }

4
core/src/main/java/google/registry/module/backend/BackendComponent.java
View file

@ -43,8 +43,6 @@ import google.registry.rde.JSchModule;
import google.registry.request.Modules.GsonModule; import google.registry.request.Modules.GsonModule;
import google.registry.request.Modules.NetHttpTransportModule; import google.registry.request.Modules.NetHttpTransportModule;
import google.registry.request.Modules.UrlConnectionServiceModule; import google.registry.request.Modules.UrlConnectionServiceModule;
import google.registry.request.Modules.UrlFetchServiceModule;
import google.registry.request.Modules.UrlFetchTransportModule;
import google.registry.request.Modules.UserServiceModule; import google.registry.request.Modules.UserServiceModule;
import google.registry.request.auth.AuthModule; import google.registry.request.auth.AuthModule;
import google.registry.util.UtilsModule; import google.registry.util.UtilsModule;
@ -80,8 +78,6 @@ import javax.inject.Singleton;
SheetsServiceModule.class, SheetsServiceModule.class,
StackdriverModule.class, StackdriverModule.class,
UrlConnectionServiceModule.class, UrlConnectionServiceModule.class,
UrlFetchServiceModule.class,
UrlFetchTransportModule.class,
UserServiceModule.class, UserServiceModule.class,
VoidDnsWriterModule.class, VoidDnsWriterModule.class,
UtilsModule.class UtilsModule.class

69
core/src/main/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java
View file

@ -14,22 +14,26 @@
package google.registry.rdap; package google.registry.rdap;
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK;
import static com.google.common.net.HttpHeaders.ACCEPT_ENCODING;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import com.google.common.flogger.FluentLogger; import com.google.common.flogger.FluentLogger;
import com.google.common.io.ByteStreams;
import google.registry.model.registrar.Registrar; import google.registry.model.registrar.Registrar;
import google.registry.request.Action; import google.registry.request.Action;
import google.registry.request.HttpException.InternalServerErrorException;
import google.registry.request.UrlConnectionService;
import google.registry.request.UrlConnectionUtils;
import google.registry.request.auth.Auth; import google.registry.request.auth.Auth;
import google.registry.util.UrlConnectionException;
import java.io.IOException; import java.io.IOException;
import java.io.StringReader; import java.io.StringReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.GeneralSecurityException;
import javax.inject.Inject; import javax.inject.Inject;
import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVParser;
@ -41,9 +45,9 @@ import org.apache.commons.csv.CSVRecord;
* <p>This will update ALL the REAL registrars. If a REAL registrar doesn't have an RDAP entry in * <p>This will update ALL the REAL registrars. If a REAL registrar doesn't have an RDAP entry in
* MoSAPI, we'll delete any BaseUrls it has. * MoSAPI, we'll delete any BaseUrls it has.
* *
* <p>The ICANN base website that provides this information can be found at * <p>The ICANN base website that provides this information can be found at <a
* https://www.iana.org/assignments/registrar-ids/registrar-ids.xhtml. The provided CSV endpoint * href=https://www.iana.org/assignments/registrar-ids/registrar-ids.xhtml>here</a>. The provided
* requires no authentication. * CSV endpoint requires no authentication.
*/ */
@Action( @Action(
service = Action.Service.BACKEND, service = Action.Service.BACKEND,
@ -52,22 +56,26 @@ import org.apache.commons.csv.CSVRecord;
auth = Auth.AUTH_API_ADMIN) auth = Auth.AUTH_API_ADMIN)
public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable { public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable {
private static final GenericUrl RDAP_IDS_URL = private static final String RDAP_IDS_URL =
new GenericUrl("https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv"); "https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv";
private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@Inject HttpTransport httpTransport; @Inject UrlConnectionService urlConnectionService;
@Inject @Inject
UpdateRegistrarRdapBaseUrlsAction() {} UpdateRegistrarRdapBaseUrlsAction() {}
@Override @Override
public void run() { public void run() {
ImmutableMap<String, String> ianaIdsToUrls = getIanaIdsToUrls(); try {
tm().transact(() -> processAllRegistrars(ianaIdsToUrls)); ImmutableMap<String, String> ianaIdsToUrls = getIanaIdsToUrls();
tm().transact(() -> processAllRegistrars(ianaIdsToUrls));
} catch (Exception e) {
throw new InternalServerErrorException("Error when retrieving RDAP base URL CSV file", e);
}
} }
private void processAllRegistrars(ImmutableMap<String, String> ianaIdsToUrls) { private static void processAllRegistrars(ImmutableMap<String, String> ianaIdsToUrls) {
int nonUpdatedRegistrars = 0; int nonUpdatedRegistrars = 0;
for (Registrar registrar : Registrar.loadAll()) { for (Registrar registrar : Registrar.loadAll()) {
// Only update REAL registrars // Only update REAL registrars
@ -95,23 +103,28 @@ public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable {
logger.atInfo().log("No change in RDAP base URLs for %d registrars", nonUpdatedRegistrars); logger.atInfo().log("No change in RDAP base URLs for %d registrars", nonUpdatedRegistrars);
} }
private ImmutableMap<String, String> getIanaIdsToUrls() { private ImmutableMap<String, String> getIanaIdsToUrls()
throws IOException, GeneralSecurityException {
CSVParser csv; CSVParser csv;
HttpURLConnection connection = urlConnectionService.createConnection(new URL(RDAP_IDS_URL));
// Explictly set the accepted encoding, as we know Brotli causes us problems when talking to
// ICANN.
connection.setRequestProperty(ACCEPT_ENCODING, "gzip");
String csvString;
try { try {
HttpRequest request = httpTransport.createRequestFactory().buildGetRequest(RDAP_IDS_URL); if (connection.getResponseCode() != STATUS_CODE_OK) {
// AppEngine might insert accept-encodings for us if we use the default gzip, so remove it throw new UrlConnectionException("Failed to load RDAP base URLs from ICANN", connection);
request.getHeaders().setAcceptEncoding(null); }
HttpResponse response = request.execute(); csvString = new String(UrlConnectionUtils.getResponseBytes(connection), UTF_8);
String csvString = new String(ByteStreams.toByteArray(response.getContent()), UTF_8); } finally {
csv = connection.disconnect();
CSVFormat.Builder.create(CSVFormat.DEFAULT)
.setHeader()
.setSkipHeaderRecord(true)
.build()
.parse(new StringReader(csvString));
} catch (IOException e) {
throw new RuntimeException("Error when retrieving RDAP base URL CSV file", e);
} }
csv =
CSVFormat.Builder.create(CSVFormat.DEFAULT)
.setHeader()
.setSkipHeaderRecord(true)
.build()
.parse(new StringReader(csvString));
ImmutableMap.Builder<String, String> result = new ImmutableMap.Builder<>(); ImmutableMap.Builder<String, String> result = new ImmutableMap.Builder<>();
for (CSVRecord record : csv) { for (CSVRecord record : csv) {
String ianaIdentifierString = record.get("ID"); String ianaIdentifierString = record.get("ID");

88
core/src/main/java/google/registry/rde/RdeReporter.java
View file

@ -14,25 +14,23 @@
package google.registry.rde; package google.registry.rde;
import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate; import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_BAD_REQUEST;
import static com.google.appengine.api.urlfetch.HTTPMethod.PUT; import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK;
import static com.google.common.io.BaseEncoding.base64; import static google.registry.request.UrlConnectionUtils.getResponseBytes;
import static com.google.common.net.HttpHeaders.AUTHORIZATION; import static google.registry.request.UrlConnectionUtils.setBasicAuth;
import static com.google.common.net.HttpHeaders.CONTENT_TYPE; import static google.registry.request.UrlConnectionUtils.setPayload;
import static google.registry.util.DomainNameUtils.canonicalizeHostname; import static google.registry.util.DomainNameUtils.canonicalizeHostname;
import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.appengine.api.urlfetch.HTTPHeader; import com.google.api.client.http.HttpMethods;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse; import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.common.flogger.FluentLogger; import com.google.common.flogger.FluentLogger;
import com.google.common.net.MediaType;
import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryConfig.Config;
import google.registry.keyring.api.KeyModule.Key; import google.registry.keyring.api.KeyModule.Key;
import google.registry.request.HttpException.InternalServerErrorException; import google.registry.request.HttpException.InternalServerErrorException;
import google.registry.util.Retrier; import google.registry.request.UrlConnectionService;
import google.registry.util.UrlConnectionException;
import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.XjcXmlTransformer;
import google.registry.xjc.iirdea.XjcIirdeaResponseElement; import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
import google.registry.xjc.iirdea.XjcIirdeaResult; import google.registry.xjc.iirdea.XjcIirdeaResult;
@ -40,10 +38,11 @@ import google.registry.xjc.rdeheader.XjcRdeHeader;
import google.registry.xjc.rdereport.XjcRdeReportReport; import google.registry.xjc.rdereport.XjcRdeReportReport;
import google.registry.xml.XmlException; import google.registry.xml.XmlException;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URL; import java.net.URL;
import java.util.Arrays; import java.security.GeneralSecurityException;
import javax.inject.Inject; import javax.inject.Inject;
/** /**
@ -59,49 +58,47 @@ public class RdeReporter {
* @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4"> * @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
* ICANN Registry Interfaces - Interface details</a> * ICANN Registry Interfaces - Interface details</a>
*/ */
private static final String REPORT_MIME = "text/xml"; private static final MediaType MEDIA_TYPE = MediaType.XML_UTF_8;
@Inject Retrier retrier; @Inject UrlConnectionService urlConnectionService;
@Inject URLFetchService urlFetchService;
@Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix; @Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix;
@Inject @Key("icannReportingPassword") String password; @Inject @Key("icannReportingPassword") String password;
@Inject RdeReporter() {} @Inject RdeReporter() {}
/** Uploads {@code reportBytes} to ICANN. */ /** Uploads {@code reportBytes} to ICANN. */
public void send(byte[] reportBytes) throws XmlException { public void send(byte[] reportBytes) throws XmlException, GeneralSecurityException, IOException {
XjcRdeReportReport report = XjcXmlTransformer.unmarshal( XjcRdeReportReport report =
XjcRdeReportReport.class, new ByteArrayInputStream(reportBytes)); XjcXmlTransformer.unmarshal(
XjcRdeReportReport.class, new ByteArrayInputStream(reportBytes));
XjcRdeHeader header = report.getHeader().getValue(); XjcRdeHeader header = report.getHeader().getValue();
// Send a PUT request to ICANN's HTTPS server. // Send a PUT request to ICANN's HTTPS server.
URL url = makeReportUrl(header.getTld(), report.getId()); URL url = makeReportUrl(header.getTld(), report.getId());
String username = header.getTld() + "_ry"; String username = header.getTld() + "_ry";
String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8));
final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d));
req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME));
req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
req.setPayload(reportBytes);
logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8)); logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8));
HTTPResponse rsp = HttpURLConnection connection = urlConnectionService.createConnection(url);
retrier.callWithRetry( connection.setRequestMethod(HttpMethods.PUT);
() -> { setBasicAuth(connection, username, password);
HTTPResponse rsp1 = urlFetchService.fetch(req); setPayload(connection, reportBytes, MEDIA_TYPE.toString());
int responseCode = rsp1.getResponseCode(); int responseCode;
if (responseCode != SC_OK && responseCode != SC_BAD_REQUEST) { byte[] responseBytes;
logger.atSevere().log(
"Failure when trying to PUT RDE report to ICANN server: %d\n%s",
responseCode, Arrays.toString(rsp1.getContent()));
throw new RuntimeException("Error uploading deposits to ICANN");
}
return rsp1;
},
SocketTimeoutException.class);
// Ensure the XML response is valid. The EPP result code would not be 1000 if we get an try {
// SC_BAD_REQUEST as the HTTP response code. responseCode = connection.getResponseCode();
XjcIirdeaResult result = parseResult(rsp.getContent()); if (responseCode != STATUS_CODE_OK && responseCode != STATUS_CODE_BAD_REQUEST) {
if (result.getCode().getValue() != 1000) { logger.atWarning().log("Connection to RDE report server failed: %d", responseCode);
throw new UrlConnectionException("PUT failed", connection);
}
responseBytes = getResponseBytes(connection);
} finally {
connection.disconnect();
}
// We know that an HTTP 200 response can only contain a result code of
// 1000 (i. e. success), there is no need to parse it.
if (responseCode != STATUS_CODE_OK) {
XjcIirdeaResult result = parseResult(responseBytes);
logger.atWarning().log( logger.atWarning().log(
"Rejected when trying to PUT RDE report to ICANN server: %d %s\n%s", "Rejected when trying to PUT RDE report to ICANN server: %d %s\n%s",
result.getCode().getValue(), result.getMsg(), result.getDescription()); result.getCode().getValue(), result.getMsg(), result.getDescription());
@ -116,10 +113,11 @@ public class RdeReporter {
* href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1"> * href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1">
* ICANN Registry Interfaces - IIRDEA Result Object</a> * ICANN Registry Interfaces - IIRDEA Result Object</a>
*/ */
private XjcIirdeaResult parseResult(byte[] responseBytes) throws XmlException { private static XjcIirdeaResult parseResult(byte[] responseBytes) throws XmlException {
logger.atInfo().log("Received response:\n%s", new String(responseBytes, UTF_8)); logger.atInfo().log("Received response:\n%s", new String(responseBytes, UTF_8));
XjcIirdeaResponseElement response = XjcXmlTransformer.unmarshal( XjcIirdeaResponseElement response =
XjcIirdeaResponseElement.class, new ByteArrayInputStream(responseBytes)); XjcXmlTransformer.unmarshal(
XjcIirdeaResponseElement.class, new ByteArrayInputStream(responseBytes));
return response.getResult(); return response.getResult();
} }

138
core/src/main/java/google/registry/reporting/icann/IcannHttpReporter.java
View file

@ -14,33 +14,31 @@
package google.registry.reporting.icann; package google.registry.reporting.icann;
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_BAD_REQUEST;
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK;
import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.net.MediaType.CSV_UTF_8; import static com.google.common.net.MediaType.CSV_UTF_8;
import static google.registry.model.tld.Tlds.assertTldExists; import static google.registry.model.tld.Tlds.assertTldExists;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.HttpMethods;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpStatusCodes;
import com.google.api.client.http.HttpTransport;
import com.google.common.base.Ascii; import com.google.common.base.Ascii;
import com.google.common.base.Splitter; import com.google.common.base.Splitter;
import com.google.common.flogger.FluentLogger; import com.google.common.flogger.FluentLogger;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryConfig.Config;
import google.registry.keyring.api.KeyModule.Key; import google.registry.keyring.api.KeyModule.Key;
import google.registry.reporting.icann.IcannReportingModule.ReportType; import google.registry.reporting.icann.IcannReportingModule.ReportType;
import google.registry.request.UrlConnectionService;
import google.registry.request.UrlConnectionUtils;
import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.XjcXmlTransformer;
import google.registry.xjc.iirdea.XjcIirdeaResponseElement; import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
import google.registry.xjc.iirdea.XjcIirdeaResult; import google.registry.xjc.iirdea.XjcIirdeaResult;
import google.registry.xml.XmlException; import google.registry.xml.XmlException;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.GeneralSecurityException;
import java.util.List; import java.util.List;
import javax.inject.Inject; import javax.inject.Inject;
import org.joda.time.YearMonth; import org.joda.time.YearMonth;
@ -62,78 +60,64 @@ public class IcannHttpReporter {
private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@Inject HttpTransport httpTransport; @Inject UrlConnectionService urlConnectionService;
@Inject @Key("icannReportingPassword") String password;
@Inject @Config("icannTransactionsReportingUploadUrl") String icannTransactionsUrl; @Inject
@Inject @Config("icannActivityReportingUploadUrl") String icannActivityUrl; @Key("icannReportingPassword")
@Inject IcannHttpReporter() {} String password;
@Inject
@Config("icannTransactionsReportingUploadUrl")
String icannTransactionsUrl;
@Inject
@Config("icannActivityReportingUploadUrl")
String icannActivityUrl;
@Inject
IcannHttpReporter() {}
/** Uploads {@code reportBytes} to ICANN, returning whether or not it succeeded. */ /** Uploads {@code reportBytes} to ICANN, returning whether or not it succeeded. */
public boolean send(byte[] reportBytes, String reportFilename) throws XmlException, IOException { public boolean send(byte[] reportBytes, String reportFilename)
throws GeneralSecurityException, XmlException, IOException {
validateReportFilename(reportFilename); validateReportFilename(reportFilename);
GenericUrl uploadUrl = new GenericUrl(makeUrl(reportFilename)); URL uploadUrl = makeUrl(reportFilename);
HttpRequest request =
httpTransport
.createRequestFactory()
.buildPutRequest(uploadUrl, new ByteArrayContent(CSV_UTF_8.toString(), reportBytes));
HttpHeaders headers = request.getHeaders();
headers.setBasicAuthentication(getTld(reportFilename) + "_ry", password);
headers.setContentType(CSV_UTF_8.toString());
request.setHeaders(headers);
request.setFollowRedirects(false);
request.setThrowExceptionOnExecuteError(false);
HttpResponse response = null;
logger.atInfo().log( logger.atInfo().log(
"Sending report to %s with content length %d.", "Sending report to %s with content length %d.", uploadUrl, reportBytes.length);
uploadUrl, request.getContent().getLength()); HttpURLConnection connection = urlConnectionService.createConnection(uploadUrl);
boolean success = true; connection.setRequestMethod(HttpMethods.PUT);
UrlConnectionUtils.setBasicAuth(connection, getTld(reportFilename) + "_ry", password);
UrlConnectionUtils.setPayload(connection, reportBytes, CSV_UTF_8.toString());
connection.setInstanceFollowRedirects(false);
int responseCode;
byte[] content;
try { try {
response = request.execute(); responseCode = connection.getResponseCode();
// Only responses with a 200 or 400 status have a body. For everything else, throw so that // Only responses with a 200 or 400 status have a body. For everything else, we can return
// the caller catches it and prints the stack trace. // false early.
if (response.getStatusCode() != HttpStatusCodes.STATUS_CODE_OK if (responseCode != STATUS_CODE_OK && responseCode != STATUS_CODE_BAD_REQUEST) {
&& response.getStatusCode() != HttpStatusCodes.STATUS_CODE_BAD_REQUEST) { logger.atWarning().log("Connection to ICANN server failed", connection);
throw new HttpResponseException(response); return false;
}
byte[] content;
try {
content = ByteStreams.toByteArray(response.getContent());
} finally {
response.getContent().close();
}
logger.atInfo().log(
"Received response code %d\n\n"
+ "Response headers: %s\n\n"
+ "Response content in UTF-8: %s\n\n"
+ "Response content in HEX: %s",
response.getStatusCode(),
response.getHeaders(),
new String(content, UTF_8),
BaseEncoding.base16().encode(content));
// For reasons unclear at the moment, when we parse the response content using UTF-8 we get
// garbled texts. Since we know that an HTTP 200 response can only contain a result code of
// 1000 (i. e. success), there is no need to parse it.
if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_BAD_REQUEST) {
success = false;
XjcIirdeaResult result = parseResult(content);
logger.atWarning().log(
"PUT rejected, status code %s:\n%s\n%s",
result.getCode().getValue(), result.getMsg(), result.getDescription());
} }
content = UrlConnectionUtils.getResponseBytes(connection);
} finally { } finally {
if (response != null) { connection.disconnect();
response.disconnect();
} else {
success = false;
logger.atWarning().log("Received null response from ICANN server at %s", uploadUrl);
}
} }
return success; // We know that an HTTP 200 response can only contain a result code of
// 1000 (i. e. success), there is no need to parse it.
// See: https://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-13#page-16
if (responseCode != STATUS_CODE_OK) {
XjcIirdeaResult result = parseResult(content);
logger.atWarning().log(
"PUT rejected, status code %s:\n%s\n%s",
result.getCode().getValue(), result.getMsg(), result.getDescription());
return false;
}
return true;
} }
private XjcIirdeaResult parseResult(byte[] content) throws XmlException { private static XjcIirdeaResult parseResult(byte[] content) throws XmlException {
XjcIirdeaResponseElement response = XjcIirdeaResponseElement response =
XjcXmlTransformer.unmarshal( XjcXmlTransformer.unmarshal(
XjcIirdeaResponseElement.class, new ByteArrayInputStream(content)); XjcIirdeaResponseElement.class, new ByteArrayInputStream(content));
@ -141,7 +125,7 @@ public class IcannHttpReporter {
} }
/** Verifies a given report filename matches the pattern tld-reportType-yyyyMM.csv. */ /** Verifies a given report filename matches the pattern tld-reportType-yyyyMM.csv. */
private void validateReportFilename(String filename) { private static void validateReportFilename(String filename) {
checkArgument( checkArgument(
filename.matches("[a-z0-9.\\-]+-((activity)|(transactions))-[0-9]{6}\\.csv"), filename.matches("[a-z0-9.\\-]+-((activity)|(transactions))-[0-9]{6}\\.csv"),
"Expected file format: tld-reportType-yyyyMM.csv, got %s instead", "Expected file format: tld-reportType-yyyyMM.csv, got %s instead",
@ -149,12 +133,12 @@ public class IcannHttpReporter {
assertTldExists(getTld(filename)); assertTldExists(getTld(filename));
} }
private String getTld(String filename) { private static String getTld(String filename) {
// Extract the TLD, up to second-to-last hyphen in the filename (works with international TLDs) // Extract the TLD, up to second-to-last hyphen in the filename (works with international TLDs)
return filename.substring(0, filename.lastIndexOf('-', filename.lastIndexOf('-') - 1)); return filename.substring(0, filename.lastIndexOf('-', filename.lastIndexOf('-') - 1));
} }
private String makeUrl(String filename) { private URL makeUrl(String filename) throws MalformedURLException {
// Filename is in the format tld-reportType-yearMonth.csv // Filename is in the format tld-reportType-yearMonth.csv
String tld = getTld(filename); String tld = getTld(filename);
// Remove the tld- prefix and csv suffix // Remove the tld- prefix and csv suffix
@ -164,7 +148,7 @@ public class IcannHttpReporter {
// Re-add hyphen between year and month, because ICANN is inconsistent between filename and URL // Re-add hyphen between year and month, because ICANN is inconsistent between filename and URL
String yearMonth = String yearMonth =
YearMonth.parse(elements.get(1), DateTimeFormat.forPattern("yyyyMM")).toString("yyyy-MM"); YearMonth.parse(elements.get(1), DateTimeFormat.forPattern("yyyyMM")).toString("yyyy-MM");
return String.format("%s/%s/%s", getUrlPrefix(reportType), tld, yearMonth); return new URL(String.format("%s/%s/%s", getUrlPrefix(reportType), tld, yearMonth));
} }
private String getUrlPrefix(ReportType reportType) { private String getUrlPrefix(ReportType reportType) {

39
core/src/main/java/google/registry/request/Modules.java
View file

@ -14,20 +14,18 @@
package google.registry.request; package google.registry.request;
import com.google.api.client.extensions.appengine.http.UrlFetchTransport;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory; import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.gson.GsonFactory; import com.google.api.client.json.gson.GsonFactory;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
import com.google.appengine.api.users.UserService; import com.google.appengine.api.users.UserService;
import com.google.appengine.api.users.UserServiceFactory; import com.google.appengine.api.users.UserServiceFactory;
import dagger.Module; import dagger.Module;
import dagger.Provides; import dagger.Provides;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import javax.inject.Singleton; import javax.inject.Singleton;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
/** Dagger modules for App Engine services and other vendor classes. */ /** Dagger modules for App Engine services and other vendor classes. */
public final class Modules { public final class Modules {
@ -37,18 +35,16 @@ public final class Modules {
public static final class UrlConnectionServiceModule { public static final class UrlConnectionServiceModule {
@Provides @Provides
static UrlConnectionService provideUrlConnectionService() { static UrlConnectionService provideUrlConnectionService() {
return url -> (HttpURLConnection) url.openConnection(); return url -> {
} HttpURLConnection connection = (HttpURLConnection) url.openConnection();
} if (connection instanceof HttpsURLConnection) {
HttpsURLConnection httpsConnection = (HttpsURLConnection) connection;
/** Dagger module for {@link URLFetchService}. */ SSLContext tls13Context = SSLContext.getInstance("TLSv1.3");
@Module tls13Context.init(null, null, null);
public static final class UrlFetchServiceModule { httpsConnection.setSSLSocketFactory(tls13Context.getSocketFactory());
private static final URLFetchService fetchService = URLFetchServiceFactory.getURLFetchService(); }
return connection;
@Provides };
static URLFetchService provideUrlFetchService() {
return fetchService;
} }
} }
@ -72,17 +68,6 @@ public final class Modules {
} }
} }
/** Dagger module that causes the App Engine's URL fetcher to be used for Google APIs requests. */
@Module
public static final class UrlFetchTransportModule {
private static final UrlFetchTransport HTTP_TRANSPORT = new UrlFetchTransport();
@Provides
static HttpTransport provideHttpTransport() {
return HTTP_TRANSPORT;
}
}
/** /**
* Dagger module that provides standard {@link NetHttpTransport}. Used in non App Engine * Dagger module that provides standard {@link NetHttpTransport}. Used in non App Engine
* environment. * environment.

11
core/src/main/java/google/registry/request/UrlConnectionUtils.java
View file

@ -27,15 +27,20 @@ import com.google.common.io.ByteStreams;
import com.google.common.net.MediaType; import com.google.common.net.MediaType;
import java.io.DataOutputStream; import java.io.DataOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.net.URLConnection; import java.net.URLConnection;
import java.util.Random; import java.util.Random;
/** Utilities for common functionality relating to {@link java.net.URLConnection}s. */ /** Utilities for common functionality relating to {@link URLConnection}s. */
public class UrlConnectionUtils { public final class UrlConnectionUtils {
private UrlConnectionUtils() {}
/** Retrieves the response from the given connection as a byte array. */ /** Retrieves the response from the given connection as a byte array. */
public static byte[] getResponseBytes(URLConnection connection) throws IOException { public static byte[] getResponseBytes(URLConnection connection) throws IOException {
return ByteStreams.toByteArray(connection.getInputStream()); try (InputStream is = connection.getInputStream()) {
return ByteStreams.toByteArray(is);
}
} }
/** Sets auth on the given connection with the given username/password. */ /** Sets auth on the given connection with the given username/password. */

2
core/src/main/java/google/registry/tools/RegistryToolComponent.java
View file

@ -39,7 +39,6 @@ import google.registry.privileges.secretmanager.SecretManagerModule;
import google.registry.rde.RdeModule; import google.registry.rde.RdeModule;
import google.registry.request.Modules.GsonModule; import google.registry.request.Modules.GsonModule;
import google.registry.request.Modules.UrlConnectionServiceModule; import google.registry.request.Modules.UrlConnectionServiceModule;
import google.registry.request.Modules.UrlFetchServiceModule;
import google.registry.request.Modules.UserServiceModule; import google.registry.request.Modules.UserServiceModule;
import google.registry.tools.AuthModule.LocalCredentialModule; import google.registry.tools.AuthModule.LocalCredentialModule;
import google.registry.util.UtilsModule; import google.registry.util.UtilsModule;
@ -77,7 +76,6 @@ import javax.inject.Singleton;
SecretManagerKeyringModule.class, SecretManagerKeyringModule.class,
SecretManagerModule.class, SecretManagerModule.class,
UrlConnectionServiceModule.class, UrlConnectionServiceModule.class,
UrlFetchServiceModule.class,
UserServiceModule.class, UserServiceModule.class,
UtilsModule.class, UtilsModule.class,
VoidDnsWriterModule.class, VoidDnsWriterModule.class,

106
core/src/test/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java
View file

@ -18,20 +18,27 @@ import static com.google.common.truth.Truth.assertThat;
import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.createTld;
import static google.registry.testing.DatabaseHelper.loadRegistrar; import static google.registry.testing.DatabaseHelper.loadRegistrar;
import static google.registry.testing.DatabaseHelper.persistSimpleResource; import static google.registry.testing.DatabaseHelper.persistSimpleResource;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpStatusCodes;
import com.google.api.client.http.LowLevelHttpRequest;
import com.google.api.client.testing.http.MockHttpTransport;
import com.google.api.client.testing.http.MockLowLevelHttpRequest;
import com.google.api.client.testing.http.MockLowLevelHttpResponse;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import google.registry.model.registrar.Registrar; import google.registry.model.registrar.Registrar;
import google.registry.model.registrar.RegistrarAddress; import google.registry.model.registrar.RegistrarAddress;
import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions;
import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
import google.registry.request.HttpException.InternalServerErrorException;
import google.registry.testing.FakeUrlConnectionService;
import google.registry.util.UrlConnectionException;
import java.io.ByteArrayInputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.extension.RegisterExtension;
@ -61,44 +68,26 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
public JpaIntegrationTestExtension jpa = public JpaIntegrationTestExtension jpa =
new JpaTestExtensions.Builder().buildIntegrationTestExtension(); new JpaTestExtensions.Builder().buildIntegrationTestExtension();
private static class TestHttpTransport extends MockHttpTransport { private final HttpURLConnection connection = mock(HttpURLConnection.class);
private MockLowLevelHttpRequest requestSent; private final FakeUrlConnectionService urlConnectionService =
private MockLowLevelHttpResponse response; new FakeUrlConnectionService(connection);
void setResponse(MockLowLevelHttpResponse response) {
this.response = response;
}
MockLowLevelHttpRequest getRequestSent() {
return requestSent;
}
@Override
public LowLevelHttpRequest buildRequest(String method, String url) {
assertThat(method).isEqualTo("GET");
MockLowLevelHttpRequest httpRequest = new MockLowLevelHttpRequest(url);
httpRequest.setResponse(response);
requestSent = httpRequest;
return httpRequest;
}
}
private TestHttpTransport httpTransport;
private UpdateRegistrarRdapBaseUrlsAction action; private UpdateRegistrarRdapBaseUrlsAction action;
@BeforeEach @BeforeEach
void beforeEach() { void beforeEach() throws Exception {
action = new UpdateRegistrarRdapBaseUrlsAction(); action = new UpdateRegistrarRdapBaseUrlsAction();
httpTransport = new TestHttpTransport(); action.urlConnectionService = urlConnectionService;
action.httpTransport = httpTransport; when(connection.getResponseCode()).thenReturn(SC_OK);
setValidResponse(); when(connection.getInputStream())
.thenReturn(new ByteArrayInputStream(CSV_REPLY.getBytes(StandardCharsets.UTF_8)));
createTld("tld"); createTld("tld");
} }
private void assertCorrectRequestSent() { private void assertCorrectRequestSent() throws Exception {
assertThat(httpTransport.getRequestSent().getUrl()) assertThat(urlConnectionService.getConnectedUrls())
.isEqualTo("https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv"); .containsExactly(
assertThat(httpTransport.getRequestSent().getHeaders().get("accept-encoding")).isNull(); new URL("https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv"));
verify(connection).setRequestProperty("Accept-Encoding", "gzip");
} }
private static void persistRegistrar( private static void persistRegistrar(
@ -119,14 +108,8 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
.build()); .build());
} }
private void setValidResponse() {
MockLowLevelHttpResponse csvResponse = new MockLowLevelHttpResponse();
csvResponse.setContent(CSV_REPLY);
httpTransport.setResponse(csvResponse);
}
@Test @Test
void testUnknownIana_cleared() { void testUnknownIana_cleared() throws Exception {
// The IANA ID isn't in the CSV reply // The IANA ID isn't in the CSV reply
persistRegistrar("someRegistrar", 4123L, Registrar.Type.REAL, "http://rdap.example/blah"); persistRegistrar("someRegistrar", 4123L, Registrar.Type.REAL, "http://rdap.example/blah");
action.run(); action.run();
@ -135,7 +118,7 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
} }
@Test @Test
void testKnownIana_changed() { void testKnownIana_changed() throws Exception {
// The IANA ID is in the CSV reply // The IANA ID is in the CSV reply
persistRegistrar("someRegistrar", 1448L, Registrar.Type.REAL, "http://rdap.example/blah"); persistRegistrar("someRegistrar", 1448L, Registrar.Type.REAL, "http://rdap.example/blah");
action.run(); action.run();
@ -145,7 +128,7 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
} }
@Test @Test
void testKnownIana_notReal_noChange() { void testKnownIana_notReal_noChange() throws Exception {
// The IANA ID is in the CSV reply // The IANA ID is in the CSV reply
persistRegistrar("someRegistrar", 9999L, Registrar.Type.INTERNAL, "http://rdap.example/blah"); persistRegistrar("someRegistrar", 9999L, Registrar.Type.INTERNAL, "http://rdap.example/blah");
// Real registrars should actually change // Real registrars should actually change
@ -159,7 +142,7 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
} }
@Test @Test
void testKnownIana_notReal_nullIANA_noChange() { void testKnownIana_notReal_nullIANA_noChange() throws Exception {
persistRegistrar("someRegistrar", null, Registrar.Type.TEST, "http://rdap.example/blah"); persistRegistrar("someRegistrar", null, Registrar.Type.TEST, "http://rdap.example/blah");
action.run(); action.run();
assertCorrectRequestSent(); assertCorrectRequestSent();
@ -168,29 +151,30 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
} }
@Test @Test
void testFailure_serverErrorResponse() { void testFailure_serverErrorResponse() throws Exception {
MockLowLevelHttpResponse badResponse = new MockLowLevelHttpResponse(); when(connection.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
badResponse.setZeroContent(); when(connection.getInputStream())
badResponse.setStatusCode(HttpStatusCodes.STATUS_CODE_SERVER_ERROR); .thenReturn(new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)));
httpTransport.setResponse(badResponse); InternalServerErrorException thrown =
assertThrows(InternalServerErrorException.class, action::run);
RuntimeException thrown = assertThrows(RuntimeException.class, action::run); verify(connection, times(0)).getInputStream();
assertThat(thrown).hasMessageThat().isEqualTo("Error when retrieving RDAP base URL CSV file"); assertThat(thrown).hasMessageThat().isEqualTo("Error when retrieving RDAP base URL CSV file");
Throwable cause = thrown.getCause(); Throwable cause = thrown.getCause();
assertThat(cause).isInstanceOf(HttpResponseException.class); assertThat(cause).isInstanceOf(UrlConnectionException.class);
assertThat(cause) assertThat(cause)
.hasMessageThat() .hasMessageThat()
.isEqualTo("500\nGET https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv"); .contains("https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv");
} }
@Test @Test
void testFailure_invalidCsv() { void testFailure_invalidCsv() throws Exception {
MockLowLevelHttpResponse csvResponse = new MockLowLevelHttpResponse(); when(connection.getInputStream())
csvResponse.setContent("foo,bar\nbaz,foo"); .thenReturn(new ByteArrayInputStream("foo,bar\nbaz,foo".getBytes(StandardCharsets.UTF_8)));
httpTransport.setResponse(csvResponse);
IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, action::run); InternalServerErrorException thrown =
assertThrows(InternalServerErrorException.class, action::run);
assertThat(thrown) assertThat(thrown)
.hasCauseThat()
.hasMessageThat() .hasMessageThat()
.isEqualTo("Mapping for ID not found, expected one of [foo, bar]"); .isEqualTo("Mapping for ID not found, expected one of [foo, bar]");
} }

127
core/src/test/java/google/registry/rde/RdeReportActionTest.java
View file

@ -14,7 +14,9 @@
package google.registry.rde; package google.registry.rde;
import static com.google.appengine.api.urlfetch.HTTPMethod.PUT; import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_BAD_REQUEST;
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK;
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_UNAUTHORIZED;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.common.Cursor.CursorType.RDE_REPORT; import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
@ -24,25 +26,17 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.createTld;
import static google.registry.testing.DatabaseHelper.loadByKey; import static google.registry.testing.DatabaseHelper.loadByKey;
import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.testing.DatabaseHelper.persistResource;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import static org.joda.time.Duration.standardDays; import static org.joda.time.Duration.standardDays;
import static org.joda.time.Duration.standardSeconds; import static org.joda.time.Duration.standardSeconds;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.ByteSource; import com.google.common.io.ByteSource;
import google.registry.gcs.GcsUtils; import google.registry.gcs.GcsUtils;
import google.registry.model.common.Cursor; import google.registry.model.common.Cursor;
@ -53,25 +47,23 @@ import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationT
import google.registry.request.HttpException.InternalServerErrorException; import google.registry.request.HttpException.InternalServerErrorException;
import google.registry.request.HttpException.NoContentException; import google.registry.request.HttpException.NoContentException;
import google.registry.testing.BouncyCastleProviderExtension; import google.registry.testing.BouncyCastleProviderExtension;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeKeyringModule; import google.registry.testing.FakeKeyringModule;
import google.registry.testing.FakeResponse; import google.registry.testing.FakeResponse;
import google.registry.testing.FakeSleeper; import google.registry.testing.FakeUrlConnectionService;
import google.registry.util.Retrier; import google.registry.util.UrlConnectionException;
import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.XjcXmlTransformer;
import google.registry.xjc.rdereport.XjcRdeReportReport; import google.registry.xjc.rdereport.XjcRdeReportReport;
import google.registry.xml.XmlException; import google.registry.xml.XmlException;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.net.SocketTimeoutException; import java.io.ByteArrayOutputStream;
import java.net.HttpURLConnection;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Optional; import java.util.Optional;
import org.bouncycastle.openpgp.PGPPublicKey; import org.bouncycastle.openpgp.PGPPublicKey;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.ArgumentCaptor;
/** Unit tests for {@link RdeReportAction}. */ /** Unit tests for {@link RdeReportAction}. */
public class RdeReportActionTest { public class RdeReportActionTest {
@ -89,9 +81,11 @@ public class RdeReportActionTest {
private final FakeResponse response = new FakeResponse(); private final FakeResponse response = new FakeResponse();
private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class); private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class);
private final URLFetchService urlFetchService = mock(URLFetchService.class); private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
private final ArgumentCaptor<HTTPRequest> request = ArgumentCaptor.forClass(HTTPRequest.class); private final FakeUrlConnectionService urlConnectionService =
private final HTTPResponse httpResponse = mock(HTTPResponse.class); new FakeUrlConnectionService(httpUrlConnection);
private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
private final PGPPublicKey encryptKey = private final PGPPublicKey encryptKey =
new FakeKeyringModule().get().getRdeStagingEncryptionKey(); new FakeKeyringModule().get().getRdeStagingEncryptionKey();
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
@ -102,9 +96,8 @@ public class RdeReportActionTest {
private RdeReportAction createAction() { private RdeReportAction createAction() {
RdeReporter reporter = new RdeReporter(); RdeReporter reporter = new RdeReporter();
reporter.reportUrlPrefix = "https://rde-report.example"; reporter.reportUrlPrefix = "https://rde-report.example";
reporter.urlFetchService = urlFetchService; reporter.urlConnectionService = urlConnectionService;
reporter.password = "foo"; reporter.password = "foo";
reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
RdeReportAction action = new RdeReportAction(); RdeReportAction action = new RdeReportAction();
action.gcsUtils = gcsUtils; action.gcsUtils = gcsUtils;
action.response = response; action.response = response;
@ -126,6 +119,9 @@ public class RdeReportActionTest {
persistResource(Cursor.createScoped(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), registry)); persistResource(Cursor.createScoped(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), registry));
gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey)); gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
when(httpUrlConnection.getResponseCode()).thenReturn(STATUS_CODE_OK);
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openBufferedStream());
} }
@Test @Test
@ -142,24 +138,20 @@ public class RdeReportActionTest {
@Test @Test
void testRunWithLock() throws Exception { void testRunWithLock() throws Exception {
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
createAction().runWithLock(loadRdeReportCursor()); createAction().runWithLock(loadRdeReportCursor());
assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getStatus()).isEqualTo(200);
assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8); assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n"); assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
// Verify the HTTP request was correct. // Verify the HTTP request was correct.
assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT); verify(httpUrlConnection).setRequestMethod("PUT");
assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https"); assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001"); assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders()); verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml"); verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
// Verify the payload XML was the same as what's in testdata/report.xml. // Verify the payload XML was the same as what's in testdata/report.xml.
XjcRdeReportReport report = parseReport(request.getValue().getPayload()); XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
assertThat(report.getId()).isEqualTo("20101017001"); assertThat(report.getId()).isEqualTo("20101017001");
assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z")); assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z")); assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@ -167,9 +159,6 @@ public class RdeReportActionTest {
@Test @Test
void testRunWithLock_withPrefix() throws Exception { void testRunWithLock_withPrefix() throws Exception {
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
RdeReportAction action = createAction(); RdeReportAction action = createAction();
action.runWithLock(loadRdeReportCursor()); action.runWithLock(loadRdeReportCursor());
assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getStatus()).isEqualTo(200);
@ -177,15 +166,14 @@ public class RdeReportActionTest {
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n"); assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
// Verify the HTTP request was correct. // Verify the HTTP request was correct.
assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT); verify(httpUrlConnection).setRequestMethod("PUT");
assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https"); assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001"); assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders()); verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml"); verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
// Verify the payload XML was the same as what's in testdata/report.xml. // Verify the payload XML was the same as what's in testdata/report.xml.
XjcRdeReportReport report = parseReport(request.getValue().getPayload()); XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
assertThat(report.getId()).isEqualTo("20101017001"); assertThat(report.getId()).isEqualTo("20101017001");
assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z")); assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z")); assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@ -200,9 +188,6 @@ public class RdeReportActionTest {
@Test @Test
void testRunWithLock_withoutPrefix() throws Exception { void testRunWithLock_withoutPrefix() throws Exception {
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
RdeReportAction action = createAction(); RdeReportAction action = createAction();
action.prefix = Optional.empty(); action.prefix = Optional.empty();
gcsUtils.delete(reportFile); gcsUtils.delete(reportFile);
@ -225,15 +210,14 @@ public class RdeReportActionTest {
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n"); assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
// Verify the HTTP request was correct. // Verify the HTTP request was correct.
assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT); verify(httpUrlConnection).setRequestMethod("PUT");
assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https"); assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001"); assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders()); verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml"); verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
// Verify the payload XML was the same as what's in testdata/report.xml. // Verify the payload XML was the same as what's in testdata/report.xml.
XjcRdeReportReport report = parseReport(request.getValue().getPayload()); XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
assertThat(report.getId()).isEqualTo("20101017001"); assertThat(report.getId()).isEqualTo("20101017001");
assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z")); assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z")); assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@ -246,9 +230,6 @@ public class RdeReportActionTest {
PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey(); PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey)); gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
createAction().runWithLock(loadRdeReportCursor()); createAction().runWithLock(loadRdeReportCursor());
assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getStatus()).isEqualTo(200);
} }
@ -281,9 +262,8 @@ public class RdeReportActionTest {
@Test @Test
void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception { void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception {
when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST); when(httpUrlConnection.getResponseCode()).thenReturn(STATUS_CODE_BAD_REQUEST);
when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read()); when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openBufferedStream());
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
InternalServerErrorException thrown = InternalServerErrorException thrown =
assertThrows( assertThrows(
InternalServerErrorException.class, InternalServerErrorException.class,
@ -292,38 +272,19 @@ public class RdeReportActionTest {
} }
@Test @Test
void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception { void testRunWithLock_notAuthorized() throws Exception {
class ExpectedThrownException extends RuntimeException {} when(httpUrlConnection.getResponseCode()).thenReturn(STATUS_CODE_UNAUTHORIZED);
when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException()); UrlConnectionException thrown =
assertThrows( assertThrows(
ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor())); UrlConnectionException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
} verify(httpUrlConnection, times(0)).getInputStream();
assertThat(thrown).hasMessageThat().contains("PUT failed");
@Test
void testRunWithLock_socketTimeout_doesRetry() throws Exception {
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture()))
.thenThrow(new SocketTimeoutException())
.thenReturn(httpResponse);
createAction().runWithLock(loadRdeReportCursor());
assertThat(response.getStatus()).isEqualTo(200);
assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
} }
private DateTime loadRdeReportCursor() { private DateTime loadRdeReportCursor() {
return loadByKey(Cursor.createScopedVKey(RDE_REPORT, registry)).getCursorTime(); return loadByKey(Cursor.createScopedVKey(RDE_REPORT, registry)).getCursorTime();
} }
private static ImmutableMap<String, String> mapifyHeaders(Iterable<HTTPHeader> headers) {
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
for (HTTPHeader header : headers) {
builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue());
}
return builder.build();
}
private static XjcRdeReportReport parseReport(byte[] data) { private static XjcRdeReportReport parseReport(byte[] data) {
try { try {
return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data)); return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data));

118
core/src/test/java/google/registry/reporting/icann/IcannHttpReporterTest.java
View file

@ -14,28 +14,27 @@
package google.registry.reporting.icann; package google.registry.reporting.icann;
import static com.google.common.net.MediaType.CSV_UTF_8; import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_BAD_REQUEST;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK;
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_SERVER_ERROR;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.createTld;
import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpStatusCodes;
import com.google.api.client.http.LowLevelHttpRequest;
import com.google.api.client.http.LowLevelHttpResponse;
import com.google.api.client.testing.http.MockHttpTransport;
import com.google.api.client.testing.http.MockLowLevelHttpRequest;
import com.google.api.client.testing.http.MockLowLevelHttpResponse;
import com.google.api.client.util.Base64;
import com.google.api.client.util.StringUtils; import com.google.api.client.util.StringUtils;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteSource; import com.google.common.io.ByteSource;
import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions;
import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
import java.io.IOException; import google.registry.testing.FakeUrlConnectionService;
import java.util.List; import java.io.ByteArrayOutputStream;
import java.util.Map; import java.net.HttpURLConnection;
import java.net.URL;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.extension.RegisterExtension;
@ -46,103 +45,75 @@ class IcannHttpReporterTest {
private static final ByteSource IIRDEA_GOOD_XML = ReportingTestData.loadBytes("iirdea_good.xml"); private static final ByteSource IIRDEA_GOOD_XML = ReportingTestData.loadBytes("iirdea_good.xml");
private static final ByteSource IIRDEA_BAD_XML = ReportingTestData.loadBytes("iirdea_bad.xml"); private static final ByteSource IIRDEA_BAD_XML = ReportingTestData.loadBytes("iirdea_bad.xml");
private static final byte[] FAKE_PAYLOAD = "test,csv\n1,2".getBytes(UTF_8); private static final byte[] FAKE_PAYLOAD = "test,csv\n1,2".getBytes(UTF_8);
private static final IcannHttpReporter reporter = new IcannHttpReporter();
private MockLowLevelHttpRequest mockRequest; private final HttpURLConnection connection = mock(HttpURLConnection.class);
private final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
private final FakeUrlConnectionService urlConnectionService =
new FakeUrlConnectionService(connection);
@RegisterExtension @RegisterExtension
final JpaIntegrationTestExtension jpa = final JpaIntegrationTestExtension jpa =
new JpaTestExtensions.Builder().buildIntegrationTestExtension(); new JpaTestExtensions.Builder().buildIntegrationTestExtension();
private MockHttpTransport createMockTransport(
int statusCode, final ByteSource iirdeaResponse) {
return new MockHttpTransport() {
@Override
public LowLevelHttpRequest buildRequest(String method, String url) {
mockRequest =
new MockLowLevelHttpRequest() {
@Override
public LowLevelHttpResponse execute() throws IOException {
MockLowLevelHttpResponse response = new MockLowLevelHttpResponse();
response.setStatusCode(statusCode);
response.setContentType(PLAIN_TEXT_UTF_8.toString());
response.setContent(iirdeaResponse.read());
return response;
}
};
mockRequest.setUrl(url);
return mockRequest;
}
};
}
private MockHttpTransport createMockTransport(final ByteSource iirdeaResponse) {
return createMockTransport(HttpStatusCodes.STATUS_CODE_OK, iirdeaResponse);
}
@BeforeEach @BeforeEach
void beforeEach() { void beforeEach() throws Exception {
createTld("test"); createTld("test");
createTld("xn--abc123"); createTld("xn--abc123");
} when(connection.getOutputStream()).thenReturn(outputStream);
when(connection.getResponseCode()).thenReturn(STATUS_CODE_OK);
private IcannHttpReporter createReporter() { when(connection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openBufferedStream());
IcannHttpReporter reporter = new IcannHttpReporter(); reporter.urlConnectionService = urlConnectionService;
reporter.httpTransport = createMockTransport(IIRDEA_GOOD_XML);
reporter.password = "fakePass"; reporter.password = "fakePass";
reporter.icannTransactionsUrl = "https://fake-transactions.url"; reporter.icannTransactionsUrl = "https://fake-transactions.url";
reporter.icannActivityUrl = "https://fake-activity.url"; reporter.icannActivityUrl = "https://fake-activity.url";
return reporter;
} }
@Test @Test
void testSuccess() throws Exception { void testSuccess() throws Exception {
IcannHttpReporter reporter = createReporter(); assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isTrue();
reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv");
assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2017-06"); assertThat(urlConnectionService.getConnectedUrls())
Map<String, List<String>> headers = mockRequest.getHeaders(); .containsExactly(new URL("https://fake-transactions.url/test/2017-06"));
String userPass = "test_ry:fakePass"; String userPass = "test_ry:fakePass";
String expectedAuth = String expectedAuth =
String.format("Basic %s", Base64.encodeBase64String(StringUtils.getBytesUtf8(userPass))); String.format("Basic %s", BaseEncoding.base64().encode(StringUtils.getBytesUtf8(userPass)));
assertThat(headers.get("authorization")).containsExactly(expectedAuth); verify(connection).setRequestProperty("Authorization", expectedAuth);
assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString()); verify(connection).setRequestProperty("Content-Type", "text/csv; charset=utf-8");
assertThat(outputStream.toByteArray()).isEqualTo(FAKE_PAYLOAD);
} }
@Test @Test
void testSuccess_internationalTld() throws Exception { void testSuccess_internationalTld() throws Exception {
IcannHttpReporter reporter = createReporter(); assertThat(reporter.send(FAKE_PAYLOAD, "xn--abc123-transactions-201706.csv")).isTrue();
reporter.send(FAKE_PAYLOAD, "xn--abc123-transactions-201706.csv");
assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/xn--abc123/2017-06"); assertThat(urlConnectionService.getConnectedUrls())
Map<String, List<String>> headers = mockRequest.getHeaders(); .containsExactly(new URL("https://fake-transactions.url/xn--abc123/2017-06"));
String userPass = "xn--abc123_ry:fakePass"; String userPass = "xn--abc123_ry:fakePass";
String expectedAuth = String expectedAuth =
String.format("Basic %s", Base64.encodeBase64String(StringUtils.getBytesUtf8(userPass))); String.format("Basic %s", BaseEncoding.base64().encode(StringUtils.getBytesUtf8(userPass)));
assertThat(headers.get("authorization")).containsExactly(expectedAuth); verify(connection).setRequestProperty("Authorization", expectedAuth);
assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString()); verify(connection).setRequestProperty("Content-Type", "text/csv; charset=utf-8");
assertThat(outputStream.toByteArray()).isEqualTo(FAKE_PAYLOAD);
} }
@Test @Test
void testFail_BadIirdeaResponse() throws Exception { void testFail_BadIirdeaResponse() throws Exception {
IcannHttpReporter reporter = createReporter(); when(connection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openBufferedStream());
reporter.httpTransport = when(connection.getResponseCode()).thenReturn(STATUS_CODE_BAD_REQUEST);
createMockTransport(HttpStatusCodes.STATUS_CODE_BAD_REQUEST, IIRDEA_BAD_XML);
assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isFalse(); assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isFalse();
verify(connection).getInputStream();
} }
@Test @Test
void testFail_transportException() { void testFail_OtherBadHttpResponse() throws Exception {
IcannHttpReporter reporter = createReporter(); when(connection.getResponseCode()).thenReturn(STATUS_CODE_SERVER_ERROR);
reporter.httpTransport = assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isFalse();
createMockTransport(HttpStatusCodes.STATUS_CODE_FORBIDDEN, ByteSource.empty()); verify(connection, times(0)).getInputStream();
assertThrows(
HttpResponseException.class,
() -> reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv"));
} }
@Test @Test
void testFail_invalidFilename_nonSixDigitYearMonth() { void testFail_invalidFilename_nonSixDigitYearMonth() {
IcannHttpReporter reporter = createReporter();
IllegalArgumentException thrown = IllegalArgumentException thrown =
assertThrows( assertThrows(
IllegalArgumentException.class, IllegalArgumentException.class,
@ -156,7 +127,6 @@ class IcannHttpReporterTest {
@Test @Test
void testFail_invalidFilename_notActivityOrTransactions() { void testFail_invalidFilename_notActivityOrTransactions() {
IcannHttpReporter reporter = createReporter();
IllegalArgumentException thrown = IllegalArgumentException thrown =
assertThrows( assertThrows(
IllegalArgumentException.class, IllegalArgumentException.class,
@ -169,7 +139,6 @@ class IcannHttpReporterTest {
@Test @Test
void testFail_invalidFilename_invalidTldName() { void testFail_invalidFilename_invalidTldName() {
IcannHttpReporter reporter = createReporter();
IllegalArgumentException thrown = IllegalArgumentException thrown =
assertThrows( assertThrows(
IllegalArgumentException.class, IllegalArgumentException.class,
@ -183,7 +152,6 @@ class IcannHttpReporterTest {
@Test @Test
void testFail_invalidFilename_tldDoesntExist() { void testFail_invalidFilename_tldDoesntExist() {
IcannHttpReporter reporter = createReporter();
IllegalArgumentException thrown = IllegalArgumentException thrown =
assertThrows( assertThrows(
IllegalArgumentException.class, IllegalArgumentException.class,

12
core/src/test/java/google/registry/testing/FakeUrlConnectionService.java
View file

@ -16,6 +16,7 @@ package google.registry.testing;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableList;
import google.registry.request.UrlConnectionService; import google.registry.request.UrlConnectionService;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
@ -26,15 +27,10 @@ import java.util.List;
public class FakeUrlConnectionService implements UrlConnectionService { public class FakeUrlConnectionService implements UrlConnectionService {
private final HttpURLConnection mockConnection; private final HttpURLConnection mockConnection;
private final List<URL> connectedUrls; private final List<URL> connectedUrls = new ArrayList<>();
public FakeUrlConnectionService(HttpURLConnection mockConnection) { public FakeUrlConnectionService(HttpURLConnection mockConnection) {
this(mockConnection, new ArrayList<>());
}
public FakeUrlConnectionService(HttpURLConnection mockConnection, List<URL> connectedUrls) {
this.mockConnection = mockConnection; this.mockConnection = mockConnection;
this.connectedUrls = connectedUrls;
} }
@Override @Override
@ -43,4 +39,8 @@ public class FakeUrlConnectionService implements UrlConnectionService {
when(mockConnection.getURL()).thenReturn(url); when(mockConnection.getURL()).thenReturn(url);
return mockConnection; return mockConnection;
} }
public ImmutableList<URL> getConnectedUrls() {
return ImmutableList.copyOf(connectedUrls);
}
} }

5
core/src/test/java/google/registry/tmch/TmchActionTestCase.java
View file

@ -25,9 +25,7 @@ import google.registry.testing.FakeClock;
import google.registry.testing.FakeUrlConnectionService; import google.registry.testing.FakeUrlConnectionService;
import google.registry.testing.TestCacheExtension; import google.registry.testing.TestCacheExtension;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL;
import java.time.Duration; import java.time.Duration;
import java.util.ArrayList;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.extension.RegisterExtension;
@ -55,9 +53,8 @@ abstract class TmchActionTestCase {
final Marksdb marksdb = new Marksdb(); final Marksdb marksdb = new Marksdb();
protected final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class); protected final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
protected final ArrayList<URL> connectedUrls = new ArrayList<>();
protected FakeUrlConnectionService urlConnectionService = protected FakeUrlConnectionService urlConnectionService =
new FakeUrlConnectionService(httpUrlConnection, connectedUrls); new FakeUrlConnectionService(httpUrlConnection);
@BeforeEach @BeforeEach
public void beforeEachTmchActionTestCase() throws Exception { public void beforeEachTmchActionTestCase() throws Exception {

3
core/src/test/java/google/registry/tmch/TmchCrlActionTest.java
View file

@ -56,7 +56,8 @@ class TmchCrlActionTest extends TmchActionTestCase {
readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read())); readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read()));
newTmchCrlAction(TmchCaMode.PILOT).run(); newTmchCrlAction(TmchCaMode.PILOT).run();
verify(httpUrlConnection).getInputStream(); verify(httpUrlConnection).getInputStream();
assertThat(connectedUrls).containsExactly(new URL("https://sloth.lol/tmch.crl")); assertThat(urlConnectionService.getConnectedUrls())
.containsExactly(new URL("https://sloth.lol/tmch.crl"));
} }
@Test @Test

5
core/src/test/java/google/registry/tmch/TmchDnlActionTest.java
View file

@ -49,7 +49,10 @@ class TmchDnlActionTest extends TmchActionTestCase {
.thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl/dnl-latest.sig").read())); .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl/dnl-latest.sig").read()));
newTmchDnlAction().run(); newTmchDnlAction().run();
verify(httpUrlConnection, times(2)).getInputStream(); verify(httpUrlConnection, times(2)).getInputStream();
assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList())) assertThat(
urlConnectionService.getConnectedUrls().stream()
.map(URL::toString)
.collect(toImmutableList()))
.containsExactly(MARKSDB_URL + "/dnl/dnl-latest.csv", MARKSDB_URL + "/dnl/dnl-latest.sig"); .containsExactly(MARKSDB_URL + "/dnl/dnl-latest.csv", MARKSDB_URL + "/dnl/dnl-latest.sig");
// Make sure the contents of testdata/dnl-latest.csv got inserted into the database. // Make sure the contents of testdata/dnl-latest.csv got inserted into the database.

5
core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java
View file

@ -50,7 +50,10 @@ class TmchSmdrlActionTest extends TmchActionTestCase {
.thenReturn(new ByteArrayInputStream(loadBytes("smdrl/smdrl-latest.sig").read())); .thenReturn(new ByteArrayInputStream(loadBytes("smdrl/smdrl-latest.sig").read()));
newTmchSmdrlAction().run(); newTmchSmdrlAction().run();
verify(httpUrlConnection, times(2)).getInputStream(); verify(httpUrlConnection, times(2)).getInputStream();
assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList())) assertThat(
urlConnectionService.getConnectedUrls().stream()
.map(URL::toString)
.collect(toImmutableList()))
.containsExactly( .containsExactly(
MARKSDB_URL + "/smdrl/smdrl-latest.csv", MARKSDB_URL + "/smdrl/smdrl-latest.sig"); MARKSDB_URL + "/smdrl/smdrl-latest.csv", MARKSDB_URL + "/smdrl/smdrl-latest.sig");
smdrl = SignedMarkRevocationList.get(); smdrl = SignedMarkRevocationList.get();

2
core/src/test/resources/google/registry/module/backend/backend_routing.txt
View file

@ -6,7 +6,7 @@ PATH CLASS
/_dr/task/deleteExpiredDomains DeleteExpiredDomainsAction GET n API APP ADMIN /_dr/task/deleteExpiredDomains DeleteExpiredDomainsAction GET n API APP ADMIN
/_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n API APP ADMIN /_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n API APP ADMIN
/_dr/task/deleteProberData DeleteProberDataAction POST n API APP ADMIN /_dr/task/deleteProberData DeleteProberDataAction POST n API APP ADMIN
/_dr/task/executeCannedScript CannedScriptExecutionAction POST y API APP ADMIN /_dr/task/executeCannedScript CannedScriptExecutionAction POST,GET y API APP ADMIN
/_dr/task/expandBillingRecurrences ExpandBillingRecurrencesAction GET n API APP ADMIN /_dr/task/expandBillingRecurrences ExpandBillingRecurrencesAction GET n API APP ADMIN
/_dr/task/exportDomainLists ExportDomainListsAction POST n API APP ADMIN /_dr/task/exportDomainLists ExportDomainListsAction POST n API APP ADMIN
/_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n API APP ADMIN /_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n API APP ADMIN