diff --git a/core/src/main/java/google/registry/batch/BatchModule.java b/core/src/main/java/google/registry/batch/BatchModule.java index ece51133d..521b5573a 100644 --- a/core/src/main/java/google/registry/batch/BatchModule.java +++ b/core/src/main/java/google/registry/batch/BatchModule.java @@ -43,6 +43,12 @@ public class BatchModule { public static final String PARAM_DRY_RUN = "dryRun"; public static final String PARAM_FAST = "fast"; + @Provides + @Parameter("url") + static String provideUrl(HttpServletRequest req) { + return extractRequiredParameter(req, "url"); + } + @Provides @Parameter("jobName") static Optional provideJobName(HttpServletRequest req) { diff --git a/core/src/main/java/google/registry/batch/CannedScriptExecutionAction.java b/core/src/main/java/google/registry/batch/CannedScriptExecutionAction.java index eb1357af5..9745ae6e3 100644 --- a/core/src/main/java/google/registry/batch/CannedScriptExecutionAction.java +++ b/core/src/main/java/google/registry/batch/CannedScriptExecutionAction.java @@ -14,26 +14,20 @@ package google.registry.batch; -import static com.google.common.collect.ImmutableList.toImmutableList; +import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.POST; -import static google.registry.util.RegistrarUtils.normalizeRegistrarId; +import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Streams; import com.google.common.flogger.FluentLogger; -import google.registry.config.RegistryConfig.Config; -import google.registry.groups.GmailClient; -import google.registry.groups.GroupsConnection; -import google.registry.model.registrar.Registrar; -import google.registry.model.registrar.RegistrarPoc; import google.registry.request.Action; +import google.registry.request.Parameter; +import google.registry.request.Response; +import google.registry.request.UrlConnectionService; +import google.registry.request.UrlConnectionUtils; import google.registry.request.auth.Auth; -import google.registry.util.EmailMessage; -import java.io.IOException; -import java.util.Set; +import java.net.URL; import javax.inject.Inject; -import javax.mail.internet.AddressException; -import javax.mail.internet.InternetAddress; +import javax.net.ssl.HttpsURLConnection; /** * Action that executes a canned script specified by the caller. @@ -50,88 +44,45 @@ import javax.mail.internet.InternetAddress; @Action( service = Action.Service.BACKEND, path = "/_dr/task/executeCannedScript", - method = POST, + method = {POST, GET}, automaticallyPrintOk = true, auth = Auth.AUTH_API_ADMIN) public class CannedScriptExecutionAction implements Runnable { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private final GroupsConnection groupsConnection; - private final GmailClient gmailClient; - - private final InternetAddress senderAddress; - - private final InternetAddress recipientAddress; - - private final String gSuiteDomainName; + @Inject UrlConnectionService urlConnectionService; + @Inject Response response; @Inject - CannedScriptExecutionAction( - GroupsConnection groupsConnection, - GmailClient gmailClient, - @Config("projectId") String projectId, - @Config("gSuiteDomainName") String gSuiteDomainName, - @Config("newAlertRecipientEmailAddress") InternetAddress recipientAddress) { - this.groupsConnection = groupsConnection; - this.gmailClient = gmailClient; - this.gSuiteDomainName = gSuiteDomainName; - try { - this.senderAddress = new InternetAddress(String.format("%s@%s", projectId, gSuiteDomainName)); - } catch (AddressException e) { - throw new RuntimeException(e); - } - this.recipientAddress = recipientAddress; - logger.atInfo().log("Sender:%s; Recipient: %s.", this.senderAddress, this.recipientAddress); - } + @Parameter("url") + String url; + + @Inject + CannedScriptExecutionAction() {} @Override public void run() { + Integer responseCode = null; + String responseContent = null; try { - // Invoke canned scripts here. - checkGroupApi(); - EmailMessage message = createEmail(); - this.gmailClient.sendEmail(message); - logger.atInfo().log("Finished running scripts."); - } catch (Throwable t) { - logger.atWarning().withCause(t).log("Error executing scripts."); - throw new RuntimeException("Execution failed."); - } - } - - // Checks if Directory and GroupSettings still work after GWorkspace changes. - void checkGroupApi() { - ImmutableList registrars = - Streams.stream(Registrar.loadAllCached()) - .filter(registrar -> registrar.isLive() && registrar.getType() == Registrar.Type.REAL) - .collect(toImmutableList()); - logger.atInfo().log("Found %s registrars.", registrars.size()); - for (Registrar registrar : registrars) { - for (final RegistrarPoc.Type type : RegistrarPoc.Type.values()) { - String groupKey = - String.format( - "%s-%s-contacts@%s", - normalizeRegistrarId(registrar.getRegistrarId()), - type.getDisplayName(), - gSuiteDomainName); - try { - Set currentMembers = groupsConnection.getMembersOfGroup(groupKey); - logger.atInfo().log("%s has %s members.", groupKey, currentMembers.size()); - // One success is enough for validation. - return; - } catch (IOException e) { - logger.atWarning().withCause(e).log("Failed to check %s", groupKey); - } + logger.atInfo().log("Connecting to: %s", url); + HttpsURLConnection connection = + (HttpsURLConnection) urlConnectionService.createConnection(new URL(url)); + responseCode = connection.getResponseCode(); + logger.atInfo().log("Code: %d", responseCode); + logger.atInfo().log("Headers: %s", connection.getHeaderFields()); + responseContent = new String(UrlConnectionUtils.getResponseBytes(connection), UTF_8); + logger.atInfo().log("Response: %s", responseContent); + } catch (Exception e) { + logger.atWarning().withCause(e).log("Connection to %s failed", url); + throw new RuntimeException(e); + } finally { + if (responseCode != null) { + response.setStatus(responseCode); + } + if (responseContent != null) { + response.setPayload(responseContent); } } - logger.atInfo().log("Finished checking GroupApis."); - } - - EmailMessage createEmail() { - return EmailMessage.newBuilder() - .setFrom(senderAddress) - .setSubject("Test: Please ignore.") - .setRecipients(ImmutableList.of(recipientAddress)) - .setBody("Sent from Nomulus through Google Workspace.") - .build(); } } diff --git a/core/src/main/java/google/registry/module/backend/BackendComponent.java b/core/src/main/java/google/registry/module/backend/BackendComponent.java index b17493243..a8415c79d 100644 --- a/core/src/main/java/google/registry/module/backend/BackendComponent.java +++ b/core/src/main/java/google/registry/module/backend/BackendComponent.java @@ -43,8 +43,6 @@ import google.registry.rde.JSchModule; import google.registry.request.Modules.GsonModule; import google.registry.request.Modules.NetHttpTransportModule; import google.registry.request.Modules.UrlConnectionServiceModule; -import google.registry.request.Modules.UrlFetchServiceModule; -import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UserServiceModule; import google.registry.request.auth.AuthModule; import google.registry.util.UtilsModule; @@ -80,8 +78,6 @@ import javax.inject.Singleton; SheetsServiceModule.class, StackdriverModule.class, UrlConnectionServiceModule.class, - UrlFetchServiceModule.class, - UrlFetchTransportModule.class, UserServiceModule.class, VoidDnsWriterModule.class, UtilsModule.class diff --git a/core/src/main/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java b/core/src/main/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java index a330e85ff..da5220888 100644 --- a/core/src/main/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java +++ b/core/src/main/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java @@ -14,22 +14,26 @@ package google.registry.rdap; +import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK; +import static com.google.common.net.HttpHeaders.ACCEPT_ENCODING; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.api.client.http.GenericUrl; -import com.google.api.client.http.HttpRequest; -import com.google.api.client.http.HttpResponse; -import com.google.api.client.http.HttpTransport; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.flogger.FluentLogger; -import com.google.common.io.ByteStreams; import google.registry.model.registrar.Registrar; import google.registry.request.Action; +import google.registry.request.HttpException.InternalServerErrorException; +import google.registry.request.UrlConnectionService; +import google.registry.request.UrlConnectionUtils; import google.registry.request.auth.Auth; +import google.registry.util.UrlConnectionException; import java.io.IOException; import java.io.StringReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.security.GeneralSecurityException; import javax.inject.Inject; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -41,9 +45,9 @@ import org.apache.commons.csv.CSVRecord; *

This will update ALL the REAL registrars. If a REAL registrar doesn't have an RDAP entry in * MoSAPI, we'll delete any BaseUrls it has. * - *

The ICANN base website that provides this information can be found at - * https://www.iana.org/assignments/registrar-ids/registrar-ids.xhtml. The provided CSV endpoint - * requires no authentication. + *

The ICANN base website that provides this information can be found at here. The provided + * CSV endpoint requires no authentication. */ @Action( service = Action.Service.BACKEND, @@ -52,22 +56,26 @@ import org.apache.commons.csv.CSVRecord; auth = Auth.AUTH_API_ADMIN) public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable { - private static final GenericUrl RDAP_IDS_URL = - new GenericUrl("https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv"); + private static final String RDAP_IDS_URL = + "https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv"; private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - @Inject HttpTransport httpTransport; + @Inject UrlConnectionService urlConnectionService; @Inject UpdateRegistrarRdapBaseUrlsAction() {} @Override public void run() { - ImmutableMap ianaIdsToUrls = getIanaIdsToUrls(); - tm().transact(() -> processAllRegistrars(ianaIdsToUrls)); + try { + ImmutableMap ianaIdsToUrls = getIanaIdsToUrls(); + tm().transact(() -> processAllRegistrars(ianaIdsToUrls)); + } catch (Exception e) { + throw new InternalServerErrorException("Error when retrieving RDAP base URL CSV file", e); + } } - private void processAllRegistrars(ImmutableMap ianaIdsToUrls) { + private static void processAllRegistrars(ImmutableMap ianaIdsToUrls) { int nonUpdatedRegistrars = 0; for (Registrar registrar : Registrar.loadAll()) { // Only update REAL registrars @@ -95,23 +103,28 @@ public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable { logger.atInfo().log("No change in RDAP base URLs for %d registrars", nonUpdatedRegistrars); } - private ImmutableMap getIanaIdsToUrls() { + private ImmutableMap getIanaIdsToUrls() + throws IOException, GeneralSecurityException { CSVParser csv; + HttpURLConnection connection = urlConnectionService.createConnection(new URL(RDAP_IDS_URL)); + // Explictly set the accepted encoding, as we know Brotli causes us problems when talking to + // ICANN. + connection.setRequestProperty(ACCEPT_ENCODING, "gzip"); + String csvString; try { - HttpRequest request = httpTransport.createRequestFactory().buildGetRequest(RDAP_IDS_URL); - // AppEngine might insert accept-encodings for us if we use the default gzip, so remove it - request.getHeaders().setAcceptEncoding(null); - HttpResponse response = request.execute(); - String csvString = new String(ByteStreams.toByteArray(response.getContent()), UTF_8); - csv = - CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader() - .setSkipHeaderRecord(true) - .build() - .parse(new StringReader(csvString)); - } catch (IOException e) { - throw new RuntimeException("Error when retrieving RDAP base URL CSV file", e); + if (connection.getResponseCode() != STATUS_CODE_OK) { + throw new UrlConnectionException("Failed to load RDAP base URLs from ICANN", connection); + } + csvString = new String(UrlConnectionUtils.getResponseBytes(connection), UTF_8); + } finally { + connection.disconnect(); } + csv = + CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader() + .setSkipHeaderRecord(true) + .build() + .parse(new StringReader(csvString)); ImmutableMap.Builder result = new ImmutableMap.Builder<>(); for (CSVRecord record : csv) { String ianaIdentifierString = record.get("ID"); diff --git a/core/src/main/java/google/registry/rde/RdeReporter.java b/core/src/main/java/google/registry/rde/RdeReporter.java index 9a78fd91c..ad08eba83 100644 --- a/core/src/main/java/google/registry/rde/RdeReporter.java +++ b/core/src/main/java/google/registry/rde/RdeReporter.java @@ -14,25 +14,23 @@ package google.registry.rde; -import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate; -import static com.google.appengine.api.urlfetch.HTTPMethod.PUT; -import static com.google.common.io.BaseEncoding.base64; -import static com.google.common.net.HttpHeaders.AUTHORIZATION; -import static com.google.common.net.HttpHeaders.CONTENT_TYPE; +import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_BAD_REQUEST; +import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK; +import static google.registry.request.UrlConnectionUtils.getResponseBytes; +import static google.registry.request.UrlConnectionUtils.setBasicAuth; +import static google.registry.request.UrlConnectionUtils.setPayload; import static google.registry.util.DomainNameUtils.canonicalizeHostname; import static java.nio.charset.StandardCharsets.UTF_8; -import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; -import static javax.servlet.http.HttpServletResponse.SC_OK; -import com.google.appengine.api.urlfetch.HTTPHeader; -import com.google.appengine.api.urlfetch.HTTPRequest; +import com.google.api.client.http.HttpMethods; import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; import com.google.common.flogger.FluentLogger; +import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.keyring.api.KeyModule.Key; import google.registry.request.HttpException.InternalServerErrorException; -import google.registry.util.Retrier; +import google.registry.request.UrlConnectionService; +import google.registry.util.UrlConnectionException; import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.iirdea.XjcIirdeaResponseElement; import google.registry.xjc.iirdea.XjcIirdeaResult; @@ -40,10 +38,11 @@ import google.registry.xjc.rdeheader.XjcRdeHeader; import google.registry.xjc.rdereport.XjcRdeReportReport; import google.registry.xml.XmlException; import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.HttpURLConnection; import java.net.MalformedURLException; -import java.net.SocketTimeoutException; import java.net.URL; -import java.util.Arrays; +import java.security.GeneralSecurityException; import javax.inject.Inject; /** @@ -59,49 +58,47 @@ public class RdeReporter { * @see * ICANN Registry Interfaces - Interface details */ - private static final String REPORT_MIME = "text/xml"; + private static final MediaType MEDIA_TYPE = MediaType.XML_UTF_8; - @Inject Retrier retrier; - @Inject URLFetchService urlFetchService; + @Inject UrlConnectionService urlConnectionService; @Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix; @Inject @Key("icannReportingPassword") String password; @Inject RdeReporter() {} /** Uploads {@code reportBytes} to ICANN. */ - public void send(byte[] reportBytes) throws XmlException { - XjcRdeReportReport report = XjcXmlTransformer.unmarshal( - XjcRdeReportReport.class, new ByteArrayInputStream(reportBytes)); + public void send(byte[] reportBytes) throws XmlException, GeneralSecurityException, IOException { + XjcRdeReportReport report = + XjcXmlTransformer.unmarshal( + XjcRdeReportReport.class, new ByteArrayInputStream(reportBytes)); XjcRdeHeader header = report.getHeader().getValue(); // Send a PUT request to ICANN's HTTPS server. URL url = makeReportUrl(header.getTld(), report.getId()); String username = header.getTld() + "_ry"; - String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8)); - final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d)); - req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME)); - req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token)); - req.setPayload(reportBytes); logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8)); - HTTPResponse rsp = - retrier.callWithRetry( - () -> { - HTTPResponse rsp1 = urlFetchService.fetch(req); - int responseCode = rsp1.getResponseCode(); - if (responseCode != SC_OK && responseCode != SC_BAD_REQUEST) { - logger.atSevere().log( - "Failure when trying to PUT RDE report to ICANN server: %d\n%s", - responseCode, Arrays.toString(rsp1.getContent())); - throw new RuntimeException("Error uploading deposits to ICANN"); - } - return rsp1; - }, - SocketTimeoutException.class); + HttpURLConnection connection = urlConnectionService.createConnection(url); + connection.setRequestMethod(HttpMethods.PUT); + setBasicAuth(connection, username, password); + setPayload(connection, reportBytes, MEDIA_TYPE.toString()); + int responseCode; + byte[] responseBytes; - // Ensure the XML response is valid. The EPP result code would not be 1000 if we get an - // SC_BAD_REQUEST as the HTTP response code. - XjcIirdeaResult result = parseResult(rsp.getContent()); - if (result.getCode().getValue() != 1000) { + try { + responseCode = connection.getResponseCode(); + if (responseCode != STATUS_CODE_OK && responseCode != STATUS_CODE_BAD_REQUEST) { + logger.atWarning().log("Connection to RDE report server failed: %d", responseCode); + throw new UrlConnectionException("PUT failed", connection); + } + responseBytes = getResponseBytes(connection); + } finally { + connection.disconnect(); + } + + // We know that an HTTP 200 response can only contain a result code of + // 1000 (i. e. success), there is no need to parse it. + if (responseCode != STATUS_CODE_OK) { + XjcIirdeaResult result = parseResult(responseBytes); logger.atWarning().log( "Rejected when trying to PUT RDE report to ICANN server: %d %s\n%s", result.getCode().getValue(), result.getMsg(), result.getDescription()); @@ -116,10 +113,11 @@ public class RdeReporter { * href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1"> * ICANN Registry Interfaces - IIRDEA Result Object */ - private XjcIirdeaResult parseResult(byte[] responseBytes) throws XmlException { + private static XjcIirdeaResult parseResult(byte[] responseBytes) throws XmlException { logger.atInfo().log("Received response:\n%s", new String(responseBytes, UTF_8)); - XjcIirdeaResponseElement response = XjcXmlTransformer.unmarshal( - XjcIirdeaResponseElement.class, new ByteArrayInputStream(responseBytes)); + XjcIirdeaResponseElement response = + XjcXmlTransformer.unmarshal( + XjcIirdeaResponseElement.class, new ByteArrayInputStream(responseBytes)); return response.getResult(); } diff --git a/core/src/main/java/google/registry/reporting/icann/IcannHttpReporter.java b/core/src/main/java/google/registry/reporting/icann/IcannHttpReporter.java index b1b97b9cb..9f4e78208 100644 --- a/core/src/main/java/google/registry/reporting/icann/IcannHttpReporter.java +++ b/core/src/main/java/google/registry/reporting/icann/IcannHttpReporter.java @@ -14,33 +14,31 @@ package google.registry.reporting.icann; +import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_BAD_REQUEST; +import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.net.MediaType.CSV_UTF_8; import static google.registry.model.tld.Tlds.assertTldExists; -import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.api.client.http.ByteArrayContent; -import com.google.api.client.http.GenericUrl; -import com.google.api.client.http.HttpHeaders; -import com.google.api.client.http.HttpRequest; -import com.google.api.client.http.HttpResponse; -import com.google.api.client.http.HttpResponseException; -import com.google.api.client.http.HttpStatusCodes; -import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.HttpMethods; import com.google.common.base.Ascii; import com.google.common.base.Splitter; import com.google.common.flogger.FluentLogger; -import com.google.common.io.BaseEncoding; -import com.google.common.io.ByteStreams; import google.registry.config.RegistryConfig.Config; import google.registry.keyring.api.KeyModule.Key; import google.registry.reporting.icann.IcannReportingModule.ReportType; +import google.registry.request.UrlConnectionService; +import google.registry.request.UrlConnectionUtils; import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.iirdea.XjcIirdeaResponseElement; import google.registry.xjc.iirdea.XjcIirdeaResult; import google.registry.xml.XmlException; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.GeneralSecurityException; import java.util.List; import javax.inject.Inject; import org.joda.time.YearMonth; @@ -62,78 +60,64 @@ public class IcannHttpReporter { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - @Inject HttpTransport httpTransport; - @Inject @Key("icannReportingPassword") String password; - @Inject @Config("icannTransactionsReportingUploadUrl") String icannTransactionsUrl; - @Inject @Config("icannActivityReportingUploadUrl") String icannActivityUrl; - @Inject IcannHttpReporter() {} + @Inject UrlConnectionService urlConnectionService; + + @Inject + @Key("icannReportingPassword") + String password; + + @Inject + @Config("icannTransactionsReportingUploadUrl") + String icannTransactionsUrl; + + @Inject + @Config("icannActivityReportingUploadUrl") + String icannActivityUrl; + + @Inject + IcannHttpReporter() {} /** Uploads {@code reportBytes} to ICANN, returning whether or not it succeeded. */ - public boolean send(byte[] reportBytes, String reportFilename) throws XmlException, IOException { + public boolean send(byte[] reportBytes, String reportFilename) + throws GeneralSecurityException, XmlException, IOException { validateReportFilename(reportFilename); - GenericUrl uploadUrl = new GenericUrl(makeUrl(reportFilename)); - HttpRequest request = - httpTransport - .createRequestFactory() - .buildPutRequest(uploadUrl, new ByteArrayContent(CSV_UTF_8.toString(), reportBytes)); - - HttpHeaders headers = request.getHeaders(); - headers.setBasicAuthentication(getTld(reportFilename) + "_ry", password); - headers.setContentType(CSV_UTF_8.toString()); - request.setHeaders(headers); - request.setFollowRedirects(false); - request.setThrowExceptionOnExecuteError(false); - - HttpResponse response = null; + URL uploadUrl = makeUrl(reportFilename); logger.atInfo().log( - "Sending report to %s with content length %d.", - uploadUrl, request.getContent().getLength()); - boolean success = true; + "Sending report to %s with content length %d.", uploadUrl, reportBytes.length); + HttpURLConnection connection = urlConnectionService.createConnection(uploadUrl); + connection.setRequestMethod(HttpMethods.PUT); + UrlConnectionUtils.setBasicAuth(connection, getTld(reportFilename) + "_ry", password); + UrlConnectionUtils.setPayload(connection, reportBytes, CSV_UTF_8.toString()); + connection.setInstanceFollowRedirects(false); + + int responseCode; + byte[] content; try { - response = request.execute(); - // Only responses with a 200 or 400 status have a body. For everything else, throw so that - // the caller catches it and prints the stack trace. - if (response.getStatusCode() != HttpStatusCodes.STATUS_CODE_OK - && response.getStatusCode() != HttpStatusCodes.STATUS_CODE_BAD_REQUEST) { - throw new HttpResponseException(response); - } - byte[] content; - try { - content = ByteStreams.toByteArray(response.getContent()); - } finally { - response.getContent().close(); - } - logger.atInfo().log( - "Received response code %d\n\n" - + "Response headers: %s\n\n" - + "Response content in UTF-8: %s\n\n" - + "Response content in HEX: %s", - response.getStatusCode(), - response.getHeaders(), - new String(content, UTF_8), - BaseEncoding.base16().encode(content)); - // For reasons unclear at the moment, when we parse the response content using UTF-8 we get - // garbled texts. Since we know that an HTTP 200 response can only contain a result code of - // 1000 (i. e. success), there is no need to parse it. - if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_BAD_REQUEST) { - success = false; - XjcIirdeaResult result = parseResult(content); - logger.atWarning().log( - "PUT rejected, status code %s:\n%s\n%s", - result.getCode().getValue(), result.getMsg(), result.getDescription()); + responseCode = connection.getResponseCode(); + // Only responses with a 200 or 400 status have a body. For everything else, we can return + // false early. + if (responseCode != STATUS_CODE_OK && responseCode != STATUS_CODE_BAD_REQUEST) { + logger.atWarning().log("Connection to ICANN server failed", connection); + return false; } + content = UrlConnectionUtils.getResponseBytes(connection); } finally { - if (response != null) { - response.disconnect(); - } else { - success = false; - logger.atWarning().log("Received null response from ICANN server at %s", uploadUrl); - } + connection.disconnect(); } - return success; + // We know that an HTTP 200 response can only contain a result code of + // 1000 (i. e. success), there is no need to parse it. + // See: https://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-13#page-16 + if (responseCode != STATUS_CODE_OK) { + XjcIirdeaResult result = parseResult(content); + logger.atWarning().log( + "PUT rejected, status code %s:\n%s\n%s", + result.getCode().getValue(), result.getMsg(), result.getDescription()); + return false; + } + return true; } - private XjcIirdeaResult parseResult(byte[] content) throws XmlException { + private static XjcIirdeaResult parseResult(byte[] content) throws XmlException { XjcIirdeaResponseElement response = XjcXmlTransformer.unmarshal( XjcIirdeaResponseElement.class, new ByteArrayInputStream(content)); @@ -141,7 +125,7 @@ public class IcannHttpReporter { } /** Verifies a given report filename matches the pattern tld-reportType-yyyyMM.csv. */ - private void validateReportFilename(String filename) { + private static void validateReportFilename(String filename) { checkArgument( filename.matches("[a-z0-9.\\-]+-((activity)|(transactions))-[0-9]{6}\\.csv"), "Expected file format: tld-reportType-yyyyMM.csv, got %s instead", @@ -149,12 +133,12 @@ public class IcannHttpReporter { assertTldExists(getTld(filename)); } - private String getTld(String filename) { + private static String getTld(String filename) { // Extract the TLD, up to second-to-last hyphen in the filename (works with international TLDs) return filename.substring(0, filename.lastIndexOf('-', filename.lastIndexOf('-') - 1)); } - private String makeUrl(String filename) { + private URL makeUrl(String filename) throws MalformedURLException { // Filename is in the format tld-reportType-yearMonth.csv String tld = getTld(filename); // Remove the tld- prefix and csv suffix @@ -164,7 +148,7 @@ public class IcannHttpReporter { // Re-add hyphen between year and month, because ICANN is inconsistent between filename and URL String yearMonth = YearMonth.parse(elements.get(1), DateTimeFormat.forPattern("yyyyMM")).toString("yyyy-MM"); - return String.format("%s/%s/%s", getUrlPrefix(reportType), tld, yearMonth); + return new URL(String.format("%s/%s/%s", getUrlPrefix(reportType), tld, yearMonth)); } private String getUrlPrefix(ReportType reportType) { diff --git a/core/src/main/java/google/registry/request/Modules.java b/core/src/main/java/google/registry/request/Modules.java index cc7e18c4c..ebd5712fc 100644 --- a/core/src/main/java/google/registry/request/Modules.java +++ b/core/src/main/java/google/registry/request/Modules.java @@ -14,20 +14,18 @@ package google.registry.request; -import com.google.api.client.extensions.appengine.http.UrlFetchTransport; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.gson.GsonFactory; -import com.google.appengine.api.urlfetch.URLFetchService; -import com.google.appengine.api.urlfetch.URLFetchServiceFactory; import com.google.appengine.api.users.UserService; import com.google.appengine.api.users.UserServiceFactory; import dagger.Module; import dagger.Provides; import java.net.HttpURLConnection; import javax.inject.Singleton; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; /** Dagger modules for App Engine services and other vendor classes. */ public final class Modules { @@ -37,18 +35,16 @@ public final class Modules { public static final class UrlConnectionServiceModule { @Provides static UrlConnectionService provideUrlConnectionService() { - return url -> (HttpURLConnection) url.openConnection(); - } - } - - /** Dagger module for {@link URLFetchService}. */ - @Module - public static final class UrlFetchServiceModule { - private static final URLFetchService fetchService = URLFetchServiceFactory.getURLFetchService(); - - @Provides - static URLFetchService provideUrlFetchService() { - return fetchService; + return url -> { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + if (connection instanceof HttpsURLConnection) { + HttpsURLConnection httpsConnection = (HttpsURLConnection) connection; + SSLContext tls13Context = SSLContext.getInstance("TLSv1.3"); + tls13Context.init(null, null, null); + httpsConnection.setSSLSocketFactory(tls13Context.getSocketFactory()); + } + return connection; + }; } } @@ -72,17 +68,6 @@ public final class Modules { } } - /** Dagger module that causes the App Engine's URL fetcher to be used for Google APIs requests. */ - @Module - public static final class UrlFetchTransportModule { - private static final UrlFetchTransport HTTP_TRANSPORT = new UrlFetchTransport(); - - @Provides - static HttpTransport provideHttpTransport() { - return HTTP_TRANSPORT; - } - } - /** * Dagger module that provides standard {@link NetHttpTransport}. Used in non App Engine * environment. diff --git a/core/src/main/java/google/registry/request/UrlConnectionUtils.java b/core/src/main/java/google/registry/request/UrlConnectionUtils.java index 62759f006..aacba19f0 100644 --- a/core/src/main/java/google/registry/request/UrlConnectionUtils.java +++ b/core/src/main/java/google/registry/request/UrlConnectionUtils.java @@ -27,15 +27,20 @@ import com.google.common.io.ByteStreams; import com.google.common.net.MediaType; import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.net.URLConnection; import java.util.Random; -/** Utilities for common functionality relating to {@link java.net.URLConnection}s. */ -public class UrlConnectionUtils { +/** Utilities for common functionality relating to {@link URLConnection}s. */ +public final class UrlConnectionUtils { + + private UrlConnectionUtils() {} /** Retrieves the response from the given connection as a byte array. */ public static byte[] getResponseBytes(URLConnection connection) throws IOException { - return ByteStreams.toByteArray(connection.getInputStream()); + try (InputStream is = connection.getInputStream()) { + return ByteStreams.toByteArray(is); + } } /** Sets auth on the given connection with the given username/password. */ diff --git a/core/src/main/java/google/registry/tools/RegistryToolComponent.java b/core/src/main/java/google/registry/tools/RegistryToolComponent.java index 783899b32..e0d19e562 100644 --- a/core/src/main/java/google/registry/tools/RegistryToolComponent.java +++ b/core/src/main/java/google/registry/tools/RegistryToolComponent.java @@ -39,7 +39,6 @@ import google.registry.privileges.secretmanager.SecretManagerModule; import google.registry.rde.RdeModule; import google.registry.request.Modules.GsonModule; import google.registry.request.Modules.UrlConnectionServiceModule; -import google.registry.request.Modules.UrlFetchServiceModule; import google.registry.request.Modules.UserServiceModule; import google.registry.tools.AuthModule.LocalCredentialModule; import google.registry.util.UtilsModule; @@ -77,7 +76,6 @@ import javax.inject.Singleton; SecretManagerKeyringModule.class, SecretManagerModule.class, UrlConnectionServiceModule.class, - UrlFetchServiceModule.class, UserServiceModule.class, UtilsModule.class, VoidDnsWriterModule.class, diff --git a/core/src/test/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java b/core/src/test/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java index 78cadc312..19a9e3240 100644 --- a/core/src/test/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java +++ b/core/src/test/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java @@ -18,20 +18,27 @@ import static com.google.common.truth.Truth.assertThat; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.loadRegistrar; import static google.registry.testing.DatabaseHelper.persistSimpleResource; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_OK; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import com.google.api.client.http.HttpResponseException; -import com.google.api.client.http.HttpStatusCodes; -import com.google.api.client.http.LowLevelHttpRequest; -import com.google.api.client.testing.http.MockHttpTransport; -import com.google.api.client.testing.http.MockLowLevelHttpRequest; -import com.google.api.client.testing.http.MockLowLevelHttpResponse; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.RegistrarAddress; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; +import google.registry.request.HttpException.InternalServerErrorException; +import google.registry.testing.FakeUrlConnectionService; +import google.registry.util.UrlConnectionException; +import java.io.ByteArrayInputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -61,44 +68,26 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest { public JpaIntegrationTestExtension jpa = new JpaTestExtensions.Builder().buildIntegrationTestExtension(); - private static class TestHttpTransport extends MockHttpTransport { - private MockLowLevelHttpRequest requestSent; - private MockLowLevelHttpResponse response; - - void setResponse(MockLowLevelHttpResponse response) { - this.response = response; - } - - MockLowLevelHttpRequest getRequestSent() { - return requestSent; - } - - @Override - public LowLevelHttpRequest buildRequest(String method, String url) { - assertThat(method).isEqualTo("GET"); - MockLowLevelHttpRequest httpRequest = new MockLowLevelHttpRequest(url); - httpRequest.setResponse(response); - requestSent = httpRequest; - return httpRequest; - } - } - - private TestHttpTransport httpTransport; + private final HttpURLConnection connection = mock(HttpURLConnection.class); + private final FakeUrlConnectionService urlConnectionService = + new FakeUrlConnectionService(connection); private UpdateRegistrarRdapBaseUrlsAction action; @BeforeEach - void beforeEach() { + void beforeEach() throws Exception { action = new UpdateRegistrarRdapBaseUrlsAction(); - httpTransport = new TestHttpTransport(); - action.httpTransport = httpTransport; - setValidResponse(); + action.urlConnectionService = urlConnectionService; + when(connection.getResponseCode()).thenReturn(SC_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(CSV_REPLY.getBytes(StandardCharsets.UTF_8))); createTld("tld"); } - private void assertCorrectRequestSent() { - assertThat(httpTransport.getRequestSent().getUrl()) - .isEqualTo("https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv"); - assertThat(httpTransport.getRequestSent().getHeaders().get("accept-encoding")).isNull(); + private void assertCorrectRequestSent() throws Exception { + assertThat(urlConnectionService.getConnectedUrls()) + .containsExactly( + new URL("https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv")); + verify(connection).setRequestProperty("Accept-Encoding", "gzip"); } private static void persistRegistrar( @@ -119,14 +108,8 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest { .build()); } - private void setValidResponse() { - MockLowLevelHttpResponse csvResponse = new MockLowLevelHttpResponse(); - csvResponse.setContent(CSV_REPLY); - httpTransport.setResponse(csvResponse); - } - @Test - void testUnknownIana_cleared() { + void testUnknownIana_cleared() throws Exception { // The IANA ID isn't in the CSV reply persistRegistrar("someRegistrar", 4123L, Registrar.Type.REAL, "http://rdap.example/blah"); action.run(); @@ -135,7 +118,7 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest { } @Test - void testKnownIana_changed() { + void testKnownIana_changed() throws Exception { // The IANA ID is in the CSV reply persistRegistrar("someRegistrar", 1448L, Registrar.Type.REAL, "http://rdap.example/blah"); action.run(); @@ -145,7 +128,7 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest { } @Test - void testKnownIana_notReal_noChange() { + void testKnownIana_notReal_noChange() throws Exception { // The IANA ID is in the CSV reply persistRegistrar("someRegistrar", 9999L, Registrar.Type.INTERNAL, "http://rdap.example/blah"); // Real registrars should actually change @@ -159,7 +142,7 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest { } @Test - void testKnownIana_notReal_nullIANA_noChange() { + void testKnownIana_notReal_nullIANA_noChange() throws Exception { persistRegistrar("someRegistrar", null, Registrar.Type.TEST, "http://rdap.example/blah"); action.run(); assertCorrectRequestSent(); @@ -168,29 +151,30 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest { } @Test - void testFailure_serverErrorResponse() { - MockLowLevelHttpResponse badResponse = new MockLowLevelHttpResponse(); - badResponse.setZeroContent(); - badResponse.setStatusCode(HttpStatusCodes.STATUS_CODE_SERVER_ERROR); - httpTransport.setResponse(badResponse); - - RuntimeException thrown = assertThrows(RuntimeException.class, action::run); + void testFailure_serverErrorResponse() throws Exception { + when(connection.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8))); + InternalServerErrorException thrown = + assertThrows(InternalServerErrorException.class, action::run); + verify(connection, times(0)).getInputStream(); assertThat(thrown).hasMessageThat().isEqualTo("Error when retrieving RDAP base URL CSV file"); Throwable cause = thrown.getCause(); - assertThat(cause).isInstanceOf(HttpResponseException.class); + assertThat(cause).isInstanceOf(UrlConnectionException.class); assertThat(cause) .hasMessageThat() - .isEqualTo("500\nGET https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv"); + .contains("https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv"); } @Test - void testFailure_invalidCsv() { - MockLowLevelHttpResponse csvResponse = new MockLowLevelHttpResponse(); - csvResponse.setContent("foo,bar\nbaz,foo"); - httpTransport.setResponse(csvResponse); + void testFailure_invalidCsv() throws Exception { + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream("foo,bar\nbaz,foo".getBytes(StandardCharsets.UTF_8))); - IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, action::run); + InternalServerErrorException thrown = + assertThrows(InternalServerErrorException.class, action::run); assertThat(thrown) + .hasCauseThat() .hasMessageThat() .isEqualTo("Mapping for ID not found, expected one of [foo, bar]"); } diff --git a/core/src/test/java/google/registry/rde/RdeReportActionTest.java b/core/src/test/java/google/registry/rde/RdeReportActionTest.java index 394798ad5..a927c402f 100644 --- a/core/src/test/java/google/registry/rde/RdeReportActionTest.java +++ b/core/src/test/java/google/registry/rde/RdeReportActionTest.java @@ -14,7 +14,9 @@ package google.registry.rde; -import static com.google.appengine.api.urlfetch.HTTPMethod.PUT; +import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_BAD_REQUEST; +import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK; +import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_UNAUTHORIZED; import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; import static com.google.common.truth.Truth.assertThat; import static google.registry.model.common.Cursor.CursorType.RDE_REPORT; @@ -24,25 +26,17 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.loadByKey; import static google.registry.testing.DatabaseHelper.persistResource; -import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; -import static javax.servlet.http.HttpServletResponse.SC_OK; import static org.joda.time.Duration.standardDays; import static org.joda.time.Duration.standardSeconds; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import com.google.appengine.api.urlfetch.HTTPHeader; -import com.google.appengine.api.urlfetch.HTTPRequest; -import com.google.appengine.api.urlfetch.HTTPResponse; -import com.google.appengine.api.urlfetch.URLFetchService; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; -import com.google.common.base.Ascii; -import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSource; import google.registry.gcs.GcsUtils; import google.registry.model.common.Cursor; @@ -53,25 +47,23 @@ import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationT import google.registry.request.HttpException.InternalServerErrorException; import google.registry.request.HttpException.NoContentException; import google.registry.testing.BouncyCastleProviderExtension; -import google.registry.testing.FakeClock; import google.registry.testing.FakeKeyringModule; import google.registry.testing.FakeResponse; -import google.registry.testing.FakeSleeper; -import google.registry.util.Retrier; +import google.registry.testing.FakeUrlConnectionService; +import google.registry.util.UrlConnectionException; import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.rdereport.XjcRdeReportReport; import google.registry.xml.XmlException; import java.io.ByteArrayInputStream; -import java.net.SocketTimeoutException; +import java.io.ByteArrayOutputStream; +import java.net.HttpURLConnection; import java.nio.charset.StandardCharsets; -import java.util.Map; import java.util.Optional; import org.bouncycastle.openpgp.PGPPublicKey; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.ArgumentCaptor; /** Unit tests for {@link RdeReportAction}. */ public class RdeReportActionTest { @@ -89,9 +81,11 @@ public class RdeReportActionTest { private final FakeResponse response = new FakeResponse(); private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class); - private final URLFetchService urlFetchService = mock(URLFetchService.class); - private final ArgumentCaptor request = ArgumentCaptor.forClass(HTTPRequest.class); - private final HTTPResponse httpResponse = mock(HTTPResponse.class); + private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class); + private final FakeUrlConnectionService urlConnectionService = + new FakeUrlConnectionService(httpUrlConnection); + private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream(); + private final PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey(); private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); @@ -102,9 +96,8 @@ public class RdeReportActionTest { private RdeReportAction createAction() { RdeReporter reporter = new RdeReporter(); reporter.reportUrlPrefix = "https://rde-report.example"; - reporter.urlFetchService = urlFetchService; + reporter.urlConnectionService = urlConnectionService; reporter.password = "foo"; - reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); RdeReportAction action = new RdeReportAction(); action.gcsUtils = gcsUtils; action.response = response; @@ -126,6 +119,9 @@ public class RdeReportActionTest { persistResource(Cursor.createScoped(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), registry)); gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0)); + when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream); + when(httpUrlConnection.getResponseCode()).thenReturn(STATUS_CODE_OK); + when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openBufferedStream()); } @Test @@ -142,24 +138,20 @@ public class RdeReportActionTest { @Test void testRunWithLock() throws Exception { - when(httpResponse.getResponseCode()).thenReturn(SC_OK); - when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); - when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); createAction().runWithLock(loadRdeReportCursor()); assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8); assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n"); // Verify the HTTP request was correct. - assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT); - assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https"); - assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001"); - Map headers = mapifyHeaders(request.getValue().getHeaders()); - assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml"); - assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28="); + verify(httpUrlConnection).setRequestMethod("PUT"); + assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https"); + assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001"); + verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8"); + verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28="); // Verify the payload XML was the same as what's in testdata/report.xml. - XjcRdeReportReport report = parseReport(request.getValue().getPayload()); + XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray()); assertThat(report.getId()).isEqualTo("20101017001"); assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z")); assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z")); @@ -167,9 +159,6 @@ public class RdeReportActionTest { @Test void testRunWithLock_withPrefix() throws Exception { - when(httpResponse.getResponseCode()).thenReturn(SC_OK); - when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); - when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); RdeReportAction action = createAction(); action.runWithLock(loadRdeReportCursor()); assertThat(response.getStatus()).isEqualTo(200); @@ -177,15 +166,14 @@ public class RdeReportActionTest { assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n"); // Verify the HTTP request was correct. - assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT); - assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https"); - assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001"); - Map headers = mapifyHeaders(request.getValue().getHeaders()); - assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml"); - assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28="); + verify(httpUrlConnection).setRequestMethod("PUT"); + assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https"); + assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001"); + verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8"); + verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28="); // Verify the payload XML was the same as what's in testdata/report.xml. - XjcRdeReportReport report = parseReport(request.getValue().getPayload()); + XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray()); assertThat(report.getId()).isEqualTo("20101017001"); assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z")); assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z")); @@ -200,9 +188,6 @@ public class RdeReportActionTest { @Test void testRunWithLock_withoutPrefix() throws Exception { - when(httpResponse.getResponseCode()).thenReturn(SC_OK); - when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); - when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); RdeReportAction action = createAction(); action.prefix = Optional.empty(); gcsUtils.delete(reportFile); @@ -225,15 +210,14 @@ public class RdeReportActionTest { assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n"); // Verify the HTTP request was correct. - assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT); - assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https"); - assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001"); - Map headers = mapifyHeaders(request.getValue().getHeaders()); - assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml"); - assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28="); + verify(httpUrlConnection).setRequestMethod("PUT"); + assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https"); + assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001"); + verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8"); + verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28="); // Verify the payload XML was the same as what's in testdata/report.xml. - XjcRdeReportReport report = parseReport(request.getValue().getPayload()); + XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray()); assertThat(report.getId()).isEqualTo("20101017001"); assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z")); assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z")); @@ -246,9 +230,6 @@ public class RdeReportActionTest { PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey(); gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1)); - when(httpResponse.getResponseCode()).thenReturn(SC_OK); - when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); - when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); createAction().runWithLock(loadRdeReportCursor()); assertThat(response.getStatus()).isEqualTo(200); } @@ -281,9 +262,8 @@ public class RdeReportActionTest { @Test void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception { - when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST); - when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read()); - when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); + when(httpUrlConnection.getResponseCode()).thenReturn(STATUS_CODE_BAD_REQUEST); + when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openBufferedStream()); InternalServerErrorException thrown = assertThrows( InternalServerErrorException.class, @@ -292,38 +272,19 @@ public class RdeReportActionTest { } @Test - void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception { - class ExpectedThrownException extends RuntimeException {} - when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException()); - assertThrows( - ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor())); - } - - @Test - void testRunWithLock_socketTimeout_doesRetry() throws Exception { - when(httpResponse.getResponseCode()).thenReturn(SC_OK); - when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); - when(urlFetchService.fetch(request.capture())) - .thenThrow(new SocketTimeoutException()) - .thenReturn(httpResponse); - createAction().runWithLock(loadRdeReportCursor()); - assertThat(response.getStatus()).isEqualTo(200); - assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8); - assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n"); + void testRunWithLock_notAuthorized() throws Exception { + when(httpUrlConnection.getResponseCode()).thenReturn(STATUS_CODE_UNAUTHORIZED); + UrlConnectionException thrown = + assertThrows( + UrlConnectionException.class, () -> createAction().runWithLock(loadRdeReportCursor())); + verify(httpUrlConnection, times(0)).getInputStream(); + assertThat(thrown).hasMessageThat().contains("PUT failed"); } private DateTime loadRdeReportCursor() { return loadByKey(Cursor.createScopedVKey(RDE_REPORT, registry)).getCursorTime(); } - private static ImmutableMap mapifyHeaders(Iterable headers) { - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - for (HTTPHeader header : headers) { - builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue()); - } - return builder.build(); - } - private static XjcRdeReportReport parseReport(byte[] data) { try { return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data)); diff --git a/core/src/test/java/google/registry/reporting/icann/IcannHttpReporterTest.java b/core/src/test/java/google/registry/reporting/icann/IcannHttpReporterTest.java index 34bd24980..b95cb3279 100644 --- a/core/src/test/java/google/registry/reporting/icann/IcannHttpReporterTest.java +++ b/core/src/test/java/google/registry/reporting/icann/IcannHttpReporterTest.java @@ -14,28 +14,27 @@ package google.registry.reporting.icann; -import static com.google.common.net.MediaType.CSV_UTF_8; -import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; +import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_BAD_REQUEST; +import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK; +import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_SERVER_ERROR; import static com.google.common.truth.Truth.assertThat; import static google.registry.testing.DatabaseHelper.createTld; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import com.google.api.client.http.HttpResponseException; -import com.google.api.client.http.HttpStatusCodes; -import com.google.api.client.http.LowLevelHttpRequest; -import com.google.api.client.http.LowLevelHttpResponse; -import com.google.api.client.testing.http.MockHttpTransport; -import com.google.api.client.testing.http.MockLowLevelHttpRequest; -import com.google.api.client.testing.http.MockLowLevelHttpResponse; -import com.google.api.client.util.Base64; import com.google.api.client.util.StringUtils; +import com.google.common.io.BaseEncoding; import com.google.common.io.ByteSource; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import java.io.IOException; -import java.util.List; -import java.util.Map; +import google.registry.testing.FakeUrlConnectionService; +import java.io.ByteArrayOutputStream; +import java.net.HttpURLConnection; +import java.net.URL; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -46,103 +45,75 @@ class IcannHttpReporterTest { private static final ByteSource IIRDEA_GOOD_XML = ReportingTestData.loadBytes("iirdea_good.xml"); private static final ByteSource IIRDEA_BAD_XML = ReportingTestData.loadBytes("iirdea_bad.xml"); private static final byte[] FAKE_PAYLOAD = "test,csv\n1,2".getBytes(UTF_8); + private static final IcannHttpReporter reporter = new IcannHttpReporter(); - private MockLowLevelHttpRequest mockRequest; + private final HttpURLConnection connection = mock(HttpURLConnection.class); + private final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + private final FakeUrlConnectionService urlConnectionService = + new FakeUrlConnectionService(connection); @RegisterExtension final JpaIntegrationTestExtension jpa = new JpaTestExtensions.Builder().buildIntegrationTestExtension(); - private MockHttpTransport createMockTransport( - int statusCode, final ByteSource iirdeaResponse) { - return new MockHttpTransport() { - @Override - public LowLevelHttpRequest buildRequest(String method, String url) { - mockRequest = - new MockLowLevelHttpRequest() { - @Override - public LowLevelHttpResponse execute() throws IOException { - MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); - response.setStatusCode(statusCode); - response.setContentType(PLAIN_TEXT_UTF_8.toString()); - response.setContent(iirdeaResponse.read()); - return response; - } - }; - mockRequest.setUrl(url); - return mockRequest; - } - }; - } - - private MockHttpTransport createMockTransport(final ByteSource iirdeaResponse) { - return createMockTransport(HttpStatusCodes.STATUS_CODE_OK, iirdeaResponse); - } - @BeforeEach - void beforeEach() { + void beforeEach() throws Exception { createTld("test"); createTld("xn--abc123"); - } - - private IcannHttpReporter createReporter() { - IcannHttpReporter reporter = new IcannHttpReporter(); - reporter.httpTransport = createMockTransport(IIRDEA_GOOD_XML); + when(connection.getOutputStream()).thenReturn(outputStream); + when(connection.getResponseCode()).thenReturn(STATUS_CODE_OK); + when(connection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openBufferedStream()); + reporter.urlConnectionService = urlConnectionService; reporter.password = "fakePass"; reporter.icannTransactionsUrl = "https://fake-transactions.url"; reporter.icannActivityUrl = "https://fake-activity.url"; - return reporter; } @Test void testSuccess() throws Exception { - IcannHttpReporter reporter = createReporter(); - reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv"); + assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isTrue(); - assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2017-06"); - Map> headers = mockRequest.getHeaders(); + assertThat(urlConnectionService.getConnectedUrls()) + .containsExactly(new URL("https://fake-transactions.url/test/2017-06")); String userPass = "test_ry:fakePass"; String expectedAuth = - String.format("Basic %s", Base64.encodeBase64String(StringUtils.getBytesUtf8(userPass))); - assertThat(headers.get("authorization")).containsExactly(expectedAuth); - assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString()); + String.format("Basic %s", BaseEncoding.base64().encode(StringUtils.getBytesUtf8(userPass))); + verify(connection).setRequestProperty("Authorization", expectedAuth); + verify(connection).setRequestProperty("Content-Type", "text/csv; charset=utf-8"); + assertThat(outputStream.toByteArray()).isEqualTo(FAKE_PAYLOAD); } @Test void testSuccess_internationalTld() throws Exception { - IcannHttpReporter reporter = createReporter(); - reporter.send(FAKE_PAYLOAD, "xn--abc123-transactions-201706.csv"); + assertThat(reporter.send(FAKE_PAYLOAD, "xn--abc123-transactions-201706.csv")).isTrue(); - assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/xn--abc123/2017-06"); - Map> headers = mockRequest.getHeaders(); + assertThat(urlConnectionService.getConnectedUrls()) + .containsExactly(new URL("https://fake-transactions.url/xn--abc123/2017-06")); String userPass = "xn--abc123_ry:fakePass"; String expectedAuth = - String.format("Basic %s", Base64.encodeBase64String(StringUtils.getBytesUtf8(userPass))); - assertThat(headers.get("authorization")).containsExactly(expectedAuth); - assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString()); + String.format("Basic %s", BaseEncoding.base64().encode(StringUtils.getBytesUtf8(userPass))); + verify(connection).setRequestProperty("Authorization", expectedAuth); + verify(connection).setRequestProperty("Content-Type", "text/csv; charset=utf-8"); + assertThat(outputStream.toByteArray()).isEqualTo(FAKE_PAYLOAD); } @Test void testFail_BadIirdeaResponse() throws Exception { - IcannHttpReporter reporter = createReporter(); - reporter.httpTransport = - createMockTransport(HttpStatusCodes.STATUS_CODE_BAD_REQUEST, IIRDEA_BAD_XML); + when(connection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openBufferedStream()); + when(connection.getResponseCode()).thenReturn(STATUS_CODE_BAD_REQUEST); assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isFalse(); + verify(connection).getInputStream(); } @Test - void testFail_transportException() { - IcannHttpReporter reporter = createReporter(); - reporter.httpTransport = - createMockTransport(HttpStatusCodes.STATUS_CODE_FORBIDDEN, ByteSource.empty()); - assertThrows( - HttpResponseException.class, - () -> reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")); + void testFail_OtherBadHttpResponse() throws Exception { + when(connection.getResponseCode()).thenReturn(STATUS_CODE_SERVER_ERROR); + assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isFalse(); + verify(connection, times(0)).getInputStream(); } @Test void testFail_invalidFilename_nonSixDigitYearMonth() { - IcannHttpReporter reporter = createReporter(); IllegalArgumentException thrown = assertThrows( IllegalArgumentException.class, @@ -156,7 +127,6 @@ class IcannHttpReporterTest { @Test void testFail_invalidFilename_notActivityOrTransactions() { - IcannHttpReporter reporter = createReporter(); IllegalArgumentException thrown = assertThrows( IllegalArgumentException.class, @@ -169,7 +139,6 @@ class IcannHttpReporterTest { @Test void testFail_invalidFilename_invalidTldName() { - IcannHttpReporter reporter = createReporter(); IllegalArgumentException thrown = assertThrows( IllegalArgumentException.class, @@ -183,7 +152,6 @@ class IcannHttpReporterTest { @Test void testFail_invalidFilename_tldDoesntExist() { - IcannHttpReporter reporter = createReporter(); IllegalArgumentException thrown = assertThrows( IllegalArgumentException.class, diff --git a/core/src/test/java/google/registry/testing/FakeUrlConnectionService.java b/core/src/test/java/google/registry/testing/FakeUrlConnectionService.java index de4bd5c81..5c1065ad0 100644 --- a/core/src/test/java/google/registry/testing/FakeUrlConnectionService.java +++ b/core/src/test/java/google/registry/testing/FakeUrlConnectionService.java @@ -16,6 +16,7 @@ package google.registry.testing; import static org.mockito.Mockito.when; +import com.google.common.collect.ImmutableList; import google.registry.request.UrlConnectionService; import java.net.HttpURLConnection; import java.net.URL; @@ -26,15 +27,10 @@ import java.util.List; public class FakeUrlConnectionService implements UrlConnectionService { private final HttpURLConnection mockConnection; - private final List connectedUrls; + private final List connectedUrls = new ArrayList<>(); public FakeUrlConnectionService(HttpURLConnection mockConnection) { - this(mockConnection, new ArrayList<>()); - } - - public FakeUrlConnectionService(HttpURLConnection mockConnection, List connectedUrls) { this.mockConnection = mockConnection; - this.connectedUrls = connectedUrls; } @Override @@ -43,4 +39,8 @@ public class FakeUrlConnectionService implements UrlConnectionService { when(mockConnection.getURL()).thenReturn(url); return mockConnection; } + + public ImmutableList getConnectedUrls() { + return ImmutableList.copyOf(connectedUrls); + } } diff --git a/core/src/test/java/google/registry/tmch/TmchActionTestCase.java b/core/src/test/java/google/registry/tmch/TmchActionTestCase.java index 23498f094..c82d697df 100644 --- a/core/src/test/java/google/registry/tmch/TmchActionTestCase.java +++ b/core/src/test/java/google/registry/tmch/TmchActionTestCase.java @@ -25,9 +25,7 @@ import google.registry.testing.FakeClock; import google.registry.testing.FakeUrlConnectionService; import google.registry.testing.TestCacheExtension; import java.net.HttpURLConnection; -import java.net.URL; import java.time.Duration; -import java.util.ArrayList; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -55,9 +53,8 @@ abstract class TmchActionTestCase { final Marksdb marksdb = new Marksdb(); protected final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class); - protected final ArrayList connectedUrls = new ArrayList<>(); protected FakeUrlConnectionService urlConnectionService = - new FakeUrlConnectionService(httpUrlConnection, connectedUrls); + new FakeUrlConnectionService(httpUrlConnection); @BeforeEach public void beforeEachTmchActionTestCase() throws Exception { diff --git a/core/src/test/java/google/registry/tmch/TmchCrlActionTest.java b/core/src/test/java/google/registry/tmch/TmchCrlActionTest.java index 66e8a0071..9b67085d3 100644 --- a/core/src/test/java/google/registry/tmch/TmchCrlActionTest.java +++ b/core/src/test/java/google/registry/tmch/TmchCrlActionTest.java @@ -56,7 +56,8 @@ class TmchCrlActionTest extends TmchActionTestCase { readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read())); newTmchCrlAction(TmchCaMode.PILOT).run(); verify(httpUrlConnection).getInputStream(); - assertThat(connectedUrls).containsExactly(new URL("https://sloth.lol/tmch.crl")); + assertThat(urlConnectionService.getConnectedUrls()) + .containsExactly(new URL("https://sloth.lol/tmch.crl")); } @Test diff --git a/core/src/test/java/google/registry/tmch/TmchDnlActionTest.java b/core/src/test/java/google/registry/tmch/TmchDnlActionTest.java index 9253ee2c2..f642495fc 100644 --- a/core/src/test/java/google/registry/tmch/TmchDnlActionTest.java +++ b/core/src/test/java/google/registry/tmch/TmchDnlActionTest.java @@ -49,7 +49,10 @@ class TmchDnlActionTest extends TmchActionTestCase { .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl/dnl-latest.sig").read())); newTmchDnlAction().run(); verify(httpUrlConnection, times(2)).getInputStream(); - assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList())) + assertThat( + urlConnectionService.getConnectedUrls().stream() + .map(URL::toString) + .collect(toImmutableList())) .containsExactly(MARKSDB_URL + "/dnl/dnl-latest.csv", MARKSDB_URL + "/dnl/dnl-latest.sig"); // Make sure the contents of testdata/dnl-latest.csv got inserted into the database. diff --git a/core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java b/core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java index 8349ecf09..85c87517d 100644 --- a/core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java +++ b/core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java @@ -50,7 +50,10 @@ class TmchSmdrlActionTest extends TmchActionTestCase { .thenReturn(new ByteArrayInputStream(loadBytes("smdrl/smdrl-latest.sig").read())); newTmchSmdrlAction().run(); verify(httpUrlConnection, times(2)).getInputStream(); - assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList())) + assertThat( + urlConnectionService.getConnectedUrls().stream() + .map(URL::toString) + .collect(toImmutableList())) .containsExactly( MARKSDB_URL + "/smdrl/smdrl-latest.csv", MARKSDB_URL + "/smdrl/smdrl-latest.sig"); smdrl = SignedMarkRevocationList.get(); diff --git a/core/src/test/resources/google/registry/module/backend/backend_routing.txt b/core/src/test/resources/google/registry/module/backend/backend_routing.txt index de7797ae0..a6b820400 100644 --- a/core/src/test/resources/google/registry/module/backend/backend_routing.txt +++ b/core/src/test/resources/google/registry/module/backend/backend_routing.txt @@ -6,7 +6,7 @@ PATH CLASS /_dr/task/deleteExpiredDomains DeleteExpiredDomainsAction GET n API APP ADMIN /_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n API APP ADMIN /_dr/task/deleteProberData DeleteProberDataAction POST n API APP ADMIN -/_dr/task/executeCannedScript CannedScriptExecutionAction POST y API APP ADMIN +/_dr/task/executeCannedScript CannedScriptExecutionAction POST,GET y API APP ADMIN /_dr/task/expandBillingRecurrences ExpandBillingRecurrencesAction GET n API APP ADMIN /_dr/task/exportDomainLists ExportDomainListsAction POST n API APP ADMIN /_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n API APP ADMIN