mirror of
https://github.com/google/nomulus.git
synced 2025-07-25 20:18:34 +02:00
Use built-in Java URL connections instead of UrlFetchService (#1535)
- Use the standard HttpsURLConnection to write/read data - Rewrite RdeReporter, Nordn*Action, and Marksdb classes and related tests to conform to the new format - Remove FakeURLFetchService and ForwardingUrlFetchService as they weren't used - Refactor UrlFetchException to UrlConnectionException - Refactor UrlFetchUtils to UrlConnectionUtils I will need to test this on Alpha. Fortunately the connections that don't require auth (e.g. TMDB downloading) should be testable.
This commit is contained in:
gbrodman
GitHub
parent
cc4f2c0c52
commit
0d62ac0410
26 changed files with 507 additions and 584 deletions
|
|
@ -0,0 +1,71 @@
|
|||
// Copyright 2022 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.util;
|
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.common.io.ByteStreams;
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
|
||||
/**
|
||||
* Used when HTTP requests return a bad response, with troubleshooting info.
|
||||
*
|
||||
* <p>This class displays lots of helpful troubleshooting information.
|
||||
*/
|
||||
public class UrlConnectionException extends RuntimeException {
|
||||
|
||||
private final HttpURLConnection connection;
|
||||
|
||||
public UrlConnectionException(String message, HttpURLConnection connection) {
|
||||
super(message);
|
||||
this.connection = connection;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getMessage() {
|
||||
byte[] resultContent;
|
||||
int responseCode;
|
||||
try {
|
||||
resultContent = ByteStreams.toByteArray(connection.getInputStream());
|
||||
responseCode = connection.getResponseCode();
|
||||
} catch (IOException e) {
|
||||
resultContent = new byte[] {};
|
||||
responseCode = 0;
|
||||
}
|
||||
StringBuilder result =
|
||||
new StringBuilder(2048 + resultContent.length)
|
||||
.append(
|
||||
String.format(
|
||||
"%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\n",
|
||||
getClass().getSimpleName(),
|
||||
super.getMessage(),
|
||||
responseCode,
|
||||
connection.getURL().toString()));
|
||||
connection
|
||||
.getRequestProperties()
|
||||
.forEach(
|
||||
(key, value) -> {
|
||||
result.append(key);
|
||||
result.append(": ");
|
||||
result.append(value);
|
||||
result.append('\n');
|
||||
});
|
||||
result.append(">>>\n");
|
||||
result.append(new String(resultContent, UTF_8));
|
||||
result.append("\n<<<");
|
||||
return result.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.util;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.api.urlfetch.HTTPHeader;
|
||||
import com.google.appengine.api.urlfetch.HTTPRequest;
|
||||
import com.google.appengine.api.urlfetch.HTTPResponse;
|
||||
|
||||
/**
|
||||
* Exception for when App Engine HTTP requests return a bad response.
|
||||
*
|
||||
* <p>This class displays lots of helpful troubleshooting information.
|
||||
*/
|
||||
public class UrlFetchException extends RuntimeException {
|
||||
|
||||
private final HTTPRequest req;
|
||||
private final HTTPResponse rsp;
|
||||
|
||||
public UrlFetchException(String message, HTTPRequest req, HTTPResponse rsp) {
|
||||
super(message);
|
||||
this.req = checkNotNull(req, "req");
|
||||
this.rsp = checkNotNull(rsp, "rsp");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getMessage() {
|
||||
StringBuilder res =
|
||||
new StringBuilder(2048 + rsp.getContent().length)
|
||||
.append(
|
||||
String.format(
|
||||
"%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\nX-Final-URL: %s\n",
|
||||
getClass().getSimpleName(),
|
||||
super.getMessage(),
|
||||
rsp.getResponseCode(),
|
||||
req.getURL().toString(),
|
||||
rsp.getFinalUrl()));
|
||||
for (HTTPHeader header : rsp.getHeadersUncombined()) {
|
||||
res.append(header.getName());
|
||||
res.append(": ");
|
||||
res.append(header.getValue());
|
||||
res.append('\n');
|
||||
}
|
||||
res.append(">>>\n");
|
||||
res.append(new String(rsp.getContent(), UTF_8));
|
||||
res.append("\n<<<");
|
||||
return res.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.util;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.io.BaseEncoding.base64;
|
||||
import static com.google.common.net.HttpHeaders.AUTHORIZATION;
|
||||
import static com.google.common.net.HttpHeaders.CONTENT_DISPOSITION;
|
||||
import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
|
||||
import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.api.urlfetch.HTTPHeader;
|
||||
import com.google.appengine.api.urlfetch.HTTPRequest;
|
||||
import com.google.appengine.api.urlfetch.HTTPResponse;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.net.MediaType;
|
||||
import java.util.Optional;
|
||||
import java.util.Random;
|
||||
|
||||
/** Helper methods for the App Engine URL fetch service. */
|
||||
public final class UrlFetchUtils {
|
||||
|
||||
/** Returns value of first header matching {@code name}. */
|
||||
public static Optional<String> getHeaderFirst(HTTPResponse rsp, String name) {
|
||||
return getHeaderFirstInternal(rsp.getHeadersUncombined(), name);
|
||||
}
|
||||
|
||||
/** Returns value of first header matching {@code name}. */
|
||||
public static Optional<String> getHeaderFirst(HTTPRequest req, String name) {
|
||||
return getHeaderFirstInternal(req.getHeaders(), name);
|
||||
}
|
||||
|
||||
private static Optional<String> getHeaderFirstInternal(Iterable<HTTPHeader> hdrs, String name) {
|
||||
name = Ascii.toLowerCase(name);
|
||||
for (HTTPHeader header : hdrs) {
|
||||
if (Ascii.toLowerCase(header.getName()).equals(name)) {
|
||||
return Optional.of(header.getValue());
|
||||
}
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets payload on request as a {@code multipart/form-data} request.
|
||||
*
|
||||
* <p>This is equivalent to running the command: {@code curl -F fieldName=@payload.txt URL}
|
||||
*
|
||||
* @see <a href="http://www.ietf.org/rfc/rfc2388.txt">RFC2388 - Returning Values from Forms</a>
|
||||
*/
|
||||
public static void setPayloadMultipart(
|
||||
HTTPRequest request,
|
||||
String name,
|
||||
String filename,
|
||||
MediaType contentType,
|
||||
String data,
|
||||
Random random) {
|
||||
String boundary = createMultipartBoundary(random);
|
||||
checkState(
|
||||
!data.contains(boundary),
|
||||
"Multipart data contains autogenerated boundary: %s", boundary);
|
||||
String multipart =
|
||||
String.format("--%s\r\n", boundary)
|
||||
+ String.format(
|
||||
"%s: form-data; name=\"%s\"; filename=\"%s\"\r\n",
|
||||
CONTENT_DISPOSITION, name, filename)
|
||||
+ String.format("%s: %s\r\n", CONTENT_TYPE, contentType)
|
||||
+ "\r\n"
|
||||
+ data
|
||||
+ "\r\n"
|
||||
+ String.format("--%s--\r\n", boundary);
|
||||
byte[] payload = multipart.getBytes(UTF_8);
|
||||
request.addHeader(
|
||||
new HTTPHeader(
|
||||
CONTENT_TYPE, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary)));
|
||||
request.addHeader(new HTTPHeader(CONTENT_LENGTH, Integer.toString(payload.length)));
|
||||
request.setPayload(payload);
|
||||
}
|
||||
|
||||
private static String createMultipartBoundary(Random random) {
|
||||
// Generate 192 random bits (24 bytes) to produce 192/log_2(64) = 192/6 = 32 base64 digits.
|
||||
byte[] rand = new byte[24];
|
||||
random.nextBytes(rand);
|
||||
// Boundary strings can be up to 70 characters long, so use 30 hyphens plus 32 random digits.
|
||||
// See https://tools.ietf.org/html/rfc2046#section-5.1.1
|
||||
return Strings.repeat("-", 30) + base64().encode(rand);
|
||||
}
|
||||
|
||||
/** Sets the HTTP Basic Authentication header on an {@link HTTPRequest}. */
|
||||
public static void setAuthorizationHeader(HTTPRequest req, Optional<String> login) {
|
||||
if (login.isPresent()) {
|
||||
String token = base64().encode(login.get().getBytes(UTF_8));
|
||||
req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue