Retry RDE report on SocketTimeoutException

It's likely that either 1) we should be doing this for more than a
SocketTimeoutException - we probably want to do this in the case of a number
of different transient failures 2) we don't want to do this at all because it
happens in a background task that will get re-run anyway.

In any case, this seems like the right fix and it addresses a problem we see
occassionally.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=139935190
This commit is contained in:
mmuller 2016-11-22 11:40:41 -08:00 committed by Ben McIlwain
parent b3907d5d90
commit ddacd33c5e
3 changed files with 40 additions and 11 deletions

View file

@ -33,6 +33,7 @@ import google.registry.config.RegistryConfig;
import google.registry.keyring.api.KeyModule.Key; import google.registry.keyring.api.KeyModule.Key;
import google.registry.request.HttpException.InternalServerErrorException; import google.registry.request.HttpException.InternalServerErrorException;
import google.registry.util.FormattingLogger; import google.registry.util.FormattingLogger;
import google.registry.util.Retrier;
import google.registry.util.UrlFetchException; import google.registry.util.UrlFetchException;
import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.XjcXmlTransformer;
import google.registry.xjc.iirdea.XjcIirdeaResponseElement; import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
@ -43,7 +44,9 @@ import google.registry.xml.XmlException;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URL; import java.net.URL;
import java.util.concurrent.Callable;
import javax.inject.Inject; import javax.inject.Inject;
/** /**
@ -60,6 +63,7 @@ public class RdeReporter {
private static final String REPORT_MIME = "text/xml"; private static final String REPORT_MIME = "text/xml";
@Inject RegistryConfig config; @Inject RegistryConfig config;
@Inject Retrier retrier;
@Inject URLFetchService urlFetchService; @Inject URLFetchService urlFetchService;
@Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix; @Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix;
@Inject @Key("icannReportingPassword") String password; @Inject @Key("icannReportingPassword") String password;
@ -75,19 +79,26 @@ public class RdeReporter {
URL url = makeReportUrl(header.getTld(), report.getId()); URL url = makeReportUrl(header.getTld(), report.getId());
String username = header.getTld() + "_ry"; String username = header.getTld() + "_ry";
String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8)); String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8));
HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d)); final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d));
req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME)); req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME));
req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token)); req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
req.setPayload(reportBytes); req.setPayload(reportBytes);
logger.infofmt("Sending report:\n%s", new String(reportBytes, UTF_8)); logger.infofmt("Sending report:\n%s", new String(reportBytes, UTF_8));
HTTPResponse rsp = urlFetchService.fetch(req); HTTPResponse rsp = retrier.callWithRetry(
switch (rsp.getResponseCode()) { new Callable<HTTPResponse>() {
case SC_OK: @Override
case SC_BAD_REQUEST: public HTTPResponse call() throws Exception {
break; HTTPResponse rsp = urlFetchService.fetch(req);
default: switch (rsp.getResponseCode()) {
throw new UrlFetchException("PUT failed", req, rsp); case SC_OK:
} case SC_BAD_REQUEST:
break;
default:
throw new UrlFetchException("PUT failed", req, rsp);
}
return rsp;
}
}, SocketTimeoutException.class);
// Ensure the XML response is valid. // Ensure the XML response is valid.
XjcIirdeaResult result = parseResult(rsp); XjcIirdeaResult result = parseResult(rsp);

View file

@ -93,8 +93,8 @@ public class Retrier implements Serializable {
@SafeVarargs @SafeVarargs
public final <V> V callWithRetry( public final <V> V callWithRetry(
Callable<V> callable, Callable<V> callable,
Class<? extends RuntimeException> retryableError, Class<? extends Throwable> retryableError,
Class<? extends RuntimeException>... moreRetryableErrors) { Class<? extends Throwable>... moreRetryableErrors) {
final Set<Class<?>> retryables = final Set<Class<?>> retryables =
new ImmutableSet.Builder<Class<?>>().add(retryableError).add(moreRetryableErrors).build(); new ImmutableSet.Builder<Class<?>>().add(retryableError).add(moreRetryableErrors).build();
return callWithRetry(callable, new Predicate<Throwable>() { return callWithRetry(callable, new Predicate<Throwable>() {

View file

@ -52,11 +52,15 @@ import google.registry.request.HttpException.InternalServerErrorException;
import google.registry.testing.AppEngineRule; import google.registry.testing.AppEngineRule;
import google.registry.testing.BouncyCastleProviderRule; import google.registry.testing.BouncyCastleProviderRule;
import google.registry.testing.ExceptionRule; import google.registry.testing.ExceptionRule;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeResponse; import google.registry.testing.FakeResponse;
import google.registry.testing.FakeSleeper;
import google.registry.util.Retrier;
import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.XjcXmlTransformer;
import google.registry.xjc.rdereport.XjcRdeReportReport; import google.registry.xjc.rdereport.XjcRdeReportReport;
import google.registry.xml.XmlException; import google.registry.xml.XmlException;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.net.SocketTimeoutException;
import java.util.Map; import java.util.Map;
import org.bouncycastle.openpgp.PGPPublicKey; import org.bouncycastle.openpgp.PGPPublicKey;
import org.joda.time.DateTime; import org.joda.time.DateTime;
@ -103,6 +107,7 @@ public class RdeReportActionTest {
reporter.reportUrlPrefix = "https://rde-report.example"; reporter.reportUrlPrefix = "https://rde-report.example";
reporter.urlFetchService = urlFetchService; reporter.urlFetchService = urlFetchService;
reporter.password = "foo"; reporter.password = "foo";
reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
RdeReportAction action = new RdeReportAction(); RdeReportAction action = new RdeReportAction();
action.gcsUtils = new GcsUtils(gcsService, 1024); action.gcsUtils = new GcsUtils(gcsService, 1024);
action.ghostryde = new Ghostryde(1024); action.ghostryde = new Ghostryde(1024);
@ -185,6 +190,19 @@ public class RdeReportActionTest {
createAction().runWithLock(loadRdeReportCursor()); createAction().runWithLock(loadRdeReportCursor());
} }
@Test
public void testRunWithLock_socketTimeout_doesRetry() throws Exception {
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
when(urlFetchService.fetch(request.capture()))
.thenThrow(new SocketTimeoutException())
.thenReturn(httpResponse);
createAction().runWithLock(loadRdeReportCursor());
assertThat(response.getStatus()).isEqualTo(200);
assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
}
private DateTime loadRdeReportCursor() { private DateTime loadRdeReportCursor() {
return ofy() return ofy()
.load() .load()