Improve logs in the GCP proxy

Tweaked a few logging levels to not spam error level logs. Also make it easy to debug issues in case relay retry fails.

[1] Put non-fatal exceptions that should be logged at warning in their explicit sets. Also always use the root cause to determine if an exception is non-fatal, because sometimes the actual causes are wrapped inside other exceptions.

[2] Record the cause of a relay failure, and record if a relay retry is successful. This way we can look at the log and figure out if a relay is eventually successful.

[3] Add a log when the frontend connection from the client is terminated.

[4] Alway close the relay channel when a relay has failed, which, depend on if the channel is frontend or backend, will reconnect and trigger a retry.

[5] Lastly changed failure test to use assertThrows instead of fail.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=208649916
This commit is contained in:
jianglai 2018-08-14 08:22:31 -07:00
parent b552c1d115
commit 0e64015cdf
10 changed files with 154 additions and 95 deletions

View file

@ -17,8 +17,8 @@ package google.registry.proxy;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.proxy.handler.SslInitializerTestUtils.getKeyPair;
import static google.registry.proxy.handler.SslInitializerTestUtils.signKeyPair;
import static google.registry.testing.JUnitBackports.assertThrows;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.fail;
import dagger.Component;
import dagger.Module;
@ -94,48 +94,36 @@ public class CertificateModuleTest {
public void testFailure_noPrivateKey() throws Exception {
byte[] pemBytes = getPemBytes(cert, ssc.cert());
component = createComponent(pemBytes);
try {
component.privateKey();
fail("Expect IllegalStateException.");
} catch (IllegalStateException e) {
assertThat(e).hasMessageThat().contains("0 keys are found");
}
IllegalStateException thrown =
assertThrows(IllegalStateException.class, () -> component.privateKey());
assertThat(thrown).hasMessageThat().contains("0 keys are found");
}
@Test
public void testFailure_twoPrivateKeys() throws Exception {
byte[] pemBytes = getPemBytes(cert, ssc.cert(), key, ssc.key());
component = createComponent(pemBytes);
try {
component.privateKey();
fail("Expect IllegalStateException.");
} catch (IllegalStateException e) {
assertThat(e).hasMessageThat().contains("2 keys are found");
}
IllegalStateException thrown =
assertThrows(IllegalStateException.class, () -> component.privateKey());
assertThat(thrown).hasMessageThat().contains("2 keys are found");
}
@Test
public void testFailure_certificatesOutOfOrder() throws Exception {
byte[] pemBytes = getPemBytes(ssc.cert(), cert, key);
component = createComponent(pemBytes);
try {
component.certificates();
fail("Expect IllegalStateException.");
} catch (IllegalStateException e) {
assertThat(e).hasMessageThat().contains("is not signed by");
}
IllegalStateException thrown =
assertThrows(IllegalStateException.class, () -> component.certificates());
assertThat(thrown).hasMessageThat().contains("is not signed by");
}
@Test
public void testFailure_noCertificates() throws Exception {
byte[] pemBytes = getPemBytes(key);
component = createComponent(pemBytes);
try {
component.certificates();
fail("Expect IllegalStateException.");
} catch (IllegalStateException e) {
assertThat(e).hasMessageThat().contains("No certificates");
}
IllegalStateException thrown =
assertThrows(IllegalStateException.class, () -> component.certificates());
assertThat(thrown).hasMessageThat().contains("No certificates");
}
@Module

View file

@ -17,14 +17,18 @@ package google.registry.proxy;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.proxy.handler.ProxyProtocolHandler.REMOTE_ADDRESS_KEY;
import static google.registry.proxy.handler.SslServerInitializer.CLIENT_CERTIFICATE_PROMISE_KEY;
import static google.registry.testing.JUnitBackports.assertThrows;
import static google.registry.util.ResourceUtils.readResourceBytes;
import static google.registry.util.X509Utils.getCertificateHash;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.base.Throwables;
import google.registry.proxy.handler.HttpsRelayServiceHandler.NonOkHttpResponseException;
import google.registry.testing.FakeClock;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.embedded.EmbeddedChannel;
import io.netty.handler.codec.EncoderException;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.FullHttpResponse;
import io.netty.handler.codec.http.HttpResponseStatus;
@ -108,8 +112,12 @@ public class EppProtocolModuleTest extends ProtocolModuleTest {
}
private FullHttpResponse makeEppHttpResponse(byte[] content, Cookie... cookies) {
return TestUtils.makeEppHttpResponse(
new String(content, UTF_8), HttpResponseStatus.OK, cookies);
return makeEppHttpResponse(content, HttpResponseStatus.OK, cookies);
}
private FullHttpResponse makeEppHttpResponse(
byte[] content, HttpResponseStatus status, Cookie... cookies) {
return TestUtils.makeEppHttpResponse(new String(content, UTF_8), status, cookies);
}
@Override
@ -209,6 +217,28 @@ public class EppProtocolModuleTest extends ProtocolModuleTest {
assertThat(channel.isActive()).isTrue();
}
@Test
public void testFailure_nonOkOutboundMessage() throws Exception {
// First inbound message is hello.
channel.readInbound();
byte[] outputBytes = readResourceBytes(getClass(), "testdata/login_response.xml").read();
// Verify outbound message is not written to the peer as the response is not OK.
EncoderException thrown =
assertThrows(
EncoderException.class,
() ->
channel.writeOutbound(
makeEppHttpResponse(outputBytes, HttpResponseStatus.UNAUTHORIZED)));
assertThat(Throwables.getRootCause(thrown)).isInstanceOf(NonOkHttpResponseException.class);
assertThat(thrown).hasMessageThat().contains("401 Unauthorized");
assertThat((Object) channel.readOutbound()).isNull();
// Channel is closed.
assertThat(channel.isActive()).isFalse();
}
@Test
public void testSuccess_setAndReadCookies() throws Exception {
// First inbound message is hello.

View file

@ -18,7 +18,6 @@ import static com.google.common.truth.Truth.assertThat;
import static google.registry.proxy.ProxyConfig.Environment.LOCAL;
import static google.registry.proxy.ProxyConfig.getProxyConfig;
import static google.registry.testing.JUnitBackports.assertThrows;
import static org.junit.Assert.fail;
import com.beust.jcommander.ParameterException;
import google.registry.proxy.ProxyConfig.Environment;
@ -66,12 +65,9 @@ public class ProxyModuleTest {
@Test
public void testFailure_parseArgs_wrongArguments() {
String[] args = {"--wrong_flag", "some_value"};
try {
proxyModule.parse(args);
fail("Expected ParameterException.");
} catch (ParameterException e) {
assertThat(e).hasMessageThat().contains("--wrong_flag");
}
ParameterException thrown =
assertThrows(ParameterException.class, () -> proxyModule.parse(args));
assertThat(thrown).hasMessageThat().contains("--wrong_flag");
}
@Test

View file

@ -20,10 +20,12 @@ import static google.registry.proxy.TestUtils.makeWhoisHttpResponse;
import static google.registry.testing.JUnitBackports.assertThrows;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static java.util.stream.Collectors.joining;
import static org.junit.Assert.fail;
import com.google.common.base.Throwables;
import google.registry.proxy.handler.HttpsRelayServiceHandler.NonOkHttpResponseException;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.handler.codec.EncoderException;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.FullHttpResponse;
import io.netty.handler.codec.http.HttpResponseStatus;
@ -152,13 +154,11 @@ public class WhoisProtocolModuleTest extends ProtocolModuleTest {
public void testFailure_outboundResponseStatusNotOK() {
String outputString = "line1\r\nline2\r\n";
FullHttpResponse response = makeWhoisHttpResponse(outputString, HttpResponseStatus.BAD_REQUEST);
try {
channel.writeOutbound(response);
fail("Expected failure due to non-OK HTTP response status");
} catch (Exception e) {
assertThat(e).hasCauseThat().isInstanceOf(IllegalArgumentException.class);
assertThat(e).hasMessageThat().contains("400 Bad Request");
}
EncoderException thrown =
assertThrows(EncoderException.class, () -> channel.writeOutbound(response));
assertThat(Throwables.getRootCause(thrown)).isInstanceOf(NonOkHttpResponseException.class);
assertThat(thrown).hasMessageThat().contains("400 Bad Request");
assertThat((Object) channel.readOutbound()).isNull();
assertThat(channel.isActive()).isFalse();
}
}

View file

@ -19,14 +19,16 @@ import static google.registry.proxy.TestUtils.assertHttpRequestEquivalent;
import static google.registry.proxy.TestUtils.makeEppHttpResponse;
import static google.registry.proxy.handler.ProxyProtocolHandler.REMOTE_ADDRESS_KEY;
import static google.registry.proxy.handler.SslServerInitializer.CLIENT_CERTIFICATE_PROMISE_KEY;
import static google.registry.testing.JUnitBackports.assertThrows;
import static google.registry.util.X509Utils.getCertificateHash;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import com.google.common.base.Throwables;
import google.registry.proxy.TestUtils;
import google.registry.proxy.handler.HttpsRelayServiceHandler.NonOkHttpResponseException;
import google.registry.proxy.metric.FrontendMetrics;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
@ -248,7 +250,7 @@ public class EppServiceHandlerTest {
response.headers().set("Epp-Session", "close");
channel.writeOutbound(response);
ByteBuf expectedResponse = channel.readOutbound();
assertThat(expectedResponse).isEqualTo(Unpooled.wrappedBuffer(content.getBytes(UTF_8)));
assertThat(Unpooled.wrappedBuffer(content.getBytes(UTF_8))).isEqualTo(expectedResponse);
// Nothing further to pass to the next handler.
assertThat((Object) channel.readOutbound()).isNull();
// Channel is disconnected.
@ -259,14 +261,16 @@ public class EppServiceHandlerTest {
public void testFailure_disconnectOnNonOKResponseStatus() throws Exception {
setHandshakeSuccess();
String content = "<epp>stuff</epp>";
try {
channel.writeOutbound(makeEppHttpResponse(content, HttpResponseStatus.BAD_REQUEST));
fail("Expected EncoderException");
} catch (EncoderException e) {
assertThat(e).hasCauseThat().isInstanceOf(IllegalArgumentException.class);
assertThat(e).hasMessageThat().contains(HttpResponseStatus.BAD_REQUEST.toString());
assertThat(channel.isActive()).isFalse();
}
EncoderException thrown =
assertThrows(
EncoderException.class,
() ->
channel.writeOutbound(
makeEppHttpResponse(content, HttpResponseStatus.BAD_REQUEST)));
assertThat(Throwables.getRootCause(thrown)).isInstanceOf(NonOkHttpResponseException.class);
assertThat(thrown).hasMessageThat().contains(HttpResponseStatus.BAD_REQUEST.toString());
assertThat((Object) channel.readOutbound()).isNull();
assertThat(channel.isActive()).isFalse();
}
@Test

View file

@ -85,7 +85,23 @@ public class RelayHandlerTest {
}
@Test
public void testSuccess_outboundClosed_enqueueBuffer() {
public void testSuccess_frontClosed() {
inboundChannel.attr(RELAY_BUFFER_KEY).set(null);
inboundChannel.attr(PROTOCOL_KEY).set(backendProtocol);
outboundChannel.attr(PROTOCOL_KEY).set(frontendProtocol);
ExpectedType inboundMessage = new ExpectedType();
// Outbound channel (frontend) is closed.
outboundChannel.finish();
assertThat(inboundChannel.writeInbound(inboundMessage)).isFalse();
ExpectedType relayedMessage = outboundChannel.readOutbound();
assertThat(relayedMessage).isNull();
// Inbound channel (backend) should stay open.
assertThat(inboundChannel.isActive()).isTrue();
assertThat(inboundChannel.attr(RELAY_BUFFER_KEY).get()).isNull();
}
@Test
public void testSuccess_backendClosed_enqueueBuffer() {
ExpectedType inboundMessage = new ExpectedType();
// Outbound channel (backend) is closed.
outboundChannel.finish();

View file

@ -17,17 +17,20 @@ package google.registry.proxy.handler;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.proxy.TestUtils.makeWhoisHttpRequest;
import static google.registry.proxy.TestUtils.makeWhoisHttpResponse;
import static google.registry.testing.JUnitBackports.assertThrows;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import com.google.common.base.Throwables;
import google.registry.proxy.handler.HttpsRelayServiceHandler.NonOkHttpResponseException;
import google.registry.proxy.metric.FrontendMetrics;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.DefaultChannelId;
import io.netty.channel.embedded.EmbeddedChannel;
import io.netty.handler.codec.EncoderException;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.FullHttpResponse;
import io.netty.handler.codec.http.HttpResponseStatus;
@ -116,13 +119,11 @@ public class WhoisServiceHandlerTest {
String outputString = "line1\r\nline2\r\n";
FullHttpResponse outputResponse =
makeWhoisHttpResponse(outputString, HttpResponseStatus.BAD_REQUEST);
try {
channel.writeOutbound(outputResponse);
fail("Expected failure due to non-OK HTTP response status.");
} catch (Exception e) {
assertThat(e).hasCauseThat().isInstanceOf(IllegalArgumentException.class);
assertThat(e).hasMessageThat().contains("400 Bad Request");
}
EncoderException thrown =
assertThrows(EncoderException.class, () -> channel.writeOutbound(outputResponse));
assertThat(Throwables.getRootCause(thrown)).isInstanceOf(NonOkHttpResponseException.class);
assertThat(thrown).hasMessageThat().contains("400 Bad Request");
assertThat((Object) channel.readOutbound()).isNull();
assertThat(channel.isActive()).isFalse();
}
}