From 3e473cb239caec4a9ddb83bdc56bcd9342dd9765 Mon Sep 17 00:00:00 2001 From: gbrodman Date: Wed, 3 Jul 2019 14:25:39 -0400 Subject: [PATCH] Attempt login to MosAPI via all available TLDs (#141) * Attempt login to MosAPI via all available TLDs There's no reason why we should need a TLD as input here because it doesn't actually matter which one we use (they all have the same password). * Refactor the TLD loop and change cron jobs * Re-throw the last exception if one exists * Fix tests and exception * Remove alpha cron job --- .../env/production/default/WEB-INF/cron.xml | 3 - .../UpdateRegistrarRdapBaseUrlsAction.java | 44 ++++++--- ...UpdateRegistrarRdapBaseUrlsActionTest.java | 99 +++++++++++++++---- 3 files changed, 106 insertions(+), 40 deletions(-) diff --git a/core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml b/core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml index 5a3041247..f55c5611a 100644 --- a/core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml +++ b/core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml @@ -112,8 +112,6 @@ backend - diff --git a/core/src/main/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java b/core/src/main/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java index 1529c89f5..7f8325dd0 100644 --- a/core/src/main/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java +++ b/core/src/main/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java @@ -14,6 +14,7 @@ package google.registry.rdap; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static google.registry.model.ofy.ObjectifyService.ofy; import static java.nio.charset.StandardCharsets.UTF_8; @@ -23,6 +24,7 @@ import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpTransport; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.flogger.FluentLogger; @@ -34,8 +36,9 @@ import com.google.gson.JsonObject; import com.googlecode.objectify.Key; import google.registry.keyring.api.KeyModule; import google.registry.model.registrar.Registrar; +import google.registry.model.registry.Registries; +import google.registry.model.registry.Registry.TldType; import google.registry.request.Action; -import google.registry.request.Parameter; import google.registry.request.auth.Auth; import java.io.IOException; import java.io.InputStream; @@ -56,8 +59,6 @@ import javax.inject.Inject; *

It is a "login/query/logout" system where you login using the ICANN Reporting credentials, get * a cookie you then send to get the list and finally logout. * - *

The username is [TLD]_ry. It could be any "real" TLD. - * *

For clarity, this is how one would contact this endpoint "manually", from a whitelisted IP * server: * @@ -88,18 +89,10 @@ public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable { @Inject HttpTransport httpTransport; @Inject @KeyModule.Key("icannReportingPassword") String password; - /** - * The TLD for which we make the request. - * - *

The actual value doesn't matter, as long as it's a TLD that has access to the ICANN - * Reporter. It's just used to login. - */ - @Inject @Parameter("tld") String tld; - @Inject UpdateRegistrarRdapBaseUrlsAction() {} - private String loginAndGetId(HttpRequestFactory requestFactory) { + private String loginAndGetId(HttpRequestFactory requestFactory, String tld) { try { logger.atInfo().log("Logging in to MoSAPI"); HttpRequest request = @@ -122,7 +115,7 @@ public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable { } } - private void logout(HttpRequestFactory requestFactory, String id) { + private void logout(HttpRequestFactory requestFactory, String id, String tld) { try { HttpRequest request = requestFactory.buildGetRequest(new GenericUrl(String.format(LOGOUT_URL, tld))); @@ -135,9 +128,9 @@ public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable { } } - private ImmutableSetMultimap getRdapBaseUrlsPerIanaId() { + private ImmutableSetMultimap getRdapBaseUrlsPerIanaIdWithTld(String tld) { HttpRequestFactory requestFactory = httpTransport.createRequestFactory(); - String id = loginAndGetId(requestFactory); + String id = loginAndGetId(requestFactory, tld); String content; try { HttpRequest request = @@ -152,7 +145,7 @@ public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable { throw new UncheckedIOException( "Error reading RDAP list from MoSAPI server: " + e.getMessage(), e); } finally { - logout(requestFactory, id); + logout(requestFactory, id, tld); } logger.atInfo().log("list reply: '%s'", content); @@ -173,6 +166,25 @@ public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable { return builder.build(); } + private ImmutableSetMultimap getRdapBaseUrlsPerIanaId() { + // All TLDs have the same data, so just keep trying until one works + // (the expectation is that all / any should work) + ImmutableList tlds = ImmutableList.sortedCopyOf(Registries.getTldsOfType(TldType.REAL)); + checkArgument(!tlds.isEmpty(), "There must exist at least one REAL TLD."); + Throwable finalThrowable = null; + for (String tld : tlds) { + try { + return getRdapBaseUrlsPerIanaIdWithTld(tld); + } catch (Throwable throwable) { + logger.atWarning().log(String + .format("Error retrieving RDAP urls with TLD %s: %s", tld, throwable.getMessage())); + finalThrowable = throwable; + } + } + throw new RuntimeException( + String.format("Error contacting MosAPI server. Tried TLDs %s", tlds), finalThrowable); + } + @Override public void run() { ImmutableSetMultimap ianaToBaseUrls = getRdapBaseUrlsPerIanaId(); diff --git a/core/src/test/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java b/core/src/test/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java index 7bfcfd1a7..fae5e8bd3 100644 --- a/core/src/test/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java +++ b/core/src/test/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java @@ -16,8 +16,12 @@ package google.registry.rdap; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth8.assertThat; +import static google.registry.testing.DatastoreHelper.createTld; +import static google.registry.testing.DatastoreHelper.deleteTld; import static google.registry.testing.DatastoreHelper.loadRegistrar; +import static google.registry.testing.DatastoreHelper.persistResource; import static google.registry.testing.DatastoreHelper.persistSimpleResource; +import static google.registry.testing.JUnitBackports.assertThrows; import com.google.api.client.http.LowLevelHttpRequest; import com.google.api.client.testing.http.MockHttpTransport; @@ -27,6 +31,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.RegistrarAddress; +import google.registry.model.registry.Registry; +import google.registry.model.registry.Registry.TldType; import google.registry.testing.AppEngineRule; import google.registry.testing.ShardableTestCase; import java.util.ArrayList; @@ -95,8 +101,8 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest extends ShardableTestCa } } - TestHttpTransport httpTransport; - UpdateRegistrarRdapBaseUrlsAction action; + private TestHttpTransport httpTransport; + private UpdateRegistrarRdapBaseUrlsAction action; @Before public void setUp() { @@ -104,27 +110,10 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest extends ShardableTestCa action = new UpdateRegistrarRdapBaseUrlsAction(); action.password = "myPassword"; - action.tld = "tld"; action.httpTransport = httpTransport; + addValidResponses(httpTransport); - MockLowLevelHttpResponse loginResponse = new MockLowLevelHttpResponse(); - loginResponse.addHeader( - "Set-Cookie", - "id=myAuthenticationId; " - + "Expires=Tue, 11-Jun-2019 16:34:21 GMT; Path=/mosapi/v1/app; Secure; HttpOnly"); - - MockLowLevelHttpResponse listResponse = new MockLowLevelHttpResponse(); - listResponse.setContent(JSON_LIST_REPLY); - - MockLowLevelHttpResponse logoutResponse = new MockLowLevelHttpResponse(); - loginResponse.addHeader( - "Set-Cookie", - "id=id; Expires=Thu, 01-Jan-1970 00:00:10 GMT; Path=/mosapi/v1/app; Secure; HttpOnly"); - - httpTransport.addNextResponse(loginResponse); - httpTransport.addNextResponse(listResponse); - httpTransport.addNextResponse(logoutResponse); - + createTld("tld"); } private void assertCorrectRequestsSent() { @@ -237,4 +226,72 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest extends ShardableTestCa assertThat(loadRegistrar("registrar4001").getRdapBaseUrls()) .containsExactly("https://rdap.example.com"); } + + @Test + public void testNoTlds() { + deleteTld("tld"); + assertThat(assertThrows(IllegalArgumentException.class, action::run)).hasMessageThat() + .isEqualTo("There must exist at least one REAL TLD."); + } + + @Test + public void testOnlyTestTlds() { + persistResource(Registry.get("tld").asBuilder().setTldType(TldType.TEST).build()); + assertThat(assertThrows(IllegalArgumentException.class, action::run)).hasMessageThat() + .isEqualTo("There must exist at least one REAL TLD."); + } + + @Test + public void testSecondTldSucceeds() { + createTld("secondtld"); + httpTransport = new TestHttpTransport(); + action.httpTransport = httpTransport; + + // the first TLD request will return a bad cookie but the second will succeed + MockLowLevelHttpResponse badLoginResponse = new MockLowLevelHttpResponse(); + badLoginResponse.addHeader("Set-Cookie", + "Expires=Thu, 01-Jan-1970 00:00:10 GMT; Path=/mosapi/v1/app; Secure; HttpOnly"); + + httpTransport.addNextResponse(badLoginResponse); + addValidResponses(httpTransport); + + action.run(); + } + + @Test + public void testBothFail() { + createTld("secondtld"); + httpTransport = new TestHttpTransport(); + action.httpTransport = httpTransport; + + MockLowLevelHttpResponse badLoginResponse = new MockLowLevelHttpResponse(); + badLoginResponse.addHeader("Set-Cookie", + "Expires=Thu, 01-Jan-1970 00:00:10 GMT; Path=/mosapi/v1/app; Secure; HttpOnly"); + + // it should fail for both TLDs + httpTransport.addNextResponse(badLoginResponse); + httpTransport.addNextResponse(badLoginResponse); + + assertThat(assertThrows(RuntimeException.class, action::run)).hasMessageThat() + .isEqualTo("Error contacting MosAPI server. Tried TLDs [secondtld, tld]"); + } + + private static void addValidResponses(TestHttpTransport httpTransport) { + MockLowLevelHttpResponse loginResponse = new MockLowLevelHttpResponse(); + loginResponse.addHeader( + "Set-Cookie", + "id=myAuthenticationId; " + + "Expires=Tue, 11-Jun-2019 16:34:21 GMT; Path=/mosapi/v1/app; Secure; HttpOnly"); + + MockLowLevelHttpResponse listResponse = new MockLowLevelHttpResponse(); + listResponse.setContent(JSON_LIST_REPLY); + + MockLowLevelHttpResponse logoutResponse = new MockLowLevelHttpResponse(); + logoutResponse.addHeader( + "Set-Cookie", + "id=id; Expires=Thu, 01-Jan-1970 00:00:10 GMT; Path=/mosapi/v1/app; Secure; HttpOnly"); + httpTransport.addNextResponse(loginResponse); + httpTransport.addNextResponse(listResponse); + httpTransport.addNextResponse(logoutResponse); + } }