diff --git a/java/google/registry/env/common/backend/WEB-INF/web.xml b/java/google/registry/env/common/backend/WEB-INF/web.xml index c9abe53b2..2ac301115 100644 --- a/java/google/registry/env/common/backend/WEB-INF/web.xml +++ b/java/google/registry/env/common/backend/WEB-INF/web.xml @@ -309,6 +309,12 @@ /_dr/task/resaveAllEppResources + + + backend-servlet + /_dr/task/updateRegistrarRdapBaseUrls + + backend-servlet diff --git a/java/google/registry/env/production/default/WEB-INF/cron.xml b/java/google/registry/env/production/default/WEB-INF/cron.xml index e1543d310..5a3041247 100644 --- a/java/google/registry/env/production/default/WEB-INF/cron.xml +++ b/java/google/registry/env/production/default/WEB-INF/cron.xml @@ -112,6 +112,18 @@ backend + + diff --git a/java/google/registry/module/backend/BUILD b/java/google/registry/module/backend/BUILD index d59646b24..37310df76 100644 --- a/java/google/registry/module/backend/BUILD +++ b/java/google/registry/module/backend/BUILD @@ -30,6 +30,7 @@ java_library( "//java/google/registry/model", "//java/google/registry/module", "//java/google/registry/monitoring/whitebox", + "//java/google/registry/rdap", "//java/google/registry/rde", "//java/google/registry/reporting", "//java/google/registry/reporting/billing", diff --git a/java/google/registry/module/backend/BackendRequestComponent.java b/java/google/registry/module/backend/BackendRequestComponent.java index 898bebb0b..ba70d1e84 100644 --- a/java/google/registry/module/backend/BackendRequestComponent.java +++ b/java/google/registry/module/backend/BackendRequestComponent.java @@ -53,6 +53,7 @@ import google.registry.export.sheet.SheetModule; import google.registry.export.sheet.SyncRegistrarsSheetAction; import google.registry.mapreduce.MapreduceModule; import google.registry.monitoring.whitebox.WhiteboxModule; +import google.registry.rdap.UpdateRegistrarRdapBaseUrlsAction; import google.registry.rde.BrdaCopyAction; import google.registry.rde.RdeModule; import google.registry.rde.RdeReportAction; @@ -148,6 +149,7 @@ interface BackendRequestComponent { TmchDnlAction tmchDnlAction(); TmchSmdrlAction tmchSmdrlAction(); UploadDatastoreBackupAction uploadDatastoreBackupAction(); + UpdateRegistrarRdapBaseUrlsAction updateRegistrarRdapBaseUrlsAction(); UpdateSnapshotViewAction updateSnapshotViewAction(); PublishInvoicesAction uploadInvoicesAction(); diff --git a/java/google/registry/rdap/BUILD b/java/google/registry/rdap/BUILD index f143256cc..8e1cc6b28 100644 --- a/java/google/registry/rdap/BUILD +++ b/java/google/registry/rdap/BUILD @@ -10,6 +10,7 @@ java_library( deps = [ "//java/google/registry/config", "//java/google/registry/flows", + "//java/google/registry/keyring/api", "//java/google/registry/model", "//java/google/registry/request", "//java/google/registry/request/auth", @@ -23,6 +24,7 @@ java_library( "@com_google_flogger_system_backend", "@com_google_gson", "@com_google_guava", + "@com_google_http_client", "@com_google_http_client_jackson2", "@com_google_monitoring_client_metrics", "@com_google_re2j", diff --git a/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java b/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java new file mode 100644 index 000000000..aad044037 --- /dev/null +++ b/java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java @@ -0,0 +1,199 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.rdap; + +import static com.google.common.base.Preconditions.checkState; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpTransport; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.flogger.FluentLogger; +import com.google.common.io.ByteStreams; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.googlecode.objectify.Key; +import google.registry.keyring.api.KeyModule; +import google.registry.model.registrar.Registrar; +import google.registry.request.Action; +import google.registry.request.Parameter; +import google.registry.request.auth.Auth; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.HttpCookie; +import java.util.Optional; +import javax.inject.Inject; + +/** + * Loads the current list of RDAP Base URLs from the ICANN servers. + * + *

This will update ALL the REAL registrars. If a REAL registrar doesn't have an RDAP entry in + * MoSAPI, we'll delete any BaseUrls it has. + * + *

The ICANN endpoint is described in the MoSAPI specifications, part 11: + * https://www.icann.org/en/system/files/files/mosapi-specification-30may19-en.pdf + * + *

It is a "login/query/logout" system where you login using the ICANN Reporting credentials, get + * a cookie you then send to get the list and finally logout. + * + *

The username is [TLD]_ry. It could be any "real" TLD. + */ +@Action( + service = Action.Service.BACKEND, + path = "/_dr/task/updateRegistrarRdapBaseUrls", + automaticallyPrintOk = true, + auth = Auth.AUTH_INTERNAL_OR_ADMIN) +public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable { + + private static final String MOSAPI_BASE_URL = "https://mosapi.icann.org/mosapi/v1/%s/"; + private static final String LOGIN_URL = MOSAPI_BASE_URL + "login"; + private static final String LIST_URL = MOSAPI_BASE_URL + "registrarRdapBaseUrl/list"; + private static final String LOGOUT_URL = MOSAPI_BASE_URL + "logout"; + private static final String COOKIE_ID = "id"; + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + @Inject HttpTransport httpTransport; + @Inject @KeyModule.Key("icannReportingPassword") String password; + + /** + * The TLD for which we make the request. + * + *

The actual value doesn't matter, as long as it's a TLD that has access to the ICANN + * Reporter. It's just used to login. + */ + @Inject @Parameter("tld") String tld; + + @Inject + UpdateRegistrarRdapBaseUrlsAction() {} + + private String loginAndGetId(HttpRequestFactory requestFactory) { + try { + logger.atInfo().log("Logging in to MoSAPI"); + HttpRequest request = + requestFactory.buildGetRequest(new GenericUrl(String.format(LOGIN_URL, tld))); + request.getHeaders().setBasicAuthentication(String.format("%s_ry", tld), password); + HttpResponse response = request.execute(); + + Optional idCookie = + HttpCookie.parse(response.getHeaders().getFirstHeaderStringValue("Set-Cookie")).stream() + .filter(cookie -> cookie.getName().equals(COOKIE_ID)) + .findAny(); + checkState( + idCookie.isPresent(), + "Didn't get the ID cookie from the login response. Code: %s, headers: %s", + response.getStatusCode(), + response.getHeaders()); + return idCookie.get().getValue(); + } catch (IOException e) { + throw new UncheckedIOException("Error logging in to MoSAPI server: " + e.getMessage(), e); + } + } + + private void logout(HttpRequestFactory requestFactory, String id) { + try { + HttpRequest request = + requestFactory.buildGetRequest(new GenericUrl(String.format(LOGOUT_URL, tld))); + request.getHeaders().setCookie(String.format("%s=%s", COOKIE_ID, id)); + request.execute(); + } catch (IOException e) { + logger.atWarning().withCause(e).log("Failed to log out of MoSAPI server. Continuing."); + // No need for the whole Action to fail if only the logout failed. We can just continue with + // the data we got. + } + } + + private ImmutableSetMultimap getRdapBaseUrlsPerIanaId() { + HttpRequestFactory requestFactory = httpTransport.createRequestFactory(); + String id = loginAndGetId(requestFactory); + String content; + try { + HttpRequest request = + requestFactory.buildGetRequest(new GenericUrl(String.format(LIST_URL, tld))); + request.getHeaders().setCookie(String.format("%s=%s", COOKIE_ID, id)); + HttpResponse response = request.execute(); + + try (InputStream input = response.getContent()) { + content = new String(ByteStreams.toByteArray(input), UTF_8); + } + } catch (IOException e) { + throw new UncheckedIOException( + "Error reading RDAP list from MoSAPI server: " + e.getMessage(), e); + } finally { + logout(requestFactory, id); + } + + logger.atInfo().log("list reply: '%s'", content); + JsonObject listReply = new Gson().fromJson(content, JsonObject.class); + JsonArray services = listReply.getAsJsonArray("services"); + // The format of the response "services" is an array of "ianaIDs to baseUrls", where "ianaIDs + // to baseUrls" is an array of size 2 where the first item is all the "iana IDs" and the + // second item all the "baseUrls". + ImmutableSetMultimap.Builder builder = new ImmutableSetMultimap.Builder<>(); + for (JsonElement service : services) { + for (JsonElement ianaId : service.getAsJsonArray().get(0).getAsJsonArray()) { + for (JsonElement baseUrl : service.getAsJsonArray().get(1).getAsJsonArray()) { + builder.put(ianaId.getAsString(), baseUrl.getAsString()); + } + } + } + + return builder.build(); + } + + @Override + public void run() { + ImmutableSetMultimap ianaToBaseUrls = getRdapBaseUrlsPerIanaId(); + + for (Key registrarKey : ofy().load().type(Registrar.class).keys()) { + ofy() + .transact( + () -> { + Registrar registrar = ofy().load().key(registrarKey).now(); + // Has the registrar been deleted since we loaded the key? (unlikly, especially + // given we don't delete registrars...) + if (registrar == null) { + return; + } + // Only update REAL registrars + if (registrar.getType() != Registrar.Type.REAL) { + return; + } + String ianaId = String.valueOf(registrar.getIanaIdentifier()); + ImmutableSet baseUrls = ianaToBaseUrls.get(ianaId); + // If this registrar already has these values, skip it + if (registrar.getRdapBaseUrls().equals(baseUrls)) { + logger.atInfo().log( + "No change in RdapBaseUrls for registrar %s (ianaId %s)", + registrar.getClientId(), ianaId); + return; + } + logger.atInfo().log( + "Updating RdapBaseUrls for registrar %s (ianaId %s) from %s to %s", + registrar.getClientId(), ianaId, registrar.getRdapBaseUrls(), baseUrls); + ofy() + .save() + .entity(registrar.asBuilder().setRdapBaseUrls(baseUrls).build()); + }); + } + } +} diff --git a/javatests/google/registry/module/backend/testdata/backend_routing.txt b/javatests/google/registry/module/backend/testdata/backend_routing.txt index 56592cd2b..1c52860e6 100644 --- a/javatests/google/registry/module/backend/testdata/backend_routing.txt +++ b/javatests/google/registry/module/backend/testdata/backend_routing.txt @@ -38,5 +38,6 @@ PATH CLASS METHOD /_dr/task/tmchCrl TmchCrlAction POST y INTERNAL APP IGNORED /_dr/task/tmchDnl TmchDnlAction POST y INTERNAL APP IGNORED /_dr/task/tmchSmdrl TmchSmdrlAction POST y INTERNAL APP IGNORED +/_dr/task/updateRegistrarRdapBaseUrls UpdateRegistrarRdapBaseUrlsAction GET y INTERNAL,API APP ADMIN /_dr/task/updateSnapshotView UpdateSnapshotViewAction POST n INTERNAL APP IGNORED /_dr/task/uploadDatastoreBackup UploadDatastoreBackupAction POST n INTERNAL APP IGNORED diff --git a/javatests/google/registry/rdap/BUILD b/javatests/google/registry/rdap/BUILD index 935d2de10..b12535846 100644 --- a/javatests/google/registry/rdap/BUILD +++ b/javatests/google/registry/rdap/BUILD @@ -26,6 +26,7 @@ java_library( "@com_google_dagger", "@com_google_gson", "@com_google_guava", + "@com_google_http_client", "@com_google_monitoring_client_contrib", "@com_google_truth", "@com_google_truth_extensions_truth_java8_extension", diff --git a/javatests/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java b/javatests/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java new file mode 100644 index 000000000..7bfcfd1a7 --- /dev/null +++ b/javatests/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java @@ -0,0 +1,240 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.rdap; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static google.registry.testing.DatastoreHelper.loadRegistrar; +import static google.registry.testing.DatastoreHelper.persistSimpleResource; + +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import google.registry.model.registrar.Registrar; +import google.registry.model.registrar.RegistrarAddress; +import google.registry.testing.AppEngineRule; +import google.registry.testing.ShardableTestCase; +import java.util.ArrayList; +import java.util.List; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link UpdateRegistrarRdapBaseUrlsAction}. */ +@RunWith(JUnit4.class) +public final class UpdateRegistrarRdapBaseUrlsActionTest extends ShardableTestCase { + + /** + * Example reply from the MoSAPI server. + * + *

This is the exact reply we got from the server when we tried to access it manually, with the + * addition of the 4000 and 4001 ones to test "multiple iana/servers in an element" + * + *

NOTE that 4000 has the same URL twice to make sure it doesn't break + * ImmutableSetMultimap.Builder + * + *

Also added value for IANA ID 9999, so we can check non-REAL registrars + */ + private static final String JSON_LIST_REPLY = + "{\"publication\":\"2019-06-04T13:02:06Z\"," + + "\"description\":\"ICANN-accredited Registrar's RDAP base URL list\"," + + "\"services\":[" + + "[[\"81\"],[\"https://rdap.gandi.net\"]]," + + "[[\"100\"],[\"https://yesnic.com/?_task=main&_action=whois_search\"]]," + + "[[\"134\"],[\"https://rdap.bb-online.com\"]]," + + "[[\"1316\"],[\"https://whois.35.com\"]]," + + "[[\"1448\"],[\"https://rdap.blacknight.com\"]]," + + "[[\"1463\"],[\"https://rdap.domaincostclub.com/\"]]," + + "[[\"99999\"],[\"https://rdaptest.com\"]]," + + "[[\"1556\"],[\"https://rdap.west.cn\"]]," + + "[[\"2288\"],[\"https://rdap.metaregistrar.com\"]]," + + "[[\"4000\",\"4001\"],[\"https://rdap.example.com\"]]," + + "[[\"4000\"],[\"https://rdap.example.net\",\"https://rdap.example.org\"]]," + + "[[\"4000\"],[\"https://rdap.example.net\"]]," + + "[[\"9999\"],[\"https://rdap.example.net\"]]" + + "]," + + "\"version\":\"1.0\"}"; + + @Rule public AppEngineRule appEngineRule = new AppEngineRule.Builder().withDatastore().build(); + + private static class TestHttpTransport extends MockHttpTransport { + private final ArrayList requestsSent = new ArrayList<>(); + private final ArrayList simulatedResponses = new ArrayList<>(); + + void addNextResponse(MockLowLevelHttpResponse response) { + simulatedResponses.add(response); + } + List getRequestsSent() { + return requestsSent; + } + + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + assertThat(method).isEqualTo("GET"); + MockLowLevelHttpRequest httpRequest = new MockLowLevelHttpRequest(url); + httpRequest.setResponse(simulatedResponses.get(requestsSent.size())); + requestsSent.add(httpRequest); + return httpRequest; + } + } + + TestHttpTransport httpTransport; + UpdateRegistrarRdapBaseUrlsAction action; + + @Before + public void setUp() { + httpTransport = new TestHttpTransport(); + action = new UpdateRegistrarRdapBaseUrlsAction(); + + action.password = "myPassword"; + action.tld = "tld"; + action.httpTransport = httpTransport; + + MockLowLevelHttpResponse loginResponse = new MockLowLevelHttpResponse(); + loginResponse.addHeader( + "Set-Cookie", + "id=myAuthenticationId; " + + "Expires=Tue, 11-Jun-2019 16:34:21 GMT; Path=/mosapi/v1/app; Secure; HttpOnly"); + + MockLowLevelHttpResponse listResponse = new MockLowLevelHttpResponse(); + listResponse.setContent(JSON_LIST_REPLY); + + MockLowLevelHttpResponse logoutResponse = new MockLowLevelHttpResponse(); + loginResponse.addHeader( + "Set-Cookie", + "id=id; Expires=Thu, 01-Jan-1970 00:00:10 GMT; Path=/mosapi/v1/app; Secure; HttpOnly"); + + httpTransport.addNextResponse(loginResponse); + httpTransport.addNextResponse(listResponse); + httpTransport.addNextResponse(logoutResponse); + + } + + private void assertCorrectRequestsSent() { + // Doing assertThat on the "getUrl()" of the elements to get better error message if we have the + // wrong number of requests. + // This way we'll see which URLs were hit on failure. + assertThat(httpTransport.getRequestsSent().stream().map(request -> request.getUrl())) + .hasSize(3); + + MockLowLevelHttpRequest loginRequest = httpTransport.getRequestsSent().get(0); + MockLowLevelHttpRequest listRequest = httpTransport.getRequestsSent().get(1); + MockLowLevelHttpRequest logoutRequest = httpTransport.getRequestsSent().get(2); + + assertThat(loginRequest.getUrl()).isEqualTo("https://mosapi.icann.org/mosapi/v1/tld/login"); + // base64.b64encode("tld_ry:myPassword") gives "dGxkX3J5Om15UGFzc3dvcmQ=" + assertThat(loginRequest.getHeaders()) + .containsEntry("authorization", ImmutableList.of("Basic dGxkX3J5Om15UGFzc3dvcmQ=")); + + assertThat(listRequest.getUrl()) + .isEqualTo("https://mosapi.icann.org/mosapi/v1/tld/registrarRdapBaseUrl/list"); + assertThat(listRequest.getHeaders()) + .containsEntry("cookie", ImmutableList.of("id=myAuthenticationId")); + + assertThat(logoutRequest.getUrl()).isEqualTo("https://mosapi.icann.org/mosapi/v1/tld/logout"); + assertThat(logoutRequest.getHeaders()) + .containsEntry("cookie", ImmutableList.of("id=myAuthenticationId")); + } + + private static void persistRegistrar( + String clientId, Long ianaId, Registrar.Type type, String... rdapBaseUrls) { + persistSimpleResource( + new Registrar.Builder() + .setClientId(clientId) + .setRegistrarName(clientId) + .setType(type) + .setIanaIdentifier(ianaId) + .setRdapBaseUrls(ImmutableSet.copyOf(rdapBaseUrls)) + .setLocalizedAddress( + new RegistrarAddress.Builder() + .setStreet(ImmutableList.of("123 fake st")) + .setCity("fakeCity") + .setCountryCode("XX") + .build()) + .build()); + } + + @Test + public void testUnknownIana_cleared() { + // The IANA ID isn't in the JSON_LIST_REPLY + persistRegistrar("someRegistrar", 4123L, Registrar.Type.REAL, "http://rdap.example/blah"); + + action.run(); + + assertCorrectRequestsSent(); + + assertThat(loadRegistrar("someRegistrar").getRdapBaseUrls()).isEmpty(); + } + + @Test + public void testKnownIana_changed() { + // The IANA ID is in the JSON_LIST_REPLY + persistRegistrar("someRegistrar", 1448L, Registrar.Type.REAL, "http://rdap.example/blah"); + + action.run(); + + assertCorrectRequestsSent(); + + assertThat(loadRegistrar("someRegistrar").getRdapBaseUrls()) + .containsExactly("https://rdap.blacknight.com"); + } + + @Test + public void testKnownIana_notReal_noChange() { + // The IANA ID is in the JSON_LIST_REPLY + persistRegistrar("someRegistrar", 9999L, Registrar.Type.INTERNAL, "http://rdap.example/blah"); + + action.run(); + + assertCorrectRequestsSent(); + + assertThat(loadRegistrar("someRegistrar").getRdapBaseUrls()) + .containsExactly("http://rdap.example/blah"); + } + + @Test + public void testKnownIana_notReal_nullIANA_noChange() { + persistRegistrar("someRegistrar", null, Registrar.Type.TEST, "http://rdap.example/blah"); + + action.run(); + + assertCorrectRequestsSent(); + + assertThat(loadRegistrar("someRegistrar").getRdapBaseUrls()) + .containsExactly("http://rdap.example/blah"); + } + + @Test + public void testKnownIana_multipleValues() { + // The IANA ID is in the JSON_LIST_REPLY + persistRegistrar("registrar4000", 4000L, Registrar.Type.REAL, "http://rdap.example/blah"); + persistRegistrar("registrar4001", 4001L, Registrar.Type.REAL, "http://rdap.example/blah"); + + action.run(); + + assertCorrectRequestsSent(); + + assertThat(loadRegistrar("registrar4000").getRdapBaseUrls()) + .containsExactly( + "https://rdap.example.com", "https://rdap.example.net", "https://rdap.example.org"); + assertThat(loadRegistrar("registrar4001").getRdapBaseUrls()) + .containsExactly("https://rdap.example.com"); + } +}