Add action to sync all Registrar RDAP base URLs from the ICANN endpoint

We aren't turning on the cron job yet - we want to make sure it works correctly first.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=252726789

java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -309,6 +309,12 @@
     <url-pattern>/_dr/task/resaveAllEppResources</url-pattern>
   </servlet-mapping>
 
+  <!--  Reread all Registrar RDAP Base Urls from the ICANN endpoint. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/updateRegistrarRdapBaseUrls</url-pattern>
+  </servlet-mapping>
+
   <!--  Action to re-save a given entity. -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>

java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -112,6 +112,18 @@
     <target>backend</target>
   </cron>
 
+  <!--
+    TODO(b/134576418) enable this cron job once we're sure the Action works
+  <cron>
+    <url><![CDATA[/_dr/task/updateRegistrarRdapBaseUrls]]></url>
+    <description>
+      This job reloads all registrar RDAP base URLs from ICANN.
+    </description>
+    <schedule>every day 02:34</schedule>
+    <target>backend</target>
+  </cron>
+  -->
+
   <cron>
     <url><![CDATA[/_dr/task/deleteOldCommitLogs]]></url>
     <description>

java/google/registry/module/backend/BUILD

@@ -30,6 +30,7 @@ java_library(
         "//java/google/registry/model",
         "//java/google/registry/module",
         "//java/google/registry/monitoring/whitebox",
+        "//java/google/registry/rdap",
         "//java/google/registry/rde",
         "//java/google/registry/reporting",
         "//java/google/registry/reporting/billing",

java/google/registry/module/backend/BackendRequestComponent.java

@@ -53,6 +53,7 @@ import google.registry.export.sheet.SheetModule;
 import google.registry.export.sheet.SyncRegistrarsSheetAction;
 import google.registry.mapreduce.MapreduceModule;
 import google.registry.monitoring.whitebox.WhiteboxModule;
+import google.registry.rdap.UpdateRegistrarRdapBaseUrlsAction;
 import google.registry.rde.BrdaCopyAction;
 import google.registry.rde.RdeModule;
 import google.registry.rde.RdeReportAction;
@@ -148,6 +149,7 @@ interface BackendRequestComponent {
   TmchDnlAction tmchDnlAction();
   TmchSmdrlAction tmchSmdrlAction();
   UploadDatastoreBackupAction uploadDatastoreBackupAction();
+  UpdateRegistrarRdapBaseUrlsAction updateRegistrarRdapBaseUrlsAction();
   UpdateSnapshotViewAction updateSnapshotViewAction();
   PublishInvoicesAction uploadInvoicesAction();
 

java/google/registry/rdap/BUILD

@@ -10,6 +10,7 @@ java_library(
     deps = [
         "//java/google/registry/config",
         "//java/google/registry/flows",
+        "//java/google/registry/keyring/api",
         "//java/google/registry/model",
         "//java/google/registry/request",
         "//java/google/registry/request/auth",
@@ -23,6 +24,7 @@ java_library(
         "@com_google_flogger_system_backend",
         "@com_google_gson",
         "@com_google_guava",
+        "@com_google_http_client",
         "@com_google_http_client_jackson2",
         "@com_google_monitoring_client_metrics",
         "@com_google_re2j",

java/google/registry/rdap/UpdateRegistrarRdapBaseUrlsAction.java

@@ -0,0 +1,199 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.rdap;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpRequestFactory;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpTransport;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.io.ByteStreams;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.googlecode.objectify.Key;
+import google.registry.keyring.api.KeyModule;
+import google.registry.model.registrar.Registrar;
+import google.registry.request.Action;
+import google.registry.request.Parameter;
+import google.registry.request.auth.Auth;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.net.HttpCookie;
+import java.util.Optional;
+import javax.inject.Inject;
+
+/**
+ * Loads the current list of RDAP Base URLs from the ICANN servers.
+ *
+ * <p>This will update ALL the REAL registrars. If a REAL registrar doesn't have an RDAP entry in
+ * MoSAPI, we'll delete any BaseUrls it has.
+ *
+ * <p>The ICANN endpoint is described in the MoSAPI specifications, part 11:
+ * https://www.icann.org/en/system/files/files/mosapi-specification-30may19-en.pdf
+ *
+ * <p>It is a "login/query/logout" system where you login using the ICANN Reporting credentials, get
+ * a cookie you then send to get the list and finally logout.
+ *
+ * <p>The username is [TLD]_ry. It could be any "real" TLD.
+ */
+@Action(
+    service = Action.Service.BACKEND,
+    path = "/_dr/task/updateRegistrarRdapBaseUrls",
+    automaticallyPrintOk = true,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public final class UpdateRegistrarRdapBaseUrlsAction implements Runnable {
+
+  private static final String MOSAPI_BASE_URL = "https://mosapi.icann.org/mosapi/v1/%s/";
+  private static final String LOGIN_URL = MOSAPI_BASE_URL + "login";
+  private static final String LIST_URL = MOSAPI_BASE_URL + "registrarRdapBaseUrl/list";
+  private static final String LOGOUT_URL = MOSAPI_BASE_URL + "logout";
+  private static final String COOKIE_ID = "id";
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  @Inject HttpTransport httpTransport;
+  @Inject @KeyModule.Key("icannReportingPassword") String password;
+
+  /**
+   * The TLD for which we make the request.
+   *
+   * <p>The actual value doesn't matter, as long as it's a TLD that has access to the ICANN
+   * Reporter. It's just used to login.
+   */
+  @Inject @Parameter("tld") String tld;
+
+  @Inject
+  UpdateRegistrarRdapBaseUrlsAction() {}
+
+  private String loginAndGetId(HttpRequestFactory requestFactory) {
+    try {
+      logger.atInfo().log("Logging in to MoSAPI");
+      HttpRequest request =
+          requestFactory.buildGetRequest(new GenericUrl(String.format(LOGIN_URL, tld)));
+      request.getHeaders().setBasicAuthentication(String.format("%s_ry", tld), password);
+      HttpResponse response = request.execute();
+
+      Optional<HttpCookie> idCookie =
+          HttpCookie.parse(response.getHeaders().getFirstHeaderStringValue("Set-Cookie")).stream()
+              .filter(cookie -> cookie.getName().equals(COOKIE_ID))
+              .findAny();
+      checkState(
+          idCookie.isPresent(),
+          "Didn't get the ID cookie from the login response. Code: %s, headers: %s",
+          response.getStatusCode(),
+          response.getHeaders());
+      return idCookie.get().getValue();
+    } catch (IOException e) {
+      throw new UncheckedIOException("Error logging in to MoSAPI server: " + e.getMessage(), e);
+    }
+  }
+
+  private void logout(HttpRequestFactory requestFactory, String id) {
+    try {
+      HttpRequest request =
+          requestFactory.buildGetRequest(new GenericUrl(String.format(LOGOUT_URL, tld)));
+      request.getHeaders().setCookie(String.format("%s=%s", COOKIE_ID, id));
+      request.execute();
+    } catch (IOException e) {
+      logger.atWarning().withCause(e).log("Failed to log out of MoSAPI server. Continuing.");
+      // No need for the whole Action to fail if only the logout failed. We can just continue with
+      // the data we got.
+    }
+  }
+
+  private ImmutableSetMultimap<String, String> getRdapBaseUrlsPerIanaId() {
+    HttpRequestFactory requestFactory = httpTransport.createRequestFactory();
+    String id = loginAndGetId(requestFactory);
+    String content;
+    try {
+      HttpRequest request =
+          requestFactory.buildGetRequest(new GenericUrl(String.format(LIST_URL, tld)));
+      request.getHeaders().setCookie(String.format("%s=%s", COOKIE_ID, id));
+      HttpResponse response = request.execute();
+
+      try (InputStream input = response.getContent()) {
+        content = new String(ByteStreams.toByteArray(input), UTF_8);
+      }
+    } catch (IOException e) {
+      throw new UncheckedIOException(
+          "Error reading RDAP list from MoSAPI server: " + e.getMessage(), e);
+    } finally {
+      logout(requestFactory, id);
+    }
+
+    logger.atInfo().log("list reply: '%s'", content);
+    JsonObject listReply = new Gson().fromJson(content, JsonObject.class);
+    JsonArray services = listReply.getAsJsonArray("services");
+    // The format of the response "services" is an array of "ianaIDs to baseUrls", where "ianaIDs
+    // to baseUrls" is an array of size 2 where the first item is all the "iana IDs" and the
+    // second item all the "baseUrls".
+    ImmutableSetMultimap.Builder<String, String> builder = new ImmutableSetMultimap.Builder<>();
+    for (JsonElement service : services) {
+      for (JsonElement ianaId : service.getAsJsonArray().get(0).getAsJsonArray()) {
+        for (JsonElement baseUrl : service.getAsJsonArray().get(1).getAsJsonArray()) {
+          builder.put(ianaId.getAsString(), baseUrl.getAsString());
+        }
+      }
+    }
+
+    return builder.build();
+  }
+
+  @Override
+  public void run() {
+    ImmutableSetMultimap<String, String> ianaToBaseUrls = getRdapBaseUrlsPerIanaId();
+
+    for (Key<Registrar> registrarKey : ofy().load().type(Registrar.class).keys()) {
+      ofy()
+          .transact(
+              () -> {
+                Registrar registrar = ofy().load().key(registrarKey).now();
+                // Has the registrar been deleted since we loaded the key? (unlikly, especially
+                // given we don't delete registrars...)
+                if (registrar == null) {
+                  return;
+                }
+                // Only update REAL registrars
+                if (registrar.getType() != Registrar.Type.REAL) {
+                  return;
+                }
+                String ianaId = String.valueOf(registrar.getIanaIdentifier());
+                ImmutableSet<String> baseUrls = ianaToBaseUrls.get(ianaId);
+                // If this registrar already has these values, skip it
+                if (registrar.getRdapBaseUrls().equals(baseUrls)) {
+                  logger.atInfo().log(
+                      "No change in RdapBaseUrls for registrar %s (ianaId %s)",
+                      registrar.getClientId(), ianaId);
+                  return;
+                }
+                logger.atInfo().log(
+                    "Updating RdapBaseUrls for registrar %s (ianaId %s) from %s to %s",
+                    registrar.getClientId(), ianaId, registrar.getRdapBaseUrls(), baseUrls);
+                ofy()
+                    .save()
+                    .entity(registrar.asBuilder().setRdapBaseUrls(baseUrls).build());
+              });
+    }
+  }
+}

javatests/google/registry/module/backend/testdata/backend_routing.txt

@@ -38,5 +38,6 @@ PATH                                   CLASS                              METHOD
 /_dr/task/tmchCrl                      TmchCrlAction                      POST     y  INTERNAL     APP IGNORED
 /_dr/task/tmchDnl                      TmchDnlAction                      POST     y  INTERNAL     APP IGNORED
 /_dr/task/tmchSmdrl                    TmchSmdrlAction                    POST     y  INTERNAL     APP IGNORED
+/_dr/task/updateRegistrarRdapBaseUrls  UpdateRegistrarRdapBaseUrlsAction  GET      y  INTERNAL,API APP ADMIN
 /_dr/task/updateSnapshotView           UpdateSnapshotViewAction           POST     n  INTERNAL     APP IGNORED
 /_dr/task/uploadDatastoreBackup        UploadDatastoreBackupAction        POST     n  INTERNAL     APP IGNORED

javatests/google/registry/rdap/BUILD

@@ -26,6 +26,7 @@ java_library(
         "@com_google_dagger",
         "@com_google_gson",
         "@com_google_guava",
+        "@com_google_http_client",
         "@com_google_monitoring_client_contrib",
         "@com_google_truth",
         "@com_google_truth_extensions_truth_java8_extension",

javatests/google/registry/rdap/UpdateRegistrarRdapBaseUrlsActionTest.java

@@ -0,0 +1,240 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.rdap;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.testing.DatastoreHelper.loadRegistrar;
+import static google.registry.testing.DatastoreHelper.persistSimpleResource;
+
+import com.google.api.client.http.LowLevelHttpRequest;
+import com.google.api.client.testing.http.MockHttpTransport;
+import com.google.api.client.testing.http.MockLowLevelHttpRequest;
+import com.google.api.client.testing.http.MockLowLevelHttpResponse;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarAddress;
+import google.registry.testing.AppEngineRule;
+import google.registry.testing.ShardableTestCase;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link UpdateRegistrarRdapBaseUrlsAction}. */
+@RunWith(JUnit4.class)
+public final class UpdateRegistrarRdapBaseUrlsActionTest extends ShardableTestCase {
+
+  /**
+   * Example reply from the MoSAPI server.
+   *
+   * <p>This is the exact reply we got from the server when we tried to access it manually, with the
+   * addition of the 4000 and 4001 ones to test "multiple iana/servers in an element"
+   *
+   * <p>NOTE that 4000 has the same URL twice to make sure it doesn't break
+   * ImmutableSetMultimap.Builder
+   *
+   * <p>Also added value for IANA ID 9999, so we can check non-REAL registrars
+   */
+  private static final String JSON_LIST_REPLY =
+      "{\"publication\":\"2019-06-04T13:02:06Z\","
+          + "\"description\":\"ICANN-accredited Registrar's RDAP base URL list\","
+          + "\"services\":["
+          + "[[\"81\"],[\"https://rdap.gandi.net\"]],"
+          + "[[\"100\"],[\"https://yesnic.com/?_task=main&amp;_action=whois_search\"]],"
+          + "[[\"134\"],[\"https://rdap.bb-online.com\"]],"
+          + "[[\"1316\"],[\"https://whois.35.com\"]],"
+          + "[[\"1448\"],[\"https://rdap.blacknight.com\"]],"
+          + "[[\"1463\"],[\"https://rdap.domaincostclub.com/\"]],"
+          + "[[\"99999\"],[\"https://rdaptest.com\"]],"
+          + "[[\"1556\"],[\"https://rdap.west.cn\"]],"
+          + "[[\"2288\"],[\"https://rdap.metaregistrar.com\"]],"
+          + "[[\"4000\",\"4001\"],[\"https://rdap.example.com\"]],"
+          + "[[\"4000\"],[\"https://rdap.example.net\",\"https://rdap.example.org\"]],"
+          + "[[\"4000\"],[\"https://rdap.example.net\"]],"
+          + "[[\"9999\"],[\"https://rdap.example.net\"]]"
+          + "],"
+          + "\"version\":\"1.0\"}";
+
+  @Rule public AppEngineRule appEngineRule = new AppEngineRule.Builder().withDatastore().build();
+
+  private static class TestHttpTransport extends MockHttpTransport {
+    private final ArrayList<MockLowLevelHttpRequest> requestsSent = new ArrayList<>();
+    private final ArrayList<MockLowLevelHttpResponse> simulatedResponses = new ArrayList<>();
+
+    void addNextResponse(MockLowLevelHttpResponse response) {
+      simulatedResponses.add(response);
+    }
+    List<MockLowLevelHttpRequest> getRequestsSent() {
+      return requestsSent;
+    }
+
+    @Override
+    public LowLevelHttpRequest buildRequest(String method, String url) {
+      assertThat(method).isEqualTo("GET");
+      MockLowLevelHttpRequest httpRequest = new MockLowLevelHttpRequest(url);
+      httpRequest.setResponse(simulatedResponses.get(requestsSent.size()));
+      requestsSent.add(httpRequest);
+      return httpRequest;
+    }
+  }
+
+  TestHttpTransport httpTransport;
+  UpdateRegistrarRdapBaseUrlsAction action;
+
+  @Before
+  public void setUp() {
+    httpTransport = new TestHttpTransport();
+    action = new UpdateRegistrarRdapBaseUrlsAction();
+
+    action.password = "myPassword";
+    action.tld = "tld";
+    action.httpTransport = httpTransport;
+
+    MockLowLevelHttpResponse loginResponse = new MockLowLevelHttpResponse();
+    loginResponse.addHeader(
+        "Set-Cookie",
+        "id=myAuthenticationId; "
+            + "Expires=Tue, 11-Jun-2019 16:34:21 GMT; Path=/mosapi/v1/app; Secure; HttpOnly");
+
+    MockLowLevelHttpResponse listResponse = new MockLowLevelHttpResponse();
+    listResponse.setContent(JSON_LIST_REPLY);
+
+    MockLowLevelHttpResponse logoutResponse = new MockLowLevelHttpResponse();
+    loginResponse.addHeader(
+        "Set-Cookie",
+        "id=id; Expires=Thu, 01-Jan-1970 00:00:10 GMT; Path=/mosapi/v1/app; Secure; HttpOnly");
+
+    httpTransport.addNextResponse(loginResponse);
+    httpTransport.addNextResponse(listResponse);
+    httpTransport.addNextResponse(logoutResponse);
+
+  }
+
+  private void assertCorrectRequestsSent() {
+    // Doing assertThat on the "getUrl()" of the elements to get better error message if we have the
+    // wrong number of requests.
+    // This way we'll see which URLs were hit on failure.
+    assertThat(httpTransport.getRequestsSent().stream().map(request -> request.getUrl()))
+        .hasSize(3);
+
+    MockLowLevelHttpRequest loginRequest = httpTransport.getRequestsSent().get(0);
+    MockLowLevelHttpRequest listRequest = httpTransport.getRequestsSent().get(1);
+    MockLowLevelHttpRequest logoutRequest = httpTransport.getRequestsSent().get(2);
+
+    assertThat(loginRequest.getUrl()).isEqualTo("https://mosapi.icann.org/mosapi/v1/tld/login");
+    // base64.b64encode("tld_ry:myPassword") gives "dGxkX3J5Om15UGFzc3dvcmQ="
+    assertThat(loginRequest.getHeaders())
+        .containsEntry("authorization", ImmutableList.of("Basic dGxkX3J5Om15UGFzc3dvcmQ="));
+
+    assertThat(listRequest.getUrl())
+        .isEqualTo("https://mosapi.icann.org/mosapi/v1/tld/registrarRdapBaseUrl/list");
+    assertThat(listRequest.getHeaders())
+        .containsEntry("cookie", ImmutableList.of("id=myAuthenticationId"));
+
+    assertThat(logoutRequest.getUrl()).isEqualTo("https://mosapi.icann.org/mosapi/v1/tld/logout");
+    assertThat(logoutRequest.getHeaders())
+        .containsEntry("cookie", ImmutableList.of("id=myAuthenticationId"));
+  }
+
+  private static void persistRegistrar(
+      String clientId, Long ianaId, Registrar.Type type, String... rdapBaseUrls) {
+    persistSimpleResource(
+        new Registrar.Builder()
+        .setClientId(clientId)
+        .setRegistrarName(clientId)
+        .setType(type)
+        .setIanaIdentifier(ianaId)
+        .setRdapBaseUrls(ImmutableSet.copyOf(rdapBaseUrls))
+        .setLocalizedAddress(
+            new RegistrarAddress.Builder()
+            .setStreet(ImmutableList.of("123 fake st"))
+            .setCity("fakeCity")
+            .setCountryCode("XX")
+            .build())
+        .build());
+  }
+
+  @Test
+  public void testUnknownIana_cleared() {
+    // The IANA ID isn't in the JSON_LIST_REPLY
+    persistRegistrar("someRegistrar", 4123L, Registrar.Type.REAL, "http://rdap.example/blah");
+
+    action.run();
+
+    assertCorrectRequestsSent();
+
+    assertThat(loadRegistrar("someRegistrar").getRdapBaseUrls()).isEmpty();
+  }
+
+  @Test
+  public void testKnownIana_changed() {
+    // The IANA ID is in the JSON_LIST_REPLY
+    persistRegistrar("someRegistrar", 1448L, Registrar.Type.REAL, "http://rdap.example/blah");
+
+    action.run();
+
+    assertCorrectRequestsSent();
+
+    assertThat(loadRegistrar("someRegistrar").getRdapBaseUrls())
+        .containsExactly("https://rdap.blacknight.com");
+  }
+
+  @Test
+  public void testKnownIana_notReal_noChange() {
+    // The IANA ID is in the JSON_LIST_REPLY
+    persistRegistrar("someRegistrar", 9999L, Registrar.Type.INTERNAL, "http://rdap.example/blah");
+
+    action.run();
+
+    assertCorrectRequestsSent();
+
+    assertThat(loadRegistrar("someRegistrar").getRdapBaseUrls())
+        .containsExactly("http://rdap.example/blah");
+  }
+
+  @Test
+  public void testKnownIana_notReal_nullIANA_noChange() {
+    persistRegistrar("someRegistrar", null, Registrar.Type.TEST, "http://rdap.example/blah");
+
+    action.run();
+
+    assertCorrectRequestsSent();
+
+    assertThat(loadRegistrar("someRegistrar").getRdapBaseUrls())
+        .containsExactly("http://rdap.example/blah");
+  }
+
+  @Test
+  public void testKnownIana_multipleValues() {
+    // The IANA ID is in the JSON_LIST_REPLY
+    persistRegistrar("registrar4000", 4000L, Registrar.Type.REAL, "http://rdap.example/blah");
+    persistRegistrar("registrar4001", 4001L, Registrar.Type.REAL, "http://rdap.example/blah");
+
+    action.run();
+
+    assertCorrectRequestsSent();
+
+    assertThat(loadRegistrar("registrar4000").getRdapBaseUrls())
+        .containsExactly(
+            "https://rdap.example.com", "https://rdap.example.net", "https://rdap.example.org");
+    assertThat(loadRegistrar("registrar4001").getRdapBaseUrls())
+        .containsExactly("https://rdap.example.com");
+  }
+}