Add next page navigation for RDAP domain searches

In addition, while adding the tests, I became discontented with the thoroughness of the cursor navigation tests, which checked only the number of items returned, not their proper ordering. So I updated them to be more careful, and backported the changes to the nameserver and entity search tests as well.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=179442118

java/google/registry/rdap/RdapDomainSearchAction.java

@@ -52,6 +52,7 @@ import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Stream;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -120,6 +121,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
       throw new BadRequestException(
           "You must specify either name=XXXX, nsLdhName=YYYY or nsIp=ZZZZ");
     }
+    decodeCursorToken();
     RdapSearchResults results;
     if (nameParam.isPresent()) {
       metricInformationBuilder.setSearchType(SearchType.BY_DOMAIN_NAME);
@@ -163,7 +165,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
     rdapJsonFormatter.addTopLevelEntries(
         builder,
         BoilerplateType.DOMAIN,
-        results.getIncompletenessWarnings(),
+        getNotices(results),
         ImmutableList.of(),
         fullServletPath);
     return builder.build();
@@ -241,11 +243,14 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
             .load()
             .type(DomainResource.class)
             .filter("fullyQualifiedDomainName <", partialStringQuery.getNextInitialString())
-            .filter("fullyQualifiedDomainName >=", partialStringQuery.getInitialString())
+            .filter("fullyQualifiedDomainName >=", partialStringQuery.getInitialString());
-            .limit(querySizeLimit);
+    if (cursorString.isPresent()) {
+      query = query.filter("fullyQualifiedDomainName >", cursorString.get());
+    }
     if (partialStringQuery.getSuffix() != null) {
       query = query.filter("tld", partialStringQuery.getSuffix());
     }
+    query = query.limit(querySizeLimit);
     // Always check for visibility, because we couldn't look at the deletionTime in the query.
     return makeSearchResults(getMatchingResources(query, true, now, querySizeLimit), now);
   }
@@ -261,9 +266,11 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
         ofy()
             .load()
             .type(DomainResource.class)
-            .filter("tld", tld)
+            .filter("tld", tld);
-            .order("fullyQualifiedDomainName")
+    if (cursorString.isPresent()) {
-            .limit(querySizeLimit);
+      query = query.filter("fullyQualifiedDomainName >", cursorString.get());
+    }
+    query = query.order("fullyQualifiedDomainName").limit(querySizeLimit);
     return makeSearchResults(getMatchingResources(query, true, now, querySizeLimit), now);
   }
 
@@ -459,10 +466,19 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
           .filter("nsHosts in", chunk);
       if (!shouldIncludeDeleted()) {
         query = query.filter("deletionTime >", now);
+      // If we are not performing an inequality query, we can filter on the cursor in the query.
+      // Otherwise, we will need to filter the results afterward.
+      } else if (cursorString.isPresent()) {
+        query = query.filter("fullyQualifiedDomainName >", cursorString.get());
       }
-      Streams.stream(query)
+      Stream<DomainResource> stream =
-          .filter(domain -> isAuthorized(domain, now))
+          Streams.stream(query).filter(domain -> isAuthorized(domain, now));
-          .forEach(domainSetBuilder::add);
+      if (cursorString.isPresent()) {
+        stream =
+            stream.filter(
+                domain -> (domain.getFullyQualifiedDomainName().compareTo(cursorString.get()) > 0));
+      }
+      stream.forEach(domainSetBuilder::add);
     }
     List<DomainResource> domains = domainSetBuilder.build().asList();
     metricInformationBuilder.setNumHostsRetrieved(numHostKeysSearched);
@@ -519,7 +535,9 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
         (domains.size() > 1) ? OutputDataType.SUMMARY : OutputDataType.FULL;
     RdapAuthorization authorization = getAuthorization();
     List<ImmutableMap<String, Object>> jsonList = new ArrayList<>();
+    Optional<String> newCursor = Optional.empty();
     for (DomainResource domain : domains) {
+      newCursor = Optional.of(domain.getFullyQualifiedDomainName());
       jsonList.add(
           rdapJsonFormatter.makeRdapJsonForDomain(
               domain, false, fullServletPath, rdapWhoisServer, now, outputDataType, authorization));
@@ -533,6 +551,10 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
             : incompletenessWarningType;
     metricInformationBuilder.setIncompletenessWarningType(finalIncompletenessWarningType);
     return RdapSearchResults.create(
-        ImmutableList.copyOf(jsonList), finalIncompletenessWarningType, Optional.empty());
+        ImmutableList.copyOf(jsonList),
+        finalIncompletenessWarningType,
+        (finalIncompletenessWarningType == IncompletenessWarningType.TRUNCATED)
+            ? newCursor
+            : Optional.empty());
   }
 }

javatests/google/registry/rdap/RdapDomainSearchActionTest.java

@@ -36,6 +36,7 @@ import static org.mockito.Mockito.when;
 
 import com.google.appengine.api.users.User;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Range;
@@ -64,6 +65,7 @@ import google.registry.testing.InjectRule;
 import google.registry.ui.server.registrar.SessionUtils;
 import google.registry.util.Idn;
 import java.net.IDN;
+import java.net.URLDecoder;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -71,6 +73,7 @@ import java.util.Optional;
 import javax.annotation.Nullable;
 import javax.servlet.http.HttpServletRequest;
 import org.joda.time.DateTime;
+import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.json.simple.JSONValue;
 import org.junit.Before;
@@ -93,7 +96,6 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
   public final InjectRule inject = new InjectRule();
 
   private final HttpServletRequest request = mock(HttpServletRequest.class);
-  private final FakeResponse response = new FakeResponse();
   private final FakeClock clock = new FakeClock(DateTime.parse("2000-01-01T00:00:00Z"));
   private final SessionUtils sessionUtils = mock(SessionUtils.class);
   private final User user = new User("rdap.user@example.com", "gmail.com", "12345");
@@ -101,6 +103,8 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
   private final UserAuthInfo adminUserAuthInfo = UserAuthInfo.create(user, true);
   private final RdapDomainSearchAction action = new RdapDomainSearchAction();
 
+  private FakeResponse response = new FakeResponse();
+
   private Registrar registrar;
   private DomainResource domainCatLol;
   private DomainResource domainCatLol2;
@@ -118,30 +122,49 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
   enum RequestType { NONE, NAME, NS_LDH_NAME, NS_IP }
 
   private Object generateActualJson(RequestType requestType, String paramValue) {
+    return generateActualJson(requestType, paramValue, null);
+  }
+
+  private Object generateActualJson(
+      RequestType requestType, String paramValue, String cursor) {
     action.requestPath = RdapDomainSearchAction.PATH;
+    String requestTypeParam = null;
     switch (requestType) {
       case NAME:
         action.nameParam = Optional.of(paramValue);
         action.nsLdhNameParam = Optional.empty();
         action.nsIpParam = Optional.empty();
+        requestTypeParam = "name";
         break;
       case NS_LDH_NAME:
         action.nameParam = Optional.empty();
         action.nsLdhNameParam = Optional.of(paramValue);
         action.nsIpParam = Optional.empty();
+        requestTypeParam = "nsLdhName";
         break;
       case NS_IP:
         action.nameParam = Optional.empty();
         action.nsLdhNameParam = Optional.empty();
         action.nsIpParam = Optional.of(paramValue);
+        requestTypeParam = "nsIp";
         break;
       default:
         action.nameParam = Optional.empty();
         action.nsLdhNameParam = Optional.empty();
         action.nsIpParam = Optional.empty();
+        requestTypeParam = "";
         break;
     }
-    action.rdapResultSetMaxSize = 4;
+    if (paramValue != null) {
+      if (cursor == null) {
+        action.parameterMap = ImmutableListMultimap.of(requestTypeParam, paramValue);
+        action.cursorTokenParam = Optional.empty();
+      } else {
+        action.parameterMap =
+            ImmutableListMultimap.of(requestTypeParam, paramValue, "cursor", cursor);
+        action.cursorTokenParam = Optional.of(cursor);
+      }
+    }
     action.run();
     return JSONValue.parse(response.getPayload());
   }
@@ -371,6 +394,8 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
     action.request = request;
     action.requestMethod = Action.Method.GET;
     action.fullServletPath = "https://example.com/rdap";
+    action.requestUrl = "https://example.com/rdap/domains";
+    action.parameterMap = ImmutableListMultimap.of();
     action.requestMethod = POST;
     action.response = response;
     action.registrarParam = Optional.empty();
@@ -381,6 +406,8 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
     action.sessionUtils = sessionUtils;
     action.authResult = AuthResult.create(AuthLevel.USER, userAuthInfo);
     action.rdapMetrics = rdapMetrics;
+    action.cursorTokenParam = Optional.empty();
+    action.rdapResultSetMaxSize = 4;
   }
 
   private void login(String clientId) {
@@ -426,6 +453,30 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
       String domain4Name,
       String domain4Handle,
       String expectedOutputFile) {
+    return generateExpectedJsonForFourDomains(
+        domain1Name,
+        domain1Handle,
+        domain2Name,
+        domain2Handle,
+        domain3Name,
+        domain3Handle,
+        domain4Name,
+        domain4Handle,
+        "none",
+        expectedOutputFile);
+  }
+
+  private Object generateExpectedJsonForFourDomains(
+      String domain1Name,
+      String domain1Handle,
+      String domain2Name,
+      String domain2Handle,
+      String domain3Name,
+      String domain3Handle,
+      String domain4Name,
+      String domain4Handle,
+      String nextQuery,
+      String expectedOutputFile) {
     return JSONValue.parse(
         loadFile(
             this.getClass(),
@@ -444,6 +495,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
                 .put("DOMAINPUNYCODENAME4", domain4Name)
                 .put("DOMAINNAME4", IDN.toUnicode(domain4Name))
                 .put("DOMAINHANDLE4", domain4Handle)
+                .put("NEXT_QUERY", nextQuery)
                 .build()));
   }
 
@@ -567,6 +619,30 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
       String domainHandle3,
       String domainName4,
       String domainHandle4) {
+    return readMultiDomainFile(
+        fileName,
+        domainName1,
+        domainHandle1,
+        domainName2,
+        domainHandle2,
+        domainName3,
+        domainHandle3,
+        domainName4,
+        domainHandle4,
+        "none");
+  }
+
+  private Object readMultiDomainFile(
+      String fileName,
+      String domainName1,
+      String domainHandle1,
+      String domainName2,
+      String domainHandle2,
+      String domainName3,
+      String domainHandle3,
+      String domainName4,
+      String domainHandle4,
+      String nextQuery) {
     return JSONValue.parse(loadFile(
         this.getClass(),
         fileName,
@@ -579,6 +655,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
             .put("DOMAINHANDLE3", domainHandle3)
             .put("DOMAINNAME4", domainName4)
             .put("DOMAINHANDLE4", domainHandle4)
+            .put("NEXT_QUERY", nextQuery)
             .build()));
   }
 
@@ -645,6 +722,26 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
       String domainRoid3,
       String domainRoid4,
       String fileName) {
+    runSuccessfulTestWithFourDomains(
+        requestType,
+        queryString,
+        domainRoid1,
+        domainRoid2,
+        domainRoid3,
+        domainRoid4,
+        "none",
+        fileName);
+  }
+
+  private void runSuccessfulTestWithFourDomains(
+      RequestType requestType,
+      String queryString,
+      String domainRoid1,
+      String domainRoid2,
+      String domainRoid3,
+      String domainRoid4,
+      String nextQuery,
+      String fileName) {
     rememberWildcardType(queryString);
     assertThat(generateActualJson(requestType, queryString))
         .isEqualTo(
@@ -657,7 +754,8 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
                 "domain3.lol",
                 domainRoid3,
                 "domain4.lol",
-                domainRoid4));
+                domainRoid4,
+                nextQuery));
     assertThat(response.getStatus()).isEqualTo(200);
   }
 
@@ -731,6 +829,50 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
     verifyMetrics(searchType, numDomainsRetrieved, numHostsRetrieved);
   }
 
+  /**
+   * Checks multi-page result set navigation using the cursor.
+   *
+   * <p>If there are more results than the max result set size, the RDAP code returns a cursor token
+   * which can be used in a subsequent call to get the next chunk of results.
+   *
+   * @param requestType the type of query (name, nameserver name or nameserver address)
+   * @param paramValue the query string
+   * @param expectedNames an immutable list of the domain names we expect to retrieve
+   */
+  private void checkCursorNavigation(
+      RequestType requestType, String paramValue, ImmutableList<String> expectedNames)
+      throws Exception {
+    String cursor = null;
+    int expectedNameOffset = 0;
+    int expectedPageCount =
+        (expectedNames.size() + action.rdapResultSetMaxSize - 1) / action.rdapResultSetMaxSize;
+    for (int pageNum = 0; pageNum < expectedPageCount; pageNum++) {
+      Object results = generateActualJson(requestType, paramValue, cursor);
+      assertThat(response.getStatus()).isEqualTo(200);
+      String linkToNext = RdapTestHelper.getLinkToNext(results);
+      if (pageNum == expectedPageCount - 1) {
+        assertThat(linkToNext).isNull();
+      } else {
+        assertThat(linkToNext).isNotNull();
+        int pos = linkToNext.indexOf("cursor=");
+        assertThat(pos).isAtLeast(0);
+        cursor = URLDecoder.decode(linkToNext.substring(pos + 7), "UTF-8");
+        Object searchResults = ((JSONObject) results).get("domainSearchResults");
+        assertThat(searchResults).isInstanceOf(JSONArray.class);
+        assertThat(((JSONArray) searchResults)).hasSize(action.rdapResultSetMaxSize);
+        for (Object item : ((JSONArray) searchResults)) {
+          assertThat(item).isInstanceOf(JSONObject.class);
+          Object name = ((JSONObject) item).get("ldhName");
+          assertThat(name).isNotNull();
+          assertThat(name).isInstanceOf(String.class);
+          assertThat(name).isEqualTo(expectedNames.get(expectedNameOffset++));
+        }
+        response = new FakeResponse();
+        action.response = response;
+      }
+    }
+  }
+
   @Test
   public void testInvalidPath_rejected() throws Exception {
     action.requestPath = RdapDomainSearchAction.PATH + "/path";
@@ -1032,6 +1174,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
                 "cat.example", "21-EXAMPLE",
                 "cat.lol", "C-LOL",
                 "cat.xn--q9jyb4c", "2D-Q9JYB4C",
+                "name=cat*&cursor=Y2F0LnhuLS1xOWp5YjRj",
                 "rdap_domains_four_with_one_unicode_truncated.json"));
     assertThat(response.getStatus()).isEqualTo(200);
     verifyMetrics(SearchType.BY_DOMAIN_NAME, Optional.of(5L), IncompletenessWarningType.TRUNCATED);
@@ -1192,6 +1335,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
         "46-LOL",
         "45-LOL",
         "44-LOL",
+        "name=domain*.lol&cursor=ZG9tYWluNC5sb2w%3D",
         "rdap_domains_four_truncated.json");
     verifyMetrics(SearchType.BY_DOMAIN_NAME, Optional.of(5L), IncompletenessWarningType.TRUNCATED);
   }
@@ -1210,7 +1354,8 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
             "domain1.lol",
             "46-LOL",
             "domain2.lol",
-            "45-LOL"));
+            "45-LOL",
+            "name=*.lol&cursor=ZG9tYWluMi5sb2w%3D"));
     verifyMetrics(SearchType.BY_DOMAIN_NAME, Optional.of(5L), IncompletenessWarningType.TRUNCATED);
   }
 
@@ -1226,6 +1371,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
         "4A-LOL",
         "49-LOL",
         "48-LOL",
+        "name=domain*.lol&cursor=ZG9tYWluNC5sb2w%3D",
         "rdap_domains_four_truncated.json");
     verifyMetrics(SearchType.BY_DOMAIN_NAME, Optional.of(5L), IncompletenessWarningType.TRUNCATED);
   }
@@ -1244,11 +1390,54 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
             "domain24.lol",
             "49-LOL",
             "domain30.lol",
-            "43-LOL"));
+            "43-LOL",
+            "name=domain*.lol&cursor=ZG9tYWluMzAubG9s"));
     assertThat(response.getStatus()).isEqualTo(200);
     verifyMetrics(SearchType.BY_DOMAIN_NAME, Optional.of(27L), IncompletenessWarningType.TRUNCATED);
   }
 
+  @Test
+  public void testDomainMatch_cursorNavigationWithInitialString() throws Exception {
+    createManyDomainsAndHosts(11, 1, 2);
+    checkCursorNavigation(
+        RequestType.NAME,
+        "domain*.lol",
+        ImmutableList.of(
+            "domain1.lol",
+            "domain10.lol",
+            "domain11.lol",
+            "domain2.lol",
+            "domain3.lol",
+            "domain4.lol",
+            "domain5.lol",
+            "domain6.lol",
+            "domain7.lol",
+            "domain8.lol",
+            "domain9.lol"));
+  }
+
+  @Test
+  public void testDomainMatch_cursorNavigationWithTldSuffix() throws Exception {
+    createManyDomainsAndHosts(11, 1, 2);
+    checkCursorNavigation(
+        RequestType.NAME,
+        "*.lol",
+        ImmutableList.of(
+            "cat.lol",
+            "cat2.lol",
+            "domain1.lol",
+            "domain10.lol",
+            "domain11.lol",
+            "domain2.lol",
+            "domain3.lol",
+            "domain4.lol",
+            "domain5.lol",
+            "domain6.lol",
+            "domain7.lol",
+            "domain8.lol",
+            "domain9.lol"));
+  }
+
   @Test
   public void testNameserverMatch_foundMultiple() throws Exception {
     rememberWildcardType("ns1.cat.lol");
@@ -1595,6 +1784,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
         "46-LOL",
         "45-LOL",
         "44-LOL",
+        "nsLdhName=ns1.domain1.lol&cursor=ZG9tYWluNC5sb2w%3D",
         "rdap_domains_four_truncated.json");
     verifyMetrics(
         SearchType.BY_NAMESERVER_NAME,
@@ -1613,6 +1803,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
         "4A-LOL",
         "49-LOL",
         "48-LOL",
+        "nsLdhName=ns1.domain1.lol&cursor=ZG9tYWluNC5sb2w%3D",
         "rdap_domains_four_truncated.json");
     verifyMetrics(
         SearchType.BY_NAMESERVER_NAME,
@@ -1666,6 +1857,23 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
         IncompletenessWarningType.MIGHT_BE_INCOMPLETE);
   }
 
+  @Test
+  public void testNameserverMatch_cursorNavigation() throws Exception {
+    createManyDomainsAndHosts(8, 1, 2);
+    checkCursorNavigation(
+        RequestType.NS_LDH_NAME,
+        "ns*.domain1.lol",
+        ImmutableList.of(
+            "domain1.lol",
+            "domain2.lol",
+            "domain3.lol",
+            "domain4.lol",
+            "domain5.lol",
+            "domain6.lol",
+            "domain7.lol",
+            "domain8.lol"));
+  }
+
   @Test
   public void testAddressMatchV4Address_invalidAddress() throws Exception {
     rememberWildcardType("1.2.3.4.5.6.7.8.9");
@@ -1819,6 +2027,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
         "46-LOL",
         "45-LOL",
         "44-LOL",
+        "nsIp=5.5.5.1&cursor=ZG9tYWluNC5sb2w%3D",
         "rdap_domains_four_truncated.json");
     verifyMetrics(
         SearchType.BY_NAMESERVER_ADDRESS,
@@ -1837,6 +2046,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
         "4A-LOL",
         "49-LOL",
         "48-LOL",
+        "nsIp=5.5.5.1&cursor=ZG9tYWluNC5sb2w%3D",
         "rdap_domains_four_truncated.json");
     verifyMetrics(
         SearchType.BY_NAMESERVER_ADDRESS,
@@ -1844,4 +2054,21 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase {
         Optional.of(1L),
         IncompletenessWarningType.TRUNCATED);
   }
+
+  @Test
+  public void testAddressMatch_cursorNavigation() throws Exception {
+    createManyDomainsAndHosts(7, 1, 2);
+    checkCursorNavigation(
+        RequestType.NS_IP,
+        "5.5.5.1",
+        ImmutableList.of(
+            "domain1.lol",
+            "domain2.lol",
+            "domain3.lol",
+            "domain4.lol",
+            "domain5.lol",
+            "domain6.lol",
+            "domain7.lol",
+            "domain8.lol"));
+  }
 }

javatests/google/registry/rdap/RdapEntitySearchActionTest.java

@@ -419,35 +419,49 @@ public class RdapEntitySearchActionTest extends RdapSearchActionTestCase {
    * Checks multi-page result set navigation using the cursor.
    *
    * <p>If there are more results than the max result set size, the RDAP code returns a cursor token
-   * which can be used in a subsequent call to get the next chunk of results. This method starts by
+   * which can be used in a subsequent call to get the next chunk of results.
-   * making the query without a cursor, then follows the chain of pages using each returned cursor
-   * to ask for the next one, and makes sure that the expected number of pages are fetched.
    *
    * @param queryType type of query being run
-   * @param queryString the full name or handle query string
+   * @param paramValue the query string
-   * @param expectedPageCount how many pages we expect to retrieve; all but the last will have a
+   * @param expectedNames an immutable list of the entity names we expect to retrieve
-   *     cursor
    */
-  private void checkCursorNavigation(QueryType queryType, String queryString, int expectedPageCount)
+  private void checkCursorNavigation(
+      QueryType queryType, String paramValue, ImmutableList<String> expectedNames)
       throws Exception {
     String cursor = null;
-    for (int i = 0; i < expectedPageCount; i++) {
+    int expectedNameOffset = 0;
+    int expectedPageCount =
+        (expectedNames.size() + action.rdapResultSetMaxSize - 1) / action.rdapResultSetMaxSize;
+    for (int pageNum = 0; pageNum < expectedPageCount; pageNum++) {
       Object results =
           (queryType == QueryType.FULL_NAME)
-              ? generateActualJsonWithFullName(queryString, cursor)
+              ? generateActualJsonWithFullName(paramValue, cursor)
-              : generateActualJsonWithHandle(queryString, cursor);
+              : generateActualJsonWithHandle(paramValue, cursor);
       assertThat(response.getStatus()).isEqualTo(200);
       String linkToNext = RdapTestHelper.getLinkToNext(results);
-      if (i == expectedPageCount - 1) {
+      if (pageNum == expectedPageCount - 1) {
         assertThat(linkToNext).isNull();
       } else {
         assertThat(linkToNext).isNotNull();
         int pos = linkToNext.indexOf("cursor=");
         assertThat(pos).isAtLeast(0);
         cursor = URLDecoder.decode(linkToNext.substring(pos + 7), "UTF-8");
-        Object nameserverSearchResults = ((JSONObject) results).get("entitySearchResults");
+        Object searchResults = ((JSONObject) results).get("entitySearchResults");
-        assertThat(nameserverSearchResults).isInstanceOf(JSONArray.class);
+        assertThat(searchResults).isInstanceOf(JSONArray.class);
-        assertThat(((JSONArray) nameserverSearchResults)).hasSize(action.rdapResultSetMaxSize);
+        assertThat(((JSONArray) searchResults)).hasSize(action.rdapResultSetMaxSize);
+        for (Object item : ((JSONArray) searchResults)) {
+          assertThat(item).isInstanceOf(JSONObject.class);
+          Object vcardArray = ((JSONObject) item).get("vcardArray");
+          assertThat(vcardArray).isInstanceOf(JSONArray.class);
+          Object vcardData = ((JSONArray) vcardArray).get(1);
+          assertThat(vcardData).isInstanceOf(JSONArray.class);
+          Object vcardFn = ((JSONArray) vcardData).get(1);
+          assertThat(vcardFn).isInstanceOf(JSONArray.class);
+          Object name = ((JSONArray) vcardFn).get(3);
+          assertThat(name).isNotNull();
+          assertThat(name).isInstanceOf(String.class);
+          assertThat(name).isEqualTo(expectedNames.get(expectedNameOffset++));
+        }
         response = new FakeResponse();
         action.response = response;
       }
@@ -696,7 +710,19 @@ public class RdapEntitySearchActionTest extends RdapSearchActionTestCase {
   public void testNameMatchContacts_cursorNavigation() throws Exception {
     login("2-RegistrarTest");
     createManyContactsAndRegistrars(9, 0, registrarTest);
-    checkCursorNavigation(QueryType.FULL_NAME, "Entity *", 3);
+    checkCursorNavigation(
+        QueryType.FULL_NAME,
+        "Entity *",
+        ImmutableList.of(
+            "Entity 1",
+            "Entity 2",
+            "Entity 3",
+            "Entity 4",
+            "Entity 5",
+            "Entity 6",
+            "Entity 7",
+            "Entity 8",
+            "Entity 9"));
   }
 
   @Test
@@ -736,7 +762,23 @@ public class RdapEntitySearchActionTest extends RdapSearchActionTestCase {
   @Test
   public void testNameMatchRegistrars_cursorNavigation() throws Exception {
     createManyContactsAndRegistrars(0, 13, registrarTest);
-    checkCursorNavigation(QueryType.FULL_NAME, "Entity *", 4);
+    checkCursorNavigation(
+        QueryType.FULL_NAME,
+        "Entity *",
+        ImmutableList.of(
+            "Entity 1",
+            "Entity 10",
+            "Entity 11",
+            "Entity 12",
+            "Entity 13",
+            "Entity 2",
+            "Entity 3",
+            "Entity 4",
+            "Entity 5",
+            "Entity 6",
+            "Entity 7",
+            "Entity 8",
+            "Entity 9"));
   }
 
   @Test
@@ -756,7 +798,16 @@ public class RdapEntitySearchActionTest extends RdapSearchActionTestCase {
   public void testNameMatchMix_cursorNavigation() throws Exception {
     login("2-RegistrarTest");
     createManyContactsAndRegistrars(3, 3, registrarTest);
-    checkCursorNavigation(QueryType.FULL_NAME, "Entity *", 2);
+    checkCursorNavigation(
+        QueryType.FULL_NAME,
+        "Entity *",
+        ImmutableList.of(
+            "Entity 1",
+            "Entity 2",
+            "Entity 3",
+            "Entity 4",
+            "Entity 5",
+            "Entity 6"));
   }
 
   @Test
@@ -1002,14 +1053,49 @@ public class RdapEntitySearchActionTest extends RdapSearchActionTestCase {
 
   @Test
   public void testHandleMatchContact_cursorNavigationWithFullLastPage() throws Exception {
+    login("2-RegistrarTest");
     createManyContactsAndRegistrars(12, 0, registrarTest);
-    checkCursorNavigation(QueryType.HANDLE, "00*", 3);
+    checkCursorNavigation(
+        QueryType.HANDLE,
+        "00*",
+        // Contacts are returned in ROID order, not name order, by handle searches.
+        ImmutableList.of(
+            "Entity 1",
+            "Entity 2",
+            "Entity 3",
+            "Entity 4",
+            "Entity 5",
+            "Entity 6",
+            "Entity 7",
+            "Entity 8",
+            "Entity 9",
+            "Entity 10",
+            "Entity 11",
+            "Entity 12"));
   }
 
   @Test
   public void testHandleMatchContact_cursorNavigationWithPartialLastPage() throws Exception {
+    login("2-RegistrarTest");
     createManyContactsAndRegistrars(13, 0, registrarTest);
-    checkCursorNavigation(QueryType.HANDLE, "00*", 4);
+    checkCursorNavigation(
+        QueryType.HANDLE,
+        "00*",
+        // Contacts are returned in ROID order, not name order, by handle searches.
+        ImmutableList.of(
+            "Entity 1",
+            "Entity 2",
+            "Entity 3",
+            "Entity 4",
+            "Entity 5",
+            "Entity 6",
+            "Entity 7",
+            "Entity 8",
+            "Entity 9",
+            "Entity 10",
+            "Entity 11",
+            "Entity 12",
+            "Entity 13"));
   }
 
   @Test

javatests/google/registry/rdap/RdapNameserverSearchActionTest.java

@@ -272,7 +272,7 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
     ImmutableList.Builder<HostResource> hostsBuilder = new ImmutableList.Builder<>();
     ImmutableSet.Builder<String> subordinateHostsBuilder = new ImmutableSet.Builder<>();
     for (int i = 1; i <= numHosts; i++) {
-      String hostName = String.format("ns%d.cat.lol", i);
+      String hostName = String.format("nsx%d.cat.lol", i);
       subordinateHostsBuilder.add(hostName);
       hostsBuilder.add(makeHostResource(hostName, "5.5.5.1", "5.5.5.2"));
     }
@@ -575,7 +575,7 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   @Test
   public void testNameMatch_nontruncatedResultSet() throws Exception {
     createManyHosts(4);
-    assertThat(generateActualJsonWithName("ns*.cat.lol"))
+    assertThat(generateActualJsonWithName("nsx*.cat.lol"))
       .isEqualTo(generateExpectedJson("rdap_nontruncated_hosts.json"));
     assertThat(response.getStatus()).isEqualTo(200);
     verifyMetrics(4);
@@ -584,10 +584,10 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   @Test
   public void testNameMatch_truncatedResultSet() throws Exception {
     createManyHosts(5);
-    assertThat(generateActualJsonWithName("ns*.cat.lol"))
+    assertThat(generateActualJsonWithName("nsx*.cat.lol"))
         .isEqualTo(
             generateExpectedJson(
-                "name=ns*.cat.lol&cursor=bnM0LmNhdC5sb2w%3D", "rdap_truncated_hosts.json"));
+                "name=nsx*.cat.lol&cursor=bnN4NC5jYXQubG9s", "rdap_truncated_hosts.json"));
     assertThat(response.getStatus()).isEqualTo(200);
     verifyMetrics(5);
   }
@@ -595,10 +595,10 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   @Test
   public void testNameMatch_reallyTruncatedResultSet() throws Exception {
     createManyHosts(9);
-    assertThat(generateActualJsonWithName("ns*.cat.lol"))
+    assertThat(generateActualJsonWithName("nsx*.cat.lol"))
         .isEqualTo(
             generateExpectedJson(
-                "name=ns*.cat.lol&cursor=bnM0LmNhdC5sb2w%3D", "rdap_truncated_hosts.json"));
+                "name=nsx*.cat.lol&cursor=bnN4NC5jYXQubG9s", "rdap_truncated_hosts.json"));
     assertThat(response.getStatus()).isEqualTo(200);
     // When searching names, we look for additional matches, in case some are not visible.
     verifyMetrics(9);
@@ -716,30 +716,40 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
    * which can be used in a subsequent call to get the next chunk of results.
    *
    * @param byName true if we are searching by name; false if we are searching by address
-   * @param queryString the name or address query string
+   * @param paramValue the query string
-   * @param expectedPageCount how many pages we expect to retrieve; all but the last will have a
+   * @param expectedNames an immutable list of the host names we expect to retrieve
-   *     cursor
    */
-  private void checkCursorNavigation(boolean byName, String queryString, int expectedPageCount)
+  private void checkCursorNavigation(
+      boolean byName, String paramValue, ImmutableList<String> expectedNames)
       throws Exception {
     String cursor = null;
-    for (int i = 0; i < expectedPageCount; i++) {
+    int expectedNameOffset = 0;
+    int expectedPageCount =
+        (expectedNames.size() + action.rdapResultSetMaxSize - 1) / action.rdapResultSetMaxSize;
+    for (int pageNum = 0; pageNum < expectedPageCount; pageNum++) {
       Object results =
           byName
-              ? generateActualJsonWithName(queryString, cursor)
+              ? generateActualJsonWithName(paramValue, cursor)
-              : generateActualJsonWithIp(queryString, cursor);
+              : generateActualJsonWithIp(paramValue, cursor);
       assertThat(response.getStatus()).isEqualTo(200);
       String linkToNext = RdapTestHelper.getLinkToNext(results);
-      if (i == expectedPageCount - 1) {
+      if (pageNum == expectedPageCount - 1) {
         assertThat(linkToNext).isNull();
       } else {
         assertThat(linkToNext).isNotNull();
         int pos = linkToNext.indexOf("cursor=");
         assertThat(pos).isAtLeast(0);
         cursor = URLDecoder.decode(linkToNext.substring(pos + 7), "UTF-8");
-        Object nameserverSearchResults = ((JSONObject) results).get("nameserverSearchResults");
+        Object searchResults = ((JSONObject) results).get("nameserverSearchResults");
-        assertThat(nameserverSearchResults).isInstanceOf(JSONArray.class);
+        assertThat(searchResults).isInstanceOf(JSONArray.class);
-        assertThat(((JSONArray) nameserverSearchResults)).hasSize(action.rdapResultSetMaxSize);
+        assertThat(((JSONArray) searchResults)).hasSize(action.rdapResultSetMaxSize);
+        for (Object item : ((JSONArray) searchResults)) {
+          assertThat(item).isInstanceOf(JSONObject.class);
+          Object name = ((JSONObject) item).get("ldhName");
+          assertThat(name).isNotNull();
+          assertThat(name).isInstanceOf(String.class);
+          assertThat(name).isEqualTo(expectedNames.get(expectedNameOffset++));
+        }
         response = new FakeResponse();
         action.response = response;
       }
@@ -749,13 +759,43 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   @Test
   public void testNameMatch_cursorNavigationWithSuperordinateDomain() throws Exception {
     createManyHosts(9);
-    checkCursorNavigation(true, "ns*.cat.lol", 3);
+    checkCursorNavigation(
+        true,
+        "ns*.cat.lol",
+        ImmutableList.of(
+            "nsx1.cat.lol",
+            "nsx2.cat.lol",
+            "nsx3.cat.lol",
+            "nsx4.cat.lol",
+            "nsx5.cat.lol",
+            "nsx6.cat.lol",
+            "nsx7.cat.lol",
+            "nsx8.cat.lol",
+            "nsx9.cat.lol"));
   }
 
   @Test
   public void testNameMatch_cursorNavigationWithPrefix() throws Exception {
     createManyHosts(9);
-    checkCursorNavigation(true, "ns*", 4);
+    checkCursorNavigation(
+        true,
+        "ns*",
+        ImmutableList.of(
+            "ns1.cat.1.test",
+            "ns1.cat.external",
+            "ns1.cat.lol",
+            "ns1.cat.xn--q9jyb4c",
+            "ns1.cat2.lol",
+            "ns2.cat.lol",
+            "nsx1.cat.lol",
+            "nsx2.cat.lol",
+            "nsx3.cat.lol",
+            "nsx4.cat.lol",
+            "nsx5.cat.lol",
+            "nsx6.cat.lol",
+            "nsx7.cat.lol",
+            "nsx8.cat.lol",
+            "nsx9.cat.lol"));
   }
 
   @Test
@@ -921,6 +961,18 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   @Test
   public void testAddressMatch_cursorNavigation() throws Exception {
     createManyHosts(9);
-    checkCursorNavigation(false, "5.5.5.1", 3);
+    checkCursorNavigation(
+        false,
+        "5.5.5.1",
+        ImmutableList.of(
+            "nsx1.cat.lol",
+            "nsx2.cat.lol",
+            "nsx3.cat.lol",
+            "nsx4.cat.lol",
+            "nsx5.cat.lol",
+            "nsx6.cat.lol",
+            "nsx7.cat.lol",
+            "nsx8.cat.lol",
+            "nsx9.cat.lol"));
   }
 }

javatests/google/registry/rdap/testdata/rdap_domains_four_truncated.json

@@ -126,6 +126,18 @@
         "Search results per query are limited."
       ]
     },
+    {
+      "title" : "Navigation Links",
+      "description" : [ "Links to related pages." ],
+      "links" :
+      [
+        {
+          "type" : "application/rdap+json",
+          "rel" : "next",
+          "href" : "https://example.com/rdap/domains?%NEXT_QUERY%"
+        }
+      ]
+    },
     {
       "title" : "RDAP Terms of Service",
       "description" :

javatests/google/registry/rdap/testdata/rdap_domains_four_with_one_unicode_truncated.json

@@ -127,6 +127,18 @@
         "Search results per query are limited."
       ]
     },
+    {
+      "title" : "Navigation Links",
+      "description" : [ "Links to related pages." ],
+      "links" :
+      [
+        {
+          "type" : "application/rdap+json",
+          "rel" : "next",
+          "href" : "https://example.com/rdap/domains?%NEXT_QUERY%"
+        }
+      ]
+    },
     {
       "title" : "RDAP Terms of Service",
       "description" :

javatests/google/registry/rdap/testdata/rdap_nontruncated_hosts.json

@@ -5,14 +5,14 @@
       "objectClassName" : "nameserver",
       "handle" : "14-ROID",
       "status" : ["active"],
-      "ldhName" : "ns1.cat.lol",
+      "ldhName" : "nsx1.cat.lol",
       "links" :
       [
         {
-          "value" : "https://example.tld/rdap/nameserver/ns1.cat.lol",
+          "value" : "https://example.tld/rdap/nameserver/nsx1.cat.lol",
           "rel" : "self",
           "type" : "application/rdap+json",
-          "href" : "https://example.tld/rdap/nameserver/ns1.cat.lol"
+          "href" : "https://example.tld/rdap/nameserver/nsx1.cat.lol"
         }
       ],
       "ipAddresses" :
@@ -33,14 +33,14 @@
       "objectClassName" : "nameserver",
       "handle" : "15-ROID",
       "status" : ["active"],
-      "ldhName" : "ns2.cat.lol",
+      "ldhName" : "nsx2.cat.lol",
       "links" :
       [
         {
-          "value" : "https://example.tld/rdap/nameserver/ns2.cat.lol",
+          "value" : "https://example.tld/rdap/nameserver/nsx2.cat.lol",
           "rel" : "self",
           "type" : "application/rdap+json",
-          "href" : "https://example.tld/rdap/nameserver/ns2.cat.lol"
+          "href" : "https://example.tld/rdap/nameserver/nsx2.cat.lol"
         }
       ],
       "ipAddresses" :
@@ -61,14 +61,14 @@
       "objectClassName" : "nameserver",
       "handle" : "16-ROID",
       "status" : ["active"],
-      "ldhName" : "ns3.cat.lol",
+      "ldhName" : "nsx3.cat.lol",
       "links" :
       [
         {
-          "value" : "https://example.tld/rdap/nameserver/ns3.cat.lol",
+          "value" : "https://example.tld/rdap/nameserver/nsx3.cat.lol",
           "rel" : "self",
           "type" : "application/rdap+json",
-          "href" : "https://example.tld/rdap/nameserver/ns3.cat.lol"
+          "href" : "https://example.tld/rdap/nameserver/nsx3.cat.lol"
         }
       ],
       "ipAddresses" :
@@ -89,14 +89,14 @@
       "objectClassName" : "nameserver",
       "handle" : "17-ROID",
       "status" : ["active"],
-      "ldhName" : "ns4.cat.lol",
+      "ldhName" : "nsx4.cat.lol",
       "links" :
       [
         {
-          "value" : "https://example.tld/rdap/nameserver/ns4.cat.lol",
+          "value" : "https://example.tld/rdap/nameserver/nsx4.cat.lol",
           "rel" : "self",
           "type" : "application/rdap+json",
-          "href" : "https://example.tld/rdap/nameserver/ns4.cat.lol"
+          "href" : "https://example.tld/rdap/nameserver/nsx4.cat.lol"
         }
       ],
       "ipAddresses" :

javatests/google/registry/rdap/testdata/rdap_truncated_hosts.json

@@ -5,14 +5,14 @@
       "objectClassName" : "nameserver",
       "handle" : "14-ROID",
       "status" : ["active"],
-      "ldhName" : "ns1.cat.lol",
+      "ldhName" : "nsx1.cat.lol",
       "links" :
       [
         {
-          "value" : "https://example.tld/rdap/nameserver/ns1.cat.lol",
+          "value" : "https://example.tld/rdap/nameserver/nsx1.cat.lol",
           "rel" : "self",
           "type" : "application/rdap+json",
-          "href" : "https://example.tld/rdap/nameserver/ns1.cat.lol"
+          "href" : "https://example.tld/rdap/nameserver/nsx1.cat.lol"
         }
       ],
       "ipAddresses" :
@@ -33,14 +33,14 @@
       "objectClassName" : "nameserver",
       "handle" : "15-ROID",
       "status" : ["active"],
-      "ldhName" : "ns2.cat.lol",
+      "ldhName" : "nsx2.cat.lol",
       "links" :
       [
         {
-          "value" : "https://example.tld/rdap/nameserver/ns2.cat.lol",
+          "value" : "https://example.tld/rdap/nameserver/nsx2.cat.lol",
           "rel" : "self",
           "type" : "application/rdap+json",
-          "href" : "https://example.tld/rdap/nameserver/ns2.cat.lol"
+          "href" : "https://example.tld/rdap/nameserver/nsx2.cat.lol"
         }
       ],
       "ipAddresses" :
@@ -61,14 +61,14 @@
       "objectClassName" : "nameserver",
       "handle" : "16-ROID",
       "status" : ["active"],
-      "ldhName" : "ns3.cat.lol",
+      "ldhName" : "nsx3.cat.lol",
       "links" :
       [
         {
-          "value" : "https://example.tld/rdap/nameserver/ns3.cat.lol",
+          "value" : "https://example.tld/rdap/nameserver/nsx3.cat.lol",
           "rel" : "self",
           "type" : "application/rdap+json",
-          "href" : "https://example.tld/rdap/nameserver/ns3.cat.lol"
+          "href" : "https://example.tld/rdap/nameserver/nsx3.cat.lol"
         }
       ],
       "ipAddresses" :
@@ -89,14 +89,14 @@
       "objectClassName" : "nameserver",
       "handle" : "17-ROID",
       "status" : ["active"],
-      "ldhName" : "ns4.cat.lol",
+      "ldhName" : "nsx4.cat.lol",
       "links" :
       [
         {
-          "value" : "https://example.tld/rdap/nameserver/ns4.cat.lol",
+          "value" : "https://example.tld/rdap/nameserver/nsx4.cat.lol",
           "rel" : "self",
           "type" : "application/rdap+json",
-          "href" : "https://example.tld/rdap/nameserver/ns4.cat.lol"
+          "href" : "https://example.tld/rdap/nameserver/nsx4.cat.lol"
         }
       ],
       "ipAddresses" :