Add next page navigation for RDAP nameserver searches

Domain and entity searches will be handled in future CLs.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=178912832

java/google/registry/rdap/RdapActionBase.java

@@ -62,7 +62,7 @@ import org.joda.time.DateTime;
 import org.json.simple.JSONValue;
 
 /**
- * Base RDAP (new WHOIS) action for single-item domain, nameserver and entity requests.
+ * Base RDAP (new WHOIS) action for all requests.
  *
  * @see <a href="https://tools.ietf.org/html/rfc7482">
  *        RFC 7482: Registration Data Access Protocol (RDAP) Query Format</a>
@@ -81,6 +81,12 @@ public abstract class RdapActionBase implements Runnable {
   private static final MediaType RESPONSE_MEDIA_TYPE =
       MediaType.create("application", "rdap+json").withCharset(UTF_8);
 
+  /** Whether to include or exclude deleted items from a query. */
+  protected enum DeletedItemHandling {
+    EXCLUDE,
+    INCLUDE
+  }
+
   @Inject HttpServletRequest request;
   @Inject Response response;
   @Inject @RequestMethod Action.Method requestMethod;
@@ -243,6 +249,10 @@ public abstract class RdapActionBase implements Runnable {
     return true;
   }
 
+  DeletedItemHandling getDeletedItemHandling() {
+    return shouldIncludeDeleted() ? DeletedItemHandling.INCLUDE : DeletedItemHandling.EXCLUDE;
+  }
+
   /**
    * Returns true if the request is authorized to see the resource.
    *
@@ -271,7 +281,9 @@ public abstract class RdapActionBase implements Runnable {
   }
 
   /**
-   * Returns true if the registrar should be visible. This is true iff:
+   * Returns true if the registrar should be visible.
+   *
+   * <p>This is true iff:
    * 1. The resource is active and publicly visible, or the request wants to see deleted items, and
    *     is authorized to do so, and:
    * 2. The request did not specify a registrar to filter on, or the registrar matches.
@@ -297,11 +309,28 @@ public abstract class RdapActionBase implements Runnable {
             : (rdapResultSetMaxSize + 1);
   }
 
+  static <T extends EppResource> Query<T> queryItems(
+      Class<T> clazz,
+      String filterField,
+      RdapSearchPattern partialStringQuery,
+      DeletedItemHandling deletedItemHandling,
+      int resultSetMaxSize) {
+    return queryItems(
+        clazz,
+        filterField,
+        partialStringQuery,
+        Optional.empty(),
+        deletedItemHandling,
+        resultSetMaxSize);
+  }
+
   /**
    * Handles prefix searches in cases where, if we need to filter out deleted items, there are no
-   * pending deletes. In such cases, it is sufficient to check whether {@code deletionTime} is equal
+   * pending deletes.
-   * to {@code END_OF_TIME}, because any other value means it has already been deleted. This allows
+   *
-   * us to use an equality query for the deletion time.
+   * <p>In such cases, it is sufficient to check whether {@code deletionTime} is equal to
+   * {@code END_OF_TIME}, because any other value means it has already been deleted. This allows us
+   * to use an equality query for the deletion time.
    *
    * @param clazz the type of resource to be queried
    * @param filterField the database field of interest
@@ -309,15 +338,18 @@ public abstract class RdapActionBase implements Runnable {
    *        equality query is used; if there is a wildcard, a range query is used instead; the
    *        initial string should not be empty, and any search suffix will be ignored, so the caller
    *        must filter the results if a suffix is specified
-   * @param includeDeleted whether to search for deleted items as well
+   * @param cursorString if a cursor is present, this parameter should specify the cursor string, to
+   *        skip any results up to and including the string; empty() if there is no cursor
+   * @param deletedItemHandling whether to include or exclude deleted items
    * @param resultSetMaxSize the maximum number of results to return
-   * @return the results of the query
+   * @return the query object
    */
   static <T extends EppResource> Query<T> queryItems(
       Class<T> clazz,
       String filterField,
       RdapSearchPattern partialStringQuery,
-      boolean includeDeleted,
+      Optional<String> cursorString,
+      DeletedItemHandling deletedItemHandling,
       int resultSetMaxSize) {
     if (partialStringQuery.getInitialString().length()
         < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) {
@@ -335,18 +367,36 @@ public abstract class RdapActionBase implements Runnable {
           .filter(filterField + " >=", partialStringQuery.getInitialString())
           .filter(filterField + " <", partialStringQuery.getNextInitialString());
     }
-    if (!includeDeleted) {
+    if (cursorString.isPresent()) {
-      query = query.filter("deletionTime", END_OF_TIME);
+      query = query.filter(filterField + " >", cursorString.get());
     }
-    return query.limit(resultSetMaxSize);
+    return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize);
   }
 
-  /** Variant of queryItems using a simple string rather than an {@link RdapSearchPattern}. */
+  /**
+   * Handles searches using a simple string rather than an {@link RdapSearchPattern}.
+   *
+   * <p>Since the filter is not an inequality, we can support also checking a cursor string against
+   * a different field (which involves an inequality on that field).
+   *
+   * @param clazz the type of resource to be queried
+   * @param filterField the database field of interest
+   * @param queryString the search string
+   * @param cursorField the field which should be compared to the cursor string, or empty() if the
+   *        key should be compared to a key created from the cursor string
+   * @param cursorString if a cursor is present, this parameter should specify the cursor string, to
+   *        skip any results up to and including the string; empty() if there is no cursor
+   * @param deletedItemHandling whether to include or exclude deleted items
+   * @param resultSetMaxSize the maximum number of results to return
+   * @return the query object
+   */
   static <T extends EppResource> Query<T> queryItems(
       Class<T> clazz,
       String filterField,
       String queryString,
-      boolean includeDeleted,
+      Optional<String> cursorField,
+      Optional<String> cursorString,
+      DeletedItemHandling deletedItemHandling,
       int resultSetMaxSize) {
     if (queryString.length() < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) {
       throw new UnprocessableEntityException(
@@ -355,14 +405,21 @@ public abstract class RdapActionBase implements Runnable {
               RdapSearchPattern.MIN_INITIAL_STRING_LENGTH));
     }
     Query<T> query = ofy().load().type(clazz).filter(filterField, queryString);
-    return setOtherQueryAttributes(query, includeDeleted, resultSetMaxSize);
+    if (cursorString.isPresent()) {
+      if (cursorField.isPresent()) {
+        query = query.filter(cursorField.get() + " >", cursorString.get());
+      } else {
+        query = query.filterKey(">", Key.create(clazz, cursorString.get()));
+      }
+    }
+    return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize);
   }
 
-  /** Variant of queryItems where the field to be searched is the key. */
+  /** Handles searches where the field to be searched is the key. */
   static <T extends EppResource> Query<T> queryItemsByKey(
       Class<T> clazz,
       RdapSearchPattern partialStringQuery,
-      boolean includeDeleted,
+      DeletedItemHandling deletedItemHandling,
       int resultSetMaxSize) {
     if (partialStringQuery.getInitialString().length()
         < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) {
@@ -380,14 +437,14 @@ public abstract class RdapActionBase implements Runnable {
           .filterKey(">=", Key.create(clazz, partialStringQuery.getInitialString()))
           .filterKey("<", Key.create(clazz, partialStringQuery.getNextInitialString()));
     }
-    return setOtherQueryAttributes(query, includeDeleted, resultSetMaxSize);
+    return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize);
   }
 
-  /** Variant of queryItems searching for a key by a simple string. */
+  /** Handles searches by key using a simple string. */
   static <T extends EppResource> Query<T> queryItemsByKey(
       Class<T> clazz,
       String queryString,
-      boolean includeDeleted,
+      DeletedItemHandling deletedItemHandling,
       int resultSetMaxSize) {
     if (queryString.length() < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) {
       throw new UnprocessableEntityException(
@@ -396,12 +453,12 @@ public abstract class RdapActionBase implements Runnable {
               RdapSearchPattern.MIN_INITIAL_STRING_LENGTH));
     }
     Query<T> query = ofy().load().type(clazz).filterKey("=", Key.create(clazz, queryString));
-    return setOtherQueryAttributes(query, includeDeleted, resultSetMaxSize);
+    return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize);
   }
 
   private static <T extends EppResource> Query<T> setOtherQueryAttributes(
-      Query<T> query, boolean includeDeleted, int resultSetMaxSize) {
+      Query<T> query, DeletedItemHandling deletedItemHandling, int resultSetMaxSize) {
-    if (!includeDeleted) {
+    if (deletedItemHandling != DeletedItemHandling.INCLUDE) {
       query = query.filter("deletionTime", END_OF_TIME);
     }
     return query.limit(resultSetMaxSize);

java/google/registry/rdap/RdapDomainSearchAction.java

@@ -70,7 +70,7 @@ import org.joda.time.DateTime;
   method = {GET, HEAD},
   auth = Auth.AUTH_PUBLIC
 )
-public class RdapDomainSearchAction extends RdapActionBase {
+public class RdapDomainSearchAction extends RdapSearchActionBase {
 
   public static final String PATH = "/rdap/domains";
 
@@ -320,7 +320,7 @@ public class RdapDomainSearchAction extends RdapActionBase {
             HostResource.class,
             "fullyQualifiedHostName",
             partialStringQuery,
-            false, /* includeDeleted */
+            DeletedItemHandling.EXCLUDE,
             MAX_NAMESERVERS_IN_FIRST_STAGE);
     Optional<String> desiredRegistrar = getDesiredRegistrar();
     if (desiredRegistrar.isPresent()) {
@@ -423,7 +423,9 @@ public class RdapDomainSearchAction extends RdapActionBase {
             HostResource.class,
             "inetAddresses",
             inetAddress.getHostAddress(),
-            false,
+            Optional.empty(),
+            Optional.empty(),
+            DeletedItemHandling.EXCLUDE,
             MAX_NAMESERVERS_IN_FIRST_STAGE);
     Optional<String> desiredRegistrar = getDesiredRegistrar();
     if (desiredRegistrar.isPresent()) {
@@ -530,6 +532,7 @@ public class RdapDomainSearchAction extends RdapActionBase {
             ? IncompletenessWarningType.TRUNCATED
             : incompletenessWarningType;
     metricInformationBuilder.setIncompletenessWarningType(finalIncompletenessWarningType);
-    return RdapSearchResults.create(ImmutableList.copyOf(jsonList), finalIncompletenessWarningType);
+    return RdapSearchResults.create(
+        ImmutableList.copyOf(jsonList), finalIncompletenessWarningType, Optional.empty());
   }
 }

java/google/registry/rdap/RdapEntitySearchAction.java

@@ -61,7 +61,7 @@ import org.joda.time.DateTime;
   method = {GET, HEAD},
   auth = Auth.AUTH_PUBLIC
 )
-public class RdapEntitySearchAction extends RdapActionBase {
+public class RdapEntitySearchAction extends RdapSearchActionBase {
 
   public static final String PATH = "/rdap/entities";
 
@@ -188,7 +188,7 @@ public class RdapEntitySearchAction extends RdapActionBase {
               ContactResource.class,
               "searchName",
               partialStringQuery,
-              false,
+              DeletedItemHandling.EXCLUDE,
               rdapResultSetMaxSize + 1);
       if (authorization.role() != RdapAuthorization.Role.ADMINISTRATOR) {
         query = query.filter("currentSponsorClientId in", authorization.clientIds());
@@ -244,7 +244,7 @@ public class RdapEntitySearchAction extends RdapActionBase {
       int querySizeLimit = getStandardQuerySizeLimit();
       Query<ContactResource> query =
           queryItemsByKey(
-              ContactResource.class, partialStringQuery, shouldIncludeDeleted(), querySizeLimit);
+              ContactResource.class, partialStringQuery, getDeletedItemHandling(), querySizeLimit);
       return makeSearchResults(
           getMatchingResources(query, shouldIncludeDeleted(), now, querySizeLimit),
           registrars,
@@ -317,7 +317,9 @@ public class RdapEntitySearchAction extends RdapActionBase {
     for (ContactResource contact : contacts) {
       if (jsonOutputList.size() >= rdapResultSetMaxSize) {
         return RdapSearchResults.create(
-            ImmutableList.copyOf(jsonOutputList), IncompletenessWarningType.TRUNCATED);
+            ImmutableList.copyOf(jsonOutputList),
+            IncompletenessWarningType.TRUNCATED,
+            Optional.empty());
       }
       // As per Andy Newton on the regext mailing list, contacts by themselves have no role, since
       // they are global, and might have different roles for different domains.
@@ -334,7 +336,9 @@ public class RdapEntitySearchAction extends RdapActionBase {
     for (Registrar registrar : registrars) {
       if (jsonOutputList.size() >= rdapResultSetMaxSize) {
         return RdapSearchResults.create(
-            ImmutableList.copyOf(jsonOutputList), IncompletenessWarningType.TRUNCATED);
+            ImmutableList.copyOf(jsonOutputList),
+            IncompletenessWarningType.TRUNCATED,
+            Optional.empty());
       }
       jsonOutputList.add(rdapJsonFormatter.makeRdapJsonForRegistrar(
           registrar, false, fullServletPath, rdapWhoisServer, now, outputDataType));
@@ -343,6 +347,7 @@ public class RdapEntitySearchAction extends RdapActionBase {
         ImmutableList.copyOf(jsonOutputList),
         (jsonOutputList.size() < rdapResultSetMaxSize)
             ? incompletenessWarningType
-            : IncompletenessWarningType.COMPLETE);
+            : IncompletenessWarningType.COMPLETE,
+        Optional.empty());
   }
 }

java/google/registry/rdap/RdapJsonFormatter.java

@@ -436,6 +436,33 @@ public class RdapJsonFormatter {
     return jsonBuilder.build();
   }
 
+  /**
+   * Creates a JSON object containing a notice with a next page navigation link, which can then be
+   * inserted into a notices array.
+   *
+   * <p>At the moment, only a next page link is supported. Other types of links (e.g. previous page)
+   * could be added in the future, but it's not clear how to generate such links, given the way we
+   * are querying the database.
+   *
+   * @param nextPageUrl URL string used to navigate to next page, or empty if there is no next
+   */
+  static ImmutableMap<String, Object> makeRdapJsonNavigationLinkNotice(
+      Optional<String> nextPageUrl) {
+    ImmutableMap.Builder<String, Object> jsonBuilder = new ImmutableMap.Builder<>();
+    jsonBuilder.put("title", "Navigation Links");
+    jsonBuilder.put("description", ImmutableList.of("Links to related pages."));
+    if (nextPageUrl.isPresent()) {
+      jsonBuilder.put(
+          "links",
+          ImmutableList.of(
+              ImmutableMap.of(
+                  "rel", "next",
+                  "href", nextPageUrl.get(),
+                  "type", "application/rdap+json")));
+    }
+    return jsonBuilder.build();
+  }
+
   /**
    * Creates a JSON object for a {@link DomainResource}.
    *

java/google/registry/rdap/RdapModule.java

@@ -78,4 +78,10 @@ public final class RdapModule {
   static Optional<Boolean> provideFormatOutput(HttpServletRequest req) {
     return RequestParameters.extractOptionalBooleanParameter(req, "formatOutput");
   }
+
+  @Provides
+  @Parameter("cursor")
+  static Optional<String> provideCursor(HttpServletRequest req) {
+    return RequestParameters.extractOptionalParameter(req, "cursor");
+  }
 }

java/google/registry/rdap/RdapNameserverSearchAction.java

@@ -62,7 +62,7 @@ import org.joda.time.DateTime;
   method = {GET, HEAD},
   auth = Auth.AUTH_PUBLIC_ANONYMOUS
 )
-public class RdapNameserverSearchAction extends RdapActionBase {
+public class RdapNameserverSearchAction extends RdapSearchActionBase {
 
   public static final String PATH = "/rdap/nameservers";
 
@@ -86,6 +86,11 @@ public class RdapNameserverSearchAction extends RdapActionBase {
     return PATH;
   }
 
+  private enum CursorType {
+    NAME,
+    ADDRESS
+  }
+
   /**
    * Parses the parameters and calls the appropriate search function.
    *
@@ -103,6 +108,7 @@ public class RdapNameserverSearchAction extends RdapActionBase {
     if (Booleans.countTrue(nameParam.isPresent(), ipParam.isPresent()) != 1) {
       throw new BadRequestException("You must specify either name=XXXX or ip=YYYY");
     }
+    decodeCursorToken();
     RdapSearchResults results;
     if (nameParam.isPresent()) {
       // syntax: /rdap/nameservers?name=exam*.com
@@ -132,12 +138,22 @@ public class RdapNameserverSearchAction extends RdapActionBase {
     }
     ImmutableMap.Builder<String, Object> jsonBuilder = new ImmutableMap.Builder<>();
     jsonBuilder.put("nameserverSearchResults", results.jsonList());
+
+    ImmutableList<ImmutableMap<String, Object>> notices = results.getIncompletenessWarnings();
+    if (results.nextCursor().isPresent()) {
+      ImmutableList.Builder<ImmutableMap<String, Object>> noticesBuilder =
+          new ImmutableList.Builder<>();
+      noticesBuilder.addAll(notices);
+      noticesBuilder.add(
+          RdapJsonFormatter.makeRdapJsonNavigationLinkNotice(
+              Optional.of(
+                  getRequestUrlWithExtraParameter(
+                      "cursor", encodeCursorToken(results.nextCursor().get())))));
+      notices = noticesBuilder.build();
+    }
+
     rdapJsonFormatter.addTopLevelEntries(
-        jsonBuilder,
+        jsonBuilder, BoilerplateType.NAMESERVER, notices, ImmutableList.of(), fullServletPath);
-        BoilerplateType.NAMESERVER,
-        results.getIncompletenessWarnings(),
-        ImmutableList.of(),
-        fullServletPath);
     return jsonBuilder.build();
   }
 
@@ -207,6 +223,9 @@ public class RdapNameserverSearchAction extends RdapActionBase {
     }
     List<HostResource> hostList = new ArrayList<>();
     for (String fqhn : ImmutableSortedSet.copyOf(domainResource.getSubordinateHosts())) {
+      if (cursorString.isPresent() && (fqhn.compareTo(cursorString.get()) <= 0)) {
+        continue;
+      }
       // We can't just check that the host name starts with the initial query string, because
       // then the query ns.exam*.example.com would match against nameserver ns.example.com.
       if (partialStringQuery.matches(fqhn)) {
@@ -223,6 +242,7 @@ public class RdapNameserverSearchAction extends RdapActionBase {
         hostList,
         IncompletenessWarningType.COMPLETE,
         domainResource.getSubordinateHosts().size(),
+        CursorType.NAME,
         now);
   }
 
@@ -240,10 +260,13 @@ public class RdapNameserverSearchAction extends RdapActionBase {
             HostResource.class,
             "fullyQualifiedHostName",
             partialStringQuery,
-            shouldIncludeDeleted(),
+            cursorString,
+            getDeletedItemHandling(),
             querySizeLimit);
     return makeSearchResults(
-        getMatchingResources(query, shouldIncludeDeleted(), now, querySizeLimit), now);
+        getMatchingResources(query, shouldIncludeDeleted(), now, querySizeLimit),
+        CursorType.NAME,
+        now);
   }
 
   /** Searches for nameservers by IP address, returning a JSON array of nameserver info maps. */
@@ -252,21 +275,27 @@ public class RdapNameserverSearchAction extends RdapActionBase {
     int querySizeLimit = getStandardQuerySizeLimit();
     Query<HostResource> query =
         queryItems(
-            HostResource.class,
+                HostResource.class,
-            "inetAddresses",
+                "inetAddresses",
-            inetAddress.getHostAddress(),
+                inetAddress.getHostAddress(),
-            shouldIncludeDeleted(),
+                Optional.empty(),
-            querySizeLimit);
+                cursorString,
+                getDeletedItemHandling(),
+                querySizeLimit);
     return makeSearchResults(
-        getMatchingResources(query, shouldIncludeDeleted(), now, querySizeLimit), now);
+        getMatchingResources(query, shouldIncludeDeleted(), now, querySizeLimit),
+        CursorType.ADDRESS,
+        now);
   }
 
   /** Output JSON for a lists of hosts contained in an {@link RdapResultSet}. */
-  private RdapSearchResults makeSearchResults(RdapResultSet<HostResource> resultSet, DateTime now) {
+  private RdapSearchResults makeSearchResults(
+      RdapResultSet<HostResource> resultSet, CursorType cursorType, DateTime now) {
     return makeSearchResults(
         resultSet.resources(),
         resultSet.incompletenessWarningType(),
         resultSet.numResourcesRetrieved(),
+        cursorType,
         now);
   }
 
@@ -275,22 +304,29 @@ public class RdapNameserverSearchAction extends RdapActionBase {
       List<HostResource> hosts,
       IncompletenessWarningType incompletenessWarningType,
       int numHostsRetrieved,
+      CursorType cursorType,
       DateTime now) {
     metricInformationBuilder.setNumHostsRetrieved(numHostsRetrieved);
     OutputDataType outputDataType =
         (hosts.size() > 1) ? OutputDataType.SUMMARY : OutputDataType.FULL;
     ImmutableList.Builder<ImmutableMap<String, Object>> jsonListBuilder =
         new ImmutableList.Builder<>();
+    Optional<String> newCursor = Optional.empty();
     for (HostResource host : Iterables.limit(hosts, rdapResultSetMaxSize)) {
+      newCursor =
+          Optional.of(
+              (cursorType == CursorType.NAME)
+                  ? host.getFullyQualifiedHostName()
+                  : host.getRepoId());
       jsonListBuilder.add(
           rdapJsonFormatter.makeRdapJsonForHost(
               host, false, fullServletPath, rdapWhoisServer, now, outputDataType));
     }
     ImmutableList<ImmutableMap<String, Object>> jsonList = jsonListBuilder.build();
-    return RdapSearchResults.create(
+    if (jsonList.size() < hosts.size()) {
-        jsonList,
+      return RdapSearchResults.create(jsonList, IncompletenessWarningType.TRUNCATED, newCursor);
-        (jsonList.size() < hosts.size())
+    } else {
-            ? IncompletenessWarningType.TRUNCATED
+      return RdapSearchResults.create(jsonList, incompletenessWarningType, Optional.empty());
-            : incompletenessWarningType);
+    }
   }
 }

java/google/registry/rdap/RdapSearchActionBase.java

@@ -0,0 +1,115 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.rdap;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
+import google.registry.request.Parameter;
+import google.registry.request.ParameterMap;
+import google.registry.request.RequestUrl;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import javax.inject.Inject;
+
+/**
+ * Base RDAP (new WHOIS) action for domain, nameserver and entity search requests.
+ *
+ * @see <a href="https://tools.ietf.org/html/rfc7482">
+ *        RFC 7482: Registration Data Access Protocol (RDAP) Query Format</a>
+ */
+public abstract class RdapSearchActionBase extends RdapActionBase {
+
+  @Inject @RequestUrl String requestUrl;
+  @Inject @ParameterMap ImmutableListMultimap<String, String> parameterMap;
+  @Inject @Parameter("cursor") Optional<String> cursorTokenParam;
+
+  protected Optional<String> cursorString;
+
+  /**
+   * Decodes the cursor token passed in the HTTP request.
+   *
+   * <p>The cursor token is just the Base 64 encoded value of the last data item returned. To fetch
+   * the next page, the code can just decode the cursor, and return only data whose value is greater
+   * than the cursor value.
+   */
+  protected void decodeCursorToken() {
+    if (!cursorTokenParam.isPresent()) {
+      cursorString = Optional.empty();
+    } else {
+      cursorString =
+          Optional.of(
+              new String(
+                  Base64.getDecoder().decode(cursorTokenParam.get().getBytes(UTF_8)), UTF_8));
+    }
+  }
+
+  /** Returns an encoded cursor token to pass back in the RDAP JSON link strings. */
+  protected String encodeCursorToken(String nextCursorString) {
+    return new String(Base64.getEncoder().encode(nextCursorString.getBytes(UTF_8)), UTF_8);
+  }
+
+  /** Returns the original request URL, but with the specified parameter added or overridden. */
+  protected String getRequestUrlWithExtraParameter(String parameterName, String parameterValue) {
+    return getRequestUrlWithExtraParameter(parameterName, ImmutableList.of(parameterValue));
+  }
+
+  /**
+   * Returns the original request URL, but with the specified parameter added or overridden.
+   *
+   * <p>This version handles a list of parameter values, all associated with the same name.
+   *
+   * <p>Example: If the original parameters were "a=w&a=x&b=y&c=z", and this method is called with
+   * parameterName = "b" and parameterValues of "p" and "q", the result will be
+   * "a=w&a=x&c=z&b=p&b=q". The new values of parameter "b" replace the old ones.
+   *
+   */
+  protected String getRequestUrlWithExtraParameter(
+      String parameterName, List<String> parameterValues) {
+    StringBuilder stringBuilder = new StringBuilder(requestUrl);
+    boolean first = true;
+    // Step one: loop through the existing parameters, copying all of them except for the parameter
+    // we want to explicitly set.
+    for (Map.Entry<String, String> entry : parameterMap.entries()) {
+      if (!entry.getKey().equals(parameterName)) {
+        appendParameter(stringBuilder, entry.getKey(), entry.getValue(), first);
+        first = false;
+      }
+    }
+    // Step two: tack on all values of the explicit parameter.
+    for (String parameterValue : parameterValues) {
+      appendParameter(stringBuilder, parameterName, parameterValue, first);
+      first = false;
+    }
+    return stringBuilder.toString();
+  }
+
+  private void appendParameter(
+      StringBuilder stringBuilder, String name, String value, boolean first) {
+    try {
+      stringBuilder.append(first ? '?' : '&');
+      stringBuilder.append(URLEncoder.encode(name, "UTF-8"));
+      stringBuilder.append('=');
+      stringBuilder.append(URLEncoder.encode(value, "UTF-8"));
+    } catch (UnsupportedEncodingException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

java/google/registry/rdap/RdapSearchResults.java

@@ -20,6 +20,7 @@ import static google.registry.rdap.RdapIcannStandardInformation.TRUNCATION_NOTIC
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import java.util.Optional;
 
 /**
  * Holds domain, nameserver and entity search results.
@@ -46,13 +47,14 @@ abstract class RdapSearchResults {
   }
 
   static RdapSearchResults create(ImmutableList<ImmutableMap<String, Object>> jsonList) {
-    return create(jsonList, IncompletenessWarningType.COMPLETE);
+    return create(jsonList, IncompletenessWarningType.COMPLETE, Optional.empty());
   }
 
   static RdapSearchResults create(
       ImmutableList<ImmutableMap<String, Object>> jsonList,
-      IncompletenessWarningType incompletenessWarningType) {
+      IncompletenessWarningType incompletenessWarningType,
-    return new AutoValue_RdapSearchResults(jsonList, incompletenessWarningType);
+      Optional<String> nextCursor) {
+    return new AutoValue_RdapSearchResults(jsonList, incompletenessWarningType, nextCursor);
   }
 
   /** List of JSON result object representations. */
@@ -61,6 +63,9 @@ abstract class RdapSearchResults {
   /** Type of warning to display regarding possible incomplete data. */
   abstract IncompletenessWarningType incompletenessWarningType();
 
+  /** Cursor for fetching the next page of results, or empty() if there are no more. */
+  abstract Optional<String> nextCursor();
+
   /** Convenience method to get the appropriate warnings for the incompleteness warning type. */
   ImmutableList<ImmutableMap<String, Object>> getIncompletenessWarnings() {
     if (incompletenessWarningType() == IncompletenessWarningType.TRUNCATED) {

java/google/registry/request/RequestModule.java

@@ -84,6 +84,12 @@ public final class RequestModule {
     return authResult;
   }
 
+  @Provides
+  @RequestUrl
+  static String provideRequestUrl(HttpServletRequest req) {
+    return req.getRequestURL().toString();
+  }
+
   @Provides
   @RequestPath
   static String provideRequestPath(HttpServletRequest req) {

java/google/registry/request/RequestUrl.java

@@ -0,0 +1,31 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.request;
+
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import javax.inject.Qualifier;
+
+/**
+ * Dagger qualifier for the HTTP request URL.
+ *
+ * @see javax.servlet.http.HttpServletRequest#getRequestURL()
+ */
+@Retention(RUNTIME)
+@Qualifier
+@Documented
+public @interface RequestUrl {}

javatests/google/registry/rdap/RdapNameserverSearchActionTest.java

@@ -34,6 +34,7 @@ import static org.mockito.Mockito.when;
 
 import com.google.appengine.api.users.User;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
@@ -54,10 +55,13 @@ import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.InjectRule;
 import google.registry.ui.server.registrar.SessionUtils;
+import java.net.URLDecoder;
 import java.util.Optional;
 import javax.annotation.Nullable;
 import javax.servlet.http.HttpServletRequest;
 import org.joda.time.DateTime;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
 import org.json.simple.JSONValue;
 import org.junit.Before;
 import org.junit.Rule;
@@ -74,7 +78,7 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   @Rule public final InjectRule inject = new InjectRule();
 
   private final HttpServletRequest request = mock(HttpServletRequest.class);
-  private final FakeResponse response = new FakeResponse();
+  private FakeResponse response = new FakeResponse();
   private final FakeClock clock = new FakeClock(DateTime.parse("2000-01-01T00:00:00Z"));
   private final SessionUtils sessionUtils = mock(SessionUtils.class);
   private final User user = new User("rdap.user@example.com", "gmail.com", "12345");
@@ -87,16 +91,39 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   private HostResource hostNs2CatLol;
 
   private Object generateActualJsonWithName(String name) {
+    return generateActualJsonWithName(name, null);
+  }
+
+  private Object generateActualJsonWithName(String name, String cursor) {
     metricSearchType = SearchType.BY_NAMESERVER_NAME;
     rememberWildcardType(name);
     action.nameParam = Optional.of(name);
+    if (cursor == null) {
+      action.parameterMap = ImmutableListMultimap.of("name", name);
+      action.cursorTokenParam = Optional.empty();
+    } else {
+      action.parameterMap = ImmutableListMultimap.of("name", name, "cursor", cursor);
+      action.cursorTokenParam = Optional.of(cursor);
+    }
     action.run();
     return JSONValue.parse(response.getPayload());
   }
 
   private Object generateActualJsonWithIp(String ipString) {
+    return generateActualJsonWithIp(ipString, null);
+  }
+
+  private Object generateActualJsonWithIp(String ipString, String cursor) {
     metricSearchType = SearchType.BY_NAMESERVER_ADDRESS;
+    action.parameterMap = ImmutableListMultimap.of("ip", ipString);
     action.ipParam = Optional.of(ipString);
+    if (cursor == null) {
+      action.parameterMap = ImmutableListMultimap.of("ip", ipString);
+      action.cursorTokenParam = Optional.empty();
+    } else {
+      action.parameterMap = ImmutableListMultimap.of("ip", ipString, "cursor", cursor);
+      action.cursorTokenParam = Optional.of(cursor);
+    }
     action.run();
     return JSONValue.parse(response.getPayload());
   }
@@ -153,7 +180,9 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
     inject.setStaticField(Ofy.class, "clock", clock);
     action.clock = clock;
     action.fullServletPath = "https://example.tld/rdap";
+    action.requestUrl = "https://example.tld/rdap/nameservers";
     action.requestPath = RdapNameserverSearchAction.PATH;
+    action.parameterMap = ImmutableListMultimap.of();
     action.request = request;
     action.requestMethod = Action.Method.GET;
     action.response = response;
@@ -168,6 +197,7 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
     action.authResult = AuthResult.create(AuthLevel.USER, userAuthInfo);
     action.sessionUtils = sessionUtils;
     action.rdapMetrics = rdapMetrics;
+    action.cursorTokenParam = Optional.empty();
   }
 
   private void login(String clientId) {
@@ -555,7 +585,9 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   public void testNameMatch_truncatedResultSet() throws Exception {
     createManyHosts(5);
     assertThat(generateActualJsonWithName("ns*.cat.lol"))
-      .isEqualTo(generateExpectedJson("rdap_truncated_hosts.json"));
+        .isEqualTo(
+            generateExpectedJson(
+                "name=ns*.cat.lol&cursor=bnM0LmNhdC5sb2w%3D", "rdap_truncated_hosts.json"));
     assertThat(response.getStatus()).isEqualTo(200);
     verifyMetrics(5);
   }
@@ -564,7 +596,9 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   public void testNameMatch_reallyTruncatedResultSet() throws Exception {
     createManyHosts(9);
     assertThat(generateActualJsonWithName("ns*.cat.lol"))
-      .isEqualTo(generateExpectedJson("rdap_truncated_hosts.json"));
+        .isEqualTo(
+            generateExpectedJson(
+                "name=ns*.cat.lol&cursor=bnM0LmNhdC5sb2w%3D", "rdap_truncated_hosts.json"));
     assertThat(response.getStatus()).isEqualTo(200);
     // When searching names, we look for additional matches, in case some are not visible.
     verifyMetrics(9);
@@ -675,6 +709,83 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
     verifyErrorMetrics();
   }
 
+  private String getLinkToNext(Object results) {
+    assertThat(results).isInstanceOf(JSONObject.class);
+    Object notices = ((JSONObject) results).get("notices");
+    assertThat(notices).isInstanceOf(JSONArray.class);
+    for (Object notice : (JSONArray) notices) {
+      assertThat(notice).isInstanceOf(JSONObject.class);
+      Object title = ((JSONObject) notice).get("title");
+      assertThat(title).isInstanceOf(String.class);
+      if (!title.equals("Navigation Links")) {
+        continue;
+      }
+      Object links = ((JSONObject) notice).get("links");
+      assertThat(links).isInstanceOf(JSONArray.class);
+      for (Object link : (JSONArray) links) {
+        assertThat(link).isInstanceOf(JSONObject.class);
+        Object rel = ((JSONObject) link).get("rel");
+        assertThat(rel).isInstanceOf(String.class);
+        if (!rel.equals("next")) {
+          continue;
+        }
+        Object href = ((JSONObject) link).get("href");
+        assertThat(href).isInstanceOf(String.class);
+        return (String) href;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Checks multi-page result set navigation using the cursor.
+   *
+   * <p>If there are more results than the max result set size, the RDAP code returns a cursor token
+   * which can be used in a subsequent call to get the next chunk of results.
+   *
+   * @param byName true if we are searching by name; false if we are searching by address
+   * @param queryString the name or address query string
+   * @param expectedPageCount how many pages we expect to retrieve; all but the last will have a
+   *     cursor
+   */
+  private void checkCursorNavigation(boolean byName, String queryString, int expectedPageCount)
+      throws Exception {
+    String cursor = null;
+    for (int i = 0; i < expectedPageCount; i++) {
+      Object results =
+          byName
+              ? generateActualJsonWithName(queryString, cursor)
+              : generateActualJsonWithIp(queryString, cursor);
+      assertThat(response.getStatus()).isEqualTo(200);
+      String linkToNext = getLinkToNext(results);
+      if (i == expectedPageCount - 1) {
+        assertThat(linkToNext).isNull();
+      } else {
+        assertThat(linkToNext).isNotNull();
+        int pos = linkToNext.indexOf("cursor=");
+        assertThat(pos).isAtLeast(0);
+        cursor = URLDecoder.decode(linkToNext.substring(pos + 7), "UTF-8");
+        Object nameserverSearchResults = ((JSONObject) results).get("nameserverSearchResults");
+        assertThat(nameserverSearchResults).isInstanceOf(JSONArray.class);
+        assertThat(((JSONArray) nameserverSearchResults)).hasSize(action.rdapResultSetMaxSize);
+        response = new FakeResponse();
+        action.response = response;
+      }
+    }
+  }
+
+  @Test
+  public void testNameMatch_cursorNavigationWithSuperordinateDomain() throws Exception {
+    createManyHosts(9);
+    checkCursorNavigation(true, "ns*.cat.lol", 3);
+  }
+
+  @Test
+  public void testNameMatch_cursorNavigationWithPrefix() throws Exception {
+    createManyHosts(9);
+    checkCursorNavigation(true, "ns*", 4);
+  }
+
   @Test
   public void testAddressMatch_invalidAddress() throws Exception {
     generateActualJsonWithIp("It is to laugh");
@@ -736,7 +847,7 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   public void testAddressMatch_nontruncatedResultSet() throws Exception {
     createManyHosts(4);
     assertThat(generateActualJsonWithIp("5.5.5.1"))
-      .isEqualTo(generateExpectedJson("rdap_nontruncated_hosts.json"));
+        .isEqualTo(generateExpectedJson("rdap_nontruncated_hosts.json"));
     assertThat(response.getStatus()).isEqualTo(200);
     verifyMetrics(4);
   }
@@ -745,7 +856,9 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   public void testAddressMatch_truncatedResultSet() throws Exception {
     createManyHosts(5);
     assertThat(generateActualJsonWithIp("5.5.5.1"))
-      .isEqualTo(generateExpectedJson("rdap_truncated_hosts.json"));
+    .isEqualTo(
+        generateExpectedJson(
+            "ip=5.5.5.1&cursor=MTctUk9JRA%3D%3D", "rdap_truncated_hosts.json"));
     assertThat(response.getStatus()).isEqualTo(200);
     verifyMetrics(5);
   }
@@ -754,7 +867,9 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
   public void testAddressMatch_reallyTruncatedResultSet() throws Exception {
     createManyHosts(9);
     assertThat(generateActualJsonWithIp("5.5.5.1"))
-      .isEqualTo(generateExpectedJson("rdap_truncated_hosts.json"));
+        .isEqualTo(
+            generateExpectedJson(
+                "ip=5.5.5.1&cursor=MTctUk9JRA%3D%3D", "rdap_truncated_hosts.json"));
     assertThat(response.getStatus()).isEqualTo(200);
     // When searching by address and not including deleted, we don't need to search for extra
     // matches.
@@ -830,4 +945,10 @@ public class RdapNameserverSearchActionTest extends RdapSearchActionTestCase {
     assertThat(response.getStatus()).isEqualTo(404);
     verifyErrorMetrics();
   }
+
+  @Test
+  public void testAddressMatch_cursorNavigation() throws Exception {
+    createManyHosts(9);
+    checkCursorNavigation(false, "5.5.5.1", 3);
+  }
 }

javatests/google/registry/rdap/testdata/rdap_truncated_hosts.json

@@ -125,6 +125,18 @@
         "Search results per query are limited."
       ]
     },
+    {
+      "title" : "Navigation Links",
+      "description" : [ "Links to related pages." ],
+      "links" :
+      [
+        {
+          "type" : "application/rdap+json",
+          "rel" : "next",
+          "href" : "https://example.tld/rdap/nameservers?%NAME%"
+        }
+      ]
+    },
     {
       "title" : "RDAP Terms of Service",
       "description" :
@@ -149,7 +161,7 @@
           "type" : "text/html"
         }
       ]
-    }
+    },
   ],
   "remarks" :
   [