RDAP: Display truncation notice for large domain result sets

The ICAAN Operational Profile dictates that a notice be added to the RDAP search results response when there are more objects than the server's chosen result set size. This CL handles the fixes for domain searches.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=135407203

java/google/registry/rdap/RdapDomainSearchAction.java

@@ -17,6 +17,7 @@ package google.registry.rdap;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.rdap.RdapIcannStandardInformation.TRUNCATION_NOTICES;
 import static google.registry.request.Action.Method.GET;
 import static google.registry.request.Action.Method.HEAD;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
@@ -63,8 +64,7 @@ public class RdapDomainSearchAction extends RdapActionBase {
 
   public static final String PATH = "/rdap/domains";
 
-  public static final int CHUNK_SIZE_SCALING_FACTOR = 5;
+  public static final int RESULT_SET_SIZE_SCALING_FACTOR = 30;
-  public static final int MAX_CHUNK_FETCHES = 20;
 
   @Inject Clock clock;
   @Inject @Parameter("name") Optional<String> nameParam;
@@ -98,7 +98,7 @@ public class RdapDomainSearchAction extends RdapActionBase {
       throw new BadRequestException(
           "You must specify either name=XXXX, nsLdhName=YYYY or nsIp=ZZZZ");
     }
-    ImmutableList<ImmutableMap<String, Object>> results;
+    RdapSearchResults results;
     if (nameParam.isPresent()) {
       // syntax: /rdap/domains?name=exam*.com
       String asciiName;
@@ -121,86 +121,65 @@ public class RdapDomainSearchAction extends RdapActionBase {
       // syntax: /rdap/domains?nsIp=1.2.3.4
       results = searchByNameserverIp(nsIpParam.get(), now);
     }
-    if (results.isEmpty()) {
+    if (results.jsonList().isEmpty()) {
       throw new NotFoundException("No domains found");
     }
     ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
-    builder.put("domainSearchResults", results);
+    builder.put("domainSearchResults", results.jsonList());
     RdapJsonFormatter.addTopLevelEntries(
         builder,
         BoilerplateType.DOMAIN,
-        ImmutableList.<ImmutableMap<String, Object>>of(),
+        results.isTruncated()
+            ? TRUNCATION_NOTICES : ImmutableList.<ImmutableMap<String, Object>>of(),
         ImmutableList.<ImmutableMap<String, Object>>of(),
         rdapLinkBase);
     return builder.build();
   }
 
   /** Searches for domains by domain name, returning a JSON array of domain info maps. */
-  private ImmutableList<ImmutableMap<String, Object>>
+  private RdapSearchResults searchByDomainName(
-      searchByDomainName(final RdapSearchPattern partialStringQuery, final DateTime now) {
+      final RdapSearchPattern partialStringQuery, final DateTime now) {
     // Handle queries without a wildcard -- just load by foreign key.
     if (!partialStringQuery.getHasWildcard()) {
       DomainResource domainResource =
           loadByForeignKey(DomainResource.class, partialStringQuery.getInitialString(), now);
-      if (domainResource == null) {
+      return makeSearchResults(
-        return ImmutableList.of();
+          (domainResource == null) ? ImmutableList.of() : ImmutableList.of(domainResource),
-      }
+          false,
-      return makeSearchResults(ImmutableList.of(domainResource), now);
+          now);
     // Handle queries with a wildcard.
     } else {
       // We can't query for undeleted domains as part of the query itself; that would require an
       // inequality query on deletion time, and we are already using inequality queries on
-      // fullyQualifiedDomainName. So we need another way to limit the result set to the desired
+      // fullyQualifiedDomainName. So we instead pick an arbitrary limit of
-      // number of undeleted domains, which we do as follows. We query a batch of domains up to five
+      // RESULT_SET_SIZE_SCALING_FACTOR times the result set size limit, fetch up to that many, and
-      // times the size of the result set size limit (a factor picked out of thin air), and weed out
+      // weed out all deleted domains. If there still isn't a full result set's worth of domains, we
-      // all deleted domains. If we still have space in the result set (because there were an
+      // give up and return just the ones we found.
-      // incredibly large number of deleted domains), we go back and query some more domains to try
+      // TODO(b/31546493): Add metrics to figure out how well this.
-      // and find more results. We try this 20 times (meaning we search for 100 times as many
-      // domains as the result set size limit), then give up and return a result set that is smaller
-      // than the limit. Ugly? You bet!
-      // TODO(b/31546493): Add metrics to figure out how well this algorithm works.
       List<DomainResource> domainList = new ArrayList<>();
-      String previousChunkEndString = null;
+      Query<DomainResource> query = ofy().load()
-      for (int numChunkFetches = 0;
+          .type(DomainResource.class)
-          (numChunkFetches < MAX_CHUNK_FETCHES) && (domainList.size() < rdapResultSetMaxSize);
+          .filter("fullyQualifiedDomainName <", partialStringQuery.getNextInitialString())
-          numChunkFetches++) {
+          .filter("fullyQualifiedDomainName >=", partialStringQuery.getInitialString());
-        // Construct the query.
+      if (partialStringQuery.getSuffix() != null) {
-        Query<DomainResource> query = ofy().load()
+        query = query.filter("tld", partialStringQuery.getSuffix());
-            .type(DomainResource.class)
+      }
-            .filter("fullyQualifiedDomainName <", partialStringQuery.getNextInitialString());
+      // TODO(mountford): Investigate fetching by foreign key instead of the domain itself.
-        if (previousChunkEndString == null) {
+      for (DomainResource domain :
-          query = query.filter(
+          query.limit(RESULT_SET_SIZE_SCALING_FACTOR * rdapResultSetMaxSize)) {
-              "fullyQualifiedDomainName >=", partialStringQuery.getInitialString());
+        if (EppResourceUtils.isActive(domain, now)) {
-        } else {
+          if (domainList.size() >= rdapResultSetMaxSize) {
-          query = query.filter("fullyQualifiedDomainName >", previousChunkEndString);
+            return makeSearchResults(ImmutableList.copyOf(domainList), true, now);
-        }
-        if (partialStringQuery.getSuffix() != null) {
-          query = query.filter("tld", partialStringQuery.getSuffix());
-        }
-        // Perform the query and weed out deleted domains.
-        previousChunkEndString = null;
-        int numDomainsInChunk = 0;
-        for (DomainResource domain :
-            query.limit(rdapResultSetMaxSize * CHUNK_SIZE_SCALING_FACTOR)) {
-          previousChunkEndString = domain.getFullyQualifiedDomainName();
-          numDomainsInChunk++;
-          if (EppResourceUtils.isActive(domain, now)) {
-            domainList.add(domain);
-            if (domainList.size() >= rdapResultSetMaxSize) {
-              break;
-            }
           }
-        }
+          domainList.add(domain);
-        if (numDomainsInChunk < rdapResultSetMaxSize * CHUNK_SIZE_SCALING_FACTOR) {
-          break;
         }
       }
-      return makeSearchResults(domainList, now);
+      return makeSearchResults(ImmutableList.copyOf(domainList), false, now);
     }
   }
 
   /** Searches for domains by nameserver name, returning a JSON array of domain info maps. */
-  private ImmutableList<ImmutableMap<String, Object>> searchByNameserverLdhName(
+  private RdapSearchResults searchByNameserverLdhName(
       final RdapSearchPattern partialStringQuery, final DateTime now) {
     Iterable<Key<HostResource>> hostKeys = getNameserverRefsByLdhName(partialStringQuery, now);
     if (Iterables.isEmpty(hostKeys)) {
@@ -210,8 +189,8 @@ public class RdapDomainSearchAction extends RdapActionBase {
   }
 
   /** Assembles a list of {@link HostResource} keys by name. */
-  private Iterable<Key<HostResource>>
+  private Iterable<Key<HostResource>> getNameserverRefsByLdhName(
-      getNameserverRefsByLdhName(final RdapSearchPattern partialStringQuery, final DateTime now) {
+      final RdapSearchPattern partialStringQuery, final DateTime now) {
     // Handle queries without a wildcard; just load the host by foreign key in the usual way.
     if (!partialStringQuery.getHasWildcard()) {
       Key<HostResource> hostKey = loadAndGetKey(
@@ -255,8 +234,8 @@ public class RdapDomainSearchAction extends RdapActionBase {
   }
 
   /** Searches for domains by nameserver address, returning a JSON array of domain info maps. */
-  private ImmutableList<ImmutableMap<String, Object>>
+  private RdapSearchResults searchByNameserverIp(
-      searchByNameserverIp(final InetAddress inetAddress, final DateTime now) {
+      final InetAddress inetAddress, final DateTime now) {
     // In theory, we could filter on the deletion time being in the future. But we can't do that in
     // the query on nameserver name (because we're already using an inequality query), and it seems
     // dangerous and confusing to filter on deletion time differently between the two queries.
@@ -274,36 +253,44 @@ public class RdapDomainSearchAction extends RdapActionBase {
   }
 
   /**
-   * Locates all domains which are linked to a set of host keys. This method is called by
+   * Locates all domains which are linked to a set of host keys.
-   * {@link #searchByNameserverLdhName} and {@link #searchByNameserverIp} after they assemble the
+   * 
-   * relevant host keys.
+   * <p>This method is called by {@link #searchByNameserverLdhName} and
+   * {@link #searchByNameserverIp} after they assemble the relevant host keys.
    */
-  private ImmutableList<ImmutableMap<String, Object>>
+  private RdapSearchResults searchByNameserverRefs(
-      searchByNameserverRefs(final Iterable<Key<HostResource>> hostKeys, final DateTime now) {
+      final Iterable<Key<HostResource>> hostKeys, final DateTime now) {
     // We must break the query up into chunks, because the in operator is limited to 30 subqueries.
     // Since it is possible for the same domain to show up more than once in our result list (if
     // we do a wildcard nameserver search that returns multiple nameservers used by the same
     // domain), we must create a set of resulting {@link DomainResource} objects. But we use a
     // LinkedHashSet to preserve the order in which we found the domains.
-    LinkedHashSet<DomainResource> domainResources = new LinkedHashSet<>();
+    LinkedHashSet<DomainResource> domains = new LinkedHashSet<>();
     for (List<Key<HostResource>> chunk : Iterables.partition(hostKeys, 30)) {
-      domainResources.addAll(
+      for (DomainResource domain : ofy().load()
-          ofy().load()
+          .type(DomainResource.class)
-              .type(DomainResource.class)
+          .filter("nameservers.linked in", chunk)
-              .filter("nameservers.linked in", chunk)
+          .filter("deletionTime >", now)
-              .filter("deletionTime >", now)
+          .limit(rdapResultSetMaxSize + 1)) {
-              .limit(rdapResultSetMaxSize - domainResources.size())
+        if (!domains.contains(domain)) {
-              .list());
+          if (domains.size() >= rdapResultSetMaxSize) {
-      if (domainResources.size() >= rdapResultSetMaxSize) {
+            return makeSearchResults(ImmutableList.copyOf(domains), true, now);
-        break;
+          }
+          domains.add(domain);
+        }
       }
     }
-    return makeSearchResults(ImmutableList.copyOf(domainResources), now);
+    return makeSearchResults(ImmutableList.copyOf(domains), false, now);
   }
 
-  /** Output JSON for a list of domains. */
+  /**
-  private ImmutableList<ImmutableMap<String, Object>> makeSearchResults(
+   * Output JSON for a list of domains.
-      List<DomainResource> domains, DateTime now) {
+   * 
+   * <p>The isTruncated parameter should be true if the search found more results than are in the
+   * list, meaning that the truncation notice should be added.
+   */
+  private RdapSearchResults makeSearchResults(
+      ImmutableList<DomainResource> domains, boolean isTruncated, DateTime now) {
     OutputDataType outputDataType =
         (domains.size() > 1) ? OutputDataType.SUMMARY : OutputDataType.FULL;
     ImmutableList.Builder<ImmutableMap<String, Object>> jsonBuilder = new ImmutableList.Builder<>();
@@ -312,6 +299,6 @@ public class RdapDomainSearchAction extends RdapActionBase {
           RdapJsonFormatter.makeRdapJsonForDomain(
               domain, false, rdapLinkBase, rdapWhoisServer, now, outputDataType));
     }
-    return jsonBuilder.build();
+    return RdapSearchResults.create(jsonBuilder.build(), isTruncated);
   }
 }

java/google/registry/rdap/RdapIcannStandardInformation.java

@@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableMap;
 /**
  * This file contains boilerplate required by the ICANN RDAP Profile.
  *
- * @see "https://whois.icann.org/sites/default/files/files/gtld-rdap-operational-profile-draft-03dec15-en.pdf"
+ * @see "https://www.icann.org/resources/pages/rdap-operational-profile-2016-07-26-en"
  */
 
 public class RdapIcannStandardInformation {
@@ -72,7 +72,7 @@ public class RdapIcannStandardInformation {
   static final ImmutableList<ImmutableMap<String, Object>> nameserverAndEntityBoilerplateRemarks =
       ImmutableList.of(CONFORMANCE_REMARK);
 
-  /** Required by ICANN RDAP Profile section 1.1.18. */
+  /** Required by ICANN RDAP Profile section 1.4.8 / 1.4.9, as interpreted. */
   static final ImmutableMap<String, Object> SUMMARY_DATA_REMARK =
       ImmutableMap.<String, Object> of(
           "title",
@@ -82,4 +82,20 @@ public class RdapIcannStandardInformation {
               "Summary data only. For complete data, send a specific query for the object."),
           "type",
           "object truncated due to unexplainable reasons");
+
+
+  /** Required by ICANN RDAP Profile section 1.4.8 / 1.4.9, as interpreted. */
+  static final ImmutableMap<String, Object> TRUNCATED_RESULT_SET_NOTICE =
+      ImmutableMap.<String, Object> of(
+          "title",
+          "Search Policy",
+          "description",
+          ImmutableList.of("Search results per query are limited."),
+          "type",
+          "result set truncated due to unexplainable reasons");
+  
+  /** Truncation notice as a singleton list, for easy use. */
+  static final ImmutableList<ImmutableMap<String, Object>> TRUNCATION_NOTICES =
+      ImmutableList.of(TRUNCATED_RESULT_SET_NOTICE);
 }
+

java/google/registry/rdap/RdapSearchResults.java

@@ -0,0 +1,44 @@
+// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.rdap;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Holds domain, nameserver and entity search results.
+ * 
+ * <p>We need to know not only the list of things we found, but also whether the result set was
+ * truncated to the limit. If it is, we must add the ICANN-mandated notice to that effect.
+ */
+@AutoValue
+abstract class RdapSearchResults {
+  
+  static RdapSearchResults create(ImmutableList<ImmutableMap<String, Object>> jsonList) {
+    return create(jsonList, false);
+  }
+  
+  static RdapSearchResults create(
+      ImmutableList<ImmutableMap<String, Object>> jsonList, boolean isTruncated) {
+    return new AutoValue_RdapSearchResults(jsonList, isTruncated);
+  }
+
+  /** List of JSON result object representations. */
+  abstract ImmutableList<ImmutableMap<String, Object>> jsonList();
+  
+  /** True if the result set was truncated to the maximum size limit. */
+  abstract boolean isTruncated();
+}

javatests/google/registry/rdap/RdapDomainSearchActionTest.java

@@ -552,6 +552,31 @@ public class RdapDomainSearchActionTest {
     persistResources(domainsBuilder.build());
   }
 
+  private Object readMultiDomainFile(
+      String fileName,
+      String domainName1,
+      String domainHandle1,
+      String domainName2,
+      String domainHandle2,
+      String domainName3,
+      String domainHandle3,
+      String domainName4,
+      String domainHandle4) {
+    return JSONValue.parse(loadFileWithSubstitutions(
+        this.getClass(),
+        fileName,
+        new ImmutableMap.Builder<String, String>()
+            .put("DOMAINNAME1", domainName1)
+            .put("DOMAINHANDLE1", domainHandle1)
+            .put("DOMAINNAME2", domainName2)
+            .put("DOMAINHANDLE2", domainHandle2)
+            .put("DOMAINNAME3", domainName3)
+            .put("DOMAINHANDLE3", domainHandle3)
+            .put("DOMAINNAME4", domainName4)
+            .put("DOMAINHANDLE4", domainHandle4)
+            .build()));
+  }
+
   private void checkNumberOfDomainsInResult(Object obj, int expected) {
     assertThat(obj).isInstanceOf(Map.class);
 
@@ -577,7 +602,7 @@ public class RdapDomainSearchActionTest {
   public void testDomainMatch_manyDeletedDomains_partialResultSetDueToInsufficientDomains()
       throws Exception {
     // There are not enough domains to fill a full result set.
-    createManyDomainsAndHosts(3, 100, 2);
+    createManyDomainsAndHosts(3, 20, 2);
     Object obj = generateActualJson(RequestType.NAME, "domain*.lol");
     assertThat(response.getStatus()).isEqualTo(200);
     checkNumberOfDomainsInResult(obj, 3);
@@ -589,12 +614,82 @@ public class RdapDomainSearchActionTest {
     // This is not exactly desired behavior, but expected: There are enough domains to fill a full
     // result set, but there are so many deleted domains that we run out of patience before we work
     // our way through all of them.
-    createManyDomainsAndHosts(4, 150, 2);
+    createManyDomainsAndHosts(4, 50, 2);
     Object obj = generateActualJson(RequestType.NAME, "domain*.lol");
     assertThat(response.getStatus()).isEqualTo(200);
     checkNumberOfDomainsInResult(obj, 3);
   }
 
+  @Test
+  public void testDomainMatch_nontruncatedResultsSet() throws Exception {
+    createManyDomainsAndHosts(4, 1, 2);
+    assertThat(generateActualJson(RequestType.NAME, "domain*.lol"))
+        .isEqualTo(readMultiDomainFile(
+            "rdap_nontruncated_domains.json",
+            "domain1.lol",
+            "41-LOL",
+            "domain2.lol",
+            "42-LOL",
+            "domain3.lol",
+            "43-LOL",
+            "domain4.lol",
+            "44-LOL"));
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
+  @Test
+  public void testDomainMatch_truncatedResultsSet() throws Exception {
+    createManyDomainsAndHosts(5, 1, 2);
+    assertThat(generateActualJson(RequestType.NAME, "domain*.lol"))
+        .isEqualTo(readMultiDomainFile(
+            "rdap_truncated_domains.json",
+            "domain1.lol",
+            "41-LOL",
+            "domain2.lol",
+            "42-LOL",
+            "domain3.lol",
+            "43-LOL",
+            "domain4.lol",
+            "44-LOL"));
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
+  @Test
+  public void testDomainMatch_reallyTruncatedResultsSet() throws Exception {
+    // Don't use 10 or more domains for this test, because domain10.lol will come before
+    // domain2.lol, and you'll get the wrong domains in the result set.
+    createManyDomainsAndHosts(9, 1, 2);
+    assertThat(generateActualJson(RequestType.NAME, "domain*.lol"))
+        .isEqualTo(readMultiDomainFile(
+            "rdap_truncated_domains.json",
+            "domain1.lol",
+            "41-LOL",
+            "domain2.lol",
+            "42-LOL",
+            "domain3.lol",
+            "43-LOL",
+            "domain4.lol",
+            "44-LOL"));
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
+  @Test
+  public void testDomainMatch_truncatedResultsAfterMultipleChunks() throws Exception {
+    createManyDomainsAndHosts(5, 6, 2);
+    assertThat(generateActualJson(RequestType.NAME, "domain*.lol"))
+        .isEqualTo(readMultiDomainFile(
+            "rdap_truncated_domains.json",
+            "domain12.lol",
+            "4C-LOL",
+            "domain18.lol",
+            "52-LOL",
+            "domain24.lol",
+            "58-LOL",
+            "domain30.lol",
+            "5E-LOL"));
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
   @Test
   public void testNameserverMatch_foundMultiple() throws Exception {
     assertThat(generateActualJson(RequestType.NS_LDH_NAME, "ns1.cat.lol"))
@@ -766,6 +861,77 @@ public class RdapDomainSearchActionTest {
     checkNumberOfDomainsInResult(obj, 3);
   }
 
+  @Test
+  public void testNameserverMatch_nontruncatedResultsSet() throws Exception {
+    createManyDomainsAndHosts(4, 1, 2);
+    assertThat(generateActualJson(RequestType.NS_LDH_NAME, "ns1.domain1.lol"))
+        .isEqualTo(readMultiDomainFile(
+            "rdap_nontruncated_domains.json",
+            "domain1.lol",
+            "41-LOL",
+            "domain2.lol",
+            "42-LOL",
+            "domain3.lol",
+            "43-LOL",
+            "domain4.lol",
+            "44-LOL"));
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
+  @Test
+  public void testNameserverMatch_truncatedResultsSet() throws Exception {
+    createManyDomainsAndHosts(5, 1, 2);
+    assertThat(generateActualJson(RequestType.NS_LDH_NAME, "ns1.domain1.lol"))
+        .isEqualTo(readMultiDomainFile(
+            "rdap_truncated_domains.json",
+            "domain1.lol",
+            "41-LOL",
+            "domain2.lol",
+            "42-LOL",
+            "domain3.lol",
+            "43-LOL",
+            "domain4.lol",
+            "44-LOL"));
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
+  @Test
+  public void testNameserverMatch_reallyTruncatedResultsSet() throws Exception {
+    createManyDomainsAndHosts(9, 1, 2);
+    assertThat(generateActualJson(RequestType.NS_LDH_NAME, "ns1.domain1.lol"))
+        .isEqualTo(readMultiDomainFile(
+            "rdap_truncated_domains.json",
+            "domain1.lol",
+            "41-LOL",
+            "domain2.lol",
+            "42-LOL",
+            "domain3.lol",
+            "43-LOL",
+            "domain4.lol",
+            "44-LOL"));
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
+  @Test
+  public void testNameserverMatch_duplicatesNotTruncated() throws Exception {
+    // 60 nameservers for each of 4 domains; these should translate into 2 30-nameserver domain
+    // fetches, which should _not_ trigger the truncation warning because all the domains will be
+    // duplicates.
+    createManyDomainsAndHosts(4, 1, 60);
+    assertThat(generateActualJson(RequestType.NS_LDH_NAME, "ns*.domain1.lol"))
+        .isEqualTo(readMultiDomainFile(
+            "rdap_nontruncated_domains.json",
+            "domain1.lol",
+            "B5-LOL",
+            "domain2.lol",
+            "B6-LOL",
+            "domain3.lol",
+            "B7-LOL",
+            "domain4.lol",
+            "B8-LOL"));
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
   @Test
   public void testAddressMatchV4Address_foundMultiple() throws Exception {
     assertThat(generateActualJson(RequestType.NS_IP, "1.2.3.4"))
@@ -821,4 +987,55 @@ public class RdapDomainSearchActionTest {
         .isEqualTo(generateExpectedJson("No domains found", null, null, "rdap_error_404.json"));
     assertThat(response.getStatus()).isEqualTo(404);
   }
+
+  @Test
+  public void testAddressMatch_nontruncatedResultsSet() throws Exception {
+    createManyDomainsAndHosts(4, 1, 2);
+    assertThat(generateActualJson(RequestType.NS_IP, "5.5.5.1"))
+        .isEqualTo(readMultiDomainFile(
+            "rdap_nontruncated_domains.json",
+            "domain1.lol",
+            "41-LOL",
+            "domain2.lol",
+            "42-LOL",
+            "domain3.lol",
+            "43-LOL",
+            "domain4.lol",
+            "44-LOL"));
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
+  @Test
+  public void testAddressMatch_truncatedResultsSet() throws Exception {
+    createManyDomainsAndHosts(5, 1, 2);
+    assertThat(generateActualJson(RequestType.NS_IP, "5.5.5.1"))
+        .isEqualTo(readMultiDomainFile(
+            "rdap_truncated_domains.json",
+            "domain1.lol",
+            "41-LOL",
+            "domain2.lol",
+            "42-LOL",
+            "domain3.lol",
+            "43-LOL",
+            "domain4.lol",
+            "44-LOL"));
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
+  @Test
+  public void testAddressMatch_reallyTruncatedResultsSet() throws Exception {
+    createManyDomainsAndHosts(9, 1, 2);
+    assertThat(generateActualJson(RequestType.NS_IP, "5.5.5.1"))
+        .isEqualTo(readMultiDomainFile(
+            "rdap_truncated_domains.json",
+            "domain1.lol",
+            "41-LOL",
+            "domain2.lol",
+            "42-LOL",
+            "domain3.lol",
+            "43-LOL",
+            "domain4.lol",
+            "44-LOL"));
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
 }

javatests/google/registry/rdap/testdata/rdap_nontruncated_domains.json

@@ -0,0 +1,187 @@
+{
+  "domainSearchResults": [
+    {
+      "status": [
+        "client delete prohibited",
+        "client renew prohibited",
+        "client transfer prohibited",
+        "server update prohibited"
+      ],
+      "handle": "%DOMAINHANDLE1%",
+      "links": [
+        {
+          "href": "https://example.com/rdap/domain/%DOMAINNAME1%",
+          "type": "application/rdap+json",
+          "rel": "self",
+          "value": "https://example.com/rdap/domain/%DOMAINNAME1%"
+        }
+      ],
+      "ldhName": "%DOMAINNAME1%",
+      "objectClassName": "domain",
+      "remarks": [
+        {
+          "title": "Incomplete Data",
+          "description": [
+            "Summary data only. For complete data, send a specific query for the object."
+          ],
+          "type": "object truncated due to unexplainable reasons"
+        }
+      ]
+    },
+    {
+      "status": [
+        "client delete prohibited",
+        "client renew prohibited",
+        "client transfer prohibited",
+        "server update prohibited"
+      ],
+      "handle": "%DOMAINHANDLE2%",
+      "links": [
+        {
+          "href": "https://example.com/rdap/domain/%DOMAINNAME2%",
+          "type": "application/rdap+json",
+          "rel": "self",
+          "value": "https://example.com/rdap/domain/%DOMAINNAME2%"
+        }
+      ],
+      "ldhName": "%DOMAINNAME2%",
+      "objectClassName": "domain",
+      "remarks": [
+        {
+          "title": "Incomplete Data",
+          "description": [
+            "Summary data only. For complete data, send a specific query for the object."
+          ],
+          "type": "object truncated due to unexplainable reasons"
+        }
+      ]
+    },
+    {
+      "status": [
+        "client delete prohibited",
+        "client renew prohibited",
+        "client transfer prohibited",
+        "server update prohibited"
+      ],
+      "handle": "%DOMAINHANDLE3%",
+      "links": [
+        {
+          "href": "https://example.com/rdap/domain/%DOMAINNAME3%",
+          "type": "application/rdap+json",
+          "rel": "self",
+          "value": "https://example.com/rdap/domain/%DOMAINNAME3%"
+        }
+      ],
+      "ldhName": "%DOMAINNAME3%",
+      "objectClassName": "domain",
+      "remarks": [
+        {
+          "title": "Incomplete Data",
+          "description": [
+            "Summary data only. For complete data, send a specific query for the object."
+          ],
+          "type": "object truncated due to unexplainable reasons"
+        }
+      ]
+    },
+    {
+      "status": [
+        "client delete prohibited",
+        "client renew prohibited",
+        "client transfer prohibited",
+        "server update prohibited"
+      ],
+      "handle": "%DOMAINHANDLE4%",
+      "links": [
+        {
+          "href": "https://example.com/rdap/domain/%DOMAINNAME4%",
+          "type": "application/rdap+json",
+          "rel": "self",
+          "value": "https://example.com/rdap/domain/%DOMAINNAME4%"
+        }
+      ],
+      "ldhName": "%DOMAINNAME4%",
+      "objectClassName": "domain",
+      "remarks": [
+        {
+          "title": "Incomplete Data",
+          "description": [
+            "Summary data only. For complete data, send a specific query for the object."
+          ],
+          "type": "object truncated due to unexplainable reasons"
+        }
+      ]
+    }
+  ],
+  "rdapConformance": [
+    "rdap_level_0"
+  ],
+  "notices" :
+  [
+    {
+      "title" : "RDAP Terms of Service",
+      "description" :
+      [
+        "By querying our Domain Database, you are agreeing to comply with these terms so please read them carefully.",
+        "Any information provided is 'as is' without any guarantee of accuracy.",
+        "Please do not misuse the Domain Database. It is intended solely for query-based access.",
+        "Don't use the Domain Database to allow, enable, or otherwise support the transmission of mass unsolicited, commercial advertising or solicitations.",
+        "Don't access our Domain Database through the use of high volume, automated electronic processes that send queries or data to the systems of Charleston Road Registry or any ICANN-accredited registrar.",
+        "You may only use the information contained in the Domain Database for lawful purposes.",
+        "Do not compile, repackage, disseminate, or otherwise use the information contained in the Domain Database in its entirety, or in any substantial portion, without our prior written permission.",
+        "We may retain certain details about queries to our Domain Database for the purposes of detecting and preventing misuse.",
+        "We reserve the right to restrict or deny your access to the database if we suspect that you have failed to comply with these terms.",
+        "We reserve the right to modify this agreement at any time."
+      ],
+      "links" :
+      [
+        {
+          "value" : "https://example.com/rdap/help/tos",
+          "rel" : "alternate",
+          "href" : "https://www.registry.google/about/rdap/tos.html",
+          "type" : "text/html"
+        }
+      ]
+    }
+  ],
+  "remarks" :
+  [
+    {
+      "description" :
+      [
+        "This response conforms to the RDAP Operational Profile for gTLD Registries and Registrars version 1.0"
+      ]
+    },
+    {
+      "title" : "EPP Status Codes",
+      "description" :
+      [
+        "For more information on domain status codes, please visit https://icann.org/epp"
+      ],
+      "links" :
+      [
+        {
+          "value" :  "https://icann.org/epp",
+          "rel" : "alternate",
+          "href" : "https://icann.org/epp",
+          "type" : "text/html"
+        }
+      ]
+    },
+    {
+      "description" :
+      [
+        "URL of the ICANN Whois Inaccuracy Complaint Form: https://www.icann.org/wicf"
+      ],
+      "links" :
+      [
+        {
+          "value" : "https://www.icann.org/wicf",
+          "rel" : "alternate",
+          "href" : "https://www.icann.org/wicf",
+          "type" : "text/html"
+        }
+      ]
+    }
+  ]
+}

javatests/google/registry/rdap/testdata/rdap_truncated_domains.json

@@ -0,0 +1,195 @@
+{
+  "domainSearchResults": [
+    {
+      "status": [
+        "client delete prohibited",
+        "client renew prohibited",
+        "client transfer prohibited",
+        "server update prohibited"
+      ],
+      "handle": "%DOMAINHANDLE1%",
+      "links": [
+        {
+          "href": "https://example.com/rdap/domain/%DOMAINNAME1%",
+          "type": "application/rdap+json",
+          "rel": "self",
+          "value": "https://example.com/rdap/domain/%DOMAINNAME1%"
+        }
+      ],
+      "ldhName": "%DOMAINNAME1%",
+      "objectClassName": "domain",
+      "remarks": [
+        {
+          "title": "Incomplete Data",
+          "description": [
+            "Summary data only. For complete data, send a specific query for the object."
+          ],
+          "type": "object truncated due to unexplainable reasons"
+        }
+      ]
+    },
+    {
+      "status": [
+        "client delete prohibited",
+        "client renew prohibited",
+        "client transfer prohibited",
+        "server update prohibited"
+      ],
+      "handle": "%DOMAINHANDLE2%",
+      "links": [
+        {
+          "href": "https://example.com/rdap/domain/%DOMAINNAME2%",
+          "type": "application/rdap+json",
+          "rel": "self",
+          "value": "https://example.com/rdap/domain/%DOMAINNAME2%"
+        }
+      ],
+      "ldhName": "%DOMAINNAME2%",
+      "objectClassName": "domain",
+      "remarks": [
+        {
+          "title": "Incomplete Data",
+          "description": [
+            "Summary data only. For complete data, send a specific query for the object."
+          ],
+          "type": "object truncated due to unexplainable reasons"
+        }
+      ]
+    },
+    {
+      "status": [
+        "client delete prohibited",
+        "client renew prohibited",
+        "client transfer prohibited",
+        "server update prohibited"
+      ],
+      "handle": "%DOMAINHANDLE3%",
+      "links": [
+        {
+          "href": "https://example.com/rdap/domain/%DOMAINNAME3%",
+          "type": "application/rdap+json",
+          "rel": "self",
+          "value": "https://example.com/rdap/domain/%DOMAINNAME3%"
+        }
+      ],
+      "ldhName": "%DOMAINNAME3%",
+      "objectClassName": "domain",
+      "remarks": [
+        {
+          "title": "Incomplete Data",
+          "description": [
+            "Summary data only. For complete data, send a specific query for the object."
+          ],
+          "type": "object truncated due to unexplainable reasons"
+        }
+      ]
+    },
+    {
+      "status": [
+        "client delete prohibited",
+        "client renew prohibited",
+        "client transfer prohibited",
+        "server update prohibited"
+      ],
+      "handle": "%DOMAINHANDLE4%",
+      "links": [
+        {
+          "href": "https://example.com/rdap/domain/%DOMAINNAME4%",
+          "type": "application/rdap+json",
+          "rel": "self",
+          "value": "https://example.com/rdap/domain/%DOMAINNAME4%"
+        }
+      ],
+      "ldhName": "%DOMAINNAME4%",
+      "objectClassName": "domain",
+      "remarks": [
+        {
+          "title": "Incomplete Data",
+          "description": [
+            "Summary data only. For complete data, send a specific query for the object."
+          ],
+          "type": "object truncated due to unexplainable reasons"
+        }
+      ]
+    }
+  ],
+  "rdapConformance": [
+    "rdap_level_0"
+  ],
+  "notices" :
+  [
+    {
+      "title" : "Search Policy",
+      "type" : "result set truncated due to unexplainable reasons",
+      "description" :
+      [
+        "Search results per query are limited."
+      ]
+    },
+    {
+      "title" : "RDAP Terms of Service",
+      "description" :
+      [
+        "By querying our Domain Database, you are agreeing to comply with these terms so please read them carefully.",
+        "Any information provided is 'as is' without any guarantee of accuracy.",
+        "Please do not misuse the Domain Database. It is intended solely for query-based access.",
+        "Don't use the Domain Database to allow, enable, or otherwise support the transmission of mass unsolicited, commercial advertising or solicitations.",
+        "Don't access our Domain Database through the use of high volume, automated electronic processes that send queries or data to the systems of Charleston Road Registry or any ICANN-accredited registrar.",
+        "You may only use the information contained in the Domain Database for lawful purposes.",
+        "Do not compile, repackage, disseminate, or otherwise use the information contained in the Domain Database in its entirety, or in any substantial portion, without our prior written permission.",
+        "We may retain certain details about queries to our Domain Database for the purposes of detecting and preventing misuse.",
+        "We reserve the right to restrict or deny your access to the database if we suspect that you have failed to comply with these terms.",
+        "We reserve the right to modify this agreement at any time."
+      ],
+      "links" :
+      [
+        {
+          "value" : "https://example.com/rdap/help/tos",
+          "rel" : "alternate",
+          "href" : "https://www.registry.google/about/rdap/tos.html",
+          "type" : "text/html"
+        }
+      ]
+    }
+  ],
+  "remarks" :
+  [
+    {
+      "description" :
+      [
+        "This response conforms to the RDAP Operational Profile for gTLD Registries and Registrars version 1.0"
+      ]
+    },
+    {
+      "title" : "EPP Status Codes",
+      "description" :
+      [
+        "For more information on domain status codes, please visit https://icann.org/epp"
+      ],
+      "links" :
+      [
+        {
+          "value" :  "https://icann.org/epp",
+          "rel" : "alternate",
+          "href" : "https://icann.org/epp",
+          "type" : "text/html"
+        }
+      ]
+    },
+    {
+      "description" :
+      [
+        "URL of the ICANN Whois Inaccuracy Complaint Form: https://www.icann.org/wicf"
+      ],
+      "links" :
+      [
+        {
+          "value" : "https://www.icann.org/wicf",
+          "rel" : "alternate",
+          "href" : "https://www.icann.org/wicf",
+          "type" : "text/html"
+        }
+      ]
+    }
+  ]
+}