diff --git a/java/google/registry/rdap/RdapDomainSearchAction.java b/java/google/registry/rdap/RdapDomainSearchAction.java index c99fdb8f0..8049b4bda 100644 --- a/java/google/registry/rdap/RdapDomainSearchAction.java +++ b/java/google/registry/rdap/RdapDomainSearchAction.java @@ -17,7 +17,6 @@ package google.registry.rdap; import static google.registry.model.EppResourceUtils.loadByForeignKey; import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey; import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.rdap.RdapIcannStandardInformation.TRUNCATION_NOTICES; import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.HEAD; import static google.registry.util.DateTimeUtils.END_OF_TIME; @@ -36,6 +35,7 @@ import google.registry.model.domain.DomainResource; import google.registry.model.host.HostResource; import google.registry.rdap.RdapJsonFormatter.BoilerplateType; import google.registry.rdap.RdapJsonFormatter.OutputDataType; +import google.registry.rdap.RdapSearchResults.IncompletenessWarningType; import google.registry.request.Action; import google.registry.request.HttpException.BadRequestException; import google.registry.request.HttpException.NotFoundException; @@ -73,6 +73,8 @@ public class RdapDomainSearchAction extends RdapActionBase { public static final int RESULT_SET_SIZE_SCALING_FACTOR = 30; + public static final int MAX_NAMESERVERS_IN_FIRST_STAGE = 1000; + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); @Inject Clock clock; @@ -142,8 +144,7 @@ public class RdapDomainSearchAction extends RdapActionBase { rdapJsonFormatter.addTopLevelEntries( builder, BoilerplateType.DOMAIN, - results.isTruncated() - ? TRUNCATION_NOTICES : ImmutableList.>of(), + results.getIncompletenessWarnings(), ImmutableList.>of(), rdapLinkBase); return builder.build(); @@ -167,7 +168,7 @@ public class RdapDomainSearchAction extends RdapActionBase { ImmutableList results = (domainResource == null) ? ImmutableList.of() : ImmutableList.of(domainResource); - return makeSearchResults(results, false /* isTruncated */, now); + return makeSearchResults(results, now); // Handle queries with a wildcard and no initial string. } else if (partialStringQuery.getInitialString().isEmpty()) { if (partialStringQuery.getSuffix() == null) { @@ -181,7 +182,7 @@ public class RdapDomainSearchAction extends RdapActionBase { .filter("tld", partialStringQuery.getSuffix()) .filter("deletionTime >", now) .limit(rdapResultSetMaxSize + 1); - return makeSearchResults(query.list(), false /* isTruncated */, now); + return makeSearchResults(query.list(), now); // Handle queries with a wildcard and an initial string. } else { if ((partialStringQuery.getSuffix() == null) @@ -214,12 +215,13 @@ public class RdapDomainSearchAction extends RdapActionBase { query.limit(RESULT_SET_SIZE_SCALING_FACTOR * rdapResultSetMaxSize)) { if (EppResourceUtils.isActive(domain, now)) { if (domainList.size() >= rdapResultSetMaxSize) { - return makeSearchResults(ImmutableList.copyOf(domainList), true /* isTruncated */, now); + return makeSearchResults( + ImmutableList.copyOf(domainList), IncompletenessWarningType.TRUNCATED, now); } domainList.add(domain); } } - return makeSearchResults(domainList, false /* isTruncated */, now); + return makeSearchResults(domainList, now); } } @@ -293,10 +295,15 @@ public class RdapDomainSearchAction extends RdapActionBase { // pending deletes for hosts, so we can call queryUndeleted. In this case, the initial string // must be present, to avoid querying every host in the system. This restriction is enforced // by queryUndeleted(). - // TODO (b/24463238): figure out how to limit the size of these queries effectively } else { + // Only return the first 1000 nameservers. This could result in an incomplete result set if + // a search asks for something like "ns*", but we need to enforce a limit in order to avoid + // arbitrarily long-running queries. return queryUndeleted( - HostResource.class, "fullyQualifiedHostName", partialStringQuery, 1000) + HostResource.class, + "fullyQualifiedHostName", + partialStringQuery, + MAX_NAMESERVERS_IN_FIRST_STAGE) .keys(); } } @@ -307,21 +314,23 @@ public class RdapDomainSearchAction extends RdapActionBase { * *

This is a two-step process: get a list of host references by IP address, and then look up * domains by host reference. + * + *

In theory, we could have any number of hosts using the same IP address. To make sure we get + * all the associated domains, we have to retrieve all of them, and use them to look up domains. + * This could open us up to a kind of DoS attack if huge number of hosts are defined on a single + * IP. To avoid this, fetch only the first 1000 nameservers. In all normal circumstances, this + * should be orders of magnitude more than there actually are. But it could result in us missing + * some domains. */ private RdapSearchResults searchByNameserverIp( final InetAddress inetAddress, final DateTime now) { - // In theory, we could filter on the deletion time being in the future. But we can't do that in - // the query on nameserver name (because we're already using an inequality query), and it seems - // dangerous and confusing to filter on deletion time differently between the two queries. - // Find all domains that link to any of these hosts, and return information about them. - // TODO (b/24463238): figure out how to limit the size of these queries effectively return searchByNameserverRefs( ofy() .load() .type(HostResource.class) .filter("inetAddresses", inetAddress.getHostAddress()) .filter("deletionTime", END_OF_TIME) - .limit(1000) + .limit(MAX_NAMESERVERS_IN_FIRST_STAGE) .keys(), now); } @@ -329,8 +338,8 @@ public class RdapDomainSearchAction extends RdapActionBase { /** * Locates all domains which are linked to a set of host keys. * - *

This method is called by {@link #searchByNameserverLdhName} and - * {@link #searchByNameserverIp} after they assemble the relevant host keys. + *

This method is called by {@link #searchByNameserverLdhName} and {@link + * #searchByNameserverIp} after they assemble the relevant host keys. */ private RdapSearchResults searchByNameserverRefs( final Iterable> hostKeys, final DateTime now) { @@ -340,7 +349,9 @@ public class RdapDomainSearchAction extends RdapActionBase { // domain), we must create a set of resulting {@link DomainResource} objects. But we use a // LinkedHashSet to preserve the order in which we found the domains. LinkedHashSet domains = new LinkedHashSet<>(); + int numHostKeysSearched = 0; for (List> chunk : Iterables.partition(hostKeys, 30)) { + numHostKeysSearched += chunk.size(); for (DomainResource domain : ofy().load() .type(DomainResource.class) .filter("nsHosts in", chunk) @@ -348,23 +359,37 @@ public class RdapDomainSearchAction extends RdapActionBase { .limit(rdapResultSetMaxSize + 1)) { if (!domains.contains(domain)) { if (domains.size() >= rdapResultSetMaxSize) { - return makeSearchResults(ImmutableList.copyOf(domains), true /* isTruncated */, now); + return makeSearchResults( + ImmutableList.copyOf(domains), IncompletenessWarningType.TRUNCATED, now); } domains.add(domain); } } } - return makeSearchResults(ImmutableList.copyOf(domains), false /* isTruncated */, now); + return makeSearchResults( + ImmutableList.copyOf(domains), + (numHostKeysSearched >= MAX_NAMESERVERS_IN_FIRST_STAGE) + ? IncompletenessWarningType.MIGHT_BE_INCOMPLETE + : IncompletenessWarningType.NONE, + now); + } + + /** Output JSON for a list of domains, with no incompleteness warnings. */ + private RdapSearchResults makeSearchResults(List domains, DateTime now) { + return makeSearchResults(domains, IncompletenessWarningType.NONE, now); } /** * Output JSON for a list of domains. * - *

The isTruncated parameter should be true if the search found more results than are in the - * list, meaning that the truncation notice should be added. + *

The incompletenessWarningType should be set to TRUNCATED if the search found more results + * than are in the list, or MIGHT_BE_INCOMPLETE if a search for domains by nameserver returned the + * maximum number of nameservers in the first stage query. */ private RdapSearchResults makeSearchResults( - List domains, boolean isTruncated, DateTime now) { + List domains, + IncompletenessWarningType incompletenessWarningType, + DateTime now) { OutputDataType outputDataType = (domains.size() > 1) ? OutputDataType.SUMMARY : OutputDataType.FULL; RdapAuthorization authorization = getAuthorization(); @@ -374,6 +399,6 @@ public class RdapDomainSearchAction extends RdapActionBase { rdapJsonFormatter.makeRdapJsonForDomain( domain, false, rdapLinkBase, rdapWhoisServer, now, outputDataType, authorization)); } - return RdapSearchResults.create(jsonBuilder.build(), isTruncated); + return RdapSearchResults.create(jsonBuilder.build(), incompletenessWarningType); } } diff --git a/java/google/registry/rdap/RdapEntitySearchAction.java b/java/google/registry/rdap/RdapEntitySearchAction.java index e3c5eb39b..0cef13c7f 100644 --- a/java/google/registry/rdap/RdapEntitySearchAction.java +++ b/java/google/registry/rdap/RdapEntitySearchAction.java @@ -15,7 +15,6 @@ package google.registry.rdap; import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.rdap.RdapIcannStandardInformation.TRUNCATION_NOTICES; import static google.registry.rdap.RdapUtils.getRegistrarByIanaIdentifier; import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.HEAD; @@ -35,6 +34,7 @@ import google.registry.model.domain.DesignatedContact; import google.registry.model.registrar.Registrar; import google.registry.rdap.RdapJsonFormatter.BoilerplateType; import google.registry.rdap.RdapJsonFormatter.OutputDataType; +import google.registry.rdap.RdapSearchResults.IncompletenessWarningType; import google.registry.request.Action; import google.registry.request.HttpException.BadRequestException; import google.registry.request.HttpException.NotFoundException; @@ -113,8 +113,7 @@ public class RdapEntitySearchAction extends RdapActionBase { rdapJsonFormatter.addTopLevelEntries( jsonBuilder, BoilerplateType.ENTITY, - results.isTruncated() - ? TRUNCATION_NOTICES : ImmutableList.>of(), + results.getIncompletenessWarnings(), ImmutableList.>of(), rdapLinkBase); return jsonBuilder.build(); @@ -258,7 +257,8 @@ public class RdapEntitySearchAction extends RdapActionBase { List> jsonOutputList = new ArrayList<>(); for (ContactResource contact : contacts) { if (jsonOutputList.size() >= rdapResultSetMaxSize) { - return RdapSearchResults.create(ImmutableList.copyOf(jsonOutputList), true); + return RdapSearchResults.create( + ImmutableList.copyOf(jsonOutputList), IncompletenessWarningType.TRUNCATED); } // As per Andy Newton on the regext mailing list, contacts by themselves have no role, since // they are global, and might have different roles for different domains. @@ -275,7 +275,8 @@ public class RdapEntitySearchAction extends RdapActionBase { for (Registrar registrar : registrars) { if (registrar.isActiveAndPubliclyVisible()) { if (jsonOutputList.size() >= rdapResultSetMaxSize) { - return RdapSearchResults.create(ImmutableList.copyOf(jsonOutputList), true); + return RdapSearchResults.create( + ImmutableList.copyOf(jsonOutputList), IncompletenessWarningType.TRUNCATED); } jsonOutputList.add(rdapJsonFormatter.makeRdapJsonForRegistrar( registrar, false, rdapLinkBase, rdapWhoisServer, now, outputDataType)); diff --git a/java/google/registry/rdap/RdapIcannStandardInformation.java b/java/google/registry/rdap/RdapIcannStandardInformation.java index 6f0f99281..dab4a4f7d 100644 --- a/java/google/registry/rdap/RdapIcannStandardInformation.java +++ b/java/google/registry/rdap/RdapIcannStandardInformation.java @@ -105,6 +105,24 @@ public class RdapIcannStandardInformation { static final ImmutableList> TRUNCATION_NOTICES = ImmutableList.of(TRUNCATED_RESULT_SET_NOTICE); + /** + * Used when a search for domains by nameserver may have returned incomplete information because + * there were too many nameservers in the first stage results. + */ + static final ImmutableMap POSSIBLY_INCOMPLETE_RESULT_SET_NOTICE = + ImmutableMap.of( + "title", + "Search Policy", + "description", + ImmutableList.of( + "Search results may contain incomplete information due to first-stage query limits."), + "type", + "result set truncated due to unexplainable reasons"); + + /** Possibly incomplete notice as a singleton list, for easy use. */ + static final ImmutableList> POSSIBLY_INCOMPLETE_NOTICES = + ImmutableList.of(POSSIBLY_INCOMPLETE_RESULT_SET_NOTICE); + /** Included when the requester is not logged in as the owner of the domain being returned. */ static final ImmutableMap DOMAIN_CONTACTS_HIDDEN_DATA_REMARK = ImmutableMap. of( diff --git a/java/google/registry/rdap/RdapNameserverSearchAction.java b/java/google/registry/rdap/RdapNameserverSearchAction.java index 5cc3d416f..0feafe49a 100644 --- a/java/google/registry/rdap/RdapNameserverSearchAction.java +++ b/java/google/registry/rdap/RdapNameserverSearchAction.java @@ -16,7 +16,6 @@ package google.registry.rdap; import static google.registry.model.EppResourceUtils.loadByForeignKey; import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.rdap.RdapIcannStandardInformation.TRUNCATION_NOTICES; import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.HEAD; import static google.registry.util.DateTimeUtils.END_OF_TIME; @@ -32,6 +31,7 @@ import google.registry.model.domain.DomainResource; import google.registry.model.host.HostResource; import google.registry.rdap.RdapJsonFormatter.BoilerplateType; import google.registry.rdap.RdapJsonFormatter.OutputDataType; +import google.registry.rdap.RdapSearchResults.IncompletenessWarningType; import google.registry.request.Action; import google.registry.request.HttpException.BadRequestException; import google.registry.request.HttpException.NotFoundException; @@ -119,8 +119,7 @@ public class RdapNameserverSearchAction extends RdapActionBase { rdapJsonFormatter.addTopLevelEntries( jsonBuilder, BoilerplateType.NAMESERVER, - results.isTruncated() - ? TRUNCATION_NOTICES : ImmutableList.>of(), + results.getIncompletenessWarnings(), ImmutableList.>of(), rdapLinkBase); return jsonBuilder.build(); @@ -215,6 +214,10 @@ public class RdapNameserverSearchAction extends RdapActionBase { host, false, rdapLinkBase, rdapWhoisServer, now, outputDataType)); } ImmutableList> jsonList = jsonListBuilder.build(); - return RdapSearchResults.create(jsonList, jsonList.size() < hosts.size()); + return RdapSearchResults.create( + jsonList, + (jsonList.size() < hosts.size()) + ? IncompletenessWarningType.TRUNCATED + : IncompletenessWarningType.NONE); } } diff --git a/java/google/registry/rdap/RdapSearchResults.java b/java/google/registry/rdap/RdapSearchResults.java index 11c4e5e63..84f02a70f 100644 --- a/java/google/registry/rdap/RdapSearchResults.java +++ b/java/google/registry/rdap/RdapSearchResults.java @@ -14,31 +14,61 @@ package google.registry.rdap; +import static google.registry.rdap.RdapIcannStandardInformation.POSSIBLY_INCOMPLETE_NOTICES; +import static google.registry.rdap.RdapIcannStandardInformation.TRUNCATION_NOTICES; + import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; /** * Holds domain, nameserver and entity search results. - * + * *

We need to know not only the list of things we found, but also whether the result set was * truncated to the limit. If it is, we must add the ICANN-mandated notice to that effect. */ @AutoValue abstract class RdapSearchResults { - - static RdapSearchResults create(ImmutableList> jsonList) { - return create(jsonList, false); + + enum IncompletenessWarningType { + + /** Result set is complete. */ + NONE, + + /** Result set has been limited to the maximum size. */ + TRUNCATED, + + /** + * Result set might be missing data because the first step of a two-step query returned a data + * set that was limited in size. + */ + MIGHT_BE_INCOMPLETE } - + + static RdapSearchResults create(ImmutableList> jsonList) { + return create(jsonList, IncompletenessWarningType.NONE); + } + static RdapSearchResults create( - ImmutableList> jsonList, boolean isTruncated) { - return new AutoValue_RdapSearchResults(jsonList, isTruncated); + ImmutableList> jsonList, + IncompletenessWarningType incompletenessWarningType) { + return new AutoValue_RdapSearchResults(jsonList, incompletenessWarningType); } /** List of JSON result object representations. */ abstract ImmutableList> jsonList(); - - /** True if the result set was truncated to the maximum size limit. */ - abstract boolean isTruncated(); + + /** Type of warning to display regarding possible incomplete data. */ + abstract IncompletenessWarningType incompletenessWarningType(); + + /** Convenience method to get the appropriate warnings for the incompleteness warning type. */ + ImmutableList> getIncompletenessWarnings() { + if (incompletenessWarningType() == IncompletenessWarningType.TRUNCATED) { + return TRUNCATION_NOTICES; + } + if (incompletenessWarningType() == IncompletenessWarningType.MIGHT_BE_INCOMPLETE) { + return POSSIBLY_INCOMPLETE_NOTICES; + } + return ImmutableList.>of(); + } } diff --git a/javatests/google/registry/rdap/RdapDomainSearchActionTest.java b/javatests/google/registry/rdap/RdapDomainSearchActionTest.java index 2b54e2e3f..84fcb9748 100644 --- a/javatests/google/registry/rdap/RdapDomainSearchActionTest.java +++ b/javatests/google/registry/rdap/RdapDomainSearchActionTest.java @@ -631,7 +631,7 @@ public class RdapDomainSearchActionTest { String hostName = String.format("ns%d.%s", i, mainDomainName); subordinateHostsBuilder.add(hostName); HostResource host = makeAndPersistHostResource( - hostName, String.format("5.5.5.%d", i), clock.nowUtc().minusYears(1)); + hostName, String.format("5.5.%d.%d", 5 + i / 250, i % 250), clock.nowUtc().minusYears(1)); hostKeysBuilder.add(Key.create(host)); } ImmutableSet> hostKeys = hostKeysBuilder.build(); @@ -1066,6 +1066,23 @@ public class RdapDomainSearchActionTest { assertThat(response.getStatus()).isEqualTo(200); } + @Test + public void testNameserverMatch_incompleteResultsSet() throws Exception { + createManyDomainsAndHosts(2, 1, 2500); + assertThat(generateActualJson(RequestType.NS_LDH_NAME, "ns*.domain1.lol")) + .isEqualTo(readMultiDomainFile( + "rdap_incomplete_domains.json", + "domain1.lol", + "41-LOL", + "domain2.lol", + "42-LOL", + "domain3.lol", + "43-LOL", + "domain4.lol", + "44-LOL")); + assertThat(response.getStatus()).isEqualTo(200); + } + @Test public void testAddressMatchV4Address_foundMultiple() throws Exception { assertThat(generateActualJson(RequestType.NS_IP, "1.2.3.4")) diff --git a/javatests/google/registry/rdap/testdata/rdap_incomplete_domains.json b/javatests/google/registry/rdap/testdata/rdap_incomplete_domains.json new file mode 100644 index 000000000..d97ab0945 --- /dev/null +++ b/javatests/google/registry/rdap/testdata/rdap_incomplete_domains.json @@ -0,0 +1,129 @@ +{ + "domainSearchResults":[ + { + "ldhName":"domain1.lol", + "status":[ + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" + ], + "remarks":[ + { + "title":"Incomplete Data", + "type":"object truncated due to unexplainable reasons", + "description":[ + "Summary data only. For complete data, send a specific query for the object." + ] + } + ], + "handle":"13C5-LOL", + "links":[ + { + "value":"https://example.com/rdap/domain/domain1.lol", + "type":"application/rdap+json", + "rel":"self", + "href":"https://example.com/rdap/domain/domain1.lol" + } + ], + "objectClassName":"domain" + }, + { + "ldhName":"domain2.lol", + "status":[ + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" + ], + "remarks":[ + { + "title":"Incomplete Data", + "type":"object truncated due to unexplainable reasons", + "description":[ + "Summary data only. For complete data, send a specific query for the object." + ] + } + ], + "handle":"13C6-LOL", + "links":[ + { + "value":"https://example.com/rdap/domain/domain2.lol", + "type":"application/rdap+json", + "rel":"self", + "href":"https://example.com/rdap/domain/domain2.lol" + } + ], + "objectClassName":"domain" + } + ], + "remarks":[ + { + "description":[ + "This response conforms to the RDAP Operational Profile for gTLD Registries and Registrars version 1.0" + ] + }, + { + "title":"EPP Status Codes", + "description":[ + "For more information on domain status codes, please visit https://icann.org/epp" + ], + "links":[ + { + "value":"https://icann.org/epp", + "type":"text/html", + "rel":"alternate", + "href":"https://icann.org/epp" + } + ] + }, + { + "description":[ + "URL of the ICANN Whois Inaccuracy Complaint Form: https://www.icann.org/wicf" + ], + "links":[ + { + "value":"https://www.icann.org/wicf", + "type":"text/html", + "rel":"alternate", + "href":"https://www.icann.org/wicf" + } + ] + } + ], + "rdapConformance":[ + "rdap_level_0" + ], + "notices":[ + { + "title":"Search Policy", + "type":"result set truncated due to unexplainable reasons", + "description":[ + "Search results may contain incomplete information due to first-stage query limits." + ] + }, + { + "title":"RDAP Terms of Service", + "description":[ + "By querying our Domain Database, you are agreeing to comply with these terms so please read them carefully.", + "Any information provided is 'as is' without any guarantee of accuracy.", + "Please do not misuse the Domain Database. It is intended solely for query-based access.", + "Don't use the Domain Database to allow, enable, or otherwise support the transmission of mass unsolicited, commercial advertising or solicitations.", + "Don't access our Domain Database through the use of high volume, automated electronic processes that send queries or data to the systems of any ICANN-accredited registrar.", + "You may only use the information contained in the Domain Database for lawful purposes.", + "Do not compile, repackage, disseminate, or otherwise use the information contained in the Domain Database in its entirety, or in any substantial portion, without our prior written permission.", + "We may retain certain details about queries to our Domain Database for the purposes of detecting and preventing misuse.", + "We reserve the right to restrict or deny your access to the database if we suspect that you have failed to comply with these terms.", + "We reserve the right to modify this agreement at any time." + ], + "links":[ + { + "value":"https://example.com/rdap/help/tos", + "type":"text/html", + "rel":"alternate", + "href":"https://www.registry.tld/about/rdap/tos.html" + } + ] + } + ] +}