diff --git a/java/google/registry/rdap/RdapActionBase.java b/java/google/registry/rdap/RdapActionBase.java index 1ea5f53cb..e95c0a16a 100644 --- a/java/google/registry/rdap/RdapActionBase.java +++ b/java/google/registry/rdap/RdapActionBase.java @@ -17,9 +17,7 @@ package google.registry.rdap; import static com.google.common.base.Charsets.UTF_8; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.net.HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN; -import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.request.Actions.getPathForAction; -import static google.registry.util.DateTimeUtils.END_OF_TIME; import static google.registry.util.DomainNameUtils.canonicalizeDomainName; import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; @@ -29,30 +27,22 @@ import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.re2j.Pattern; -import com.googlecode.objectify.Key; -import com.googlecode.objectify.cmd.Query; import google.registry.config.RegistryConfig.Config; import google.registry.model.EppResource; import google.registry.model.registrar.Registrar; import google.registry.rdap.RdapMetrics.EndpointType; -import google.registry.rdap.RdapMetrics.WildcardType; import google.registry.rdap.RdapObjectClasses.ErrorResponse; import google.registry.rdap.RdapObjectClasses.ReplyPayloadBase; import google.registry.rdap.RdapObjectClasses.TopLevelReplyObject; import google.registry.rdap.RdapSearchResults.BaseSearchResponse; -import google.registry.rdap.RdapSearchResults.IncompletenessWarningType; import google.registry.request.Action; import google.registry.request.HttpException; -import google.registry.request.HttpException.UnprocessableEntityException; import google.registry.request.Parameter; import google.registry.request.RequestMethod; import google.registry.request.RequestPath; import google.registry.request.Response; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; import java.util.Optional; import javax.inject.Inject; import org.joda.time.DateTime; @@ -67,13 +57,6 @@ public abstract class RdapActionBase implements Runnable { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - /** - * Pattern for checking LDH names, which must officially contains only alphanumeric plus dots and - * hyphens. In this case, allow the wildcard asterisk as well. - */ - static final Pattern LDH_PATTERN = Pattern.compile("[-.a-zA-Z0-9*]+"); - private static final int RESULT_SET_SIZE_SCALING_FACTOR = 30; - private static final MediaType RESPONSE_MEDIA_TYPE = MediaType.create("application", "rdap+json").withCharset(UTF_8); @@ -88,7 +71,6 @@ public abstract class RdapActionBase implements Runnable { @Inject @RequestPath String requestPath; @Inject RdapAuthorization rdapAuthorization; @Inject RdapJsonFormatter rdapJsonFormatter; - @Inject @Parameter("registrar") Optional registrarParam; @Inject @Parameter("includeDeleted") Optional includeDeletedParam; @Inject @Parameter("formatOutput") Optional formatOutputParam; @Inject @Config("rdapResultSetMaxSize") int rdapResultSetMaxSize; @@ -121,6 +103,8 @@ public abstract class RdapActionBase implements Runnable { /** * Does the actual search and returns an RDAP JSON object. * + * RFC7480 4.1 - we have to support GET and HEAD. + * * @param pathSearchString the search string in the URL path * @param isHeadRequest whether the returned map will actually be used. HTTP HEAD requests don't * actually return anything. However, we usually still want to go through the process of @@ -135,30 +119,37 @@ public abstract class RdapActionBase implements Runnable { @Override public void run() { metricInformationBuilder.setIncludeDeleted(includeDeletedParam.orElse(false)); - metricInformationBuilder.setRegistrarSpecified(registrarParam.isPresent()); metricInformationBuilder.setRole(rdapAuthorization.role()); metricInformationBuilder.setRequestMethod(requestMethod); metricInformationBuilder.setEndpointType(endpointType); + // RFC7480 4.2 - servers receiving an RDAP request return an entity with a Content-Type header + // containing the RDAP-specific JSON media type. + response.setContentType(RESPONSE_MEDIA_TYPE); + // RDAP Technical Implementation Guide 1.13 - when responding to RDAP valid requests, we MUST + // include the Access-Control-Allow-Origin, which MUST be "*" unless otherwise specified. + response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*"); try { // Extract what we're searching for from the request path. Some RDAP commands use trailing // data in the path itself (e.g. /rdap/domain/mydomain.com), and some use the query string // (e.g. /rdap/domains?name=mydomain); the query parameters are extracted by the subclasses // directly as needed. - response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*"); URI uri = new URI(requestPath); String pathProper = uri.getPath(); checkArgument( pathProper.startsWith(getActionPath()), "%s doesn't start with %s", pathProper, getActionPath()); + String pathSearchString = pathProper.substring(getActionPath().length()); + logger.atInfo().log("path search string: '%s'", pathSearchString); + ReplyPayloadBase replyObject = - getJsonObjectForResource( - pathProper.substring(getActionPath().length()), requestMethod == Action.Method.HEAD); + getJsonObjectForResource(pathSearchString, requestMethod == Action.Method.HEAD); if (replyObject instanceof BaseSearchResponse) { metricInformationBuilder.setIncompletenessWarningType( ((BaseSearchResponse) replyObject).incompletenessWarningType()); } + // RFC7480 5.1 - if the server has the information requested and wishes to respond, it returns + // that answer in the body of a 200 (OK) response response.setStatus(SC_OK); - response.setContentType(RESPONSE_MEDIA_TYPE); setPayload(replyObject); metricInformationBuilder.setStatusCode(SC_OK); } catch (HttpException e) { @@ -177,7 +168,6 @@ public abstract class RdapActionBase implements Runnable { void setError(int status, String title, String description) { metricInformationBuilder.setStatusCode(status); response.setStatus(status); - response.setContentType(RESPONSE_MEDIA_TYPE); try { setPayload(ErrorResponse.create(status, title, description)); } catch (Exception ex) { @@ -204,11 +194,6 @@ public abstract class RdapActionBase implements Runnable { response.setPayload(gson.toJson(topLevelObject.toJson())); } - /** Returns the registrar on which results should be filtered, or absent(). */ - Optional getDesiredRegistrar() { - return registrarParam; - } - /** * Returns true if the query should include deleted items. * @@ -247,43 +232,16 @@ public abstract class RdapActionBase implements Runnable { eppResource.getPersistedCurrentSponsorClientId())); } - /** - * Returns true if the EPP resource should be visible. - * - *

This is true iff: 1. The resource is not deleted, or the request wants to see deleted items, - * and is authorized to do so, and: 2. The request did not specify a registrar to filter on, or - * the registrar matches. - */ - boolean shouldBeVisible(EppResource eppResource) { - return isAuthorized(eppResource) - && (!registrarParam.isPresent() - || registrarParam.get().equals(eppResource.getPersistedCurrentSponsorClientId())); - } - - /** - * Returns true if the EPP resource should be visible. - * - *

This is true iff: 1. The passed in resource exists and is not deleted (deleted ones will - * have been projected forward in time to empty), 2. The request did not specify a registrar to - * filter on, or the registrar matches. - */ - boolean shouldBeVisible(Optional eppResource) { - return eppResource.isPresent() && shouldBeVisible(eppResource.get()); - } - /** * Returns true if the registrar should be visible. * - *

This is true iff: - * 1. The resource is active and publicly visible, or the request wants to see deleted items, and - * is authorized to do so, and: - * 2. The request did not specify a registrar to filter on, or the registrar matches. + *

This is true iff: The resource is active and publicly visible, or the request wants to see + * deleted items, and is authorized to do so */ - boolean shouldBeVisible(Registrar registrar) { + boolean isAuthorized(Registrar registrar) { return (registrar.isLiveAndPubliclyVisible() || (shouldIncludeDeleted() - && rdapAuthorization.isAuthorizedForClientId(registrar.getClientId()))) - && (!registrarParam.isPresent() || registrarParam.get().equals(registrar.getClientId())); + && rdapAuthorization.isAuthorizedForClientId(registrar.getClientId()))); } String canonicalizeName(String name) { @@ -294,250 +252,6 @@ public abstract class RdapActionBase implements Runnable { return name; } - int getStandardQuerySizeLimit() { - return shouldIncludeDeleted() - ? (RESULT_SET_SIZE_SCALING_FACTOR * (rdapResultSetMaxSize + 1)) - : (rdapResultSetMaxSize + 1); - } - - static Query queryItems( - Class clazz, - String filterField, - RdapSearchPattern partialStringQuery, - DeletedItemHandling deletedItemHandling, - int resultSetMaxSize) { - return queryItems( - clazz, - filterField, - partialStringQuery, - Optional.empty(), - deletedItemHandling, - resultSetMaxSize); - } - - /** - * Handles prefix searches in cases where, if we need to filter out deleted items, there are no - * pending deletes. - * - *

In such cases, it is sufficient to check whether {@code deletionTime} is equal to - * {@code END_OF_TIME}, because any other value means it has already been deleted. This allows us - * to use an equality query for the deletion time. - * - * @param clazz the type of resource to be queried - * @param filterField the database field of interest - * @param partialStringQuery the details of the search string; if there is no wildcard, an - * equality query is used; if there is a wildcard, a range query is used instead; the - * initial string should not be empty, and any search suffix will be ignored, so the caller - * must filter the results if a suffix is specified - * @param cursorString if a cursor is present, this parameter should specify the cursor string, to - * skip any results up to and including the string; empty() if there is no cursor - * @param deletedItemHandling whether to include or exclude deleted items - * @param resultSetMaxSize the maximum number of results to return - * @return the query object - */ - static Query queryItems( - Class clazz, - String filterField, - RdapSearchPattern partialStringQuery, - Optional cursorString, - DeletedItemHandling deletedItemHandling, - int resultSetMaxSize) { - if (partialStringQuery.getInitialString().length() - < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { - throw new UnprocessableEntityException( - String.format( - "Initial search string must be at least %d characters", - RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); - } - Query query = ofy().load().type(clazz); - if (!partialStringQuery.getHasWildcard()) { - query = query.filter(filterField, partialStringQuery.getInitialString()); - } else { - // Ignore the suffix; the caller will need to filter on the suffix, if any. - query = query - .filter(filterField + " >=", partialStringQuery.getInitialString()) - .filter(filterField + " <", partialStringQuery.getNextInitialString()); - } - if (cursorString.isPresent()) { - query = query.filter(filterField + " >", cursorString.get()); - } - return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize); - } - - /** - * Handles searches using a simple string rather than an {@link RdapSearchPattern}. - * - *

Since the filter is not an inequality, we can support also checking a cursor string against - * a different field (which involves an inequality on that field). - * - * @param clazz the type of resource to be queried - * @param filterField the database field of interest - * @param queryString the search string - * @param cursorField the field which should be compared to the cursor string, or empty() if the - * key should be compared to a key created from the cursor string - * @param cursorString if a cursor is present, this parameter should specify the cursor string, to - * skip any results up to and including the string; empty() if there is no cursor - * @param deletedItemHandling whether to include or exclude deleted items - * @param resultSetMaxSize the maximum number of results to return - * @return the query object - */ - static Query queryItems( - Class clazz, - String filterField, - String queryString, - Optional cursorField, - Optional cursorString, - DeletedItemHandling deletedItemHandling, - int resultSetMaxSize) { - if (queryString.length() < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { - throw new UnprocessableEntityException( - String.format( - "Initial search string must be at least %d characters", - RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); - } - Query query = ofy().load().type(clazz).filter(filterField, queryString); - if (cursorString.isPresent()) { - if (cursorField.isPresent()) { - query = query.filter(cursorField.get() + " >", cursorString.get()); - } else { - query = query.filterKey(">", Key.create(clazz, cursorString.get())); - } - } - return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize); - } - - /** Handles searches where the field to be searched is the key. */ - static Query queryItemsByKey( - Class clazz, - RdapSearchPattern partialStringQuery, - Optional cursorString, - DeletedItemHandling deletedItemHandling, - int resultSetMaxSize) { - if (partialStringQuery.getInitialString().length() - < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { - throw new UnprocessableEntityException( - String.format( - "Initial search string must be at least %d characters", - RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); - } - Query query = ofy().load().type(clazz); - if (!partialStringQuery.getHasWildcard()) { - query = query.filterKey("=", Key.create(clazz, partialStringQuery.getInitialString())); - } else { - // Ignore the suffix; the caller will need to filter on the suffix, if any. - query = query - .filterKey(">=", Key.create(clazz, partialStringQuery.getInitialString())) - .filterKey("<", Key.create(clazz, partialStringQuery.getNextInitialString())); - } - if (cursorString.isPresent()) { - query = query.filterKey(">", Key.create(clazz, cursorString.get())); - } - return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize); - } - - /** Handles searches by key using a simple string. */ - static Query queryItemsByKey( - Class clazz, - String queryString, - DeletedItemHandling deletedItemHandling, - int resultSetMaxSize) { - if (queryString.length() < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { - throw new UnprocessableEntityException( - String.format( - "Initial search string must be at least %d characters", - RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); - } - Query query = ofy().load().type(clazz).filterKey("=", Key.create(clazz, queryString)); - return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize); - } - - private static Query setOtherQueryAttributes( - Query query, DeletedItemHandling deletedItemHandling, int resultSetMaxSize) { - if (deletedItemHandling != DeletedItemHandling.INCLUDE) { - query = query.filter("deletionTime", END_OF_TIME); - } - return query.limit(resultSetMaxSize); - } - - /** - * Runs the given query, and checks for permissioning if necessary. - * - * @param query an already-defined query to be run; a filter on currentSponsorClientId will be - * added if appropriate - * @param checkForVisibility true if the results should be checked to make sure they are visible; - * normally this should be equal to the shouldIncludeDeleted setting, but in cases where the - * query could not check deletion status (due to Datastore limitations such as the limit of - * one field queried for inequality, for instance), it may need to be set to true even when - * not including deleted records - * @param querySizeLimit the maximum number of items the query is expected to return, usually - * because the limit has been set - * @return an {@link RdapResultSet} object containing the list of resources and an incompleteness - * warning flag, which is set to MIGHT_BE_INCOMPLETE iff any resources were excluded due to - * lack of visibility, and the resulting list of resources is less than the maximum allowable, - * and the number of items returned by the query is greater than or equal to the maximum - * number we might have expected - */ - RdapResultSet getMatchingResources( - Query query, boolean checkForVisibility, int querySizeLimit) { - Optional desiredRegistrar = getDesiredRegistrar(); - if (desiredRegistrar.isPresent()) { - query = query.filter("currentSponsorClientId", desiredRegistrar.get()); - } - if (!checkForVisibility) { - return RdapResultSet.create(query.list()); - } - // If we are including deleted resources, we need to check that we're authorized for each one. - List resources = new ArrayList<>(); - int numResourcesQueried = 0; - boolean someExcluded = false; - for (T resource : query) { - if (shouldBeVisible(resource)) { - resources.add(resource); - } else { - someExcluded = true; - } - numResourcesQueried++; - if (resources.size() > rdapResultSetMaxSize) { - break; - } - } - // The incompleteness problem comes about because we don't know how many items to fetch. We want - // to return rdapResultSetMaxSize worth of items, but some might be excluded, so we fetch more - // just in case. But how many more? That's the potential problem, addressed with the three way - // AND statement: - // 1. If we didn't exclude any items, then we can't have the incompleteness problem. - // 2. If have a full result set batch (rdapResultSetMaxSize items), we must by definition be - // giving the user a complete result set. - // 3. If we started with fewer than querySizeLimit items, then there weren't any more items that - // we missed. Even if we return fewer than rdapResultSetMaxSize items, it isn't because we - // didn't fetch enough to start. - // Only if all three conditions are true might things be incomplete. In other words, we fetched - // as many as our limit allowed, but then excluded so many that we wound up with less than a - // full result set's worth of results. - return RdapResultSet.create( - resources, - (someExcluded - && (resources.size() < rdapResultSetMaxSize) - && (numResourcesQueried >= querySizeLimit)) - ? IncompletenessWarningType.MIGHT_BE_INCOMPLETE - : IncompletenessWarningType.COMPLETE, - numResourcesQueried); - } - - RdapSearchPattern recordWildcardType(RdapSearchPattern partialStringQuery) { - if (!partialStringQuery.getHasWildcard()) { - metricInformationBuilder.setWildcardType(WildcardType.NO_WILDCARD); - } else if (partialStringQuery.getSuffix() == null) { - metricInformationBuilder.setWildcardType(WildcardType.PREFIX); - } else if (partialStringQuery.getInitialString().isEmpty()) { - metricInformationBuilder.setWildcardType(WildcardType.SUFFIX); - } else { - metricInformationBuilder.setWildcardType(WildcardType.PREFIX_AND_SUFFIX); - } - metricInformationBuilder.setPrefixLength(partialStringQuery.getInitialString().length()); - return partialStringQuery; - } - /** Returns the DateTime this request took place. */ DateTime getRequestTime() { return rdapJsonFormatter.getRequestTime(); diff --git a/java/google/registry/rdap/RdapDataStructures.java b/java/google/registry/rdap/RdapDataStructures.java index d65934158..df83d2523 100644 --- a/java/google/registry/rdap/RdapDataStructures.java +++ b/java/google/registry/rdap/RdapDataStructures.java @@ -48,6 +48,11 @@ final class RdapDataStructures { // Conformance to the RDAP Response Profile V2.1 // (see section 1.3) jsonArray.add("icann_rdap_response_profile_0"); + + // Conformance to the RDAP Technical Implementation Guide V2.1 + // (see section 1.14) + jsonArray.add("icann_rdap_technical_implementation_guide_0"); + return jsonArray; } } diff --git a/java/google/registry/rdap/RdapDomainAction.java b/java/google/registry/rdap/RdapDomainAction.java index a991bbd5a..cbcf697b6 100644 --- a/java/google/registry/rdap/RdapDomainAction.java +++ b/java/google/registry/rdap/RdapDomainAction.java @@ -47,6 +47,8 @@ public class RdapDomainAction extends RdapActionBase { @Override public RdapDomain getJsonObjectForResource(String pathSearchString, boolean isHeadRequest) { + // RDAP Technical Implementation Guide 2.1.1 - we must support A-label (Punycode) and U-label + // (Unicode) formats. canonicalizeName will transform Unicode to Punycode so we support both. pathSearchString = canonicalizeName(pathSearchString); try { validateDomainName(pathSearchString); @@ -62,7 +64,12 @@ public class RdapDomainAction extends RdapActionBase { DomainBase.class, pathSearchString, shouldIncludeDeleted() ? START_OF_TIME : rdapJsonFormatter.getRequestTime()); - if (!shouldBeVisible(domainBase)) { + if (!domainBase.isPresent() || !isAuthorized(domainBase.get())) { + // RFC7480 5.3 - if the server wishes to respond that it doesn't have data satisfying the + // query, it MUST reply with 404 response code. + // + // Note we don't do RFC7480 5.3 - returning a different code if we wish to say "this info + // exists but we don't want to show it to you", because we DON'T wish to say that. throw new NotFoundException(pathSearchString + " not found"); } return rdapJsonFormatter.createRdapDomain(domainBase.get(), OutputDataType.FULL); diff --git a/java/google/registry/rdap/RdapDomainSearchAction.java b/java/google/registry/rdap/RdapDomainSearchAction.java index 8a6f46651..433379690 100644 --- a/java/google/registry/rdap/RdapDomainSearchAction.java +++ b/java/google/registry/rdap/RdapDomainSearchAction.java @@ -44,7 +44,6 @@ import google.registry.request.HttpException.NotFoundException; import google.registry.request.HttpException.UnprocessableEntityException; import google.registry.request.Parameter; import google.registry.request.auth.Auth; -import google.registry.util.Idn; import google.registry.util.NonFinalForTesting; import java.net.InetAddress; import java.util.Comparator; @@ -62,6 +61,9 @@ import javax.inject.Inject; * (RDAP) Query Format * @see RFC 7483: JSON Responses for the Registration * Data Access Protocol (RDAP) + * + * TODO(guyben):This isn't required by the RDAP Technical Implementation Guide, and hence should be + * deleted, at least until it's actually required. */ @Action( service = Action.Service.PUBAPI, @@ -90,41 +92,29 @@ public class RdapDomainSearchAction extends RdapSearchActionBase { *

The RDAP spec allows for domain search by domain name, nameserver name or nameserver IP. */ @Override - public DomainSearchResponse getJsonObjectForResource( - String pathSearchString, boolean isHeadRequest) { + public DomainSearchResponse getSearchResponse(boolean isHeadRequest) { // RDAP syntax example: /rdap/domains?name=exam*.com. - // The pathSearchString is not used by search commands. - if (pathSearchString.length() > 0) { - throw new BadRequestException("Unexpected path"); - } if (Booleans.countTrue(nameParam.isPresent(), nsLdhNameParam.isPresent(), nsIpParam.isPresent()) != 1) { throw new BadRequestException( "You must specify either name=XXXX, nsLdhName=YYYY or nsIp=ZZZZ"); } - decodeCursorToken(); DomainSearchResponse results; if (nameParam.isPresent()) { metricInformationBuilder.setSearchType(SearchType.BY_DOMAIN_NAME); // syntax: /rdap/domains?name=exam*.com - String asciiName; - try { - asciiName = Idn.toASCII(nameParam.get()); - } catch (Exception e) { - throw new BadRequestException("Invalid value of nsLdhName parameter"); - } - results = searchByDomainName(recordWildcardType(RdapSearchPattern.create(asciiName, true))); + results = + searchByDomainName( + recordWildcardType( + RdapSearchPattern.createFromLdhOrUnicodeDomainName(nameParam.get()))); } else if (nsLdhNameParam.isPresent()) { metricInformationBuilder.setSearchType(SearchType.BY_NAMESERVER_NAME); // syntax: /rdap/domains?nsLdhName=ns1.exam*.com // RFC 7482 appears to say that Unicode domains must be specified using punycode when // passed to nsLdhName, so IDN.toASCII is not called here. - if (!LDH_PATTERN.matcher(nsLdhNameParam.get()).matches()) { - throw new BadRequestException("Invalid value of nsLdhName parameter"); - } results = searchByNameserverLdhName( - recordWildcardType(RdapSearchPattern.create(nsLdhNameParam.get(), true))); + recordWildcardType(RdapSearchPattern.createFromLdhDomainName(nsLdhNameParam.get()))); } else { metricInformationBuilder.setSearchType(SearchType.BY_NAMESERVER_ADDRESS); metricInformationBuilder.setWildcardType(WildcardType.NO_WILDCARD); @@ -189,7 +179,9 @@ public class RdapDomainSearchAction extends RdapSearchActionBase { Optional domainBase = loadByForeignKey(DomainBase.class, partialStringQuery.getInitialString(), getRequestTime()); return makeSearchResults( - shouldBeVisible(domainBase) ? ImmutableList.of(domainBase.get()) : ImmutableList.of()); + shouldBeVisible(domainBase) + ? ImmutableList.of(domainBase.get()) + : ImmutableList.of()); } /** Searches for domains by domain name with an initial string, wildcard and possible suffix. */ @@ -294,6 +286,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase { HostResource.class, "fullyQualifiedHostName", partialStringQuery, + Optional.empty(), DeletedItemHandling.EXCLUDE, maxNameserversInFirstStage); Optional desiredRegistrar = getDesiredRegistrar(); diff --git a/java/google/registry/rdap/RdapEntityAction.java b/java/google/registry/rdap/RdapEntityAction.java index 9bdfefb2c..e57f786a9 100644 --- a/java/google/registry/rdap/RdapEntityAction.java +++ b/java/google/registry/rdap/RdapEntityAction.java @@ -16,6 +16,7 @@ package google.registry.rdap; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.rdap.RdapUtils.getRegistrarByIanaIdentifier; +import static google.registry.rdap.RdapUtils.getRegistrarByName; import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.HEAD; @@ -29,7 +30,6 @@ import google.registry.rdap.RdapJsonFormatter.OutputDataType; import google.registry.rdap.RdapMetrics.EndpointType; import google.registry.rdap.RdapObjectClasses.RdapEntity; import google.registry.request.Action; -import google.registry.request.HttpException.BadRequestException; import google.registry.request.HttpException.NotFoundException; import google.registry.request.auth.Auth; import java.util.Optional; @@ -63,31 +63,46 @@ public class RdapEntityAction extends RdapActionBase { public RdapEntity getJsonObjectForResource( String pathSearchString, boolean isHeadRequest) { // The query string is not used; the RDAP syntax is /rdap/entity/handle (the handle is the roid - // for contacts and the client identifier for registrars). Since RDAP's concept of an entity + // for contacts and the client identifier/fn for registrars). Since RDAP's concept of an entity // includes both contacts and registrars, search for one first, then the other. - boolean wasValidKey = false; + + // RDAP Technical Implementation Guide 2.3.1 - MUST support contact entity lookup using the + // handle if (ROID_PATTERN.matcher(pathSearchString).matches()) { - wasValidKey = true; Key contactKey = Key.create(ContactResource.class, pathSearchString); ContactResource contactResource = ofy().load().key(contactKey).now(); // As per Andy Newton on the regext mailing list, contacts by themselves have no role, since // they are global, and might have different roles for different domains. - if ((contactResource != null) && shouldBeVisible(contactResource)) { + if (contactResource != null && isAuthorized(contactResource)) { return rdapJsonFormatter.createRdapContactEntity( contactResource, ImmutableSet.of(), OutputDataType.FULL); } } + + // RDAP Technical Implementation Guide 2.4.1 - MUST support registrar entity lookup using the + // IANA ID as handle Long ianaIdentifier = Longs.tryParse(pathSearchString); if (ianaIdentifier != null) { - wasValidKey = true; Optional registrar = getRegistrarByIanaIdentifier(ianaIdentifier); - if (registrar.isPresent() && shouldBeVisible(registrar.get())) { + if (registrar.isPresent() && isAuthorized(registrar.get())) { return rdapJsonFormatter.createRdapRegistrarEntity(registrar.get(), OutputDataType.FULL); } } + + // RDAP Technical Implementation Guide 2.4.2 - MUST support registrar entity lookup using the + // fn as handle + Optional registrar = getRegistrarByName(pathSearchString); + if (registrar.isPresent() && isAuthorized(registrar.get())) { + return rdapJsonFormatter.createRdapRegistrarEntity(registrar.get(), OutputDataType.FULL); + } + // At this point, we have failed to find either a contact or a registrar. - throw wasValidKey - ? new NotFoundException(pathSearchString + " not found") - : new BadRequestException(pathSearchString + " is not a valid entity handle"); + // + // RFC7480 5.3 - if the server wishes to respond that it doesn't have data satisfying the + // query, it MUST reply with 404 response code. + // + // Note we don't do RFC7480 5.3 - returning a different code if we wish to say "this info + // exists but we don't want to show it to you", because we DON'T wish to say that. + throw new NotFoundException(pathSearchString + " not found"); } } diff --git a/java/google/registry/rdap/RdapEntitySearchAction.java b/java/google/registry/rdap/RdapEntitySearchAction.java index c1522b0be..7acaebd20 100644 --- a/java/google/registry/rdap/RdapEntitySearchAction.java +++ b/java/google/registry/rdap/RdapEntitySearchAction.java @@ -72,6 +72,9 @@ import javax.inject.Inject; * (RDAP) Query Format * @see RFC 7483: JSON Responses for the Registration * Data Access Protocol (RDAP) + * + * TODO(guyben):This isn't required by the RDAP Technical Implementation Guide, and hence should be + * deleted, at least until it's actually required. */ @Action( service = Action.Service.PUBAPI, @@ -109,14 +112,9 @@ public class RdapEntitySearchAction extends RdapSearchActionBase { /** Parses the parameters and calls the appropriate search function. */ @Override - public EntitySearchResponse getJsonObjectForResource( - String pathSearchString, boolean isHeadRequest) { + public EntitySearchResponse getSearchResponse(boolean isHeadRequest) { // RDAP syntax example: /rdap/entities?fn=Bobby%20Joe*. - // The pathSearchString is not used by search commands. - if (pathSearchString.length() > 0) { - throw new BadRequestException("Unexpected path"); - } if (Booleans.countTrue(fnParam.isPresent(), handleParam.isPresent()) != 1) { throw new BadRequestException("You must specify either fn=XXXX or handle=YYYY"); } @@ -133,8 +131,6 @@ public class RdapEntitySearchAction extends RdapSearchActionBase { throw new BadRequestException("Subtype parameter must specify contacts, registrars or all"); } - // Decode the cursor token and extract the prefix and string portions. - decodeCursorToken(); CursorType cursorType; Optional cursorQueryString; if (!cursorString.isPresent()) { @@ -162,7 +158,7 @@ public class RdapEntitySearchAction extends RdapSearchActionBase { // The name is the contact name or registrar name (not registrar contact name). results = searchByName( - recordWildcardType(RdapSearchPattern.create(fnParam.get(), false)), + recordWildcardType(RdapSearchPattern.createFromUnicodeString(fnParam.get())), cursorType, cursorQueryString, subtype); @@ -174,7 +170,7 @@ public class RdapEntitySearchAction extends RdapSearchActionBase { // The handle is either the contact roid or the registrar clientId. results = searchByHandle( - recordWildcardType(RdapSearchPattern.create(handleParam.get(), false)), + recordWildcardType(RdapSearchPattern.createFromUnicodeString(handleParam.get())), cursorType, cursorQueryString, subtype); diff --git a/java/google/registry/rdap/RdapIcannStandardInformation.java b/java/google/registry/rdap/RdapIcannStandardInformation.java index 368c63df9..401eb2860 100644 --- a/java/google/registry/rdap/RdapIcannStandardInformation.java +++ b/java/google/registry/rdap/RdapIcannStandardInformation.java @@ -82,6 +82,8 @@ public class RdapIcannStandardInformation { /** * Required by ICANN RDAP Profile section 1.4.9, as corrected by Gustavo Lozano of ICANN. * + * Also mentioned in the RDAP Technical Implementation Guide 3.6. + * * @see Questions about * the ICANN RDAP Profile */ @@ -96,6 +98,8 @@ public class RdapIcannStandardInformation { /** * Required by ICANN RDAP Profile section 1.4.8, as corrected by Gustavo Lozano of ICANN. * + * Also mentioned in the RDAP Technical Implementation Guide 3.5. + * * @see Questions about * the ICANN RDAP Profile */ diff --git a/java/google/registry/rdap/RdapMetrics.java b/java/google/registry/rdap/RdapMetrics.java index 90a631053..b698e8d3d 100644 --- a/java/google/registry/rdap/RdapMetrics.java +++ b/java/google/registry/rdap/RdapMetrics.java @@ -298,6 +298,7 @@ public class RdapMetrics { .setSearchType(SearchType.NONE) .setWildcardType(WildcardType.INVALID) .setPrefixLength(0) + .setRegistrarSpecified(false) .setIncompletenessWarningType(IncompletenessWarningType.COMPLETE); } } diff --git a/java/google/registry/rdap/RdapNameserverAction.java b/java/google/registry/rdap/RdapNameserverAction.java index 0bda72f8c..d7da3ba63 100644 --- a/java/google/registry/rdap/RdapNameserverAction.java +++ b/java/google/registry/rdap/RdapNameserverAction.java @@ -47,6 +47,8 @@ public class RdapNameserverAction extends RdapActionBase { @Override public RdapNameserver getJsonObjectForResource(String pathSearchString, boolean isHeadRequest) { + // RDAP Technical Implementation Guide 2.2.1 - we must support A-label (Punycode) and U-label + // (Unicode) formats. canonicalizeName will transform Unicode to Punycode so we support both. pathSearchString = canonicalizeName(pathSearchString); // The RDAP syntax is /rdap/nameserver/ns1.mydomain.com. try { @@ -64,7 +66,12 @@ public class RdapNameserverAction extends RdapActionBase { HostResource.class, pathSearchString, shouldIncludeDeleted() ? START_OF_TIME : getRequestTime()); - if (!shouldBeVisible(hostResource)) { + if (!hostResource.isPresent() || !isAuthorized(hostResource.get())) { + // RFC7480 5.3 - if the server wishes to respond that it doesn't have data satisfying the + // query, it MUST reply with 404 response code. + // + // Note we don't do RFC7480 5.3 - returning a different code if we wish to say "this info + // exists but we don't want to show it to you", because we DON'T wish to say that. throw new NotFoundException(pathSearchString + " not found"); } return rdapJsonFormatter.createRdapNameserver(hostResource.get(), OutputDataType.FULL); diff --git a/java/google/registry/rdap/RdapNameserverSearchAction.java b/java/google/registry/rdap/RdapNameserverSearchAction.java index 9dde55df2..62ad67532 100644 --- a/java/google/registry/rdap/RdapNameserverSearchAction.java +++ b/java/google/registry/rdap/RdapNameserverSearchAction.java @@ -36,7 +36,6 @@ import google.registry.request.HttpException.NotFoundException; import google.registry.request.HttpException.UnprocessableEntityException; import google.registry.request.Parameter; import google.registry.request.auth.Auth; -import google.registry.util.Idn; import java.net.InetAddress; import java.util.ArrayList; import java.util.List; @@ -79,30 +78,26 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase { *

The RDAP spec allows nameserver search by either name or IP address. */ @Override - public NameserverSearchResponse getJsonObjectForResource( - String pathSearchString, boolean isHeadRequest) { + public NameserverSearchResponse getSearchResponse(boolean isHeadRequest) { // RDAP syntax example: /rdap/nameservers?name=ns*.example.com. - // The pathSearchString is not used by search commands. - if (pathSearchString.length() > 0) { - throw new BadRequestException("Unexpected path"); - } if (Booleans.countTrue(nameParam.isPresent(), ipParam.isPresent()) != 1) { throw new BadRequestException("You must specify either name=XXXX or ip=YYYY"); } - decodeCursorToken(); NameserverSearchResponse results; if (nameParam.isPresent()) { + // RDAP Technical Implementation Guilde 2.2.3 - we MAY support nameserver search queries based + // on a "nameserver search pattern" as defined in RFC7482 + // // syntax: /rdap/nameservers?name=exam*.com metricInformationBuilder.setSearchType(SearchType.BY_NAMESERVER_NAME); - if (!LDH_PATTERN.matcher(nameParam.get()).matches()) { - throw new BadRequestException( - "Name parameter must contain only letters, dots" - + " and hyphens, and an optional single wildcard"); - } results = searchByName( - recordWildcardType(RdapSearchPattern.create(Idn.toASCII(nameParam.get()), true))); + recordWildcardType( + RdapSearchPattern.createFromLdhOrUnicodeDomainName(nameParam.get()))); } else { + // RDAP Technical Implementation Guide 2.2.3 - we MUST support nameserver search queries based + // on IP address as defined in RFC7482 3.2.2. Doesn't require pattern matching + // // syntax: /rdap/nameservers?ip=1.2.3.4 metricInformationBuilder.setSearchType(SearchType.BY_NAMESERVER_ADDRESS); InetAddress inetAddress; @@ -130,22 +125,22 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase { // nameservers are desired, because there may be multiple nameservers with the same name. if (!partialStringQuery.getHasWildcard() && !shouldIncludeDeleted()) { return searchByNameUsingForeignKey(partialStringQuery); + } // Handle queries with a wildcard (or including deleted entries). If there is a suffix, it // should be a domain that we manage, so we can look up the domain and search through the // subordinate hosts. This is more efficient, and lets us permit wildcard searches with no // initial string. Deleted nameservers cannot be searched using a suffix, because the logic // of the deletion status of the superordinate domain versus the deletion status of the // subordinate host gets too messy. - } else if (partialStringQuery.getSuffix() != null) { + if (partialStringQuery.getSuffix() != null) { if (shouldIncludeDeleted()) { throw new UnprocessableEntityException( "A suffix after a wildcard is not allowed when searching for deleted nameservers"); } return searchByNameUsingSuperordinateDomain(partialStringQuery); - // Handle queries with a wildcard (or deleted entries included), but no suffix. - } else { - return searchByNameUsingPrefix(partialStringQuery); } + // Handle queries with a wildcard (or deleted entries included), but no suffix. + return searchByNameUsingPrefix(partialStringQuery); } /** @@ -155,21 +150,21 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase { */ private NameserverSearchResponse searchByNameUsingForeignKey( RdapSearchPattern partialStringQuery) { - Optional hostResource = - loadByForeignKey( - HostResource.class, partialStringQuery.getInitialString(), getRequestTime()); - if (!shouldBeVisible(hostResource)) { - metricInformationBuilder.setNumHostsRetrieved(0); - throw new NotFoundException("No nameservers found"); - } - metricInformationBuilder.setNumHostsRetrieved(1); - NameserverSearchResponse.Builder builder = NameserverSearchResponse.builder() .setIncompletenessWarningType(IncompletenessWarningType.COMPLETE); - builder - .nameserverSearchResultsBuilder() - .add(rdapJsonFormatter.createRdapNameserver(hostResource.get(), OutputDataType.FULL)); + + Optional hostResource = + loadByForeignKey( + HostResource.class, partialStringQuery.getInitialString(), getRequestTime()); + + metricInformationBuilder.setNumHostsRetrieved(hostResource.isPresent() ? 1 : 0); + + if (shouldBeVisible(hostResource)) { + builder + .nameserverSearchResultsBuilder() + .add(rdapJsonFormatter.createRdapNameserver(hostResource.get(), OutputDataType.FULL)); + } return builder.build(); } @@ -215,7 +210,8 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase { /** * Searches for nameservers by name with a prefix and wildcard. * - *

There are no pending deletes for hosts, so we can call {@link RdapActionBase#queryItems}. + *

There are no pending deletes for hosts, so we can call {@link + * RdapSearchActionBase#queryItems}. */ private NameserverSearchResponse searchByNameUsingPrefix(RdapSearchPattern partialStringQuery) { // Add 1 so we can detect truncation. @@ -238,13 +234,13 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase { int querySizeLimit = getStandardQuerySizeLimit(); Query query = queryItems( - HostResource.class, - "inetAddresses", - inetAddress.getHostAddress(), - Optional.empty(), - cursorString, - getDeletedItemHandling(), - querySizeLimit); + HostResource.class, + "inetAddresses", + inetAddress.getHostAddress(), + Optional.empty(), + cursorString, + getDeletedItemHandling(), + querySizeLimit); return makeSearchResults( getMatchingResources(query, shouldIncludeDeleted(), querySizeLimit), CursorType.ADDRESS); } diff --git a/java/google/registry/rdap/RdapObjectClasses.java b/java/google/registry/rdap/RdapObjectClasses.java index fdc7191ef..b5237ab82 100644 --- a/java/google/registry/rdap/RdapObjectClasses.java +++ b/java/google/registry/rdap/RdapObjectClasses.java @@ -163,7 +163,7 @@ final class RdapObjectClasses { * *

RFC 7483 specifies that the top-level object should include an entry indicating the * conformance level. ICANN RDAP spec for 15feb19 mandates several additional entries, in sections - * 2.6.3, 2.11 of the Response Profile and 3.3.2, 3.5, of the Technical Implementation Guide. + * 2.6.3, 2.11 of the Response Profile and 3.3, 3.5, of the Technical Implementation Guide. */ @AutoValue @RestrictJsonNames({}) diff --git a/java/google/registry/rdap/RdapSearchActionBase.java b/java/google/registry/rdap/RdapSearchActionBase.java index aa5d2e4d3..66b1a42bf 100644 --- a/java/google/registry/rdap/RdapSearchActionBase.java +++ b/java/google/registry/rdap/RdapSearchActionBase.java @@ -14,17 +14,29 @@ package google.registry.rdap; -import static java.nio.charset.StandardCharsets.UTF_8; +import static com.google.common.base.Charsets.UTF_8; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.util.DateTimeUtils.END_OF_TIME; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; +import com.googlecode.objectify.Key; +import com.googlecode.objectify.cmd.Query; +import google.registry.model.EppResource; +import google.registry.model.registrar.Registrar; import google.registry.rdap.RdapMetrics.EndpointType; +import google.registry.rdap.RdapMetrics.WildcardType; +import google.registry.rdap.RdapSearchResults.BaseSearchResponse; +import google.registry.rdap.RdapSearchResults.IncompletenessWarningType; +import google.registry.request.HttpException.BadRequestException; +import google.registry.request.HttpException.UnprocessableEntityException; import google.registry.request.Parameter; import google.registry.request.ParameterMap; import google.registry.request.RequestUrl; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URLEncoder; +import java.util.ArrayList; import java.util.Base64; import java.util.List; import java.util.Map; @@ -39,9 +51,12 @@ import javax.inject.Inject; */ public abstract class RdapSearchActionBase extends RdapActionBase { + private static final int RESULT_SET_SIZE_SCALING_FACTOR = 30; + @Inject @RequestUrl String requestUrl; @Inject @ParameterMap ImmutableListMultimap parameterMap; @Inject @Parameter("cursor") Optional cursorTokenParam; + @Inject @Parameter("registrar") Optional registrarParam; protected Optional cursorString; @@ -49,6 +64,20 @@ public abstract class RdapSearchActionBase extends RdapActionBase { super(humanReadableObjectTypeName, endpointType); } + @Override + public final BaseSearchResponse getJsonObjectForResource( + String pathSearchString, boolean isHeadRequest) { + // The pathSearchString is not used by search commands. + if (pathSearchString.length() > 0) { + throw new BadRequestException("Unexpected path"); + } + decodeCursorToken(); + metricInformationBuilder.setRegistrarSpecified(registrarParam.isPresent()); + return getSearchResponse(isHeadRequest); + } + + public abstract BaseSearchResponse getSearchResponse(boolean isHeadRequest); + /** * Decodes the cursor token passed in the HTTP request. * @@ -57,14 +86,9 @@ public abstract class RdapSearchActionBase extends RdapActionBase { * than the cursor value. */ protected void decodeCursorToken() { - if (!cursorTokenParam.isPresent()) { - cursorString = Optional.empty(); - } else { - cursorString = - Optional.of( - new String( - Base64.getDecoder().decode(cursorTokenParam.get().getBytes(UTF_8)), UTF_8)); - } + cursorString = + cursorTokenParam.map( + cursor -> new String(Base64.getDecoder().decode(cursor.getBytes(UTF_8)), UTF_8)); } /** Returns an encoded cursor token to pass back in the RDAP JSON link strings. */ @@ -77,6 +101,121 @@ public abstract class RdapSearchActionBase extends RdapActionBase { return getRequestUrlWithExtraParameter(parameterName, ImmutableList.of(parameterValue)); } + /** Returns the registrar on which results should be filtered, or absent(). */ + protected Optional getDesiredRegistrar() { + return registrarParam; + } + + protected boolean shouldBeVisible(Optional eppResource) { + return eppResource.isPresent() && shouldBeVisible(eppResource.get()); + } + + /** + * Returns true if the EPP resource should be visible. + * + *

This is true iff: + * 1. The resource is not deleted, or the request wants to see deleted items, and is authorized to + * do so, and: + * 2. The request did not specify a registrar to filter on, or the registrar matches. + */ + protected boolean shouldBeVisible(EppResource eppResource) { + return isAuthorized(eppResource) + && (!registrarParam.isPresent() + || registrarParam.get().equals(eppResource.getPersistedCurrentSponsorClientId())); + } + + /** + * Returns true if the EPP resource should be visible. + * + *

This is true iff: + * 1. The resource is not deleted, or the request wants to see deleted items, and is authorized to + * do so, and: + * 2. The request did not specify a registrar to filter on, or the registrar matches. + */ + protected boolean shouldBeVisible(Registrar registrar) { + return isAuthorized(registrar) + && (!registrarParam.isPresent() || registrarParam.get().equals(registrar.getClientId())); + } + + /** + * Runs the given query, and checks for permissioning if necessary. + * + * @param query an already-defined query to be run; a filter on currentSponsorClientId will be + * added if appropriate + * @param checkForVisibility true if the results should be checked to make sure they are visible; + * normally this should be equal to the shouldIncludeDeleted setting, but in cases where the + * query could not check deletion status (due to Datastore limitations such as the limit of + * one field queried for inequality, for instance), it may need to be set to true even when + * not including deleted records + * @param querySizeLimit the maximum number of items the query is expected to return, usually + * because the limit has been set + * @return an {@link RdapResultSet} object containing the list of resources and an incompleteness + * warning flag, which is set to MIGHT_BE_INCOMPLETE iff any resources were excluded due to + * lack of visibility, and the resulting list of resources is less than the maximum allowable, + * and the number of items returned by the query is greater than or equal to the maximum + * number we might have expected + */ + RdapResultSet getMatchingResources( + Query query, boolean checkForVisibility, int querySizeLimit) { + Optional desiredRegistrar = getDesiredRegistrar(); + if (desiredRegistrar.isPresent()) { + query = query.filter("currentSponsorClientId", desiredRegistrar.get()); + } + if (!checkForVisibility) { + return RdapResultSet.create(query.list()); + } + // If we are including deleted resources, we need to check that we're authorized for each one. + List resources = new ArrayList<>(); + int numResourcesQueried = 0; + boolean someExcluded = false; + for (T resource : query) { + if (shouldBeVisible(resource)) { + resources.add(resource); + } else { + someExcluded = true; + } + numResourcesQueried++; + if (resources.size() > rdapResultSetMaxSize) { + break; + } + } + // The incompleteness problem comes about because we don't know how many items to fetch. We want + // to return rdapResultSetMaxSize worth of items, but some might be excluded, so we fetch more + // just in case. But how many more? That's the potential problem, addressed with the three way + // AND statement: + // 1. If we didn't exclude any items, then we can't have the incompleteness problem. + // 2. If have a full result set batch (rdapResultSetMaxSize items), we must by definition be + // giving the user a complete result set. + // 3. If we started with fewer than querySizeLimit items, then there weren't any more items that + // we missed. Even if we return fewer than rdapResultSetMaxSize items, it isn't because we + // didn't fetch enough to start. + // Only if all three conditions are true might things be incomplete. In other words, we fetched + // as many as our limit allowed, but then excluded so many that we wound up with less than a + // full result set's worth of results. + return RdapResultSet.create( + resources, + (someExcluded + && (resources.size() < rdapResultSetMaxSize) + && (numResourcesQueried >= querySizeLimit)) + ? IncompletenessWarningType.MIGHT_BE_INCOMPLETE + : IncompletenessWarningType.COMPLETE, + numResourcesQueried); + } + + RdapSearchPattern recordWildcardType(RdapSearchPattern partialStringQuery) { + if (!partialStringQuery.getHasWildcard()) { + metricInformationBuilder.setWildcardType(WildcardType.NO_WILDCARD); + } else if (partialStringQuery.getSuffix() == null) { + metricInformationBuilder.setWildcardType(WildcardType.PREFIX); + } else if (partialStringQuery.getInitialString().isEmpty()) { + metricInformationBuilder.setWildcardType(WildcardType.SUFFIX); + } else { + metricInformationBuilder.setWildcardType(WildcardType.PREFIX_AND_SUFFIX); + } + metricInformationBuilder.setPrefixLength(partialStringQuery.getInitialString().length()); + return partialStringQuery; + } + /** * Returns the original request URL, but with the specified parameter added or overridden. * @@ -123,4 +262,158 @@ public abstract class RdapSearchActionBase extends RdapActionBase { URI createNavigationUri(String cursor) { return URI.create(getRequestUrlWithExtraParameter("cursor", encodeCursorToken(cursor))); } + + // We want to return rdapResultSetMaxSize + 1 results, so that we know if there are "extra" + // results (in which case we'll have a "next" link in the RDAP response). + // In case that we want to return deleted results as well, we have to scale the number of results + // to be (more) sure we got everything. + int getStandardQuerySizeLimit() { + return shouldIncludeDeleted() + ? (RESULT_SET_SIZE_SCALING_FACTOR * (rdapResultSetMaxSize + 1)) + : (rdapResultSetMaxSize + 1); + } + + /** + * Handles prefix searches in cases where, if we need to filter out deleted items, there are no + * pending deletes. + * + *

In such cases, it is sufficient to check whether {@code deletionTime} is equal to + * {@code END_OF_TIME}, because any other value means it has already been deleted. This allows us + * to use an equality query for the deletion time. + * + * @param clazz the type of resource to be queried + * @param filterField the database field of interest + * @param partialStringQuery the details of the search string; if there is no wildcard, an + * equality query is used; if there is a wildcard, a range query is used instead; the + * initial string should not be empty, and any search suffix will be ignored, so the caller + * must filter the results if a suffix is specified + * @param cursorString if a cursor is present, this parameter should specify the cursor string, to + * skip any results up to and including the string; empty() if there is no cursor + * @param deletedItemHandling whether to include or exclude deleted items + * @param resultSetMaxSize the maximum number of results to return + * @return the query object + */ + static Query queryItems( + Class clazz, + String filterField, + RdapSearchPattern partialStringQuery, + Optional cursorString, + DeletedItemHandling deletedItemHandling, + int resultSetMaxSize) { + if (partialStringQuery.getInitialString().length() + < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { + throw new UnprocessableEntityException( + String.format( + "Initial search string must be at least %d characters", + RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); + } + Query query = ofy().load().type(clazz); + if (!partialStringQuery.getHasWildcard()) { + query = query.filter(filterField, partialStringQuery.getInitialString()); + } else { + // Ignore the suffix; the caller will need to filter on the suffix, if any. + query = query + .filter(filterField + " >=", partialStringQuery.getInitialString()) + .filter(filterField + " <", partialStringQuery.getNextInitialString()); + } + if (cursorString.isPresent()) { + query = query.filter(filterField + " >", cursorString.get()); + } + return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize); + } + + /** + * Handles searches using a simple string rather than an {@link RdapSearchPattern}. + * + *

Since the filter is not an inequality, we can support also checking a cursor string against + * a different field (which involves an inequality on that field). + * + * @param clazz the type of resource to be queried + * @param filterField the database field of interest + * @param queryString the search string + * @param cursorField the field which should be compared to the cursor string, or empty() if the + * key should be compared to a key created from the cursor string + * @param cursorString if a cursor is present, this parameter should specify the cursor string, to + * skip any results up to and including the string; empty() if there is no cursor + * @param deletedItemHandling whether to include or exclude deleted items + * @param resultSetMaxSize the maximum number of results to return + * @return the query object + */ + static Query queryItems( + Class clazz, + String filterField, + String queryString, + Optional cursorField, + Optional cursorString, + DeletedItemHandling deletedItemHandling, + int resultSetMaxSize) { + if (queryString.length() < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { + throw new UnprocessableEntityException( + String.format( + "Initial search string must be at least %d characters", + RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); + } + Query query = ofy().load().type(clazz).filter(filterField, queryString); + if (cursorString.isPresent()) { + if (cursorField.isPresent()) { + query = query.filter(cursorField.get() + " >", cursorString.get()); + } else { + query = query.filterKey(">", Key.create(clazz, cursorString.get())); + } + } + return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize); + } + + /** Handles searches where the field to be searched is the key. */ + static Query queryItemsByKey( + Class clazz, + RdapSearchPattern partialStringQuery, + Optional cursorString, + DeletedItemHandling deletedItemHandling, + int resultSetMaxSize) { + if (partialStringQuery.getInitialString().length() + < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { + throw new UnprocessableEntityException( + String.format( + "Initial search string must be at least %d characters", + RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); + } + Query query = ofy().load().type(clazz); + if (!partialStringQuery.getHasWildcard()) { + query = query.filterKey("=", Key.create(clazz, partialStringQuery.getInitialString())); + } else { + // Ignore the suffix; the caller will need to filter on the suffix, if any. + query = query + .filterKey(">=", Key.create(clazz, partialStringQuery.getInitialString())) + .filterKey("<", Key.create(clazz, partialStringQuery.getNextInitialString())); + } + if (cursorString.isPresent()) { + query = query.filterKey(">", Key.create(clazz, cursorString.get())); + } + return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize); + } + + /** Handles searches by key using a simple string. */ + static Query queryItemsByKey( + Class clazz, + String queryString, + DeletedItemHandling deletedItemHandling, + int resultSetMaxSize) { + if (queryString.length() < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { + throw new UnprocessableEntityException( + String.format( + "Initial search string must be at least %d characters", + RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); + } + Query query = ofy().load().type(clazz).filterKey("=", Key.create(clazz, queryString)); + return setOtherQueryAttributes(query, deletedItemHandling, resultSetMaxSize); + } + + static Query setOtherQueryAttributes( + Query query, DeletedItemHandling deletedItemHandling, int resultSetMaxSize) { + if (deletedItemHandling != DeletedItemHandling.INCLUDE) { + query = query.filter("deletionTime", END_OF_TIME); + } + return query.limit(resultSetMaxSize); + } } diff --git a/java/google/registry/rdap/RdapSearchPattern.java b/java/google/registry/rdap/RdapSearchPattern.java index cc4211bf0..b8c585fae 100644 --- a/java/google/registry/rdap/RdapSearchPattern.java +++ b/java/google/registry/rdap/RdapSearchPattern.java @@ -14,9 +14,13 @@ package google.registry.rdap; -import static google.registry.util.DomainNameUtils.ACE_PREFIX; +import com.google.common.base.Strings; +import google.registry.request.HttpException.BadRequestException; import google.registry.request.HttpException.UnprocessableEntityException; +import google.registry.util.Idn; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.annotation.Nullable; /** @@ -32,6 +36,19 @@ public final class RdapSearchPattern { static final int MIN_INITIAL_STRING_LENGTH = 2; + /** + * Pattern for allowed LDH searches. + * + *

Based on RFC7482 4.1. Must contains only alphanumeric plus dots and hyphens. A single + * whildcard asterix is allowed - but if exists must be the last character of a domain name label + * (so exam* and exam*.com are allowed, but exam*le.com isn't allowd) + * + *

The prefix is in group(1), and the suffix without the dot (if it exists) is in group(4). If + * there's no wildcard, group(2) is empty. + */ + static final Pattern LDH_PATTERN = + Pattern.compile("([-.a-zA-Z0-9]*)([*]([.]([-.a-zA-Z0-9]+))?)?"); + /** String before the wildcard character. */ private final String initialString; @@ -78,51 +95,83 @@ public final class RdapSearchPattern { } /** - * Creates a SearchPattern using the provided search pattern string. + * Creates a SearchPattern using the provided search pattern string in Unicode. * - * @param pattern the string containing the partial match pattern - * @param allowSuffix true if a suffix is allowed after the wildcard + *

The search query might end in an asterix, in which case that asterix is considered a + * wildcard and can match 0 or more characters. Without that asterix - the match will be exact. * - * @throws UnprocessableEntityException if {@code pattern} does not meet the requirements of RFC - * 7482 + * @param searchQuery the string containing the partial match pattern, optionally ending in a + * wildcard asterix + * @throws UnprocessableEntityException if {@code pattern} has a wildcard not at the end of the + * query */ - public static RdapSearchPattern create(String pattern, boolean allowSuffix) { - String initialString; - boolean hasWildcard; - String suffix; - // If there's no wildcard character, just lump everything into the initial string. - int wildcardPos = pattern.indexOf('*'); - if (wildcardPos < 0) { - initialString = pattern; - hasWildcard = false; - suffix = null; - } else if (pattern.indexOf('*', wildcardPos + 1) >= 0) { - throw new UnprocessableEntityException("Only one wildcard allowed"); - } else { - hasWildcard = true; - // Check for a suffix (e.g. exam*.com or ns*.example.com). - if (pattern.length() > wildcardPos + 1) { - if (!allowSuffix) { - throw new UnprocessableEntityException("Suffix not allowed after wildcard"); - } - if ((pattern.length() == wildcardPos + 2) || (pattern.charAt(wildcardPos + 1) != '.')) { - throw new UnprocessableEntityException( - "Suffix after wildcard must be one or more domain" - + " name labels, e.g. exam*.tld, ns*.example.tld"); - } - suffix = pattern.substring(wildcardPos + 2); - } else { - suffix = null; - } - initialString = pattern.substring(0, wildcardPos); - if (initialString.startsWith(ACE_PREFIX) && (initialString.length() < 7)) { - throw new UnprocessableEntityException( - "At least seven characters must be specified for punycode domain searches"); - } + public static RdapSearchPattern createFromUnicodeString(String searchQuery) { + int wildcardLocation = searchQuery.indexOf('*'); + if (wildcardLocation < 0) { + return new RdapSearchPattern(searchQuery, false, null); } + if (wildcardLocation == searchQuery.length() - 1) { + return new RdapSearchPattern(searchQuery.substring(0, wildcardLocation), true, null); + } + throw new UnprocessableEntityException( + String.format( + "Query can only have a single wildcard, and it must be at the end of the query, but" + + " was: '%s'", + searchQuery)); + } + + /** + * Creates a SearchPattern using the provided domain search pattern in LDH format. + * + *

The domain search pattern can have a single wildcard asterix that can match 0 or more + * charecters. If such an asterix exists - it must be at the end of a domain label. + * + * @param searchQuery the string containing the partial match pattern + * @throws UnprocessableEntityException if {@code pattern} does not meet the requirements of RFC + * 7482 + */ + public static RdapSearchPattern createFromLdhDomainName(String searchQuery) { + Matcher matcher = LDH_PATTERN.matcher(searchQuery); + if (!matcher.matches()) { + throw new UnprocessableEntityException( + String.format( + "Query can only have a single wildcard, and it must be at the end of a label," + + " but was: '%s'", + searchQuery)); + } + + String initialString = matcher.group(1); + boolean hasWildcard = !Strings.isNullOrEmpty(matcher.group(2)); + String suffix = Strings.emptyToNull(matcher.group(4)); + return new RdapSearchPattern(initialString, hasWildcard, suffix); } + /** + * Creates a SearchPattern using the provided domain search pattern in LDH or Unicode format. + * + *

The domain search pattern can have a single wildcard asterix that can match 0 or more + * charecters. If such an asterix exists - it must be at the end of a domain label. + * + *

In theory, according to RFC7482 4.1 - we should make some checks about partial matching in + * unicode queries. We don't, but we might want to just disable partial matches for unicode inputs + * (meaning if it doesn't match LDH_PATTERN, then don't allow wildcard at all). + * + * @param searchQuery the string containing the partial match pattern + * @throws UnprocessableEntityException if {@code pattern} does not meet the requirements of RFC + * 7482 + */ + public static RdapSearchPattern createFromLdhOrUnicodeDomainName(String searchQuery) { + String ldhSearchQuery; + try { + ldhSearchQuery = Idn.toASCII(searchQuery); + } catch (Exception e) { + throw new BadRequestException( + String.format("Invalid value of searchQuery: '%s'", searchQuery), e); + } + return RdapSearchPattern.createFromLdhDomainName(ldhSearchQuery); + } + /** * Checks a string to make sure that it matches the search pattern. * diff --git a/java/google/registry/rdap/RdapUtils.java b/java/google/registry/rdap/RdapUtils.java index 10a9ff271..d925d4f9c 100644 --- a/java/google/registry/rdap/RdapUtils.java +++ b/java/google/registry/rdap/RdapUtils.java @@ -14,6 +14,7 @@ package google.registry.rdap; +import com.google.common.base.Ascii; import com.google.common.collect.Streams; import google.registry.model.registrar.Registrar; import java.util.Objects; @@ -30,4 +31,17 @@ public final class RdapUtils { .filter(registrar -> Objects.equals(ianaIdentifier, registrar.getIanaIdentifier())) .findFirst(); } + + /** + * Looks up a registrar by its name. + * + *

Used for RDAP Technical Implementation Guide 2.4.2 - search of registrar by the fn element. + * + *

For convenience, we use case insensitive search. + */ + static Optional getRegistrarByName(String registrarName) { + return Streams.stream(Registrar.loadAllCached()) + .filter(registrar -> Ascii.equalsIgnoreCase(registrarName, registrar.getRegistrarName())) + .findFirst(); + } } diff --git a/javatests/google/registry/rdap/RdapActionBaseTestCase.java b/javatests/google/registry/rdap/RdapActionBaseTestCase.java index 2b415114c..5bad3c2f7 100644 --- a/javatests/google/registry/rdap/RdapActionBaseTestCase.java +++ b/javatests/google/registry/rdap/RdapActionBaseTestCase.java @@ -86,7 +86,6 @@ public class RdapActionBaseTestCase { inject.setStaticField(Ofy.class, "clock", clock); action = TypeUtils.instantiate(rdapActionClass); action.includeDeletedParam = Optional.empty(); - action.registrarParam = Optional.empty(); action.formatOutputParam = Optional.empty(); action.response = response; action.rdapJsonFormatter = RdapTestHelper.getTestRdapJsonFormatter(clock); diff --git a/javatests/google/registry/rdap/RdapDataStructuresTest.java b/javatests/google/registry/rdap/RdapDataStructuresTest.java index 89870123b..20fbd3690 100644 --- a/javatests/google/registry/rdap/RdapDataStructuresTest.java +++ b/javatests/google/registry/rdap/RdapDataStructuresTest.java @@ -45,7 +45,12 @@ public final class RdapDataStructuresTest { @Test public void testRdapConformance() { assertThat(RdapConformance.INSTANCE.toJson()) - .isEqualTo(createJson("['rdap_level_0','icann_rdap_response_profile_0']")); + .isEqualTo(createJson( + "[", + " 'rdap_level_0',", + " 'icann_rdap_response_profile_0',", + " 'icann_rdap_technical_implementation_guide_0'", + "]")); } @Test diff --git a/javatests/google/registry/rdap/RdapDomainActionTest.java b/javatests/google/registry/rdap/RdapDomainActionTest.java index f03a4fd6f..685ecf63e 100644 --- a/javatests/google/registry/rdap/RdapDomainActionTest.java +++ b/javatests/google/registry/rdap/RdapDomainActionTest.java @@ -273,20 +273,6 @@ public class RdapDomainActionTest extends RdapActionBaseTestCase", "rdap_registrar.json"); + runSuccessfulHandleTest("101", "Yes Virginia