From c13c2f403a74abfcdbca75649899e68f02bb71b9 Mon Sep 17 00:00:00 2001 From: mountford Date: Tue, 26 Sep 2017 14:07:42 -0700 Subject: [PATCH] Add RDAP support for deleted nameservers and filtering by registrar This CL adds the functionality for nameserver searches. Future CLs will handle domains and entities. Deleted items can only be seen by admins, and by registrars viewing their own deleted items. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=170106014 --- java/google/registry/rdap/RdapActionBase.java | 213 +++++++++++++- .../registry/rdap/RdapDomainSearchAction.java | 5 +- .../registry/rdap/RdapEntitySearchAction.java | 15 +- .../registry/rdap/RdapJsonFormatter.java | 73 +++-- java/google/registry/rdap/RdapModule.java | 12 + .../registry/rdap/RdapNameserverAction.java | 9 +- .../rdap/RdapNameserverSearchAction.java | 183 +++++++----- ...ResourcesAndIncompletenessWarningType.java | 42 +++ .../rdap/RdapNameserverActionTest.java | 201 ++++++++++++- .../rdap/RdapNameserverSearchActionTest.java | 268 ++++++++++++++++-- .../registry/rdap/testdata/rdap_host.json | 2 +- 11 files changed, 870 insertions(+), 153 deletions(-) create mode 100644 java/google/registry/rdap/RdapResourcesAndIncompletenessWarningType.java diff --git a/java/google/registry/rdap/RdapActionBase.java b/java/google/registry/rdap/RdapActionBase.java index 06d8d09ad..af4a073f6 100644 --- a/java/google/registry/rdap/RdapActionBase.java +++ b/java/google/registry/rdap/RdapActionBase.java @@ -15,6 +15,7 @@ package google.registry.rdap; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.net.HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.model.registry.Registries.findTldForName; @@ -30,15 +31,18 @@ import com.google.common.collect.ImmutableMap; import com.google.common.net.InternetDomainName; import com.google.common.net.MediaType; import com.google.re2j.Pattern; +import com.googlecode.objectify.Key; import com.googlecode.objectify.cmd.Query; import google.registry.config.RegistryConfig.Config; import google.registry.model.EppResource; import google.registry.model.registrar.Registrar; +import google.registry.rdap.RdapSearchResults.IncompletenessWarningType; import google.registry.request.Action; import google.registry.request.HttpException; import google.registry.request.HttpException.BadRequestException; import google.registry.request.HttpException.NotFoundException; import google.registry.request.HttpException.UnprocessableEntityException; +import google.registry.request.Parameter; import google.registry.request.RequestMethod; import google.registry.request.RequestPath; import google.registry.request.Response; @@ -48,9 +52,12 @@ import google.registry.ui.server.registrar.SessionUtils; import google.registry.util.FormattingLogger; import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; import javax.annotation.Nullable; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; +import org.joda.time.DateTime; import org.json.simple.JSONValue; /** @@ -78,8 +85,11 @@ public abstract class RdapActionBase implements Runnable { @Inject AuthResult authResult; @Inject SessionUtils sessionUtils; @Inject RdapJsonFormatter rdapJsonFormatter; + @Inject @Parameter("registrar") Optional registrarParam; + @Inject @Parameter("includeDeleted") Optional includeDeletedParam; @Inject @Config("rdapLinkBase") String rdapLinkBase; @Inject @Config("rdapWhoisServer") @Nullable String rdapWhoisServer; + @Inject @Config("rdapResultSetMaxSize") int rdapResultSetMaxSize; /** Returns a string like "domain name" or "nameserver", used for error strings. */ abstract String getHumanReadableObjectTypeName(); @@ -169,6 +179,72 @@ public abstract class RdapActionBase implements Runnable { return RdapAuthorization.create(RdapAuthorization.Role.REGISTRAR, clientId); } + /** Returns the registrar on which results should be filtered, or absent(). */ + Optional getDesiredRegistrar() { + return registrarParam; + } + + /** + * Returns true if the query should include deleted items. + * + *

This is true only if the request specified an includeDeleted parameter of true, AND is + * eligible to see deleted information. Admins can see all deleted information, while + * authenticated registrars can see only their own deleted information. Note that if this method + * returns true, it just means that some deleted information might be viewable. If this is a + * registrar request, the caller must still verify that the registrar can see each particular + * item by calling {@link RdapAuthorization#isAuthorizedForClientId}. + */ + boolean shouldIncludeDeleted() { + // If includeDeleted is not specified, or set to false, we don't need to go any further. + if (!includeDeletedParam.or(false)) { + return false; + } + if (!authResult.userAuthInfo().isPresent()) { + return false; + } + UserAuthInfo userAuthInfo = authResult.userAuthInfo().get(); + if (userAuthInfo.isUserAdmin()) { + return true; + } + if (!sessionUtils.checkRegistrarConsoleLogin(request, userAuthInfo)) { + return false; + } + String clientId = sessionUtils.getRegistrarClientId(request); + checkState( + Registrar.loadByClientIdCached(clientId).isPresent(), + "Registrar with clientId %s doesn't exist", + clientId); + return true; + } + + /** + * Returns true if the EPP resource should be visible. This is true iff: + * 1. The resource is not deleted, or the request wants to see deleted items, and is authorized to + * do so, and: + * 2. The request did not specify a registrar to filter on, or the registrar matches. + */ + boolean shouldBeVisible(EppResource eppResource, DateTime now) { + return (now.isBefore(eppResource.getDeletionTime()) + || (shouldIncludeDeleted() + && getAuthorization() + .isAuthorizedForClientId(eppResource.getPersistedCurrentSponsorClientId()))) + && (!registrarParam.isPresent() + || registrarParam.get().equals(eppResource.getPersistedCurrentSponsorClientId())); + } + + /** + * Returns true if the registrar should be visible. This is true iff: + * 1. The resource is active and publicly visible, or the request wants to see deleted items, and + * is authorized to do so, and: + * 2. The request did not specify a registrar to filter on, or the registrar matches. + */ + boolean shouldBeVisible(Registrar registrar) { + return (registrar.isActiveAndPubliclyVisible() + || (shouldIncludeDeleted() + && getAuthorization().isAuthorizedForClientId(registrar.getClientId()))) + && (!registrarParam.isPresent() || registrarParam.get().equals(registrar.getClientId())); + } + void validateDomainName(String name) { try { Optional tld = findTldForName(InternetDomainName.from(name)); @@ -190,10 +266,10 @@ public abstract class RdapActionBase implements Runnable { } /** - * Handles prefix searches in cases where there are no pending deletes. In such cases, it is - * sufficient to check whether {@code deletionTime} is equal to {@code END_OF_TIME}, because any - * other value means it has already been deleted. This allows us to use an equality query for the - * deletion time. + * Handles prefix searches in cases where, if we need to filter out deleted items, there are no + * pending deletes. In such cases, it is sufficient to check whether {@code deletionTime} is equal + * to {@code END_OF_TIME}, because any other value means it has already been deleted. This allows + * us to use an equality query for the deletion time. * * @param clazz the type of resource to be queried * @param filterField the database field of interest @@ -201,13 +277,15 @@ public abstract class RdapActionBase implements Runnable { * equality query is used; if there is a wildcard, a range query is used instead; the * initial string should not be empty, and any search suffix will be ignored, so the caller * must filter the results if a suffix is specified + * @param includeDeleted whether to search for deleted items as well * @param resultSetMaxSize the maximum number of results to return * @return the results of the query */ - static Query queryUndeleted( + static Query queryItems( Class clazz, String filterField, RdapSearchPattern partialStringQuery, + boolean includeDeleted, int resultSetMaxSize) { if (partialStringQuery.getInitialString().length() < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { @@ -216,20 +294,125 @@ public abstract class RdapActionBase implements Runnable { "Initial search string must be at least %d characters", RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); } + Query query = ofy().load().type(clazz); if (!partialStringQuery.getHasWildcard()) { - return ofy().load() - .type(clazz) - .filter(filterField, partialStringQuery.getInitialString()) - .filter("deletionTime", END_OF_TIME) - .limit(resultSetMaxSize); + query = query.filter(filterField, partialStringQuery.getInitialString()); } else { // Ignore the suffix; the caller will need to filter on the suffix, if any. - return ofy().load() - .type(clazz) + query = query .filter(filterField + " >=", partialStringQuery.getInitialString()) - .filter(filterField + " <", partialStringQuery.getNextInitialString()) - .filter("deletionTime", END_OF_TIME) - .limit(resultSetMaxSize); + .filter(filterField + " <", partialStringQuery.getNextInitialString()); } + if (!includeDeleted) { + query = query.filter("deletionTime", END_OF_TIME); + } + return query.limit(resultSetMaxSize); + } + + /** Variant of queryItems using a simple string rather than an {@link RdapSearchPattern}. */ + static Query queryItems( + Class clazz, + String filterField, + String queryString, + boolean includeDeleted, + int resultSetMaxSize) { + if (queryString.length() < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { + throw new UnprocessableEntityException( + String.format( + "Initial search string must be at least %d characters", + RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); + } + Query query = ofy().load().type(clazz).filter(filterField, queryString); + return setOtherQueryAttributes(query, includeDeleted, resultSetMaxSize); + } + + /** Variant of queryItems where the field to be searched is the key. */ + static Query queryItemsByKey( + Class clazz, + RdapSearchPattern partialStringQuery, + boolean includeDeleted, + int resultSetMaxSize) { + if (partialStringQuery.getInitialString().length() + < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { + throw new UnprocessableEntityException( + String.format( + "Initial search string must be at least %d characters", + RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); + } + Query query = ofy().load().type(clazz); + if (!partialStringQuery.getHasWildcard()) { + query = query.filterKey("=", Key.create(clazz, partialStringQuery.getInitialString())); + } else { + // Ignore the suffix; the caller will need to filter on the suffix, if any. + query = query + .filterKey(">=", Key.create(clazz, partialStringQuery.getInitialString())) + .filterKey("<", Key.create(clazz, partialStringQuery.getNextInitialString())); + } + return setOtherQueryAttributes(query, includeDeleted, resultSetMaxSize); + } + + /** Variant of queryItems searching for a key by a simple string. */ + static Query queryItemsByKey( + Class clazz, + String queryString, + boolean includeDeleted, + int resultSetMaxSize) { + if (queryString.length() < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { + throw new UnprocessableEntityException( + String.format( + "Initial search string must be at least %d characters", + RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)); + } + Query query = ofy().load().type(clazz).filterKey("=", Key.create(clazz, queryString)); + return setOtherQueryAttributes(query, includeDeleted, resultSetMaxSize); + } + + private static Query setOtherQueryAttributes( + Query query, boolean includeDeleted, int resultSetMaxSize) { + if (!includeDeleted) { + query = query.filter("deletionTime", END_OF_TIME); + } + return query.limit(resultSetMaxSize); + } + + /** + * Runs the given query, and checks for permissioning if necessary. + * + * @param query an already-defined query to be run; a filter on currentSponsorClientId will be + * added if appropriate + * @param now the time as of which to evaluate the query + * @return an {@link RdapResourcesAndIncompletenessWarningType} object containing the list of + * resources and an incompleteness warning flag, which is set to MIGHT_BE_INCOMPLETE iff + * any resources were excluded due to lack of visibility, and the resulting list of + * resources is less than the maximum allowable, which indicates that we may not have + * fetched enough resources + */ + RdapResourcesAndIncompletenessWarningType getMatchingResources( + Query query, DateTime now) { + Optional desiredRegistrar = getDesiredRegistrar(); + if (desiredRegistrar.isPresent()) { + query = query.filter("currentSponsorClientId", desiredRegistrar.get()); + } + if (!shouldIncludeDeleted()) { + return RdapResourcesAndIncompletenessWarningType.create(query.list()); + } + // If we are including deleted resources, we need to check that we're authorized for each one. + List resources = new ArrayList<>(); + boolean someExcluded = false; + for (T resource : query) { + if (shouldBeVisible(resource, now)) { + resources.add(resource); + } else { + someExcluded = true; + } + if (resources.size() > rdapResultSetMaxSize) { + break; + } + } + return RdapResourcesAndIncompletenessWarningType.create( + resources, + (someExcluded && (resources.size() < rdapResultSetMaxSize + 1)) + ? IncompletenessWarningType.MIGHT_BE_INCOMPLETE + : IncompletenessWarningType.NONE); } } diff --git a/java/google/registry/rdap/RdapDomainSearchAction.java b/java/google/registry/rdap/RdapDomainSearchAction.java index 6fc60d799..f02b792f1 100644 --- a/java/google/registry/rdap/RdapDomainSearchAction.java +++ b/java/google/registry/rdap/RdapDomainSearchAction.java @@ -29,7 +29,6 @@ import com.google.common.collect.Iterables; import com.google.common.primitives.Booleans; import com.googlecode.objectify.Key; import com.googlecode.objectify.cmd.Query; -import google.registry.config.RegistryConfig.Config; import google.registry.model.EppResourceUtils; import google.registry.model.domain.DomainResource; import google.registry.model.host.HostResource; @@ -81,7 +80,6 @@ public class RdapDomainSearchAction extends RdapActionBase { @Inject @Parameter("name") Optional nameParam; @Inject @Parameter("nsLdhName") Optional nsLdhNameParam; @Inject @Parameter("nsIp") Optional nsIpParam; - @Inject @Config("rdapResultSetMaxSize") int rdapResultSetMaxSize; @Inject RdapDomainSearchAction() {} @Override @@ -300,10 +298,11 @@ public class RdapDomainSearchAction extends RdapActionBase { // Only return the first 1000 nameservers. This could result in an incomplete result set if // a search asks for something like "ns*", but we need to enforce a limit in order to avoid // arbitrarily long-running queries. - return queryUndeleted( + return queryItems( HostResource.class, "fullyQualifiedHostName", partialStringQuery, + false, /* includeDeleted */ MAX_NAMESERVERS_IN_FIRST_STAGE) .keys(); } diff --git a/java/google/registry/rdap/RdapEntitySearchAction.java b/java/google/registry/rdap/RdapEntitySearchAction.java index 0cef13c7f..e2c9c22ef 100644 --- a/java/google/registry/rdap/RdapEntitySearchAction.java +++ b/java/google/registry/rdap/RdapEntitySearchAction.java @@ -28,7 +28,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Booleans; import com.google.common.primitives.Longs; import com.googlecode.objectify.Key; -import google.registry.config.RegistryConfig.Config; import google.registry.model.contact.ContactResource; import google.registry.model.domain.DesignatedContact; import google.registry.model.registrar.Registrar; @@ -69,7 +68,6 @@ public class RdapEntitySearchAction extends RdapActionBase { @Inject Clock clock; @Inject @Parameter("fn") Optional fnParam; @Inject @Parameter("handle") Optional handleParam; - @Inject @Config("rdapResultSetMaxSize") int rdapResultSetMaxSize; @Inject RdapEntitySearchAction() {} @Override @@ -160,10 +158,15 @@ public class RdapEntitySearchAction extends RdapActionBase { // Get the contact matches and return the results, fetching an additional contact to detect // truncation. return makeSearchResults( - queryUndeleted( - ContactResource.class, "searchName", partialStringQuery, rdapResultSetMaxSize + 1).list(), - registrarMatches, - now); + queryItems( + ContactResource.class, + "searchName", + partialStringQuery, + false /* includeDeleted */, + rdapResultSetMaxSize + 1) + .list(), + registrarMatches, + now); } /** Searches for entities by handle, returning a JSON array of entity info maps. */ diff --git a/java/google/registry/rdap/RdapJsonFormatter.java b/java/google/registry/rdap/RdapJsonFormatter.java index d32341ba9..270fc3b34 100644 --- a/java/google/registry/rdap/RdapJsonFormatter.java +++ b/java/google/registry/rdap/RdapJsonFormatter.java @@ -23,6 +23,7 @@ import static google.registry.util.DomainNameUtils.ACE_PREFIX; import com.google.common.base.Function; import com.google.common.base.Functions; import com.google.common.base.Optional; +import com.google.common.base.Predicates; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -172,8 +173,8 @@ public class RdapJsonFormatter { private static final ImmutableMap statusToRdapStatusMap = Maps.immutableEnumMap( new ImmutableMap.Builder() - // StatusValue.ADD_PERIOD not defined in our system - // StatusValue.AUTO_RENEW_PERIOD not defined in our system + // RdapStatus.ADD_PERIOD not defined in our system + // RdapStatus.AUTO_RENEW_PERIOD not defined in our system .put(StatusValue.CLIENT_DELETE_PROHIBITED, RdapStatus.CLIENT_DELETE_PROHIBITED) .put(StatusValue.CLIENT_HOLD, RdapStatus.CLIENT_HOLD) .put(StatusValue.CLIENT_RENEW_PROHIBITED, RdapStatus.CLIENT_RENEW_PROHIBITED) @@ -184,18 +185,18 @@ public class RdapJsonFormatter { .put(StatusValue.OK, RdapStatus.ACTIVE) .put(StatusValue.PENDING_CREATE, RdapStatus.PENDING_CREATE) .put(StatusValue.PENDING_DELETE, RdapStatus.PENDING_DELETE) - // StatusValue.PENDING_RENEW not defined in our system - // StatusValue.PENDING_RESTORE not defined in our system + // RdapStatus.PENDING_RENEW not defined in our system + // RdapStatus.PENDING_RESTORE not defined in our system .put(StatusValue.PENDING_TRANSFER, RdapStatus.PENDING_TRANSFER) .put(StatusValue.PENDING_UPDATE, RdapStatus.PENDING_UPDATE) - // StatusValue.REDEMPTION_PERIOD not defined in our system - // StatusValue.RENEW_PERIOD not defined in our system + // RdapStatus.REDEMPTION_PERIOD not defined in our system + // RdapStatus.RENEW_PERIOD not defined in our system .put(StatusValue.SERVER_DELETE_PROHIBITED, RdapStatus.SERVER_DELETE_PROHIBITED) .put(StatusValue.SERVER_HOLD, RdapStatus.SERVER_HOLD) .put(StatusValue.SERVER_RENEW_PROHIBITED, RdapStatus.SERVER_RENEW_PROHIBITED) .put(StatusValue.SERVER_TRANSFER_PROHIBITED, RdapStatus.SERVER_TRANSFER_PROHIBITED) .put(StatusValue.SERVER_UPDATE_PROHIBITED, RdapStatus.SERVER_UPDATE_PROHIBITED) - // StatusValue.TRANSFER_PERIOD not defined in our system + // RdapStatus.TRANSFER_PERIOD not defined in our system .build()); /** Role values specified in RFC 7483 ยง 10.2.4. */ @@ -270,6 +271,8 @@ public class RdapJsonFormatter { private static final ImmutableList STATUS_LIST_ACTIVE = ImmutableList.of(RdapStatus.ACTIVE.rfc7483String); + private static final ImmutableList STATUS_LIST_REMOVED = + ImmutableList.of(RdapStatus.REMOVED.rfc7483String); private static final ImmutableMap> PHONE_TYPE_VOICE = ImmutableMap.of("type", ImmutableList.of("voice")); private static final ImmutableMap> PHONE_TYPE_FAX = @@ -464,7 +467,10 @@ public class RdapJsonFormatter { if (hasUnicodeComponents(domainResource.getFullyQualifiedDomainName())) { jsonBuilder.put("unicodeName", Idn.toUnicode(domainResource.getFullyQualifiedDomainName())); } - jsonBuilder.put("status", makeStatusValueList(domainResource.getStatusValues())); + jsonBuilder.put( + "status", + makeStatusValueList( + domainResource.getStatusValues(), domainResource.getDeletionTime().isBefore(now))); jsonBuilder.put("links", ImmutableList.of( makeLink("domain", domainResource.getFullyQualifiedDomainName(), linkBase))); boolean displayContacts = @@ -577,7 +583,9 @@ public class RdapJsonFormatter { .contains(StatusValue.PENDING_TRANSFER)) { statuses.add(StatusValue.PENDING_TRANSFER); } - jsonBuilder.put("status", makeStatusValueList(statuses.build())); + jsonBuilder.put( + "status", + makeStatusValueList(statuses.build(), hostResource.getDeletionTime().isBefore(now))); jsonBuilder.put("links", ImmutableList.of( makeLink("nameserver", hostResource.getFullyQualifiedHostName(), linkBase))); List> remarks; @@ -661,10 +669,13 @@ public class RdapJsonFormatter { = new ImmutableList.Builder<>(); jsonBuilder.put("objectClassName", "entity"); jsonBuilder.put("handle", contactResource.getRepoId()); - jsonBuilder.put("status", makeStatusValueList( - isLinked(Key.create(contactResource), now) - ? union(contactResource.getStatusValues(), StatusValue.LINKED) - : contactResource.getStatusValues())); + jsonBuilder.put( + "status", + makeStatusValueList( + isLinked(Key.create(contactResource), now) + ? union(contactResource.getStatusValues(), StatusValue.LINKED) + : contactResource.getStatusValues(), + contactResource.getDeletionTime().isBefore(now))); if (contactType.isPresent()) { jsonBuilder.put("roles", ImmutableList.of(convertContactTypeToRdapRole(contactType.get()))); @@ -757,7 +768,7 @@ public class RdapJsonFormatter { ImmutableMap.Builder jsonBuilder = new ImmutableMap.Builder<>(); jsonBuilder.put("objectClassName", "entity"); jsonBuilder.put("handle", registrar.getIanaIdentifier().toString()); - jsonBuilder.put("status", STATUS_LIST_ACTIVE); + jsonBuilder.put("status", registrar.isActive() ? STATUS_LIST_ACTIVE : STATUS_LIST_REMOVED); jsonBuilder.put("roles", ImmutableList.of(RdapEntityRole.REGISTRAR.rfc7483String)); jsonBuilder.put("links", ImmutableList.of(makeLink("entity", registrar.getIanaIdentifier().toString(), linkBase))); @@ -1045,18 +1056,30 @@ public class RdapJsonFormatter { } /** - * Creates a string array of status values; the spec indicates that OK should be listed as - * "active". + * Creates a string array of status values. + * + *

The spec indicates that OK should be listed as "active". We use the "removed" status to + * indicate deleted objects. */ - private static ImmutableList makeStatusValueList(ImmutableSet statusValues) { - return FluentIterable - .from(statusValues) - .transform(Functions.forMap(statusToRdapStatusMap, RdapStatus.OBSCURED)) - .transform(new Function() { - @Override - public String apply(RdapStatus status) { - return status.getDisplayName(); - }}) + private static ImmutableList makeStatusValueList( + ImmutableSet statusValues, boolean isDeleted) { + FluentIterable iterable = + FluentIterable.from(statusValues) + .transform(Functions.forMap(statusToRdapStatusMap, RdapStatus.OBSCURED)); + if (isDeleted) { + iterable = + iterable + .filter(Predicates.not(Predicates.equalTo(RdapStatus.ACTIVE))) + .append(RdapStatus.REMOVED); + } + return iterable + .transform( + new Function() { + @Override + public String apply(RdapStatus status) { + return status.getDisplayName(); + } + }) .toSortedSet(Ordering.natural()) .asList(); } diff --git a/java/google/registry/rdap/RdapModule.java b/java/google/registry/rdap/RdapModule.java index d806cd821..870173377 100644 --- a/java/google/registry/rdap/RdapModule.java +++ b/java/google/registry/rdap/RdapModule.java @@ -61,4 +61,16 @@ public final class RdapModule { static Optional provideHandle(HttpServletRequest req) { return RequestParameters.extractOptionalParameter(req, "handle"); } + + @Provides + @Parameter("registrar") + static Optional provideRegistrar(HttpServletRequest req) { + return RequestParameters.extractOptionalParameter(req, "registrar"); + } + + @Provides + @Parameter("includeDeleted") + static Optional provideIncludeDeleted(HttpServletRequest req) { + return RequestParameters.extractOptionalBooleanParameter(req, "includeDeleted"); + } } diff --git a/java/google/registry/rdap/RdapNameserverAction.java b/java/google/registry/rdap/RdapNameserverAction.java index 4ec89ef7f..fa587b30b 100644 --- a/java/google/registry/rdap/RdapNameserverAction.java +++ b/java/google/registry/rdap/RdapNameserverAction.java @@ -17,6 +17,7 @@ package google.registry.rdap; import static google.registry.model.EppResourceUtils.loadByForeignKey; import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.HEAD; +import static google.registry.util.DateTimeUtils.START_OF_TIME; import com.google.common.collect.ImmutableMap; import google.registry.model.host.HostResource; @@ -59,8 +60,12 @@ public class RdapNameserverAction extends RdapActionBase { pathSearchString = canonicalizeName(pathSearchString); // The RDAP syntax is /rdap/nameserver/ns1.mydomain.com. validateDomainName(pathSearchString); - HostResource hostResource = loadByForeignKey(HostResource.class, pathSearchString, now); - if (hostResource == null) { + // If there are no undeleted nameservers with the given name, the foreign key should point to + // the most recently deleted one. + HostResource hostResource = + loadByForeignKey( + HostResource.class, pathSearchString, shouldIncludeDeleted() ? START_OF_TIME : now); + if ((hostResource == null) || !shouldBeVisible(hostResource, now)) { throw new NotFoundException(pathSearchString + " not found"); } return rdapJsonFormatter.makeRdapJsonForHost( diff --git a/java/google/registry/rdap/RdapNameserverSearchAction.java b/java/google/registry/rdap/RdapNameserverSearchAction.java index 0feafe49a..a347d0d72 100644 --- a/java/google/registry/rdap/RdapNameserverSearchAction.java +++ b/java/google/registry/rdap/RdapNameserverSearchAction.java @@ -15,10 +15,8 @@ package google.registry.rdap; import static google.registry.model.EppResourceUtils.loadByForeignKey; -import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.HEAD; -import static google.registry.util.DateTimeUtils.END_OF_TIME; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; @@ -26,7 +24,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; import com.google.common.primitives.Booleans; -import google.registry.config.RegistryConfig.Config; +import com.googlecode.objectify.cmd.Query; import google.registry.model.domain.DomainResource; import google.registry.model.host.HostResource; import google.registry.rdap.RdapJsonFormatter.BoilerplateType; @@ -64,11 +62,11 @@ import org.joda.time.DateTime; public class RdapNameserverSearchAction extends RdapActionBase { public static final String PATH = "/rdap/nameservers"; + private static final int RESULT_SET_SIZE_SCALING_FACTOR = 30; @Inject Clock clock; @Inject @Parameter("name") Optional nameParam; @Inject @Parameter("ip") Optional ipParam; - @Inject @Config("rdapResultSetMaxSize") int rdapResultSetMaxSize; @Inject RdapNameserverSearchAction() {} @Override @@ -125,85 +123,136 @@ public class RdapNameserverSearchAction extends RdapActionBase { return jsonBuilder.build(); } - /** Searches for nameservers by name, returning a JSON array of nameserver info maps. */ + /** + * Searches for nameservers by name, returning a JSON array of nameserver info maps. + * + *

When deleted nameservers are included in the search, the search is treated as if it has a + * wildcard, because multiple results can be returned. + */ private RdapSearchResults searchByName( final RdapSearchPattern partialStringQuery, final DateTime now) { - // Handle queries without a wildcard -- just load by foreign key. - if (!partialStringQuery.getHasWildcard()) { - HostResource hostResource = - loadByForeignKey(HostResource.class, partialStringQuery.getInitialString(), now); - if (hostResource == null) { - throw new NotFoundException("No nameservers found"); + // Handle queries without a wildcard -- just load by foreign key. We can't do this if deleted + // nameservers are desired, because there may be multiple nameservers with the same name. + if (!partialStringQuery.getHasWildcard() && !shouldIncludeDeleted()) { + return searchByNameUsingForeignKey(partialStringQuery, now); + // Handle queries with a wildcard (or including deleted entries). If there is a suffix, it + // should be a domain that we manage, so we can look up the domain and search through the + // subordinate hosts. This is more efficient, and lets us permit wildcard searches with no + // initial string. Deleted nameservers cannot be searched using a suffix, because the logic + // of the deletion status of the superordinate domain versus the deletion status of the + // subordinate host gets too messy. + } else if (partialStringQuery.getSuffix() != null) { + if (shouldIncludeDeleted()) { + throw new UnprocessableEntityException( + "A suffix after a wildcard is not allowed when searching for deleted nameservers"); } - return RdapSearchResults.create( - ImmutableList.of( - rdapJsonFormatter.makeRdapJsonForHost( - hostResource, false, rdapLinkBase, rdapWhoisServer, now, OutputDataType.FULL))); - // Handle queries with a wildcard. + return searchByNameUsingSuperordinateDomain(partialStringQuery, now); + // Handle queries with a wildcard (or deleted entries included), but no suffix. } else { - // If there is a suffix, it should be a domain that we manage, so we can look up the domain - // and search through the subordinate hosts. This is more efficient, and lets us permit - // wildcard searches with no initial string. - if (partialStringQuery.getSuffix() != null) { - DomainResource domainResource = - loadByForeignKey(DomainResource.class, partialStringQuery.getSuffix(), now); - if (domainResource == null) { - // Don't allow wildcards with suffixes which are not domains we manage. That would risk a - // table scan in many easily foreseeable cases. The user might ask for ns*.zombo.com, - // forcing us to query for all hosts beginning with ns, then filter for those ending in - // .zombo.com. It might well be that 80% of all hostnames begin with ns, leading to - // inefficiency. - throw new UnprocessableEntityException( - "A suffix after a wildcard in a nameserver lookup must be an in-bailiwick domain"); - } - List hostList = new ArrayList<>(); - for (String fqhn : ImmutableSortedSet.copyOf(domainResource.getSubordinateHosts())) { - // We can't just check that the host name starts with the initial query string, because - // then the query ns.exam*.example.com would match against nameserver ns.example.com. - if (partialStringQuery.matches(fqhn)) { - HostResource hostResource = loadByForeignKey(HostResource.class, fqhn, now); - if (hostResource != null) { - hostList.add(hostResource); - if (hostList.size() > rdapResultSetMaxSize) { - break; - } - } + return searchByNameUsingPrefix(partialStringQuery, now); + } + } + + /** + * Searches for nameservers by name with no wildcard or deleted names. + * + *

In this case, we can load by foreign key. + */ + private RdapSearchResults searchByNameUsingForeignKey( + final RdapSearchPattern partialStringQuery, final DateTime now) { + HostResource hostResource = + loadByForeignKey(HostResource.class, partialStringQuery.getInitialString(), now); + if ((hostResource == null) || !shouldBeVisible(hostResource, now)) { + throw new NotFoundException("No nameservers found"); + } + return RdapSearchResults.create( + ImmutableList.of( + rdapJsonFormatter.makeRdapJsonForHost( + hostResource, false, rdapLinkBase, rdapWhoisServer, now, OutputDataType.FULL))); + } + + /** Searches for nameservers by name using the superordinate domain as a suffix. */ + private RdapSearchResults searchByNameUsingSuperordinateDomain( + final RdapSearchPattern partialStringQuery, final DateTime now) { + DomainResource domainResource = + loadByForeignKey(DomainResource.class, partialStringQuery.getSuffix(), now); + if (domainResource == null) { + // Don't allow wildcards with suffixes which are not domains we manage. That would risk a + // table scan in many easily foreseeable cases. The user might ask for ns*.zombo.com, + // forcing us to query for all hosts beginning with ns, then filter for those ending in + // .zombo.com. It might well be that 80% of all hostnames begin with ns, leading to + // inefficiency. + throw new UnprocessableEntityException( + "A suffix after a wildcard in a nameserver lookup must be an in-bailiwick domain"); + } + List hostList = new ArrayList<>(); + for (String fqhn : ImmutableSortedSet.copyOf(domainResource.getSubordinateHosts())) { + // We can't just check that the host name starts with the initial query string, because + // then the query ns.exam*.example.com would match against nameserver ns.example.com. + if (partialStringQuery.matches(fqhn)) { + HostResource hostResource = loadByForeignKey(HostResource.class, fqhn, now); + if ((hostResource != null) && shouldBeVisible(hostResource, now)) { + hostList.add(hostResource); + if (hostList.size() > rdapResultSetMaxSize) { + break; } } - return makeSearchResults(hostList, now); - // Handle queries with a wildcard, but no suffix. There are no pending deletes for hosts, so - // we can call queryUndeleted. Unlike the above problem with suffixes, we can safely search - // for nameservers beginning with a particular suffix, because we need only fetch the first - // rdapResultSetMaxSize entries, and ignore the rest. - } else { - return makeSearchResults( - // Add 1 so we can detect truncation. - queryUndeleted( - HostResource.class, - "fullyQualifiedHostName", - partialStringQuery, - rdapResultSetMaxSize + 1) - .list(), - now); } } + return makeSearchResults(hostList, IncompletenessWarningType.NONE, now); + } + + /** + * Searches for nameservers by name with a prefix and wildcard. + * + *

There are no pending deletes for hosts, so we can call {@link RdapActionBase#queryItems}. + */ + private RdapSearchResults searchByNameUsingPrefix( + final RdapSearchPattern partialStringQuery, final DateTime now) { + // Add 1 so we can detect truncation. + Query query = + queryItems( + HostResource.class, + "fullyQualifiedHostName", + partialStringQuery, + shouldIncludeDeleted(), + shouldIncludeDeleted() + ? (RESULT_SET_SIZE_SCALING_FACTOR * (rdapResultSetMaxSize + 1)) + : (rdapResultSetMaxSize + 1)); + return makeSearchResults(getMatchingResources(query, now), now); } /** Searches for nameservers by IP address, returning a JSON array of nameserver info maps. */ private RdapSearchResults searchByIp(final InetAddress inetAddress, DateTime now) { + // Add 1 so we can detect truncation. + Query query = + queryItems( + HostResource.class, + "inetAddresses", + inetAddress.getHostAddress(), + shouldIncludeDeleted(), + shouldIncludeDeleted() + ? (RESULT_SET_SIZE_SCALING_FACTOR * (rdapResultSetMaxSize + 1)) + : (rdapResultSetMaxSize + 1)); + return makeSearchResults(getMatchingResources(query, now), now); + } + + /** + * Output JSON for a lists of hosts contained in an {@link + * RdapResourcesAndIncompletenessWarningType}. + */ + private RdapSearchResults makeSearchResults( + RdapResourcesAndIncompletenessWarningType resourcesAndIncompletenessWarningType, + DateTime now) { return makeSearchResults( - // Add 1 so we can detect truncation. - ofy().load() - .type(HostResource.class) - .filter("inetAddresses", inetAddress.getHostAddress()) - .filter("deletionTime", END_OF_TIME) - .limit(rdapResultSetMaxSize + 1) - .list(), + resourcesAndIncompletenessWarningType.resources(), + resourcesAndIncompletenessWarningType.incompletenessWarningType(), now); } /** Output JSON for a list of hosts. */ - private RdapSearchResults makeSearchResults(List hosts, DateTime now) { + private RdapSearchResults makeSearchResults( + List hosts, IncompletenessWarningType incompletenessWarningType, DateTime now) { OutputDataType outputDataType = (hosts.size() > 1) ? OutputDataType.SUMMARY : OutputDataType.FULL; ImmutableList.Builder> jsonListBuilder = @@ -218,6 +267,6 @@ public class RdapNameserverSearchAction extends RdapActionBase { jsonList, (jsonList.size() < hosts.size()) ? IncompletenessWarningType.TRUNCATED - : IncompletenessWarningType.NONE); + : incompletenessWarningType); } } diff --git a/java/google/registry/rdap/RdapResourcesAndIncompletenessWarningType.java b/java/google/registry/rdap/RdapResourcesAndIncompletenessWarningType.java new file mode 100644 index 000000000..0a3efe917 --- /dev/null +++ b/java/google/registry/rdap/RdapResourcesAndIncompletenessWarningType.java @@ -0,0 +1,42 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.rdap; + +import com.google.auto.value.AutoValue; +import google.registry.model.EppResource; +import google.registry.rdap.RdapSearchResults.IncompletenessWarningType; +import java.util.List; + +@AutoValue +abstract class RdapResourcesAndIncompletenessWarningType { + + static RdapResourcesAndIncompletenessWarningType create( + List resources) { + return create(resources, IncompletenessWarningType.NONE); + } + + static RdapResourcesAndIncompletenessWarningType create( + List resources, IncompletenessWarningType incompletenessWarningType) { + return new AutoValue_RdapResourcesAndIncompletenessWarningType( + resources, incompletenessWarningType); + } + + /** List of EPP resources. */ + abstract List resources(); + + /** Type of warning to display regarding possible incomplete data. */ + abstract IncompletenessWarningType incompletenessWarningType(); +} + diff --git a/javatests/google/registry/rdap/RdapNameserverActionTest.java b/javatests/google/registry/rdap/RdapNameserverActionTest.java index 686053134..525d9ef8e 100644 --- a/javatests/google/registry/rdap/RdapNameserverActionTest.java +++ b/javatests/google/registry/rdap/RdapNameserverActionTest.java @@ -16,20 +16,31 @@ package google.registry.rdap; import static com.google.common.truth.Truth.assertThat; import static google.registry.testing.DatastoreHelper.createTld; +import static google.registry.testing.DatastoreHelper.persistResource; import static google.registry.testing.FullFieldsTestEntityHelper.makeAndPersistHostResource; +import static google.registry.testing.FullFieldsTestEntityHelper.makeRegistrar; import static google.registry.testing.TestDataHelper.loadFileWithSubstitutions; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; -import com.google.appengine.api.NamespaceManager; +import com.google.appengine.api.users.User; +import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import google.registry.model.ofy.Ofy; +import google.registry.model.registrar.Registrar; +import google.registry.request.auth.AuthLevel; +import google.registry.request.auth.AuthResult; +import google.registry.request.auth.UserAuthInfo; import google.registry.testing.AppEngineRule; import google.registry.testing.FakeClock; import google.registry.testing.FakeResponse; import google.registry.testing.InjectRule; +import google.registry.ui.server.registrar.SessionUtils; import java.util.Map; import javax.annotation.Nullable; +import javax.servlet.http.HttpServletRequest; import org.joda.time.DateTime; import org.json.simple.JSONValue; import org.junit.Before; @@ -50,8 +61,13 @@ public class RdapNameserverActionTest { @Rule public final InjectRule inject = new InjectRule(); + private final HttpServletRequest request = mock(HttpServletRequest.class); private final FakeResponse response = new FakeResponse(); - final FakeClock clock = new FakeClock(DateTime.parse("2000-01-01TZ")); + private final FakeClock clock = new FakeClock(DateTime.parse("2000-01-01TZ")); + private final SessionUtils sessionUtils = mock(SessionUtils.class); + private final User user = new User("rdap.user@example.com", "gmail.com", "12345"); + private final UserAuthInfo userAuthInfo = UserAuthInfo.create(user, false); + private final UserAuthInfo adminUserAuthInfo = UserAuthInfo.create(user, true); @Before public void setUp() throws Exception { @@ -68,22 +84,50 @@ public class RdapNameserverActionTest { createTld("1.tld"); makeAndPersistHostResource( "ns1.domain.1.tld", "5.6.7.8", clock.nowUtc().minusYears(1)); - NamespaceManager.set(null); + // deleted + persistResource( + makeAndPersistHostResource("nsdeleted.cat.lol", "1.2.3.4", clock.nowUtc().minusYears(1)) + .asBuilder() + .setDeletionTime(clock.nowUtc().minusMonths(1)) + .build()); + // other registrar + persistResource( + makeRegistrar("otherregistrar", "Yes Virginia