From a5c931a1523bc9000fe9bd30ce8932f3e99085bf Mon Sep 17 00:00:00 2001 From: mountford Date: Mon, 2 Oct 2017 12:20:04 -0700 Subject: [PATCH] Add RDAP support for deleted contacts and registrars This CL adds the functionality for contact and registrar searches. A future CL will handle domains and entities. Support is also added for filtering results by registrar. Deleted items can only be seen by admins, and by registrars viewing their own deleted items. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=170734664 --- .../registry/rdap/RdapEntityAction.java | 5 +- .../registry/rdap/RdapEntitySearchAction.java | 234 +++++---- .../registry/rdap/RdapJsonFormatter.java | 22 +- .../registry/rdap/RdapEntityActionTest.java | 326 ++++++++++--- .../rdap/RdapEntitySearchActionTest.java | 457 ++++++++++++------ .../registry/rdap/testdata/rdap_contact.json | 3 +- .../rdap/testdata/rdap_contact_deleted.json | 54 +++ .../testdata/rdap_nontruncated_contacts.json | 24 +- .../rdap/testdata/rdap_registrar.json | 4 +- .../rdap/testdata/rdap_registrar_test.json | 74 +++ .../testdata/rdap_truncated_contacts.json | 24 +- .../rdap_truncated_mixed_entities.json | 18 +- .../testing/FullFieldsTestEntityHelper.java | 35 +- 13 files changed, 905 insertions(+), 375 deletions(-) create mode 100644 javatests/google/registry/rdap/testdata/rdap_contact_deleted.json create mode 100644 javatests/google/registry/rdap/testdata/rdap_registrar_test.json diff --git a/java/google/registry/rdap/RdapEntityAction.java b/java/google/registry/rdap/RdapEntityAction.java index 169656794..5e8615b90 100644 --- a/java/google/registry/rdap/RdapEntityAction.java +++ b/java/google/registry/rdap/RdapEntityAction.java @@ -85,7 +85,7 @@ public class RdapEntityAction extends RdapActionBase { ContactResource contactResource = ofy().load().key(contactKey).now(); // As per Andy Newton on the regext mailing list, contacts by themselves have no role, since // they are global, and might have different roles for different domains. - if ((contactResource != null) && now.isBefore(contactResource.getDeletionTime())) { + if ((contactResource != null) && shouldBeVisible(contactResource, now)) { return rdapJsonFormatter.makeRdapJsonForContact( contactResource, true, @@ -101,7 +101,7 @@ public class RdapEntityAction extends RdapActionBase { if (ianaIdentifier != null) { wasValidKey = true; Optional registrar = getRegistrarByIanaIdentifier(ianaIdentifier); - if ((registrar.isPresent()) && registrar.get().isLiveAndPubliclyVisible()) { + if (registrar.isPresent() && shouldBeVisible(registrar.get())) { return rdapJsonFormatter.makeRdapJsonForRegistrar( registrar.get(), true, rdapLinkBase, rdapWhoisServer, now, OutputDataType.FULL); } @@ -112,4 +112,3 @@ public class RdapEntityAction extends RdapActionBase { : new BadRequestException(pathSearchString + " is not a valid entity handle"); } } - diff --git a/java/google/registry/rdap/RdapEntitySearchAction.java b/java/google/registry/rdap/RdapEntitySearchAction.java index 20868476e..e2e7bdfd3 100644 --- a/java/google/registry/rdap/RdapEntitySearchAction.java +++ b/java/google/registry/rdap/RdapEntitySearchAction.java @@ -18,7 +18,6 @@ import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.rdap.RdapUtils.getRegistrarByIanaIdentifier; import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.HEAD; -import static google.registry.util.DateTimeUtils.END_OF_TIME; import com.google.common.base.Optional; import com.google.common.base.Predicate; @@ -27,7 +26,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Booleans; import com.google.common.primitives.Longs; -import com.googlecode.objectify.Key; +import com.googlecode.objectify.cmd.Query; import google.registry.model.contact.ContactResource; import google.registry.model.domain.DesignatedContact; import google.registry.model.registrar.Registrar; @@ -65,6 +64,8 @@ public class RdapEntitySearchAction extends RdapActionBase { public static final String PATH = "/rdap/entities"; + private static final int RESULT_SET_SIZE_SCALING_FACTOR = 30; + @Inject Clock clock; @Inject @Parameter("fn") Optional fnParam; @Inject @Parameter("handle") Optional handleParam; @@ -124,134 +125,167 @@ public class RdapEntitySearchAction extends RdapActionBase { * by registrar contact name: * *

The search is by registrar name only. The profile is supporting the functionality defined - * in the Base Registry Agreement (see 1.6 of Section 4 of the Base Registry Agreement, - * https://newgtlds.icann.org/sites/default/files/agreements/ - * agreement-approved-09jan14-en.htm). + * in the Base Registry Agreement. * *

According to RFC 7482 section 6.1, punycode is only used for domain name labels, so we can * assume that entity names are regular unicode. + * + *

Searches for deleted entities are treated like wildcard searches, because they can return + * multiple entities. + * + * @see 1.6 + * of Section 4 of the Base Registry Agreement */ private RdapSearchResults searchByName(final RdapSearchPattern partialStringQuery, DateTime now) { - // For wildcard searches, make sure the initial string is long enough, and don't allow suffixes. - if (partialStringQuery.getHasWildcard()) { - if (partialStringQuery.getSuffix() != null) { - throw new UnprocessableEntityException( - "Suffixes not allowed in wildcard entity name searches"); - } - if (partialStringQuery.getInitialString().length() - < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { - throw new UnprocessableEntityException( - "Initial search string required in wildcard entity name searches"); - } + // For wildcard searches, and searches that include deleted items, make sure the initial string + // is long enough, and don't allow suffixes. + if ((partialStringQuery.getHasWildcard() || shouldIncludeDeleted()) + && (partialStringQuery.getSuffix() != null)) { + throw new UnprocessableEntityException( + partialStringQuery.getHasWildcard() + ? "Suffixes not allowed in wildcard entity name searches" + : "Suffixes not allowed when searching for deleted entities"); } - // Get the registrar matches, depending on whether there's a wildcard. - ImmutableList registrarMatches = + if (partialStringQuery.getHasWildcard() + && (partialStringQuery.getInitialString().length() + < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH)) { + throw new UnprocessableEntityException( + partialStringQuery.getHasWildcard() + ? "Initial search string required in wildcard entity name searches" + : "Initial search string required when searching for deleted entities"); + } + // Get the registrar matches. + ImmutableList registrars = FluentIterable.from(Registrar.loadAllCached()) .filter( new Predicate() { @Override public boolean apply(Registrar registrar) { - return partialStringQuery.matches(registrar.getRegistrarName()); - }}) + return partialStringQuery.matches(registrar.getRegistrarName()) + && shouldBeVisible(registrar); + } + }) .limit(rdapResultSetMaxSize + 1) .toList(); // Get the contact matches and return the results, fetching an additional contact to detect - // truncation. - return makeSearchResults( + // truncation. If we are including deleted entries, we must fetch more entries, in case some + // get excluded due to permissioning. + Query query = queryItems( - ContactResource.class, - "searchName", - partialStringQuery, - false /* includeDeleted */, - rdapResultSetMaxSize + 1) - .list(), - registrarMatches, - now); + ContactResource.class, + "searchName", + partialStringQuery, + shouldIncludeDeleted(), + shouldIncludeDeleted() + ? (RESULT_SET_SIZE_SCALING_FACTOR * (rdapResultSetMaxSize + 1)) + : (rdapResultSetMaxSize + 1)); + return makeSearchResults(getMatchingResources(query, now), registrars, now); } - /** Searches for entities by handle, returning a JSON array of entity info maps. */ + /** + * Searches for entities by handle, returning a JSON array of entity info maps. + * + *

Searches for deleted entities are treated like wildcard searches. + * + *

We don't allow suffixes after a wildcard in entity searches. Suffixes are used in domain + * searches to specify a TLD, and in nameserver searches to specify an in-bailiwick domain name. + * In both cases, the suffix can be turned into an additional query filter field. For contacts, + * there is no equivalent string suffix that can be used as a query filter, so we disallow use. + */ private RdapSearchResults searchByHandle( final RdapSearchPattern partialStringQuery, DateTime now) { - // Handle queries without a wildcard -- load by ID. - if (!partialStringQuery.getHasWildcard()) { + if (partialStringQuery.getSuffix() != null) { + throw new UnprocessableEntityException("Suffixes not allowed in entity handle searches"); + } + // Handle queries without a wildcard (and not including deleted) -- load by ID. + if (!partialStringQuery.getHasWildcard() && !shouldIncludeDeleted()) { ContactResource contactResource = ofy().load() .type(ContactResource.class) .id(partialStringQuery.getInitialString()) .now(); - ImmutableList registrars = - getMatchingRegistrars(partialStringQuery.getInitialString()); return makeSearchResults( - ((contactResource == null) || !contactResource.getDeletionTime().isEqual(END_OF_TIME)) - ? ImmutableList.of() : ImmutableList.of(contactResource), - registrars, + ((contactResource != null) && shouldBeVisible(contactResource, now)) + ? ImmutableList.of(contactResource) + : ImmutableList.of(), + IncompletenessWarningType.NONE, + getMatchingRegistrars(partialStringQuery.getInitialString()), now); - // Handle queries with a wildcard, but no suffix. For contact resources, the deletion time will - // always be END_OF_TIME for non-deleted records; unlike domain resources, we don't need to - // worry about deletion times in the future. That allows us to use an equality query for the - // deletion time. Because the handle for registrars is the IANA identifier number, don't allow - // wildcard searches for registrars, by simply not searching for registrars if a wildcard is - // present. Fetch an extra contact to detect result set truncation. - } else if (partialStringQuery.getSuffix() == null) { - if (partialStringQuery.getInitialString().length() - < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) { - throw new UnprocessableEntityException( - "Initial search string required in wildcard entity handle searches"); - } - return makeSearchResults( - ofy().load() - .type(ContactResource.class) - .filterKey( - ">=", Key.create(ContactResource.class, partialStringQuery.getInitialString())) - .filterKey( - "<", Key.create(ContactResource.class, partialStringQuery.getNextInitialString())) - .filter("deletionTime", END_OF_TIME) - .limit(rdapResultSetMaxSize + 1) - .list(), - ImmutableList.of(), - now); - // Don't allow suffixes in entity handle search queries. + // Handle queries with a wildcard (or including deleted), but no suffix. Because the handle + // for registrars is the IANA identifier number, don't allow wildcard searches for registrars, + // by simply not searching for registrars if a wildcard is present. Fetch an extra contact to + // detect result set truncation. } else { - throw new UnprocessableEntityException("Suffixes not allowed in entity handle searches"); + ImmutableList registrars = + partialStringQuery.getHasWildcard() + ? ImmutableList.of() + : getMatchingRegistrars(partialStringQuery.getInitialString()); + // Get the contact matches and return the results, fetching an additional contact to detect + // truncation. If we are including deleted entries, we must fetch more entries, in case some + // get excluded due to permissioning. + Query query = + queryItemsByKey( + ContactResource.class, + partialStringQuery, + shouldIncludeDeleted(), + shouldIncludeDeleted() + ? (RESULT_SET_SIZE_SCALING_FACTOR * (rdapResultSetMaxSize + 1)) + : (rdapResultSetMaxSize + 1)); + return makeSearchResults(getMatchingResources(query, now), registrars, now); } } /** Looks up registrars by handle (i.e. IANA identifier). */ private ImmutableList getMatchingRegistrars(final String ianaIdentifierString) { Long ianaIdentifier = Longs.tryParse(ianaIdentifierString); - if (ianaIdentifier != null) { - Optional registrar = getRegistrarByIanaIdentifier(ianaIdentifier); - if (registrar.isPresent()) { - return ImmutableList.of(registrar.get()); - } + if (ianaIdentifier == null) { + return ImmutableList.of(); } - return ImmutableList.of(); + Optional registrar = getRegistrarByIanaIdentifier(ianaIdentifier); + return (registrar.isPresent() && shouldBeVisible(registrar.get())) + ? ImmutableList.of(registrar.get()) + : ImmutableList.of(); } - /** Builds a JSON array of entity info maps based on the specified contacts and registrars. */ + /** + * Builds a JSON array of entity info maps based on the specified contacts and registrars. + * + *

This is a convenience wrapper for the four-argument makeSearchResults; it unpacks the two + * properties of the ContactsAndIncompletenessWarningType structure and passes them as separate + * arguments. + */ private RdapSearchResults makeSearchResults( - List contacts, List registrars, DateTime now) { + RdapResourcesAndIncompletenessWarningType + resourcesAndIncompletenessWarningType, + List registrars, + DateTime now) { + return makeSearchResults( + resourcesAndIncompletenessWarningType.resources(), + resourcesAndIncompletenessWarningType.incompletenessWarningType(), + registrars, + now); + } + + /** + * Builds a JSON array of entity info maps based on the specified contacts and registrars. + * + * @param contacts the list of contacts which can be returned + * @param incompletenessWarningType MIGHT_BE_INCOMPLETE if the list of contacts might be + * incomplete; this only matters if the total count of contacts and registrars combined is + * less than a full result set's worth + * @param registrars the list of registrars which can be returned + * @param now the current date and time + * @return an {@link RdapSearchResults} object + */ + private RdapSearchResults makeSearchResults( + List contacts, + IncompletenessWarningType incompletenessWarningType, + List registrars, + DateTime now) { // Determine what output data type to use, depending on whether more than one entity will be // returned. - int numEntities = contacts.size(); - OutputDataType outputDataType; - // If there's more than one contact, then we know already we need SUMMARY mode. - if (numEntities > 1) { - outputDataType = OutputDataType.SUMMARY; - // If there are fewer than two contacts, loop through and compute the total number of contacts - // and registrars, stopping as soon as we find two. - } else { - outputDataType = OutputDataType.FULL; - for (Registrar registrar : registrars) { - if (registrar.isLiveAndPubliclyVisible()) { - numEntities++; - if (numEntities > 1) { - outputDataType = OutputDataType.SUMMARY; - break; - } - } - } - } + OutputDataType outputDataType = + (contacts.size() + registrars.size() > 1) ? OutputDataType.SUMMARY : OutputDataType.FULL; // There can be more results than our max size, partially because we have two pools to draw from // (contacts and registrars), and partially because we try to fetch one more than the max size, @@ -276,15 +310,17 @@ public class RdapEntitySearchAction extends RdapActionBase { authorization)); } for (Registrar registrar : registrars) { - if (registrar.isLiveAndPubliclyVisible()) { - if (jsonOutputList.size() >= rdapResultSetMaxSize) { - return RdapSearchResults.create( - ImmutableList.copyOf(jsonOutputList), IncompletenessWarningType.TRUNCATED); - } - jsonOutputList.add(rdapJsonFormatter.makeRdapJsonForRegistrar( - registrar, false, rdapLinkBase, rdapWhoisServer, now, outputDataType)); + if (jsonOutputList.size() >= rdapResultSetMaxSize) { + return RdapSearchResults.create( + ImmutableList.copyOf(jsonOutputList), IncompletenessWarningType.TRUNCATED); } + jsonOutputList.add(rdapJsonFormatter.makeRdapJsonForRegistrar( + registrar, false, rdapLinkBase, rdapWhoisServer, now, outputDataType)); } - return RdapSearchResults.create(ImmutableList.copyOf(jsonOutputList)); + return RdapSearchResults.create( + ImmutableList.copyOf(jsonOutputList), + (jsonOutputList.size() < rdapResultSetMaxSize) + ? incompletenessWarningType + : IncompletenessWarningType.NONE); } } diff --git a/java/google/registry/rdap/RdapJsonFormatter.java b/java/google/registry/rdap/RdapJsonFormatter.java index 88338fb2b..9a5972162 100644 --- a/java/google/registry/rdap/RdapJsonFormatter.java +++ b/java/google/registry/rdap/RdapJsonFormatter.java @@ -767,16 +767,19 @@ public class RdapJsonFormatter { OutputDataType outputDataType) { ImmutableMap.Builder jsonBuilder = new ImmutableMap.Builder<>(); jsonBuilder.put("objectClassName", "entity"); - jsonBuilder.put("handle", registrar.getIanaIdentifier().toString()); + Long ianaIdentifier = registrar.getIanaIdentifier(); + jsonBuilder.put("handle", (ianaIdentifier == null) ? "(none)" : ianaIdentifier.toString()); jsonBuilder.put("status", registrar.isLive() ? STATUS_LIST_ACTIVE : STATUS_LIST_REMOVED); jsonBuilder.put("roles", ImmutableList.of(RdapEntityRole.REGISTRAR.rfc7483String)); - jsonBuilder.put("links", - ImmutableList.of(makeLink("entity", registrar.getIanaIdentifier().toString(), linkBase))); - jsonBuilder.put("publicIds", - ImmutableList.of( - ImmutableMap.of( - "type", "IANA Registrar ID", - "identifier", registrar.getIanaIdentifier().toString()))); + if (ianaIdentifier != null) { + jsonBuilder.put("links", + ImmutableList.of(makeLink("entity", ianaIdentifier.toString(), linkBase))); + jsonBuilder.put( + "publicIds", + ImmutableList.of( + ImmutableMap.of( + "type", "IANA Registrar ID", "identifier", ianaIdentifier.toString()))); + } // Create the vCard. ImmutableList.Builder vcardBuilder = new ImmutableList.Builder<>(); vcardBuilder.add(VCARD_ENTRY_VERSION); @@ -964,9 +967,10 @@ public class RdapJsonFormatter { */ private static ImmutableList makeEvents(Registrar registrar, DateTime now) { ImmutableList.Builder eventsBuilder = new ImmutableList.Builder<>(); + Long ianaIdentifier = registrar.getIanaIdentifier(); eventsBuilder.add(makeEvent( RdapEventAction.REGISTRATION, - registrar.getIanaIdentifier().toString(), + (ianaIdentifier == null) ? "(none)" : ianaIdentifier.toString(), registrar.getCreationTime())); if ((registrar.getLastUpdateTime() != null) && registrar.getLastUpdateTime().isAfter(registrar.getCreationTime())) { diff --git a/javatests/google/registry/rdap/RdapEntityActionTest.java b/javatests/google/registry/rdap/RdapEntityActionTest.java index 53d3a55d3..f99ce9e8c 100644 --- a/javatests/google/registry/rdap/RdapEntityActionTest.java +++ b/javatests/google/registry/rdap/RdapEntityActionTest.java @@ -19,7 +19,6 @@ import static google.registry.testing.DatastoreHelper.createTld; import static google.registry.testing.DatastoreHelper.persistResource; import static google.registry.testing.DatastoreHelper.persistSimpleResources; import static google.registry.testing.FullFieldsTestEntityHelper.makeAndPersistContactResource; -import static google.registry.testing.FullFieldsTestEntityHelper.makeContactResource; import static google.registry.testing.FullFieldsTestEntityHelper.makeDomainResource; import static google.registry.testing.FullFieldsTestEntityHelper.makeHostResource; import static google.registry.testing.FullFieldsTestEntityHelper.makeRegistrar; @@ -29,6 +28,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.google.appengine.api.users.User; +import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -45,6 +45,7 @@ import google.registry.testing.FakeResponse; import google.registry.testing.InjectRule; import google.registry.ui.server.registrar.SessionUtils; import java.util.Map; +import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; import org.joda.time.DateTime; import org.json.simple.JSONValue; @@ -72,6 +73,7 @@ public class RdapEntityActionTest { private final SessionUtils sessionUtils = mock(SessionUtils.class); private final User user = new User("rdap.user@example.com", "gmail.com", "12345"); UserAuthInfo userAuthInfo = UserAuthInfo.create(user, false); + UserAuthInfo adminUserAuthInfo = UserAuthInfo.create(user, true); private RdapEntityAction action; @@ -132,32 +134,50 @@ public class RdapEntityActionTest { Registrar registrar1tld = persistResource( makeRegistrar("1tldregistrar", "Multilevel Registrar", Registrar.State.ACTIVE, 103L)); persistSimpleResources(makeRegistrarContacts(registrar1tld)); + // deleted registrar + Registrar registrarDeleted = persistResource( + makeRegistrar("deletedregistrar", "Yes Virginia