Add RDAP metrics for non-search endpoints

This CL uses the previously-defined RDAP metrics class to record basic metrics
for all RDAP endpoints, and handles testing of non-search endpoints. Searches
are more complicated, and will be handled in future CLs.

The default wildcard type is now INVALID rather than NO_WILDCARD.

A change to getMatchingResources() (adding an additional parameter) is also included in this CL, as it was needed to set the incompleteness warning type correctly.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=175719265

java/google/registry/rdap/RdapActionBase.java

@@ -34,6 +34,7 @@ import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
 import google.registry.model.registrar.Registrar;
 import google.registry.rdap.RdapMetrics.EndpointType;
+import google.registry.rdap.RdapMetrics.WildcardType;
 import google.registry.rdap.RdapSearchResults.IncompletenessWarningType;
 import google.registry.request.Action;
 import google.registry.request.FullServletPath;
@@ -73,6 +74,7 @@ public abstract class RdapActionBase implements Runnable {
    * hyphens. In this case, allow the wildcard asterisk as well.
    */
   static final Pattern LDH_PATTERN = Pattern.compile("[-.a-zA-Z0-9*]+");
+  private static final int RESULT_SET_SIZE_SCALING_FACTOR = 30;
 
   private static final MediaType RESPONSE_MEDIA_TYPE =
       MediaType.create("application", "rdap+json").withCharset(UTF_8);
@@ -89,6 +91,11 @@ public abstract class RdapActionBase implements Runnable {
   @Inject @Parameter("includeDeleted") Optional<Boolean> includeDeletedParam;
   @Inject @Config("rdapWhoisServer") @Nullable String rdapWhoisServer;
   @Inject @Config("rdapResultSetMaxSize") int rdapResultSetMaxSize;
+  @Inject RdapMetrics rdapMetrics;
+
+  /** Builder for metric recording. */
+  final RdapMetrics.RdapMetricInformation.Builder metricInformationBuilder =
+      RdapMetrics.RdapMetricInformation.builder();
 
   /** Returns a string like "domain name" or "nameserver", used for error strings. */
   abstract String getHumanReadableObjectTypeName();
@@ -115,6 +122,11 @@ public abstract class RdapActionBase implements Runnable {
 
   @Override
   public void run() {
+    metricInformationBuilder.setIncludeDeleted(includeDeletedParam.orElse(false));
+    metricInformationBuilder.setRegistrarSpecified(registrarParam.isPresent());
+    metricInformationBuilder.setRole(getAuthorization().role());
+    metricInformationBuilder.setRequestMethod(requestMethod);
+    metricInformationBuilder.setEndpointType(getEndpointType());
     try {
       // Extract what we're searching for from the request path. Some RDAP commands use trailing
       // data in the path itself (e.g. /rdap/domain/mydomain.com), and some use the query string
@@ -134,6 +146,7 @@ public abstract class RdapActionBase implements Runnable {
       if (requestMethod != Action.Method.HEAD) {
         response.setPayload(JSONValue.toJSONString(rdapJson));
       }
+      metricInformationBuilder.setStatusCode(SC_OK);
     } catch (HttpException e) {
       setError(e.getResponseCode(), e.getResponseCodeString(), e.getMessage());
     } catch (URISyntaxException | IllegalArgumentException e) {
@@ -142,9 +155,11 @@ public abstract class RdapActionBase implements Runnable {
       setError(SC_INTERNAL_SERVER_ERROR, "Internal Server Error", "An error was encountered");
       logger.severe(e, "Exception encountered while processing RDAP command");
     }
+    rdapMetrics.updateMetrics(metricInformationBuilder.build());
   }
 
   void setError(int status, String title, String description) {
+    metricInformationBuilder.setStatusCode(status);
     response.setStatus(status);
     response.setContentType(RESPONSE_MEDIA_TYPE);
     try {
@@ -264,6 +279,12 @@ public abstract class RdapActionBase implements Runnable {
     return name;
   }
 
+  int getStandardQuerySizeLimit() {
+    return shouldIncludeDeleted()
+            ? (RESULT_SET_SIZE_SCALING_FACTOR * (rdapResultSetMaxSize + 1))
+            : (rdapResultSetMaxSize + 1);
+  }
+
   /**
    * Handles prefix searches in cases where, if we need to filter out deleted items, there are no
    * pending deletes. In such cases, it is sufficient to check whether {@code deletionTime} is equal
@@ -385,14 +406,16 @@ public abstract class RdapActionBase implements Runnable {
    *        the query could not check deletion status (due to Datastore limitations such as the
    *        limit of one field queried for inequality, for instance), it may need to be set to true
    *        even when not including deleted records
+   * @param querySizeLimit the maximum number of items the query is expected to return, usually
+   *        because the limit has been set
    * @return an {@link RdapResultSet} object containing the list of
    *         resources and an incompleteness warning flag, which is set to MIGHT_BE_INCOMPLETE iff
    *         any resources were excluded due to lack of visibility, and the resulting list of
-   *         resources is less than the maximum allowable, which indicates that we may not have
-   *         fetched enough resources
+   *         resources is less than the maximum allowable, and the number of items returned by the
+   *         query is greater than or equal to the maximum number we might have expected
    */
   <T extends EppResource> RdapResultSet<T> getMatchingResources(
-      Query<T> query, boolean checkForVisibility, DateTime now) {
+      Query<T> query, boolean checkForVisibility, DateTime now, int querySizeLimit) {
     Optional<String> desiredRegistrar = getDesiredRegistrar();
     if (desiredRegistrar.isPresent()) {
       query = query.filter("currentSponsorClientId", desiredRegistrar.get());
@@ -415,11 +438,40 @@ public abstract class RdapActionBase implements Runnable {
         break;
       }
     }
+    // The incompleteness problem comes about because we don't know how many items to fetch. We want
+    // to return rdapResultSetMaxSize worth of items, but some might be excluded, so we fetch more
+    // just in case. But how many more? That's the potential problem, addressed with the three way
+    // AND statement:
+    // 1. If we didn't exclude any items, then we can't have the incompleteness problem.
+    // 2. If have a full result set batch (rdapResultSetMaxSize items), we must by definition be
+    //    giving the user a complete result set.
+    // 3. If we started with fewer than querySizeLimit items, then there weren't any more items that
+    //    we missed. Even if we return fewer than rdapResultSetMaxSize items, it isn't because we
+    // didn't fetch enough to start.
+    // Only if all three conditions are true might things be incomplete. In other words, we fetched
+    // as many as our limit allowed, but then excluded so many that we wound up with less than a
+    // full result set's worth of results.
     return RdapResultSet.create(
         resources,
-        (someExcluded && (resources.size() < rdapResultSetMaxSize + 1))
+        (someExcluded
+                && (resources.size() < rdapResultSetMaxSize)
+                && (numResourcesQueried >= querySizeLimit))
             ? IncompletenessWarningType.MIGHT_BE_INCOMPLETE
             : IncompletenessWarningType.COMPLETE,
         numResourcesQueried);
   }
+
+  RdapSearchPattern recordWildcardType(RdapSearchPattern partialStringQuery) {
+    if (!partialStringQuery.getHasWildcard()) {
+      metricInformationBuilder.setWildcardType(WildcardType.NO_WILDCARD);
+    } else if (partialStringQuery.getSuffix() == null) {
+      metricInformationBuilder.setWildcardType(WildcardType.PREFIX);
+    } else if (partialStringQuery.getInitialString().isEmpty()) {
+      metricInformationBuilder.setWildcardType(WildcardType.SUFFIX);
+    } else {
+      metricInformationBuilder.setWildcardType(WildcardType.PREFIX_AND_SUFFIX);
+    }
+    metricInformationBuilder.setPrefixLength(partialStringQuery.getInitialString().length());
+    return partialStringQuery;
+  }
 }

java/google/registry/rdap/RdapDomainSearchAction.java

@@ -239,7 +239,10 @@ public class RdapDomainSearchAction extends RdapActionBase {
       query = query.filter("tld", partialStringQuery.getSuffix());
     }
     // Always check for visibility, because we couldn't look at the deletionTime in the query.
-    return makeSearchResults(getMatchingResources(query, true, now), now);
+    return makeSearchResults(
+        getMatchingResources(
+            query, true, now, RESULT_SET_SIZE_SCALING_FACTOR * rdapResultSetMaxSize),
+        now);
   }
 
   /** Searches for domains by domain name with a TLD suffix. */
@@ -255,7 +258,10 @@ public class RdapDomainSearchAction extends RdapActionBase {
             .filter("tld", tld)
             .order("fullyQualifiedDomainName")
             .limit(RESULT_SET_SIZE_SCALING_FACTOR * rdapResultSetMaxSize);
-    return makeSearchResults(getMatchingResources(query, true, now), now);
+    return makeSearchResults(
+        getMatchingResources(
+            query, true, now, RESULT_SET_SIZE_SCALING_FACTOR * rdapResultSetMaxSize),
+        now);
   }
 
   /**

java/google/registry/rdap/RdapEntitySearchAction.java

@@ -64,8 +64,6 @@ public class RdapEntitySearchAction extends RdapActionBase {
 
   public static final String PATH = "/rdap/entities";
 
-  private static final int RESULT_SET_SIZE_SCALING_FACTOR = 30;
-
   @Inject Clock clock;
   @Inject @Parameter("fn") Optional<String> fnParam;
   @Inject @Parameter("handle") Optional<String> handleParam;
@@ -189,7 +187,7 @@ public class RdapEntitySearchAction extends RdapActionBase {
       if (authorization.role() != RdapAuthorization.Role.ADMINISTRATOR) {
         query = query.filter("currentSponsorClientId in", authorization.clientIds());
       }
-      resultSet = getMatchingResources(query, false, now);
+      resultSet = getMatchingResources(query, false, now, rdapResultSetMaxSize + 1);
     }
     return makeSearchResults(resultSet, registrars, now);
   }
@@ -234,16 +232,17 @@ public class RdapEntitySearchAction extends RdapActionBase {
       // Get the contact matches and return the results, fetching an additional contact to detect
       // truncation. If we are including deleted entries, we must fetch more entries, in case some
       // get excluded due to permissioning.
+      int querySizeLimit = getStandardQuerySizeLimit();
       Query<ContactResource> query =
           queryItemsByKey(
               ContactResource.class,
               partialStringQuery,
               shouldIncludeDeleted(),
-              shouldIncludeDeleted()
-                  ? (RESULT_SET_SIZE_SCALING_FACTOR * (rdapResultSetMaxSize + 1))
-                  : (rdapResultSetMaxSize + 1));
+              querySizeLimit);
       return makeSearchResults(
-          getMatchingResources(query, shouldIncludeDeleted(), now), registrars, now);
+          getMatchingResources(query, shouldIncludeDeleted(), now, querySizeLimit),
+          registrars,
+          now);
     }
   }
 

java/google/registry/rdap/RdapMetrics.java

@@ -294,7 +294,11 @@ public class RdapMetrics {
     }
 
     static Builder builder() {
-      return new AutoValue_RdapMetrics_RdapMetricInformation.Builder();
+      return new AutoValue_RdapMetrics_RdapMetricInformation.Builder()
+          .setSearchType(SearchType.NONE)
+          .setWildcardType(WildcardType.INVALID)
+          .setPrefixLength(0)
+          .setIncompletenessWarningType(IncompletenessWarningType.COMPLETE);
     }
   }
 }

java/google/registry/rdap/RdapNameserverSearchAction.java

@@ -64,7 +64,6 @@ import org.joda.time.DateTime;
 public class RdapNameserverSearchAction extends RdapActionBase {
 
   public static final String PATH = "/rdap/nameservers";
-  private static final int RESULT_SET_SIZE_SCALING_FACTOR = 30;
 
   @Inject Clock clock;
   @Inject @Parameter("name") Optional<String> nameParam;
@@ -223,31 +222,31 @@ public class RdapNameserverSearchAction extends RdapActionBase {
   private RdapSearchResults searchByNameUsingPrefix(
       final RdapSearchPattern partialStringQuery, final DateTime now) {
     // Add 1 so we can detect truncation.
+    int querySizeLimit = getStandardQuerySizeLimit();
     Query<HostResource> query =
         queryItems(
             HostResource.class,
             "fullyQualifiedHostName",
             partialStringQuery,
             shouldIncludeDeleted(),
-            shouldIncludeDeleted()
-                ? (RESULT_SET_SIZE_SCALING_FACTOR * (rdapResultSetMaxSize + 1))
-                : (rdapResultSetMaxSize + 1));
-    return makeSearchResults(getMatchingResources(query, shouldIncludeDeleted(), now), now);
+            querySizeLimit);
+    return makeSearchResults(
+        getMatchingResources(query, shouldIncludeDeleted(), now, querySizeLimit), now);
   }
 
   /** Searches for nameservers by IP address, returning a JSON array of nameserver info maps. */
   private RdapSearchResults searchByIp(final InetAddress inetAddress, DateTime now) {
     // Add 1 so we can detect truncation.
+    int querySizeLimit = getStandardQuerySizeLimit();
     Query<HostResource> query =
         queryItems(
             HostResource.class,
             "inetAddresses",
             inetAddress.getHostAddress(),
             shouldIncludeDeleted(),
-            shouldIncludeDeleted()
-                ? (RESULT_SET_SIZE_SCALING_FACTOR * (rdapResultSetMaxSize + 1))
-                : (rdapResultSetMaxSize + 1));
-    return makeSearchResults(getMatchingResources(query, shouldIncludeDeleted(), now), now);
+            querySizeLimit);
+    return makeSearchResults(
+        getMatchingResources(query, shouldIncludeDeleted(), now, querySizeLimit), now);
   }
 
   /** Output JSON for a lists of hosts contained in an {@link RdapResultSet}. */