mirror of
https://github.com/google/nomulus.git
synced 2025-05-22 04:09:46 +02:00
Add limit to list_domains command
This allows list_domains to continue working for large TLDs. TESTED=Deploys to alpha and it works to list the most recently created domains even on a TLD with a huge number of domains on it (much more than .app has currently). ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=196717389
This commit is contained in:
mcilwain
jianglai
parent
e4f25c08e8
commit
9c0d3b6db3
11 changed files with 243 additions and 198 deletions
|
|
@ -1,99 +1,105 @@
|
|||
<datastore-indexes autoGenerate="false">
|
||||
<!-- For finding contact resources by registrar. -->
|
||||
<datastore-index kind="ContactResource" ancestor="false" source="manual">
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
<property name="searchName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding domain resources by registrar. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding domain resources by TLD. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="tld" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding domain resources by registrar. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding host resources by registrar. -->
|
||||
<datastore-index kind="HostResource" ancestor="false" source="manual">
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
<property name="fullyQualifiedHostName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding account balance of registrar and viewing billing history. -->
|
||||
<datastore-index kind="RegistrarBillingEntry" ancestor="true" source="manual">
|
||||
<property name="currency" direction="asc"/>
|
||||
<property name="created" direction="desc"/>
|
||||
</datastore-index>
|
||||
<!-- For determining the active domains linked to a given contact. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="allContacts.contact" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For determining the active domains linked to a given host. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="nsHosts" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For RDAP searches by linked nameserver. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="nsHosts" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For WHOIS IP address lookup -->
|
||||
<datastore-index kind="HostResource" ancestor="false" source="manual">
|
||||
<property name="inetAddresses" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For Poll -->
|
||||
<datastore-index kind="PollMessage" ancestor="false" source="manual">
|
||||
<property name="clientId" direction="asc"/>
|
||||
<property name="eventTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="PollMessage" ancestor="true" source="manual">
|
||||
<property name="clientId" direction="asc"/>
|
||||
<property name="eventTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For the history viewer. -->
|
||||
<datastore-index kind="HistoryEntry" ancestor="true" source="manual">
|
||||
<property name="modificationTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For RDAP. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="fullyQualifiedDomainName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="fullyQualifiedDomainName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="tld" direction="asc"/>
|
||||
<property name="fullyQualifiedDomainName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="tld" direction="asc"/>
|
||||
<property name="fullyQualifiedDomainName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="HostResource" ancestor="false" source="manual">
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
<property name="fullyQualifiedHostName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="ContactResource" ancestor="false" source="manual">
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
<property name="searchName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding contact resources by registrar. -->
|
||||
<datastore-index kind="ContactResource" ancestor="false" source="manual">
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
<property name="searchName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding domain resources by registrar. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding domain resources by TLD. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="tld" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding domain resources by registrar. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding the most recently created domain resources. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="tld" direction="asc"/>
|
||||
<property name="creationTime" direction="desc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding host resources by registrar. -->
|
||||
<datastore-index kind="HostResource" ancestor="false" source="manual">
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
<property name="fullyQualifiedHostName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For finding account balance of registrar and viewing billing history. -->
|
||||
<datastore-index kind="RegistrarBillingEntry" ancestor="true" source="manual">
|
||||
<property name="currency" direction="asc"/>
|
||||
<property name="created" direction="desc"/>
|
||||
</datastore-index>
|
||||
<!-- For determining the active domains linked to a given contact. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="allContacts.contact" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For determining the active domains linked to a given host. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="nsHosts" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For RDAP searches by linked nameserver. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="nsHosts" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For WHOIS IP address lookup -->
|
||||
<datastore-index kind="HostResource" ancestor="false" source="manual">
|
||||
<property name="inetAddresses" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For Poll -->
|
||||
<datastore-index kind="PollMessage" ancestor="false" source="manual">
|
||||
<property name="clientId" direction="asc"/>
|
||||
<property name="eventTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="PollMessage" ancestor="true" source="manual">
|
||||
<property name="clientId" direction="asc"/>
|
||||
<property name="eventTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For the history viewer. -->
|
||||
<datastore-index kind="HistoryEntry" ancestor="true" source="manual">
|
||||
<property name="modificationTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For RDAP. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="fullyQualifiedDomainName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="fullyQualifiedDomainName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="currentSponsorClientId" direction="asc"/>
|
||||
<property name="tld" direction="asc"/>
|
||||
<property name="fullyQualifiedDomainName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="^i" direction="asc"/>
|
||||
<property name="tld" direction="asc"/>
|
||||
<property name="fullyQualifiedDomainName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="HostResource" ancestor="false" source="manual">
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
<property name="fullyQualifiedHostName" direction="asc"/>
|
||||
</datastore-index>
|
||||
<datastore-index kind="ContactResource" ancestor="false" source="manual">
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
<property name="searchName" direction="asc"/>
|
||||
</datastore-index>
|
||||
</datastore-indexes>
|
||||
|
|
|
|||
|
|
@ -83,6 +83,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
|
|||
// Map the method to XML, not the field, because if we map the field (with an adaptor class) it
|
||||
// will never be omitted from the xml even if the timestamp inside creationTime is null and we
|
||||
// return null from the adaptor. (Instead it gets written as an empty tag.)
|
||||
@Index
|
||||
CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -33,17 +33,22 @@ final class ListDomainsCommand extends ListObjectsCommand {
|
|||
required = true)
|
||||
private List<String> tlds;
|
||||
|
||||
@Parameter(
|
||||
names = {"-n", "--limit"},
|
||||
description = "Max number of domains to list, most recent first; defaults to no limit."
|
||||
)
|
||||
private int maxDomains = Integer.MAX_VALUE;
|
||||
|
||||
@Override
|
||||
String getCommandPath() {
|
||||
return ListDomainsAction.PATH;
|
||||
}
|
||||
|
||||
/** Returns a map of parameters to be sent to the server
|
||||
* (in addition to the usual ones). */
|
||||
/** Returns a map of parameters to be sent to the server (in addition to the usual ones). */
|
||||
@Override
|
||||
ImmutableMap<String, Object> getParameterMap() {
|
||||
String tldsParam = Joiner.on(',').join(tlds);
|
||||
checkArgument(tldsParam.length() < 1024, "Total length of TLDs is too long for URL parameter");
|
||||
return ImmutableMap.of("tlds", tldsParam);
|
||||
return ImmutableMap.of("tlds", tldsParam, "limit", maxDomains);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,11 +65,9 @@ abstract class ListObjectsCommand implements RemoteApiCommand, ServerSideCommand
|
|||
/** Returns the path to the servlet task. */
|
||||
abstract String getCommandPath();
|
||||
|
||||
/** Returns a map of parameters to be sent to the server
|
||||
* (in addition to the usual ones). */
|
||||
@Nullable
|
||||
/** Returns a map of parameters to be sent to the server (in addition to the usual ones). */
|
||||
ImmutableMap<String, Object> getParameterMap() {
|
||||
return null;
|
||||
return ImmutableMap.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -84,10 +82,7 @@ abstract class ListObjectsCommand implements RemoteApiCommand, ServerSideCommand
|
|||
if (fullFieldNames) {
|
||||
params.put(FULL_FIELD_NAMES_PARAM, Boolean.TRUE);
|
||||
}
|
||||
ImmutableMap<String, Object> extraParams = getParameterMap();
|
||||
if (extraParams != null) {
|
||||
params.putAll(extraParams);
|
||||
}
|
||||
params.putAll(getParameterMap());
|
||||
// Call the server and get the response data.
|
||||
String response = connection.send(
|
||||
getCommandPath(),
|
||||
|
|
|
|||
|
|
@ -15,22 +15,23 @@
|
|||
package google.registry.tools.server;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static google.registry.model.EppResourceUtils.queryNotDeleted;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.registry.Registries.assertTldsExist;
|
||||
import static google.registry.request.Action.Method.GET;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static java.util.Comparator.comparing;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSortedSet;
|
||||
import com.google.common.collect.Lists;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.EppResourceUtils;
|
||||
import google.registry.model.domain.DomainResource;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.List;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** An action that lists domains, for use by the {@code nomulus list_domains} command. */
|
||||
@Action(
|
||||
|
|
@ -45,6 +46,7 @@ public final class ListDomainsAction extends ListObjectsAction<DomainResource> {
|
|||
public static final String PATH = "/_dr/admin/list/domains";
|
||||
|
||||
@Inject @Parameter("tlds") ImmutableSet<String> tlds;
|
||||
@Inject @Parameter("limit") int limit;
|
||||
@Inject Clock clock;
|
||||
@Inject ListDomainsAction() {}
|
||||
|
||||
|
|
@ -56,12 +58,27 @@ public final class ListDomainsAction extends ListObjectsAction<DomainResource> {
|
|||
@Override
|
||||
public ImmutableSet<DomainResource> loadObjects() {
|
||||
checkArgument(!tlds.isEmpty(), "Must specify TLDs to query");
|
||||
checkArgument(
|
||||
tlds.size() <= MAX_NUM_SUBQUERIES,
|
||||
"Cannot query more than %s TLDs simultaneously",
|
||||
MAX_NUM_SUBQUERIES);
|
||||
assertTldsExist(tlds);
|
||||
ImmutableSortedSet.Builder<DomainResource> builder =
|
||||
new ImmutableSortedSet.Builder<>(comparing(DomainResource::getFullyQualifiedDomainName));
|
||||
for (List<String> batch : Lists.partition(tlds.asList(), MAX_NUM_SUBQUERIES)) {
|
||||
builder.addAll(queryNotDeleted(DomainResource.class, clock.nowUtc(), "tld in", batch));
|
||||
}
|
||||
return builder.build();
|
||||
DateTime now = clock.nowUtc();
|
||||
return ofy()
|
||||
.load()
|
||||
.type(DomainResource.class)
|
||||
.filter("tld in", tlds)
|
||||
// Get the N most recently created domains (requires ordering in descending order).
|
||||
.order("-creationTime")
|
||||
.limit(limit)
|
||||
.list()
|
||||
.stream()
|
||||
.map(EppResourceUtils.transformAtTime(now))
|
||||
// Deleted entities must be filtered out post-query because queries don't allow ordering
|
||||
// with two filters.
|
||||
.filter(d -> d.getDeletionTime().isAfter(now))
|
||||
// Sort back to ascending order for nicer display.
|
||||
.sorted(comparing(EppResource::getCreationTime))
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ public abstract class ListObjectsAction<T extends ImmutableObject> implements Ru
|
|||
// Get the object data first, so we can figure out the list of all available fields using the
|
||||
// data if necessary.
|
||||
ImmutableSet<T> objects = loadObjects();
|
||||
logger.infofmt("Loaded %d objects.", objects.size());
|
||||
// Get the list of fields we should return.
|
||||
ImmutableSet<String> fieldsToUse = getFieldsToUse(objects);
|
||||
// Convert the data into a table.
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ package google.registry.tools.server;
|
|||
|
||||
import static com.google.common.base.Strings.emptyToNull;
|
||||
import static google.registry.request.RequestParameters.extractBooleanParameter;
|
||||
import static google.registry.request.RequestParameters.extractIntParameter;
|
||||
import static google.registry.request.RequestParameters.extractOptionalParameter;
|
||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||
|
||||
|
|
@ -90,6 +91,12 @@ public class ToolsServerModule {
|
|||
return ImmutableSet.copyOf(Splitter.on(',').split(tldsString));
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter("limit")
|
||||
static int provideLimit(HttpServletRequest req) {
|
||||
return extractIntParameter(req, "limit");
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter("rawKeys")
|
||||
static String provideRawKeys(HttpServletRequest req) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue