mirror of
https://github.com/google/nomulus.git
synced 2025-05-14 00:17:20 +02:00
Allow nomulus list_domains to query any number of TLDs
This limit did not exist prior to [] which added the ability to limit the size of the list. I didn't think that we needed to be able to query more than 30 TLDs at any one time so I got rid of batching, but it turns out we do need this ability for domain_watcher. So I'm re-adding batching, which is a little bit more complicated now that we're also limiting and sorting by creation time. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=199826414
This commit is contained in:
parent
06fc89c6c5
commit
ddf55005c3
2 changed files with 113 additions and 110 deletions
|
@ -15,7 +15,7 @@
|
||||||
package google.registry.tools.server;
|
package google.registry.tools.server;
|
||||||
|
|
||||||
import static com.google.common.base.Preconditions.checkArgument;
|
import static com.google.common.base.Preconditions.checkArgument;
|
||||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||||
import static google.registry.model.registry.Registries.assertTldsExist;
|
import static google.registry.model.registry.Registries.assertTldsExist;
|
||||||
import static google.registry.request.Action.Method.GET;
|
import static google.registry.request.Action.Method.GET;
|
||||||
|
@ -23,7 +23,10 @@ import static google.registry.request.Action.Method.POST;
|
||||||
import static google.registry.request.RequestParameters.PARAM_TLDS;
|
import static google.registry.request.RequestParameters.PARAM_TLDS;
|
||||||
import static java.util.Comparator.comparing;
|
import static java.util.Comparator.comparing;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.ImmutableSet;
|
import com.google.common.collect.ImmutableSet;
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
import google.registry.model.EppResource;
|
import google.registry.model.EppResource;
|
||||||
import google.registry.model.EppResourceUtils;
|
import google.registry.model.EppResourceUtils;
|
||||||
import google.registry.model.domain.DomainResource;
|
import google.registry.model.domain.DomainResource;
|
||||||
|
@ -31,25 +34,35 @@ import google.registry.request.Action;
|
||||||
import google.registry.request.Parameter;
|
import google.registry.request.Parameter;
|
||||||
import google.registry.request.auth.Auth;
|
import google.registry.request.auth.Auth;
|
||||||
import google.registry.util.Clock;
|
import google.registry.util.Clock;
|
||||||
|
import google.registry.util.NonFinalForTesting;
|
||||||
|
import java.util.List;
|
||||||
import javax.inject.Inject;
|
import javax.inject.Inject;
|
||||||
import org.joda.time.DateTime;
|
import org.joda.time.DateTime;
|
||||||
|
|
||||||
/** An action that lists domains, for use by the {@code nomulus list_domains} command. */
|
/** An action that lists domains, for use by the {@code nomulus list_domains} command. */
|
||||||
@Action(
|
@Action(
|
||||||
path = ListDomainsAction.PATH,
|
path = ListDomainsAction.PATH,
|
||||||
method = {GET, POST},
|
method = {GET, POST},
|
||||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN
|
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||||
)
|
|
||||||
public final class ListDomainsAction extends ListObjectsAction<DomainResource> {
|
public final class ListDomainsAction extends ListObjectsAction<DomainResource> {
|
||||||
|
|
||||||
/** An App Engine limitation on how many subqueries can be used in a single query. */
|
/** An App Engine limitation on how many subqueries can be used in a single query. */
|
||||||
private static final int MAX_NUM_SUBQUERIES = 30;
|
@VisibleForTesting @NonFinalForTesting static int maxNumSubqueries = 30;
|
||||||
|
|
||||||
public static final String PATH = "/_dr/admin/list/domains";
|
public static final String PATH = "/_dr/admin/list/domains";
|
||||||
|
|
||||||
@Inject @Parameter(PARAM_TLDS) ImmutableSet<String> tlds;
|
@Inject
|
||||||
@Inject @Parameter("limit") int limit;
|
@Parameter(PARAM_TLDS)
|
||||||
|
ImmutableSet<String> tlds;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
@Parameter("limit")
|
||||||
|
int limit;
|
||||||
|
|
||||||
@Inject Clock clock;
|
@Inject Clock clock;
|
||||||
@Inject ListDomainsAction() {}
|
|
||||||
|
@Inject
|
||||||
|
ListDomainsAction() {}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ImmutableSet<String> getPrimaryKeyFields() {
|
public ImmutableSet<String> getPrimaryKeyFields() {
|
||||||
|
@ -59,27 +72,35 @@ public final class ListDomainsAction extends ListObjectsAction<DomainResource> {
|
||||||
@Override
|
@Override
|
||||||
public ImmutableSet<DomainResource> loadObjects() {
|
public ImmutableSet<DomainResource> loadObjects() {
|
||||||
checkArgument(!tlds.isEmpty(), "Must specify TLDs to query");
|
checkArgument(!tlds.isEmpty(), "Must specify TLDs to query");
|
||||||
checkArgument(
|
|
||||||
tlds.size() <= MAX_NUM_SUBQUERIES,
|
|
||||||
"Cannot query more than %s TLDs simultaneously",
|
|
||||||
MAX_NUM_SUBQUERIES);
|
|
||||||
assertTldsExist(tlds);
|
assertTldsExist(tlds);
|
||||||
DateTime now = clock.nowUtc();
|
DateTime now = clock.nowUtc();
|
||||||
return ofy()
|
ImmutableList.Builder<DomainResource> domainsBuilder = new ImmutableList.Builder<>();
|
||||||
.load()
|
for (List<String> tldsBatch : Lists.partition(tlds.asList(), maxNumSubqueries)) {
|
||||||
.type(DomainResource.class)
|
domainsBuilder.addAll(
|
||||||
.filter("tld in", tlds)
|
ofy()
|
||||||
// Get the N most recently created domains (requires ordering in descending order).
|
.load()
|
||||||
.order("-creationTime")
|
.type(DomainResource.class)
|
||||||
.limit(limit)
|
.filter("tld in", tldsBatch)
|
||||||
.list()
|
// Get the N most recently created domains (requires ordering in descending order).
|
||||||
.stream()
|
.order("-creationTime")
|
||||||
.map(EppResourceUtils.transformAtTime(now))
|
.limit(limit)
|
||||||
// Deleted entities must be filtered out post-query because queries don't allow ordering
|
.list()
|
||||||
// with two filters.
|
.stream()
|
||||||
.filter(d -> d.getDeletionTime().isAfter(now))
|
.map(EppResourceUtils.transformAtTime(now))
|
||||||
// Sort back to ascending order for nicer display.
|
// Deleted entities must be filtered out post-query because queries don't allow
|
||||||
.sorted(comparing(EppResource::getCreationTime))
|
// ordering with two filters.
|
||||||
.collect(toImmutableSet());
|
.filter(d -> d.getDeletionTime().isAfter(now))
|
||||||
|
.collect(toImmutableList()));
|
||||||
|
}
|
||||||
|
// Combine the batches together by sorting all domains together with newest first, applying the
|
||||||
|
// limit, and then reversing for display order.
|
||||||
|
return ImmutableSet.copyOf(
|
||||||
|
domainsBuilder
|
||||||
|
.build()
|
||||||
|
.stream()
|
||||||
|
.sorted(comparing(EppResource::getCreationTime).reversed())
|
||||||
|
.limit(limit)
|
||||||
|
.collect(toImmutableList())
|
||||||
|
.reverse());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,16 +27,14 @@ import org.junit.Test;
|
||||||
import org.junit.runner.RunWith;
|
import org.junit.runner.RunWith;
|
||||||
import org.junit.runners.JUnit4;
|
import org.junit.runners.JUnit4;
|
||||||
|
|
||||||
/**
|
/** Unit tests for {@link ListDomainsAction}. */
|
||||||
* Unit tests for {@link ListDomainsAction}.
|
|
||||||
*/
|
|
||||||
@RunWith(JUnit4.class)
|
@RunWith(JUnit4.class)
|
||||||
public class ListDomainsActionTest extends ListActionTestCase {
|
public class ListDomainsActionTest extends ListActionTestCase {
|
||||||
|
|
||||||
ListDomainsAction action;
|
private ListDomainsAction action;
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void init() throws Exception {
|
public void init() {
|
||||||
createTld("foo");
|
createTld("foo");
|
||||||
action = new ListDomainsAction();
|
action = new ListDomainsAction();
|
||||||
action.clock = new FakeClock(DateTime.parse("2018-01-01TZ"));
|
action.clock = new FakeClock(DateTime.parse("2018-01-01TZ"));
|
||||||
|
@ -44,92 +42,82 @@ public class ListDomainsActionTest extends ListActionTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_invalidRequest_missingTlds() throws Exception {
|
public void testRun_invalidRequest_missingTlds() {
|
||||||
action.tlds = ImmutableSet.of();
|
action.tlds = ImmutableSet.of();
|
||||||
testRunError(
|
testRunError(action, null, null, null, "^Must specify TLDs to query$");
|
||||||
action,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
"^Must specify TLDs to query$");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_invalidRequest_invalidTld() throws Exception {
|
public void testRun_invalidRequest_invalidTld() {
|
||||||
action.tlds = ImmutableSet.of("%%%badtld%%%");
|
action.tlds = ImmutableSet.of("%%%badtld%%%");
|
||||||
testRunError(
|
testRunError(action, null, null, null, "^TLDs do not exist: %%%badtld%%%$");
|
||||||
action,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
"^TLDs do not exist: %%%badtld%%%$");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_noParameters() throws Exception {
|
public void testRun_noParameters() {
|
||||||
action.tlds = ImmutableSet.of("foo");
|
action.tlds = ImmutableSet.of("foo");
|
||||||
testRunSuccess(
|
testRunSuccess(action, null, null, null);
|
||||||
action,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
null);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_twoLinesWithIdOnly() throws Exception {
|
public void testRun_twoLinesWithIdOnly() {
|
||||||
action.tlds = ImmutableSet.of("foo");
|
action.tlds = ImmutableSet.of("foo");
|
||||||
createTlds("bar", "sim");
|
createTlds("bar", "sim");
|
||||||
persistActiveDomain("dontlist.bar");
|
persistActiveDomain("dontlist.bar", DateTime.parse("2015-02-14T15:15:15Z"));
|
||||||
persistActiveDomain("example1.foo");
|
persistActiveDomain("example1.foo", DateTime.parse("2015-02-15T15:15:15Z"));
|
||||||
persistActiveDomain("example2.foo");
|
persistActiveDomain("example2.foo", DateTime.parse("2015-02-16T15:15:15Z"));
|
||||||
persistActiveDomain("notlistedaswell.sim");
|
persistActiveDomain("notlistedaswell.sim", DateTime.parse("2015-02-17T15:15:15Z"));
|
||||||
// Only list the two domains in .foo, not the .bar or .sim ones.
|
// Only list the two domains in .foo, not the .bar or .sim ones.
|
||||||
testRunSuccess(
|
testRunSuccess(action, null, null, null, "^example1.foo$", "^example2.foo$");
|
||||||
action,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
"^example1.foo$",
|
|
||||||
"^example2.foo$");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_multipleTlds() throws Exception {
|
public void testRun_multipleTlds() {
|
||||||
action.tlds = ImmutableSet.of("bar", "foo");
|
action.tlds = ImmutableSet.of("bar", "foo");
|
||||||
createTlds("bar", "sim");
|
createTlds("bar", "sim");
|
||||||
persistActiveDomain("dolist.bar");
|
persistActiveDomain("dolist.bar", DateTime.parse("2015-01-15T15:15:15Z"));
|
||||||
persistActiveDomain("example1.foo");
|
persistActiveDomain("example1.foo", DateTime.parse("2015-02-15T15:15:15Z"));
|
||||||
persistActiveDomain("example2.foo");
|
persistActiveDomain("example2.foo", DateTime.parse("2015-03-15T15:15:15Z"));
|
||||||
persistActiveDomain("notlistedaswell.sim");
|
persistActiveDomain("notlistedaswell.sim", DateTime.parse("2015-04-15T15:15:15Z"));
|
||||||
|
testRunSuccess(action, null, null, null, "^dolist.bar", "^example1.foo$", "^example2.foo$");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRun_moreTldsThanMaxNumSubqueries() {
|
||||||
|
ListDomainsAction.maxNumSubqueries = 2;
|
||||||
|
createTlds("baa", "bab", "bac", "bad");
|
||||||
|
action.tlds = ImmutableSet.of("baa", "bab", "bac", "bad");
|
||||||
|
action.limit = 4;
|
||||||
|
persistActiveDomain("domain1.baa", DateTime.parse("2010-03-04T16:00:00Z"));
|
||||||
|
persistActiveDomain("domain2.bab", DateTime.parse("2009-03-04T16:00:00Z"));
|
||||||
|
persistActiveDomain("domain3.bac", DateTime.parse("2011-03-04T16:00:00Z"));
|
||||||
|
persistActiveDomain("domain4.bad", DateTime.parse("2010-06-04T16:00:00Z"));
|
||||||
|
persistActiveDomain("domain5.baa", DateTime.parse("2008-01-04T16:00:00Z"));
|
||||||
|
// Since the limit is 4, expect all but domain5.baa (the oldest), sorted by creationTime asc.
|
||||||
testRunSuccess(
|
testRunSuccess(
|
||||||
action,
|
action,
|
||||||
null,
|
null,
|
||||||
null,
|
null,
|
||||||
null,
|
null,
|
||||||
"^dolist.bar",
|
"^domain2.bab$",
|
||||||
"^example1.foo$",
|
"^domain1.baa$",
|
||||||
"^example2.foo$");
|
"^domain4.bad$",
|
||||||
|
"^domain3.bac$");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_twoLinesWithIdOnlyNoHeader() throws Exception {
|
public void testRun_twoLinesWithIdOnlyNoHeader() {
|
||||||
action.tlds = ImmutableSet.of("foo");
|
action.tlds = ImmutableSet.of("foo");
|
||||||
persistActiveDomain("example1.foo");
|
persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
|
||||||
persistActiveDomain("example2.foo");
|
persistActiveDomain("example2.foo", DateTime.parse("2011-03-04T16:00:00Z"));
|
||||||
testRunSuccess(
|
testRunSuccess(action, null, Optional.of(false), null, "^example1.foo$", "^example2.foo$");
|
||||||
action,
|
|
||||||
null,
|
|
||||||
Optional.of(false),
|
|
||||||
null,
|
|
||||||
"^example1.foo$",
|
|
||||||
"^example2.foo$");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_twoLinesWithIdOnlyExplicitHeader() throws Exception {
|
public void testRun_twoLinesWithIdOnlyExplicitHeader() {
|
||||||
action.tlds = ImmutableSet.of("foo");
|
action.tlds = ImmutableSet.of("foo");
|
||||||
persistActiveDomain("example1.foo");
|
persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
|
||||||
persistActiveDomain("example2.foo");
|
persistActiveDomain("example2.foo", DateTime.parse("2011-03-04T16:00:00Z"));
|
||||||
testRunSuccess(
|
testRunSuccess(
|
||||||
action,
|
action,
|
||||||
null,
|
null,
|
||||||
|
@ -142,10 +130,10 @@ public class ListDomainsActionTest extends ListActionTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_twoLinesWithRepoId() throws Exception {
|
public void testRun_twoLinesWithRepoId() {
|
||||||
action.tlds = ImmutableSet.of("foo");
|
action.tlds = ImmutableSet.of("foo");
|
||||||
persistActiveDomain("example1.foo");
|
persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
|
||||||
persistActiveDomain("example3.foo");
|
persistActiveDomain("example3.foo", DateTime.parse("2011-03-04T16:00:00Z"));
|
||||||
testRunSuccess(
|
testRunSuccess(
|
||||||
action,
|
action,
|
||||||
Optional.of("repoId"),
|
Optional.of("repoId"),
|
||||||
|
@ -158,10 +146,10 @@ public class ListDomainsActionTest extends ListActionTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_twoLinesWithRepoIdNoHeader() throws Exception {
|
public void testRun_twoLinesWithRepoIdNoHeader() {
|
||||||
action.tlds = ImmutableSet.of("foo");
|
action.tlds = ImmutableSet.of("foo");
|
||||||
persistActiveDomain("example1.foo");
|
persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
|
||||||
persistActiveDomain("example3.foo");
|
persistActiveDomain("example3.foo", DateTime.parse("2011-03-04T16:00:00Z"));
|
||||||
testRunSuccess(
|
testRunSuccess(
|
||||||
action,
|
action,
|
||||||
Optional.of("repoId"),
|
Optional.of("repoId"),
|
||||||
|
@ -172,10 +160,10 @@ public class ListDomainsActionTest extends ListActionTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_twoLinesWithRepoIdExplicitHeader() throws Exception {
|
public void testRun_twoLinesWithRepoIdExplicitHeader() {
|
||||||
action.tlds = ImmutableSet.of("foo");
|
action.tlds = ImmutableSet.of("foo");
|
||||||
persistActiveDomain("example1.foo");
|
persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
|
||||||
persistActiveDomain("example3.foo");
|
persistActiveDomain("example3.foo", DateTime.parse("2011-03-04T16:00:00Z"));
|
||||||
testRunSuccess(
|
testRunSuccess(
|
||||||
action,
|
action,
|
||||||
Optional.of("repoId"),
|
Optional.of("repoId"),
|
||||||
|
@ -188,10 +176,10 @@ public class ListDomainsActionTest extends ListActionTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_twoLinesWithWildcard() throws Exception {
|
public void testRun_twoLinesWithWildcard() {
|
||||||
action.tlds = ImmutableSet.of("foo");
|
action.tlds = ImmutableSet.of("foo");
|
||||||
persistActiveDomain("example1.foo");
|
persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
|
||||||
persistActiveDomain("example3.foo");
|
persistActiveDomain("example3.foo", DateTime.parse("2010-03-05T16:00:00Z"));
|
||||||
testRunSuccess(
|
testRunSuccess(
|
||||||
action,
|
action,
|
||||||
Optional.of("*"),
|
Optional.of("*"),
|
||||||
|
@ -204,10 +192,10 @@ public class ListDomainsActionTest extends ListActionTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_twoLinesWithWildcardAndAnotherField() throws Exception {
|
public void testRun_twoLinesWithWildcardAndAnotherField() {
|
||||||
action.tlds = ImmutableSet.of("foo");
|
action.tlds = ImmutableSet.of("foo");
|
||||||
persistActiveDomain("example1.foo");
|
persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
|
||||||
persistActiveDomain("example3.foo");
|
persistActiveDomain("example3.foo", DateTime.parse("2010-03-04T17:00:00Z"));
|
||||||
testRunSuccess(
|
testRunSuccess(
|
||||||
action,
|
action,
|
||||||
Optional.of("*,repoId"),
|
Optional.of("*,repoId"),
|
||||||
|
@ -220,7 +208,7 @@ public class ListDomainsActionTest extends ListActionTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testRun_withBadField_returnsError() throws Exception {
|
public void testRun_withBadField_returnsError() {
|
||||||
action.tlds = ImmutableSet.of("foo");
|
action.tlds = ImmutableSet.of("foo");
|
||||||
persistActiveDomain("example2.foo");
|
persistActiveDomain("example2.foo");
|
||||||
persistActiveDomain("example1.foo");
|
persistActiveDomain("example1.foo");
|
||||||
|
@ -242,12 +230,6 @@ public class ListDomainsActionTest extends ListActionTestCase {
|
||||||
persistActiveDomain("example2.bar", DateTime.parse("2017-02-01TZ"));
|
persistActiveDomain("example2.bar", DateTime.parse("2017-02-01TZ"));
|
||||||
persistActiveDomain("example3.bar", DateTime.parse("2017-03-01TZ"));
|
persistActiveDomain("example3.bar", DateTime.parse("2017-03-01TZ"));
|
||||||
persistActiveDomain("example5.baz", DateTime.parse("2018-01-01TZ"));
|
persistActiveDomain("example5.baz", DateTime.parse("2018-01-01TZ"));
|
||||||
testRunSuccess(
|
testRunSuccess(action, null, null, null, "^example3.bar$", "^example4.foo$");
|
||||||
action,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
"^example3.bar$",
|
|
||||||
"^example4.foo$");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue