zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-13 16:07:15 +02:00
Code Issues Projects Releases Packages Wiki Activity

Allow nomulus list_domains to query any number of TLDs

Browse source 
This limit did not exist prior to [] which added the ability to limit
the size of the list. I didn't think that we needed to be able to query more
than 30 TLDs at any one time so I got rid of batching, but it turns out we do
need this ability for domain_watcher. So I'm re-adding batching, which is a
little bit more complicated now that we're also limiting and sorting by creation
time.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=199826414
This commit is contained in:
mcilwain 2018-06-08 11:41:54 -07:00 • committed by Ben McIlwain
parent 06fc89c6c5
commit ddf55005c3
2 changed files with 113 additions and 110 deletions
Download patch file Download diff file Expand all files Collapse all files

57
java/google/registry/tools/server/ListDomainsAction.java
View file

@ -15,7 +15,7 @@
package google.registry.tools.server; package google.registry.tools.server;
import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.ImmutableList.toImmutableList;
import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.registry.Registries.assertTldsExist; import static google.registry.model.registry.Registries.assertTldsExist;
import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.GET;
@ -23,7 +23,10 @@ import static google.registry.request.Action.Method.POST;
import static google.registry.request.RequestParameters.PARAM_TLDS; import static google.registry.request.RequestParameters.PARAM_TLDS;
import static java.util.Comparator.comparing; import static java.util.Comparator.comparing;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import google.registry.model.EppResource; import google.registry.model.EppResource;
import google.registry.model.EppResourceUtils; import google.registry.model.EppResourceUtils;
import google.registry.model.domain.DomainResource; import google.registry.model.domain.DomainResource;
@ -31,6 +34,8 @@ import google.registry.request.Action;
import google.registry.request.Parameter; import google.registry.request.Parameter;
import google.registry.request.auth.Auth; import google.registry.request.auth.Auth;
import google.registry.util.Clock; import google.registry.util.Clock;
import google.registry.util.NonFinalForTesting;
import java.util.List;
import javax.inject.Inject; import javax.inject.Inject;
import org.joda.time.DateTime; import org.joda.time.DateTime;
@ -38,18 +43,26 @@ import org.joda.time.DateTime;
@Action( @Action(
path = ListDomainsAction.PATH, path = ListDomainsAction.PATH,
method = {GET, POST}, method = {GET, POST},
auth = Auth.AUTH_INTERNAL_OR_ADMIN auth = Auth.AUTH_INTERNAL_OR_ADMIN)
)
public final class ListDomainsAction extends ListObjectsAction<DomainResource> { public final class ListDomainsAction extends ListObjectsAction<DomainResource> {
/** An App Engine limitation on how many subqueries can be used in a single query. */ /** An App Engine limitation on how many subqueries can be used in a single query. */
private static final int MAX_NUM_SUBQUERIES = 30; @VisibleForTesting @NonFinalForTesting static int maxNumSubqueries = 30;
public static final String PATH = "/_dr/admin/list/domains"; public static final String PATH = "/_dr/admin/list/domains";
@Inject @Parameter(PARAM_TLDS) ImmutableSet<String> tlds; @Inject
@Inject @Parameter("limit") int limit; @Parameter(PARAM_TLDS)
ImmutableSet<String> tlds;
@Inject
@Parameter("limit")
int limit;
@Inject Clock clock; @Inject Clock clock;
@Inject ListDomainsAction() {}
@Inject
ListDomainsAction() {}
@Override @Override
public ImmutableSet<String> getPrimaryKeyFields() { public ImmutableSet<String> getPrimaryKeyFields() {
@ -59,27 +72,35 @@ public final class ListDomainsAction extends ListObjectsAction<DomainResource> {
@Override @Override
public ImmutableSet<DomainResource> loadObjects() { public ImmutableSet<DomainResource> loadObjects() {
checkArgument(!tlds.isEmpty(), "Must specify TLDs to query"); checkArgument(!tlds.isEmpty(), "Must specify TLDs to query");
checkArgument(
tlds.size() <= MAX_NUM_SUBQUERIES,
"Cannot query more than %s TLDs simultaneously",
MAX_NUM_SUBQUERIES);
assertTldsExist(tlds); assertTldsExist(tlds);
DateTime now = clock.nowUtc(); DateTime now = clock.nowUtc();
return ofy() ImmutableList.Builder<DomainResource> domainsBuilder = new ImmutableList.Builder<>();
for (List<String> tldsBatch : Lists.partition(tlds.asList(), maxNumSubqueries)) {
domainsBuilder.addAll(
ofy()
.load() .load()
.type(DomainResource.class) .type(DomainResource.class)
.filter("tld in", tlds) .filter("tld in", tldsBatch)
// Get the N most recently created domains (requires ordering in descending order). // Get the N most recently created domains (requires ordering in descending order).
.order("-creationTime") .order("-creationTime")
.limit(limit) .limit(limit)
.list() .list()
.stream() .stream()
.map(EppResourceUtils.transformAtTime(now)) .map(EppResourceUtils.transformAtTime(now))
// Deleted entities must be filtered out post-query because queries don't allow ordering // Deleted entities must be filtered out post-query because queries don't allow
// with two filters. // ordering with two filters.
.filter(d -> d.getDeletionTime().isAfter(now)) .filter(d -> d.getDeletionTime().isAfter(now))
// Sort back to ascending order for nicer display. .collect(toImmutableList()));
.sorted(comparing(EppResource::getCreationTime)) }
.collect(toImmutableSet()); // Combine the batches together by sorting all domains together with newest first, applying the
// limit, and then reversing for display order.
return ImmutableSet.copyOf(
domainsBuilder
.build()
.stream()
.sorted(comparing(EppResource::getCreationTime).reversed())
.limit(limit)
.collect(toImmutableList())
.reverse());
} }
} }

144
javatests/google/registry/tools/server/ListDomainsActionTest.java
View file

@ -27,16 +27,14 @@ import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.junit.runners.JUnit4; import org.junit.runners.JUnit4;
/** /** Unit tests for {@link ListDomainsAction}. */
* Unit tests for {@link ListDomainsAction}.
*/
@RunWith(JUnit4.class) @RunWith(JUnit4.class)
public class ListDomainsActionTest extends ListActionTestCase { public class ListDomainsActionTest extends ListActionTestCase {
ListDomainsAction action; private ListDomainsAction action;
@Before @Before
public void init() throws Exception { public void init() {
createTld("foo"); createTld("foo");
action = new ListDomainsAction(); action = new ListDomainsAction();
action.clock = new FakeClock(DateTime.parse("2018-01-01TZ")); action.clock = new FakeClock(DateTime.parse("2018-01-01TZ"));
@ -44,92 +42,82 @@ public class ListDomainsActionTest extends ListActionTestCase {
} }
@Test @Test
public void testRun_invalidRequest_missingTlds() throws Exception { public void testRun_invalidRequest_missingTlds() {
action.tlds = ImmutableSet.of(); action.tlds = ImmutableSet.of();
testRunError( testRunError(action, null, null, null, "^Must specify TLDs to query$");
action,
null,
null,
null,
"^Must specify TLDs to query$");
} }
@Test @Test
public void testRun_invalidRequest_invalidTld() throws Exception { public void testRun_invalidRequest_invalidTld() {
action.tlds = ImmutableSet.of("%%%badtld%%%"); action.tlds = ImmutableSet.of("%%%badtld%%%");
testRunError( testRunError(action, null, null, null, "^TLDs do not exist: %%%badtld%%%$");
action,
null,
null,
null,
"^TLDs do not exist: %%%badtld%%%$");
} }
@Test @Test
public void testRun_noParameters() throws Exception { public void testRun_noParameters() {
action.tlds = ImmutableSet.of("foo"); action.tlds = ImmutableSet.of("foo");
testRunSuccess( testRunSuccess(action, null, null, null);
action,
null,
null,
null);
} }
@Test @Test
public void testRun_twoLinesWithIdOnly() throws Exception { public void testRun_twoLinesWithIdOnly() {
action.tlds = ImmutableSet.of("foo"); action.tlds = ImmutableSet.of("foo");
createTlds("bar", "sim"); createTlds("bar", "sim");
persistActiveDomain("dontlist.bar"); persistActiveDomain("dontlist.bar", DateTime.parse("2015-02-14T15:15:15Z"));
persistActiveDomain("example1.foo"); persistActiveDomain("example1.foo", DateTime.parse("2015-02-15T15:15:15Z"));
persistActiveDomain("example2.foo"); persistActiveDomain("example2.foo", DateTime.parse("2015-02-16T15:15:15Z"));
persistActiveDomain("notlistedaswell.sim"); persistActiveDomain("notlistedaswell.sim", DateTime.parse("2015-02-17T15:15:15Z"));
// Only list the two domains in .foo, not the .bar or .sim ones. // Only list the two domains in .foo, not the .bar or .sim ones.
testRunSuccess( testRunSuccess(action, null, null, null, "^example1.foo$", "^example2.foo$");
action,
null,
null,
null,
"^example1.foo$",
"^example2.foo$");
} }
@Test @Test
public void testRun_multipleTlds() throws Exception { public void testRun_multipleTlds() {
action.tlds = ImmutableSet.of("bar", "foo"); action.tlds = ImmutableSet.of("bar", "foo");
createTlds("bar", "sim"); createTlds("bar", "sim");
persistActiveDomain("dolist.bar"); persistActiveDomain("dolist.bar", DateTime.parse("2015-01-15T15:15:15Z"));
persistActiveDomain("example1.foo"); persistActiveDomain("example1.foo", DateTime.parse("2015-02-15T15:15:15Z"));
persistActiveDomain("example2.foo"); persistActiveDomain("example2.foo", DateTime.parse("2015-03-15T15:15:15Z"));
persistActiveDomain("notlistedaswell.sim"); persistActiveDomain("notlistedaswell.sim", DateTime.parse("2015-04-15T15:15:15Z"));
testRunSuccess(action, null, null, null, "^dolist.bar", "^example1.foo$", "^example2.foo$");
}
@Test
public void testRun_moreTldsThanMaxNumSubqueries() {
ListDomainsAction.maxNumSubqueries = 2;
createTlds("baa", "bab", "bac", "bad");
action.tlds = ImmutableSet.of("baa", "bab", "bac", "bad");
action.limit = 4;
persistActiveDomain("domain1.baa", DateTime.parse("2010-03-04T16:00:00Z"));
persistActiveDomain("domain2.bab", DateTime.parse("2009-03-04T16:00:00Z"));
persistActiveDomain("domain3.bac", DateTime.parse("2011-03-04T16:00:00Z"));
persistActiveDomain("domain4.bad", DateTime.parse("2010-06-04T16:00:00Z"));
persistActiveDomain("domain5.baa", DateTime.parse("2008-01-04T16:00:00Z"));
// Since the limit is 4, expect all but domain5.baa (the oldest), sorted by creationTime asc.
testRunSuccess( testRunSuccess(
action, action,
null, null,
null, null,
null, null,
"^dolist.bar", "^domain2.bab$",
"^example1.foo$", "^domain1.baa$",
"^example2.foo$"); "^domain4.bad$",
"^domain3.bac$");
} }
@Test @Test
public void testRun_twoLinesWithIdOnlyNoHeader() throws Exception { public void testRun_twoLinesWithIdOnlyNoHeader() {
action.tlds = ImmutableSet.of("foo"); action.tlds = ImmutableSet.of("foo");
persistActiveDomain("example1.foo"); persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
persistActiveDomain("example2.foo"); persistActiveDomain("example2.foo", DateTime.parse("2011-03-04T16:00:00Z"));
testRunSuccess( testRunSuccess(action, null, Optional.of(false), null, "^example1.foo$", "^example2.foo$");
action,
null,
Optional.of(false),
null,
"^example1.foo$",
"^example2.foo$");
} }
@Test @Test
public void testRun_twoLinesWithIdOnlyExplicitHeader() throws Exception { public void testRun_twoLinesWithIdOnlyExplicitHeader() {
action.tlds = ImmutableSet.of("foo"); action.tlds = ImmutableSet.of("foo");
persistActiveDomain("example1.foo"); persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
persistActiveDomain("example2.foo"); persistActiveDomain("example2.foo", DateTime.parse("2011-03-04T16:00:00Z"));
testRunSuccess( testRunSuccess(
action, action,
null, null,
@ -142,10 +130,10 @@ public class ListDomainsActionTest extends ListActionTestCase {
} }
@Test @Test
public void testRun_twoLinesWithRepoId() throws Exception { public void testRun_twoLinesWithRepoId() {
action.tlds = ImmutableSet.of("foo"); action.tlds = ImmutableSet.of("foo");
persistActiveDomain("example1.foo"); persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
persistActiveDomain("example3.foo"); persistActiveDomain("example3.foo", DateTime.parse("2011-03-04T16:00:00Z"));
testRunSuccess( testRunSuccess(
action, action,
Optional.of("repoId"), Optional.of("repoId"),
@ -158,10 +146,10 @@ public class ListDomainsActionTest extends ListActionTestCase {
} }
@Test @Test
public void testRun_twoLinesWithRepoIdNoHeader() throws Exception { public void testRun_twoLinesWithRepoIdNoHeader() {
action.tlds = ImmutableSet.of("foo"); action.tlds = ImmutableSet.of("foo");
persistActiveDomain("example1.foo"); persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
persistActiveDomain("example3.foo"); persistActiveDomain("example3.foo", DateTime.parse("2011-03-04T16:00:00Z"));
testRunSuccess( testRunSuccess(
action, action,
Optional.of("repoId"), Optional.of("repoId"),
@ -172,10 +160,10 @@ public class ListDomainsActionTest extends ListActionTestCase {
} }
@Test @Test
public void testRun_twoLinesWithRepoIdExplicitHeader() throws Exception { public void testRun_twoLinesWithRepoIdExplicitHeader() {
action.tlds = ImmutableSet.of("foo"); action.tlds = ImmutableSet.of("foo");
persistActiveDomain("example1.foo"); persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
persistActiveDomain("example3.foo"); persistActiveDomain("example3.foo", DateTime.parse("2011-03-04T16:00:00Z"));
testRunSuccess( testRunSuccess(
action, action,
Optional.of("repoId"), Optional.of("repoId"),
@ -188,10 +176,10 @@ public class ListDomainsActionTest extends ListActionTestCase {
} }
@Test @Test
public void testRun_twoLinesWithWildcard() throws Exception { public void testRun_twoLinesWithWildcard() {
action.tlds = ImmutableSet.of("foo"); action.tlds = ImmutableSet.of("foo");
persistActiveDomain("example1.foo"); persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
persistActiveDomain("example3.foo"); persistActiveDomain("example3.foo", DateTime.parse("2010-03-05T16:00:00Z"));
testRunSuccess( testRunSuccess(
action, action,
Optional.of("*"), Optional.of("*"),
@ -204,10 +192,10 @@ public class ListDomainsActionTest extends ListActionTestCase {
} }
@Test @Test
public void testRun_twoLinesWithWildcardAndAnotherField() throws Exception { public void testRun_twoLinesWithWildcardAndAnotherField() {
action.tlds = ImmutableSet.of("foo"); action.tlds = ImmutableSet.of("foo");
persistActiveDomain("example1.foo"); persistActiveDomain("example1.foo", DateTime.parse("2010-03-04T16:00:00Z"));
persistActiveDomain("example3.foo"); persistActiveDomain("example3.foo", DateTime.parse("2010-03-04T17:00:00Z"));
testRunSuccess( testRunSuccess(
action, action,
Optional.of("*,repoId"), Optional.of("*,repoId"),
@ -220,7 +208,7 @@ public class ListDomainsActionTest extends ListActionTestCase {
} }
@Test @Test
public void testRun_withBadField_returnsError() throws Exception { public void testRun_withBadField_returnsError() {
action.tlds = ImmutableSet.of("foo"); action.tlds = ImmutableSet.of("foo");
persistActiveDomain("example2.foo"); persistActiveDomain("example2.foo");
persistActiveDomain("example1.foo"); persistActiveDomain("example1.foo");
@ -242,12 +230,6 @@ public class ListDomainsActionTest extends ListActionTestCase {
persistActiveDomain("example2.bar", DateTime.parse("2017-02-01TZ")); persistActiveDomain("example2.bar", DateTime.parse("2017-02-01TZ"));
persistActiveDomain("example3.bar", DateTime.parse("2017-03-01TZ")); persistActiveDomain("example3.bar", DateTime.parse("2017-03-01TZ"));
persistActiveDomain("example5.baz", DateTime.parse("2018-01-01TZ")); persistActiveDomain("example5.baz", DateTime.parse("2018-01-01TZ"));
testRunSuccess( testRunSuccess(action, null, null, null, "^example3.bar$", "^example4.foo$");
action,
null,
null,
null,
"^example3.bar$",
"^example4.foo$");
} }
} }