mirror of
https://github.com/google/nomulus.git
synced 2025-05-12 22:38:16 +02:00
Remove empty TLD parameter when fanning out without TLDs
TldFanoutAction fans out a given endpoint to all TLDs (either TEST, REAL, or both). However, it is also used to delegate a single endpoint request that we want set in a specific queue (so we can control retries). We do that by setting the TLD list to "runInEmpty" rather than "forEachRealTld" or "forEachTestTld". Currently, using "runInEmpty" would still specify a TLD - but that TLD would be the empty string. This is a bug: it sets the TLD parameter to a bad value. It worked only because none of the endpoints called with "runInEmpty" were using the TLD parameter. However, this will (and does) break if either (a) the endpoint accepts an optional TLD parameter (like deleteProberData does), or (b) the given endpoint already has a TLD parameter in it (we want to run the endpoint with a single TLD, but still use the "fanout" to set the right queue). This CL fixes several things: - if runInEmpty is given, no TLD parameter is added - 'runInEmpty' is now mutually exclusive with 'forEach*Tld' and 'excludes' - we do some sanity checks and added logging - removed the buggy and unused "':tld' in path is replaced by TLD" - in the cron.xml, removed documentation for :tld and the broken :registrar Note that none of the endpoints that were used with runInEmpty fanout had the TLD parameter prior to deleteProberData ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=189954585
This commit is contained in:
guyben
jianglai
parent
edcb725a18
commit
63785e5149
5 changed files with 111 additions and 69 deletions
|
|
@ -16,13 +16,13 @@ package google.registry.cron;
|
|||
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Predicates.in;
|
||||
import static com.google.common.base.Predicates.not;
|
||||
import static com.google.common.base.Strings.nullToEmpty;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static com.google.common.collect.Iterables.getFirst;
|
||||
import static com.google.common.collect.Multimaps.filterKeys;
|
||||
import static com.google.common.collect.Sets.difference;
|
||||
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
|
||||
import static google.registry.model.registry.Registries.getTldsOfType;
|
||||
import static google.registry.model.registry.Registry.TldType.REAL;
|
||||
|
|
@ -42,10 +42,11 @@ import google.registry.request.ParameterMap;
|
|||
import google.registry.request.RequestParameters;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import google.registry.util.TaskEnqueuer;
|
||||
import java.util.Optional;
|
||||
import java.util.Random;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
|
|
@ -58,10 +59,10 @@ import javax.inject.Inject;
|
|||
* <li>{@code queue} (Required) Name of the App Engine push queue to which this task should be sent.
|
||||
* <li>{@code forEachRealTld} Launch the task in each real TLD namespace.
|
||||
* <li>{@code forEachTestTld} Launch the task in each test TLD namespace.
|
||||
* <li>{@code runInEmpty} Launch the task in the empty namespace.
|
||||
* <li>{@code runInEmpty} Launch the task once, without the TLD argument.
|
||||
* <li>{@code exclude} TLDs to exclude.
|
||||
* <li>{@code jitterSeconds} Randomly delay each task by up to this many seconds.
|
||||
* <li>Any other parameters specified will be passed through as POST parameters to the called task.
|
||||
* <li>Any other parameters specified will be passed through as POST parameters to the called task.
|
||||
* </ul>
|
||||
*
|
||||
* <h3>Patharg Reference</h3>
|
||||
|
|
@ -98,9 +99,10 @@ public final class TldFanoutAction implements Runnable {
|
|||
EXCLUDE_PARAM,
|
||||
JITTER_SECONDS_PARAM);
|
||||
|
||||
private static final String TLD_PATHARG = ":tld";
|
||||
private static final Random random = new Random();
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@Inject TaskEnqueuer taskEnqueuer;
|
||||
@Inject Response response;
|
||||
@Inject @Parameter(ENDPOINT_PARAM) String endpoint;
|
||||
|
|
@ -115,28 +117,40 @@ public final class TldFanoutAction implements Runnable {
|
|||
|
||||
@Override
|
||||
public void run() {
|
||||
Set<String> tlds =
|
||||
difference(
|
||||
Streams.concat(
|
||||
Streams.stream(runInEmpty ? ImmutableSet.of("") : ImmutableSet.of()),
|
||||
Streams.stream(
|
||||
forEachRealTld ? getTldsOfType(REAL) : ImmutableSet.of()),
|
||||
Streams.stream(
|
||||
forEachTestTld ? getTldsOfType(TEST) : ImmutableSet.of()))
|
||||
.collect(toImmutableSet()),
|
||||
excludes);
|
||||
checkArgument(
|
||||
!(runInEmpty && (forEachTestTld || forEachRealTld)),
|
||||
"runInEmpty and forEach*Tld are mutually exclusive");
|
||||
checkArgument(
|
||||
runInEmpty || forEachTestTld || forEachRealTld,
|
||||
"At least one of runInEmpty, forEachTestTld, forEachRealTld must be given");
|
||||
checkArgument(
|
||||
!(runInEmpty && !excludes.isEmpty()),
|
||||
"Can't specify 'exclude' with 'runInEmpty'");
|
||||
ImmutableSet<String> tlds =
|
||||
Streams.concat(
|
||||
runInEmpty ? Stream.of("") : Stream.of(),
|
||||
forEachRealTld ? getTldsOfType(REAL).stream() : Stream.of(),
|
||||
forEachTestTld ? getTldsOfType(TEST).stream() : Stream.of())
|
||||
.filter(tld -> !excludes.contains(tld))
|
||||
.collect(toImmutableSet());
|
||||
Multimap<String, String> flowThruParams = filterKeys(params, not(in(CONTROL_PARAMS)));
|
||||
Queue taskQueue = getQueue(queue);
|
||||
StringBuilder outputPayload =
|
||||
new StringBuilder(
|
||||
String.format("OK: Launched the following %d tasks in queue %s\n", tlds.size(), queue));
|
||||
logger.infofmt("Launching %d tasks in queue %s", tlds.size(), queue);
|
||||
if (tlds.isEmpty()) {
|
||||
logger.warning("No TLDs to fan-out!");
|
||||
}
|
||||
for (String tld : tlds) {
|
||||
TaskOptions taskOptions = createTaskOptions(tld, flowThruParams);
|
||||
TaskHandle taskHandle = taskEnqueuer.enqueue(taskQueue, taskOptions);
|
||||
outputPayload.append(
|
||||
String.format(
|
||||
"- Task: %s, tld: %s, endpoint: %s\n",
|
||||
"- Task: '%s', tld: '%s', endpoint: '%s'\n",
|
||||
taskHandle.getName(), tld, taskOptions.getUrl()));
|
||||
logger.infofmt("Task: '%s', tld: '%s', endpoint: '%s'",
|
||||
taskHandle.getName(), tld, taskOptions.getUrl());
|
||||
}
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
response.setPayload(outputPayload.toString());
|
||||
|
|
@ -144,12 +158,14 @@ public final class TldFanoutAction implements Runnable {
|
|||
|
||||
private TaskOptions createTaskOptions(String tld, Multimap<String, String> params) {
|
||||
TaskOptions options =
|
||||
withUrl(endpoint.replace(TLD_PATHARG, String.valueOf(tld)))
|
||||
withUrl(endpoint)
|
||||
.countdownMillis(
|
||||
jitterSeconds.isPresent()
|
||||
? random.nextInt((int) SECONDS.toMillis(jitterSeconds.get()))
|
||||
: 0);
|
||||
options.param(RequestParameters.PARAM_TLD, tld);
|
||||
jitterSeconds
|
||||
.map(seconds -> random.nextInt((int) SECONDS.toMillis(seconds)))
|
||||
.orElse(0));
|
||||
if (!tld.isEmpty()) {
|
||||
options.param(RequestParameters.PARAM_TLD, tld);
|
||||
}
|
||||
for (String param : params.keySet()) {
|
||||
// TaskOptions.param() does not accept null values.
|
||||
options.param(param, nullToEmpty((getFirst(params.get(param), null))));
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue