mirror of
https://github.com/google/nomulus.git
synced 2025-04-30 03:57:51 +02:00
5012893c1d
This change renames directories in preparation for the great package rename. The repository is now in a broken state because the code itself hasn't been updated. However this should ensure that git correctly preserves history for each file.
204 lines
8.6 KiB
Java
204 lines
8.6 KiB
Java
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package com.google.domain.registry.dns;
|
|
|
|
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
|
import static com.google.common.collect.Sets.difference;
|
|
import static com.google.domain.registry.dns.DnsConstants.DNS_PUBLISH_PUSH_QUEUE_NAME;
|
|
import static com.google.domain.registry.dns.DnsConstants.DNS_TARGET_NAME_PARAM;
|
|
import static com.google.domain.registry.dns.DnsConstants.DNS_TARGET_TYPE_PARAM;
|
|
import static com.google.domain.registry.model.registry.Registries.getTlds;
|
|
import static java.util.concurrent.TimeUnit.SECONDS;
|
|
|
|
import com.google.appengine.api.taskqueue.Queue;
|
|
import com.google.appengine.api.taskqueue.TaskHandle;
|
|
import com.google.appengine.api.taskqueue.TaskOptions;
|
|
import com.google.common.base.Optional;
|
|
import com.google.common.collect.ComparisonChain;
|
|
import com.google.common.collect.ImmutableList;
|
|
import com.google.common.collect.ImmutableMap;
|
|
import com.google.common.collect.ImmutableSet;
|
|
import com.google.common.collect.Iterables;
|
|
import com.google.common.collect.TreeMultimap;
|
|
import com.google.domain.registry.config.ConfigModule.Config;
|
|
import com.google.domain.registry.dns.DnsConstants.TargetType;
|
|
import com.google.domain.registry.model.registry.Registry;
|
|
import com.google.domain.registry.request.Action;
|
|
import com.google.domain.registry.request.Parameter;
|
|
import com.google.domain.registry.request.RequestParameters;
|
|
import com.google.domain.registry.util.FormattingLogger;
|
|
import com.google.domain.registry.util.TaskEnqueuer;
|
|
|
|
import java.io.UnsupportedEncodingException;
|
|
import java.util.Collection;
|
|
import java.util.HashSet;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Random;
|
|
import java.util.Set;
|
|
|
|
import javax.inject.Inject;
|
|
import javax.inject.Named;
|
|
|
|
/**
|
|
* Action for fanning out DNS refresh tasks by TLD, using data taken from the DNS pull queue.
|
|
*
|
|
* <h3>Parameters Reference</h3>
|
|
*
|
|
* <ul>
|
|
* <li>{@code jitterSeconds} Randomly delay each task by up to this many seconds.
|
|
* <li>{@code keepTasks} Do not delete any tasks from the pull queue, whether they are processed or
|
|
* not.
|
|
* </ul>
|
|
*/
|
|
@Action(path = "/_dr/cron/readDnsQueue", automaticallyPrintOk = true)
|
|
public final class ReadDnsQueueAction implements Runnable {
|
|
|
|
public static final String KEEP_TASKS_PARAM = "keepTasks";
|
|
|
|
private static final String JITTER_SECONDS_PARAM = "jitterSeconds";
|
|
private static final Random random = new Random();
|
|
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
|
|
|
@Inject @Config("dnsTldUpdateBatchSize") int tldUpdateBatchSize;
|
|
@Inject @Named(DNS_PUBLISH_PUSH_QUEUE_NAME) Queue dnsPublishPushQueue;
|
|
@Inject @Parameter(JITTER_SECONDS_PARAM) Optional<Integer> jitterSeconds;
|
|
@Inject @Parameter(KEEP_TASKS_PARAM) boolean keepTasks;
|
|
@Inject DnsQueue dnsQueue;
|
|
@Inject TaskEnqueuer taskEnqueuer;
|
|
@Inject ReadDnsQueueAction() {}
|
|
|
|
/** Container for items we pull out of the DNS pull queue and process for fanout. */
|
|
private class RefreshItem implements Comparable<RefreshItem> {
|
|
final TargetType type;
|
|
final String name;
|
|
|
|
public RefreshItem(final TargetType type, final String name) {
|
|
this.type = type;
|
|
this.name = name;
|
|
}
|
|
|
|
@Override
|
|
public int compareTo(RefreshItem other) {
|
|
return ComparisonChain.start()
|
|
.compare(this.type, other.type)
|
|
.compare(this.name, other.name)
|
|
.result();
|
|
}
|
|
}
|
|
|
|
/** Leases all tasks from the pull queue and creates per-tld update actions for them. */
|
|
@Override
|
|
public void run() {
|
|
Set<String> tldsOfInterest = getTlds();
|
|
|
|
List<TaskHandle> tasks = dnsQueue.leaseTasks();
|
|
if (tasks.isEmpty()) {
|
|
return;
|
|
}
|
|
logger.infofmt("leased %d tasks", tasks.size());
|
|
// Normally, all tasks will be deleted from the pull queue. But some might have to remain if
|
|
// we are not interested in the associated TLD, or if the TLD is paused. Remember which these
|
|
// are.
|
|
Set<TaskHandle> tasksToKeep = new HashSet<>();
|
|
// The paused TLDs for which we found at least one refresh request.
|
|
Set<String> pausedTlds = new HashSet<>();
|
|
// Create a sorted multimap into which we will insert the refresh items, so that the items for
|
|
// each TLD will be grouped together, and domains and hosts will be grouped within a TLD. The
|
|
// grouping and ordering of domains and hosts is not technically necessary, but a predictable
|
|
// ordering makes it possible to write detailed tests.
|
|
TreeMultimap<String, RefreshItem> refreshItemMultimap = TreeMultimap.create();
|
|
// Read all tasks on the DNS pull queue and load them into the refresh item multimap.
|
|
for (TaskHandle task : tasks) {
|
|
try {
|
|
Map<String, String> params = ImmutableMap.copyOf(task.extractParams());
|
|
// Dual-read the TLD from either the parameter (new methodology) or the tag (old way).
|
|
// TODO(b/24564175): get the TLD from the regular parameter only.
|
|
String tld = task.getTag();
|
|
if (tld == null) {
|
|
tld = params.get(RequestParameters.PARAM_TLD);
|
|
}
|
|
if (tld == null) {
|
|
logger.severe("discarding invalid DNS refresh request; no TLD specified");
|
|
} else if (!tldsOfInterest.contains(tld)) {
|
|
tasksToKeep.add(task);
|
|
} else if (Registry.get(tld).getDnsPaused()) {
|
|
tasksToKeep.add(task);
|
|
pausedTlds.add(tld);
|
|
} else {
|
|
String typeString = params.get(DNS_TARGET_TYPE_PARAM);
|
|
String name = params.get(DNS_TARGET_NAME_PARAM);
|
|
if (typeString == null) {
|
|
logger.severe("discarding invalid DNS refresh request; no type specified");
|
|
} else if (name == null) {
|
|
logger.severe("discarding invalid DNS refresh request; no name specified");
|
|
} else {
|
|
TargetType type = TargetType.valueOf(typeString);
|
|
switch (type) {
|
|
case DOMAIN:
|
|
case HOST:
|
|
refreshItemMultimap.put(tld, new RefreshItem(type, name));
|
|
break;
|
|
default:
|
|
logger.severefmt("discarding DNS refresh request of type %s", typeString);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
} catch (UnsupportedEncodingException e) {
|
|
logger.severefmt(e, "discarding invalid DNS refresh request (task %s)", task);
|
|
}
|
|
}
|
|
if (!pausedTlds.isEmpty()) {
|
|
logger.infofmt("the dns-pull queue is paused for tlds: %s", pausedTlds);
|
|
}
|
|
// Loop through the multimap by TLD and generate refresh tasks for the hosts and domains.
|
|
for (Map.Entry<String, Collection<RefreshItem>> tldRefreshItemsEntry
|
|
: refreshItemMultimap.asMap().entrySet()) {
|
|
for (List<RefreshItem> chunk : Iterables.partition(
|
|
tldRefreshItemsEntry.getValue(), tldUpdateBatchSize)) {
|
|
TaskOptions options = withUrl(PublishDnsUpdatesAction.PATH)
|
|
.countdownMillis(jitterSeconds.isPresent()
|
|
? random.nextInt((int) SECONDS.toMillis(jitterSeconds.get()))
|
|
: 0)
|
|
.param(RequestParameters.PARAM_TLD, tldRefreshItemsEntry.getKey());
|
|
for (RefreshItem refreshItem : chunk) {
|
|
options.param(
|
|
(refreshItem.type == TargetType.HOST)
|
|
? PublishDnsUpdatesAction.HOSTS_PARAM : PublishDnsUpdatesAction.DOMAINS_PARAM,
|
|
refreshItem.name);
|
|
}
|
|
taskEnqueuer.enqueue(dnsPublishPushQueue, options);
|
|
}
|
|
}
|
|
Set<TaskHandle> tasksToDelete = difference(ImmutableSet.copyOf(tasks), tasksToKeep);
|
|
// In keepTasks mode, never delete any tasks.
|
|
if (keepTasks) {
|
|
logger.infofmt("would have deleted %d tasks", tasksToDelete.size());
|
|
for (TaskHandle task : tasks) {
|
|
dnsQueue.dropTaskLease(task);
|
|
}
|
|
// Otherwise, either delete or drop the lease of each task.
|
|
} else {
|
|
logger.infofmt("deleting %d tasks", tasksToDelete.size());
|
|
dnsQueue.deleteTasks(ImmutableList.copyOf(tasksToDelete));
|
|
logger.infofmt("dropping %d tasks", tasksToKeep.size());
|
|
for (TaskHandle task : tasksToKeep) {
|
|
dnsQueue.dropTaskLease(task);
|
|
}
|
|
logger.infofmt("done");
|
|
}
|
|
}
|
|
}
|