Remove old DNS queue processing code.

The old DNS processing was performed by WriteDnsAction, which was invoked by the standard cron fanout action. The new code, which has been running for several months in production, uses ReadDnsQueueAction to do a custom fanout to PublishDnsUpdatesAction. We no longer need the old code, so it's time to remove it.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=127983115
This commit is contained in:
Brian Mountford 2016-07-20 13:44:49 -07:00 committed by Justine Tunney
parent b83b3b313f
commit 2c9567e183
13 changed files with 24 additions and 418 deletions

View file

@ -58,13 +58,11 @@ public class DnsQueue {
long writeBatchSize = QueueConstants.maxLeaseCount();
/**
* Enqueues the given task type with the given target name to the DNS queue, tagged with the
* specified TLD.
* Enqueues the given task type with the given target name to the DNS queue.
*/
private TaskHandle addToQueue(TargetType targetType, String targetName, String tld) {
return queue.add(TaskOptions.Builder
// TODO(b/24564175): don't set the tag
.withTag(tld)
.withDefaults()
.method(Method.PULL)
.param(DNS_TARGET_TYPE_PARAM, targetType.toString())
.param(DNS_TARGET_NAME_PARAM, targetName)

View file

@ -124,12 +124,7 @@ public final class ReadDnsQueueAction implements Runnable {
for (TaskHandle task : tasks) {
try {
Map<String, String> params = ImmutableMap.copyOf(task.extractParams());
// Dual-read the TLD from either the parameter (new methodology) or the tag (old way).
// TODO(b/24564175): get the TLD from the regular parameter only.
String tld = task.getTag();
if (tld == null) {
tld = params.get(RequestParameters.PARAM_TLD);
}
String tld = params.get(RequestParameters.PARAM_TLD);
if (tld == null) {
logger.severe("discarding invalid DNS refresh request; no TLD specified");
} else if (!tldsOfInterest.contains(tld)) {
@ -151,9 +146,7 @@ public final class ReadDnsQueueAction implements Runnable {
break;
}
}
} catch (RuntimeException e) {
logger.severefmt(e, "discarding invalid DNS refresh request (task %s)", task);
} catch (UnsupportedEncodingException e) {
} catch (RuntimeException | UnsupportedEncodingException e) {
logger.severefmt(e, "discarding invalid DNS refresh request (task %s)", task);
}
}

View file

@ -1,155 +0,0 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.dns;
import static com.google.common.base.Preconditions.checkNotNull;
import static google.registry.dns.DnsConstants.DNS_TARGET_NAME_PARAM;
import static google.registry.dns.DnsConstants.DNS_TARGET_TYPE_PARAM;
import static google.registry.model.server.Lock.executeWithLocks;
import static google.registry.request.Action.Method.POST;
import com.google.appengine.api.LifecycleManager;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.common.base.Throwables;
import com.google.common.net.InternetDomainName;
import google.registry.config.ConfigModule.Config;
import google.registry.dns.DnsConstants.TargetType;
import google.registry.dns.writer.api.DnsWriter;
import google.registry.model.registry.Registry;
import google.registry.request.Action;
import google.registry.request.HttpException;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.Parameter;
import google.registry.request.RequestParameters;
import google.registry.util.DomainNameUtils;
import google.registry.util.FormattingLogger;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import javax.inject.Inject;
import javax.inject.Provider;
import org.joda.time.Duration;
/** Action that consumes pull-queue for zone updates to write to the DNS server. */
@Action(path = "/_dr/task/writeDns", method = POST, automaticallyPrintOk = true)
public final class WriteDnsAction implements Runnable, Callable<Void> {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject DnsQueue dnsQueue;
@Inject Provider<DnsWriter> writerProvider;
@Inject @Config("dnsWriteLockTimeout") Duration timeout;
@Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@Inject WriteDnsAction() {}
/** Runs the task. */
@Override
public void run() {
String lockName = String.format("DNS zone %s", tld);
executeWithLocks(this, getClass(), tld, timeout, lockName);
}
/** Runs the task, with the lock. */
@Override
public Void call() {
processBatch();
return null;
}
/** Leases a batch of tasks tagged with the zone name from the pull queue and processes them. */
private void processBatch() {
if (LifecycleManager.getInstance().isShuttingDown()) {
logger.infofmt("%s: lifecycle manager is shutting down", tld);
return;
}
if (Registry.get(tld).getDnsPaused()) {
logger.infofmt("%s: the dns-pull queue is paused", tld);
return;
}
// Make a defensive copy to allow mutations.
List<TaskHandle> tasks = new ArrayList<>(dnsQueue.leaseTasks(tld));
if (tasks.isEmpty()) {
logger.infofmt("%s: no tasks in the dns-pull queue", tld);
return;
}
try (DnsWriter writer = writerProvider.get()) {
Iterator<TaskHandle> it = tasks.iterator();
while (it.hasNext()) {
TaskHandle task = it.next();
try {
processTask(writer, task, tld);
} catch (UnsupportedOperationException e) {
// Handle fatal errors by deleting the task.
logger.severefmt(e, "%s: deleting unsupported task %s", tld, task.toString());
dnsQueue.deleteTask(task);
it.remove();
}
}
} catch (RuntimeException e) {
Throwables.propagateIfInstanceOf(e, HttpException.class);
// Handle transient errors by dropping the task leases.
logger.severefmt(e, "%s: dropping leases of failed tasks", tld);
for (TaskHandle task : tasks) {
dnsQueue.dropTaskLease(task);
}
return;
}
for (TaskHandle task : tasks) {
dnsQueue.deleteTask(task);
}
logger.infofmt("%s: batch of %s tasks processed", tld, tasks.size());
}
/** Stages a write to authoritative DNS for this task. */
private static void processTask(DnsWriter writer, TaskHandle task, String tld) {
Map<String, String> params = new HashMap<>();
try {
for (Map.Entry<String, String> entry : task.extractParams()) {
params.put(entry.getKey(), entry.getValue());
}
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
TargetType type = TargetType.valueOf(params.get(DNS_TARGET_TYPE_PARAM));
String name = checkNotNull(params.get(DNS_TARGET_NAME_PARAM));
switch (type) {
case DOMAIN:
checkRequestArgument(
DomainNameUtils.isUnder(InternetDomainName.from(name), InternetDomainName.from(tld)),
"domain name %s is not under tld %s", name, tld);
writer.publishDomain(name);
break;
case HOST:
checkRequestArgument(
DomainNameUtils.isUnder(InternetDomainName.from(name), InternetDomainName.from(tld)),
"host name %s is not under tld %s", name, tld);
writer.publishHost(name);
break;
default:
// TODO(b/11592394): Write a full zone.
throw new UnsupportedOperationException(String.format("unexpected Type: %s", type));
}
}
private static void checkRequestArgument(boolean condition, String format, Object... args) {
if (!condition) {
throw new BadRequestException(String.format(format, args));
}
}
}

View file

@ -89,13 +89,6 @@
<url-pattern>/_dr/task/nordnVerify</url-pattern>
</servlet-mapping>
<!-- TODO(b/24564175): Remove this entry. -->
<!-- Write DNS updates. -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>
<url-pattern>/_dr/task/writeDns</url-pattern>
</servlet-mapping>
<!-- Reads the DNS push and pull queues and kick off the appropriate tasks to update zone. -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>

View file

@ -28,7 +28,6 @@ import google.registry.dns.DnsModule;
import google.registry.dns.PublishDnsUpdatesAction;
import google.registry.dns.ReadDnsQueueAction;
import google.registry.dns.RefreshDnsAction;
import google.registry.dns.WriteDnsAction;
import google.registry.export.BigqueryPollJobAction;
import google.registry.export.ExportDomainListsAction;
import google.registry.export.ExportRequestModule;
@ -110,6 +109,5 @@ interface BackendRequestComponent {
TmchDnlAction tmchDnlAction();
TmchSmdrlAction tmchSmdrlAction();
UpdateSnapshotViewAction updateSnapshotViewAction();
WriteDnsAction writeDnsAction();
VerifyEntityIntegrityAction verifyEntityIntegrityAction();
}

View file

@ -29,6 +29,7 @@ java_library(
"//third_party/java/servlet/servlet_api",
"//third_party/java/truth",
"//java/google/registry/config",
"//java/google/registry/cron",
"//java/google/registry/dns",
"//java/google/registry/dns:constants",
"//java/google/registry/dns/writer/api",

View file

@ -75,12 +75,12 @@ public final class DnsInjectionTest {
}
@Test
public void testWriteDnsAction_injectsAndWorks() throws Exception {
public void testReadDnsQueueAction_injectsAndWorks() throws Exception {
persistActiveSubordinateHost("ns1.example.lol", persistActiveDomain("example.lol"));
clock.advanceOneMilli();
dnsQueue.addDomainRefreshTask("example.lol");
when(req.getParameter("tld")).thenReturn("lol");
component.writeDnsAction().run();
component.readDnsQueueAction().run();
assertNoDnsTasksEnqueued();
}

View file

@ -57,7 +57,7 @@ public class DnsQueueTest {
createTld("tld");
dnsQueue.addHostRefreshTask("octopus.tld");
assertTasksEnqueued("dns-pull",
new TaskMatcher().tag("tld").payload("Target-Type=HOST&Target-Name=octopus.tld&tld=tld"));
new TaskMatcher().payload("Target-Type=HOST&Target-Name=octopus.tld&tld=tld"));
}
@Test
@ -76,7 +76,7 @@ public class DnsQueueTest {
createTld("tld");
dnsQueue.addDomainRefreshTask("octopus.tld");
assertTasksEnqueued("dns-pull",
new TaskMatcher().tag("tld").payload("Target-Type=DOMAIN&Target-Name=octopus.tld&tld=tld"));
new TaskMatcher().payload("Target-Type=DOMAIN&Target-Name=octopus.tld&tld=tld"));
}
@Test

View file

@ -16,10 +16,12 @@ package google.registry.dns;
import dagger.Component;
import google.registry.config.ConfigModule;
import google.registry.cron.CronModule;
import google.registry.dns.writer.api.VoidDnsWriterModule;
import google.registry.module.backend.BackendModule;
import google.registry.request.RequestModule;
import google.registry.util.SystemClock.SystemClockModule;
import google.registry.util.SystemSleeper.SystemSleeperModule;
@Component(modules = {
SystemClockModule.class,
@ -28,9 +30,11 @@ import google.registry.util.SystemClock.SystemClockModule;
DnsModule.class,
RequestModule.class,
VoidDnsWriterModule.class,
SystemSleeperModule.class,
CronModule.class,
})
interface DnsTestComponent {
DnsQueue dnsQueue();
RefreshDnsAction refreshDns();
WriteDnsAction writeDnsAction();
ReadDnsQueueAction readDnsQueueAction();
}

View file

@ -38,7 +38,7 @@ import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.runners.MockitoJUnitRunner;
/** Unit tests for {@link WriteDnsAction}. */
/** Unit tests for {@link PublishDnsUpdatesAction}. */
@RunWith(MockitoJUnitRunner.class)
public class PublishDnsUpdatesActionTest {

View file

@ -101,22 +101,13 @@ public class ReadDnsQueueActionTest {
action.run();
}
// TODO(b/24564175): remove
private enum RefreshTld { AS_TAG, AS_PARAM }
private static TaskOptions createRefreshTask(
String name, TargetType type, RefreshTld refreshTld) {
private static TaskOptions createRefreshTask(String name, TargetType type) {
TaskOptions options = TaskOptions.Builder
.withMethod(Method.PULL)
.param(DNS_TARGET_TYPE_PARAM, type.toString())
.param(DNS_TARGET_NAME_PARAM, name);
String tld = InternetDomainName.from(name).parts().reverse().get(0);
switch (refreshTld) {
case AS_TAG:
return options.tag(tld);
default:
return options.param(PARAM_TLD, tld);
}
return options.param(PARAM_TLD, tld);
}
private void assertTldsEnqueuedInPushQueue(String... tlds) throws Exception {
@ -164,29 +155,9 @@ public class ReadDnsQueueActionTest {
run(true);
assertTasksEnqueued(
DnsConstants.DNS_PULL_QUEUE_NAME,
new TaskMatcher().tag("com"),
new TaskMatcher().tag("net"),
new TaskMatcher().tag("example"));
assertTldsEnqueuedInPushQueue("com", "net", "example");
}
@Test
public void testSuccess_allTldsNoTag() throws Exception {
dnsQueue.queue.add(createRefreshTask("domain.com", TargetType.DOMAIN, RefreshTld.AS_PARAM));
dnsQueue.queue.add(createRefreshTask("domain.net", TargetType.DOMAIN, RefreshTld.AS_PARAM));
dnsQueue.queue.add(createRefreshTask("domain.example", TargetType.DOMAIN, RefreshTld.AS_PARAM));
run(false);
assertNoTasksEnqueued(DnsConstants.DNS_PULL_QUEUE_NAME);
assertTldsEnqueuedInPushQueue("com", "net", "example");
}
@Test
public void testSuccess_allTldsMixedOldAndNewTldStyles() throws Exception {
dnsQueue.addDomainRefreshTask("domain.com");
dnsQueue.queue.add(createRefreshTask("domain.net", TargetType.DOMAIN, RefreshTld.AS_PARAM));
dnsQueue.queue.add(createRefreshTask("domain.example", TargetType.DOMAIN, RefreshTld.AS_TAG));
run(false);
assertNoTasksEnqueued(DnsConstants.DNS_PULL_QUEUE_NAME);
new TaskMatcher().payload("Target-Type=DOMAIN&Target-Name=domain.com&tld=com"),
new TaskMatcher().payload("Target-Type=DOMAIN&Target-Name=domain.net&tld=net"),
new TaskMatcher().payload("Target-Type=DOMAIN&Target-Name=domain.example&tld=example"));
assertTldsEnqueuedInPushQueue("com", "net", "example");
}
@ -197,7 +168,7 @@ public class ReadDnsQueueActionTest {
dnsQueue.addDomainRefreshTask("domain.net");
dnsQueue.addDomainRefreshTask("domain.example");
run(false);
assertTasksEnqueued(DnsConstants.DNS_PULL_QUEUE_NAME, new TaskMatcher().tag("net"));
assertTasksEnqueued(DnsConstants.DNS_PULL_QUEUE_NAME, new TaskMatcher());
assertTldsEnqueuedInPushQueue("com", "example");
}
@ -239,13 +210,11 @@ public class ReadDnsQueueActionTest {
task.param("domains", domainName);
break;
case 1:
dnsQueue.queue.add(
createRefreshTask("ns1." + domainName, TargetType.HOST, RefreshTld.AS_TAG));
dnsQueue.queue.add(createRefreshTask("ns1." + domainName, TargetType.HOST));
task.param("hosts", "ns1." + domainName);
break;
case 2:
dnsQueue.queue.add(
createRefreshTask("ns2." + domainName, TargetType.HOST, RefreshTld.AS_PARAM));
dnsQueue.queue.add(createRefreshTask("ns2." + domainName, TargetType.HOST));
task.param("hosts", "ns2." + domainName);
break;
}

View file

@ -1,195 +0,0 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.dns;
import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.testing.DatastoreHelper.persistActiveDomain;
import static google.registry.testing.DatastoreHelper.persistActiveSubordinateHost;
import static google.registry.testing.DatastoreHelper.persistResource;
import static google.registry.testing.TaskQueueHelper.assertDnsTasksEnqueued;
import static google.registry.testing.TaskQueueHelper.assertNoDnsTasksEnqueued;
import static google.registry.testing.TaskQueueHelper.clearTaskQueue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.verifyZeroInteractions;
import com.google.appengine.api.taskqueue.QueueFactory;
import google.registry.dns.writer.api.DnsWriter;
import google.registry.model.ofy.Ofy;
import google.registry.model.registry.Registry;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock;
import google.registry.testing.InjectRule;
import javax.inject.Provider;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.runners.MockitoJUnitRunner;
/** Unit tests for {@link WriteDnsAction}. */
@RunWith(MockitoJUnitRunner.class)
public class WriteDnsActionTest {
@Rule
public final AppEngineRule appEngine = AppEngineRule.builder()
.withDatastore()
.withTaskQueue()
.build();
@Rule
public final InjectRule inject = new InjectRule();
@Rule
public final ExpectedException thrown = ExpectedException.none();
private final FakeClock clock = new FakeClock(DateTime.parse("1971-01-01TZ"));
private final DnsWriter dnsWriter = mock(DnsWriter.class);
private final DnsQueue dnsQueue = new DnsQueue();
@Before
public void setUp() throws Exception {
inject.setStaticField(Ofy.class, "clock", clock);
createTld("xn--q9jyb4c");
dnsQueue.queue = QueueFactory.getQueue(DnsConstants.DNS_PULL_QUEUE_NAME);
dnsQueue.writeBatchSize = 7;
dnsQueue.writeLockTimeout = Duration.standardSeconds(10);
}
private void run(String tld) {
WriteDnsAction action = new WriteDnsAction();
action.dnsQueue = dnsQueue;
action.timeout = Duration.standardSeconds(10);
action.tld = tld;
action.writerProvider = new Provider<DnsWriter>() {
@Override
public DnsWriter get() {
return dnsWriter;
}};
action.run();
}
@After
public void cleanUp() throws Exception {
clearTaskQueue("dns-pull");
}
@Test
public void testSuccess_host() throws Exception {
persistActiveSubordinateHost(
"ns1.example.xn--q9jyb4c", persistActiveDomain("example.xn--q9jyb4c"));
clock.advanceOneMilli();
dnsQueue.addHostRefreshTask("ns1.example.xn--q9jyb4c");
assertDnsTasksEnqueued("ns1.example.xn--q9jyb4c");
run("xn--q9jyb4c");
verify(dnsWriter).publishHost("ns1.example.xn--q9jyb4c");
verify(dnsWriter).close();
verifyNoMoreInteractions(dnsWriter);
assertNoDnsTasksEnqueued();
}
@Test
public void testSuccess_domain() throws Exception {
persistActiveSubordinateHost(
"ns1.example.xn--q9jyb4c", persistActiveDomain("example.xn--q9jyb4c"));
clock.advanceOneMilli();
dnsQueue.addDomainRefreshTask("example.xn--q9jyb4c");
assertDnsTasksEnqueued("example.xn--q9jyb4c");
run("xn--q9jyb4c");
verify(dnsWriter).publishDomain("example.xn--q9jyb4c");
verify(dnsWriter).close();
verifyNoMoreInteractions(dnsWriter);
assertNoDnsTasksEnqueued();
}
@Test
public void testSuccess_zone() throws Exception {
dnsQueue.addZoneRefreshTask("xn--q9jyb4c");
assertDnsTasksEnqueued("xn--q9jyb4c");
run("xn--q9jyb4c");
verify(dnsWriter).close();
verifyNoMoreInteractions(dnsWriter);
assertNoDnsTasksEnqueued();
}
@Test
public void testSuccess_dnsPaused() throws Exception {
persistActiveSubordinateHost(
"ns1.example.xn--q9jyb4c", persistActiveDomain("example.xn--q9jyb4c"));
clock.advanceOneMilli();
dnsQueue.addDomainRefreshTask("example.xn--q9jyb4c");
assertDnsTasksEnqueued("example.xn--q9jyb4c");
persistResource(Registry.get("xn--q9jyb4c").asBuilder().setDnsPaused(true).build());
clock.advanceOneMilli();
run("xn--q9jyb4c");
verifyZeroInteractions(dnsWriter);
assertDnsTasksEnqueued("example.xn--q9jyb4c");
}
@Test
public void testSuccess_twoTasksFromTheSameTld() throws Exception {
persistActiveSubordinateHost(
"ns1.example.xn--q9jyb4c", persistActiveDomain("example.xn--q9jyb4c"));
clock.advanceOneMilli();
dnsQueue.addDomainRefreshTask("example.xn--q9jyb4c");
persistActiveSubordinateHost(
"ns1.example2.xn--q9jyb4c", persistActiveDomain("example2.xn--q9jyb4c"));
clock.advanceOneMilli();
dnsQueue.addDomainRefreshTask("example2.xn--q9jyb4c");
assertDnsTasksEnqueued("example.xn--q9jyb4c", "example2.xn--q9jyb4c");
run("xn--q9jyb4c");
assertNoDnsTasksEnqueued();
}
@Test
public void testSuccess_twoTasksInDifferentTlds() throws Exception {
// refresh example.xn--q9jyb4c
persistActiveSubordinateHost(
"ns1.example.xn--q9jyb4c", persistActiveDomain("example.xn--q9jyb4c"));
clock.advanceOneMilli();
dnsQueue.addDomainRefreshTask("example.xn--q9jyb4c");
// refresh example.example
createTld("example");
persistActiveSubordinateHost(
"ns1.example.example", persistActiveDomain("example.example"));
clock.advanceOneMilli();
dnsQueue.addDomainRefreshTask("example.example");
// there should now be two tasks enqueued
assertDnsTasksEnqueued("example.example", "example.xn--q9jyb4c");
// process one, leaving one
run("example");
assertDnsTasksEnqueued("example.xn--q9jyb4c");
// process the other, leaving none
run("xn--q9jyb4c");
assertNoDnsTasksEnqueued();
}
@Test
public void testSuccess_domainDeleted() throws Exception {
dnsQueue.addDomainRefreshTask("example.xn--q9jyb4c");
assertDnsTasksEnqueued("example.xn--q9jyb4c");
run("xn--q9jyb4c");
assertNoDnsTasksEnqueued();
}
}

View file

@ -73,7 +73,7 @@ public final class RegistryTestServer {
google.registry.module.backend.BackendServlet.class),
// Process DNS pull queue
route("/_dr/task/writeDns",
route("/_dr/cron/readDnsQueue",
google.registry.module.backend.BackendServlet.class),
// Registrar Console