zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-14 00:17:20 +02:00
Code Issues Projects Releases Packages Wiki Activity

Delete obsoleted non-batched DNS refresh on host rename action

Browse source 
-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=135681285
This commit is contained in:
mcilwain 2016-10-10 09:02:24 -07:00 • committed by Ben McIlwain
parent f7a205975f
commit 94c549d960
7 changed files with 6 additions and 253 deletions
Download patch file Download diff file Expand all files Collapse all files

9
docs/app-engine-architecture.md
View file

@ -111,6 +111,11 @@ specification for the cron task.
Here are the task queues in use by the system. All are push queues unless Here are the task queues in use by the system. All are push queues unless
explicitly marked as otherwise. explicitly marked as otherwise.
* `async-delete-pull` and `async-host-rename-pull` -- Pull queues for tasks to
asynchronously delete contacts/hosts and to asynchronously refresh DNS
for renamed hosts, respectively. Tasks are enqueued during EPP
flows and then handled in batches by the regularly running cron tasks
`DeleteContactsAndHostsAction` and `RefreshDnsOnHostRenameAction`.
* `bigquery-streaming-metrics` -- Queue for metrics that are asynchronously * `bigquery-streaming-metrics` -- Queue for metrics that are asynchronously
streamed to BigQuery in the `Metrics` class. Tasks are enqueued during EPP streamed to BigQuery in the `Metrics` class. Tasks are enqueued during EPP
flows in `EppController`. This means that there is a lag of a few seconds to flows in `EppController`. This means that there is a lag of a few seconds to
@ -151,10 +156,6 @@ explicitly marked as otherwise.
* `export-snapshot-update-view` -- Queue for tasks to update the BigQuery * `export-snapshot-update-view` -- Queue for tasks to update the BigQuery
views to point to the most recently uploaded snapshot. Tasks are enqueued by views to point to the most recently uploaded snapshot. Tasks are enqueued by
`LoadSnapshotAction` and executed by `UpdateSnapshotViewAction`. `LoadSnapshotAction` and executed by `UpdateSnapshotViewAction`.
* `flows-async` -- Queue for asynchronous tasks that are enqueued during EPP
command flows. Currently all of these tasks correspond to invocations of any
of the following three MapReduces: `DnsRefreshForHostRenameAction`,
`DeleteHostResourceAction`, or `DeleteContactResourceAction`.
* `group-members-sync` -- Cron queue for tasks to sync registrar contacts (not * `group-members-sync` -- Cron queue for tasks to sync registrar contacts (not
domain contacts!) to Google Groups. Tasks are executed by domain contacts!) to Google Groups. Tasks are executed by
`SyncGroupMembersAction`. `SyncGroupMembersAction`.

8
java/google/registry/env/common/default/WEB-INF/queue.xml vendored
View file

@ -12,14 +12,6 @@
<mode>pull</mode> <mode>pull</mode>
</queue> </queue>
<!-- Queue for launching asynchronous actions (e.g. mapreduces) from async flows. -->
<!-- TODO(b/26140521): Remove queue once new batch async handling mapreduces are deployed. -->
<queue>
<name>flows-async</name>
<rate>1/m</rate>
<target>backend</target>
</queue>
<queue> <queue>
<name>dns-publish</name> <name>dns-publish</name>
<rate>100/s</rate> <rate>100/s</rate>

2
java/google/registry/flows/async/AsyncFlowEnqueuer.java
View file

@ -17,7 +17,7 @@ package google.registry.flows.async;
import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_IS_SUPERUSER; import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_IS_SUPERUSER;
import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_REQUESTING_CLIENT_ID; import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_REQUESTING_CLIENT_ID;
import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_RESOURCE_KEY; import static google.registry.flows.async.DeleteContactsAndHostsAction.PARAM_RESOURCE_KEY;
import static google.registry.flows.async.DnsRefreshForHostRenameAction.PARAM_HOST_KEY; import static google.registry.flows.async.RefreshDnsOnHostRenameAction.PARAM_HOST_KEY;
import static google.registry.flows.async.RefreshDnsOnHostRenameAction.QUEUE_ASYNC_HOST_RENAME; import static google.registry.flows.async.RefreshDnsOnHostRenameAction.QUEUE_ASYNC_HOST_RENAME;
import com.google.appengine.api.taskqueue.Queue; import com.google.appengine.api.taskqueue.Queue;

11
java/google/registry/flows/async/AsyncFlowsModule.java
View file

@ -15,17 +15,13 @@
package google.registry.flows.async; package google.registry.flows.async;
import static google.registry.flows.async.DeleteContactsAndHostsAction.QUEUE_ASYNC_DELETE; import static google.registry.flows.async.DeleteContactsAndHostsAction.QUEUE_ASYNC_DELETE;
import static google.registry.flows.async.DnsRefreshForHostRenameAction.PARAM_HOST_KEY;
import static google.registry.flows.async.RefreshDnsOnHostRenameAction.QUEUE_ASYNC_HOST_RENAME; import static google.registry.flows.async.RefreshDnsOnHostRenameAction.QUEUE_ASYNC_HOST_RENAME;
import static google.registry.request.RequestParameters.extractRequiredParameter;
import com.google.appengine.api.taskqueue.Queue; import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.api.taskqueue.QueueFactory;
import dagger.Module; import dagger.Module;
import dagger.Provides; import dagger.Provides;
import google.registry.request.Parameter;
import javax.inject.Named; import javax.inject.Named;
import javax.servlet.http.HttpServletRequest;
/** Dagger module for the async flows package. */ /** Dagger module for the async flows package. */
@Module @Module
@ -42,11 +38,4 @@ public final class AsyncFlowsModule {
static Queue provideAsyncHostRenamePullQueue() { static Queue provideAsyncHostRenamePullQueue() {
return QueueFactory.getQueue(QUEUE_ASYNC_HOST_RENAME); return QueueFactory.getQueue(QUEUE_ASYNC_HOST_RENAME);
} }
//TODO(b/26140521): Delete this method once non-batched DNS host refresh mapreduce is deleted.
@Provides
@Parameter(PARAM_HOST_KEY)
static String provideHostKey(HttpServletRequest req) {
return extractRequiredParameter(req, PARAM_HOST_KEY);
}
} }

105
java/google/registry/flows/async/DnsRefreshForHostRenameAction.java
View file

@ -1,105 +0,0 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.flows.async;
import static google.registry.model.EppResourceUtils.isActive;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.util.PipelineUtils.createJobPath;
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.collect.ImmutableList;
import com.googlecode.objectify.Key;
import google.registry.dns.DnsQueue;
import google.registry.mapreduce.MapreduceRunner;
import google.registry.mapreduce.inputs.EppResourceInputs;
import google.registry.model.domain.DomainResource;
import google.registry.model.host.HostResource;
import google.registry.request.Action;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.util.FormattingLogger;
import google.registry.util.NonFinalForTesting;
import javax.inject.Inject;
import org.joda.time.DateTime;
/**
* Enqueues DNS refreshes for applicable domains following a host rename.
*/
//TODO(b/26140521): Delete this once non-batched DNS host refresh mapreduce is deleted.
@Action(path = "/_dr/task/dnsRefreshForHostRename")
public class DnsRefreshForHostRenameAction implements Runnable {
/** The HTTP parameter name used to specify the websafe key of the host to rename. */
public static final String PARAM_HOST_KEY = "hostKey";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@NonFinalForTesting
static DnsQueue dnsQueue = DnsQueue.create();
@Inject @Parameter(PARAM_HOST_KEY) String hostKeyString;
@Inject MapreduceRunner mrRunner;
@Inject Response response;
@Inject DnsRefreshForHostRenameAction() {}
@Override
public void run() {
Key<HostResource> resourceKey = null;
HostResource host;
try {
resourceKey = Key.create(hostKeyString);
host = checkArgumentNotNull(ofy().load().key(resourceKey).now());
} catch (IllegalArgumentException e) {
throw new BadRequestException(resourceKey == null
? "Could not parse key string: " + hostKeyString
: "Could not load resource for key: " + resourceKey);
}
response.sendJavaScriptRedirect(createJobPath(mrRunner
.setJobName("Enqueue DNS refreshes for domains following a host rename")
.setModuleName("backend")
.runMapOnly(
new DnsRefreshForHostRenameMapper(host),
ImmutableList.of(EppResourceInputs.createEntityInput(DomainResource.class)))));
}
/** Map over domains and refresh the dns of those that referenced this host. */
public static class DnsRefreshForHostRenameMapper extends Mapper<DomainResource, Void, Void> {
private static final long serialVersionUID = -4707015136971008447L;
private final DateTime hostUpdateTime;
private final Key<HostResource> targetHostKey;
DnsRefreshForHostRenameMapper(HostResource host) {
this.targetHostKey = Key.create(host);
this.hostUpdateTime = host.getUpdateAutoTimestamp().getTimestamp();
}
@Override
public final void map(DomainResource domain) {
if (isActive(domain, hostUpdateTime)
&& domain.getNameservers().contains(targetHostKey)) {
try {
dnsQueue.addDomainRefreshTask(domain.getFullyQualifiedDomainName());
logger.infofmt("Enqueued refresh for domain %s", domain.getFullyQualifiedDomainName());
} catch (Throwable t) {
logger.severefmt(t, "Error while refreshing DNS for host rename %s", targetHostKey);
}
}
}
}
}

2
java/google/registry/module/backend/BackendRequestComponent.java
View file

@ -43,7 +43,6 @@ import google.registry.export.sheet.SheetModule;
import google.registry.export.sheet.SyncRegistrarsSheetAction; import google.registry.export.sheet.SyncRegistrarsSheetAction;
import google.registry.flows.async.AsyncFlowsModule; import google.registry.flows.async.AsyncFlowsModule;
import google.registry.flows.async.DeleteContactsAndHostsAction; import google.registry.flows.async.DeleteContactsAndHostsAction;
import google.registry.flows.async.DnsRefreshForHostRenameAction;
import google.registry.flows.async.RefreshDnsOnHostRenameAction; import google.registry.flows.async.RefreshDnsOnHostRenameAction;
import google.registry.mapreduce.MapreduceModule; import google.registry.mapreduce.MapreduceModule;
import google.registry.monitoring.whitebox.MetricsExportAction; import google.registry.monitoring.whitebox.MetricsExportAction;
@ -93,7 +92,6 @@ interface BackendRequestComponent {
DeleteContactsAndHostsAction deleteContactsAndHostsAction(); DeleteContactsAndHostsAction deleteContactsAndHostsAction();
DeleteOldCommitLogsAction deleteOldCommitLogsAction(); DeleteOldCommitLogsAction deleteOldCommitLogsAction();
DeleteProberDataAction deleteProberDataAction(); DeleteProberDataAction deleteProberDataAction();
DnsRefreshForHostRenameAction dnsRefreshForHostRenameAction();
ExpandRecurringBillingEventsAction expandRecurringBillingEventsAction(); ExpandRecurringBillingEventsAction expandRecurringBillingEventsAction();
ExportCommitLogDiffAction exportCommitLogDiffAction(); ExportCommitLogDiffAction exportCommitLogDiffAction();
ExportDomainListsAction exportDomainListsAction(); ExportDomainListsAction exportDomainListsAction();

122
javatests/google/registry/flows/async/DnsRefreshForHostRenameActionTest.java
View file

@ -1,122 +0,0 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.flows.async;
import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.testing.DatastoreHelper.newDomainApplication;
import static google.registry.testing.DatastoreHelper.newDomainResource;
import static google.registry.testing.DatastoreHelper.newHostResource;
import static google.registry.testing.DatastoreHelper.persistActiveHost;
import static google.registry.testing.DatastoreHelper.persistResource;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.verifyZeroInteractions;
import com.google.common.collect.ImmutableSet;
import com.googlecode.objectify.Key;
import google.registry.dns.DnsQueue;
import google.registry.model.host.HostResource;
import google.registry.request.HttpException.BadRequestException;
import google.registry.testing.ExceptionRule;
import google.registry.testing.FakeResponse;
import google.registry.testing.InjectRule;
import google.registry.testing.mapreduce.MapreduceTestCase;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link DnsRefreshForHostRenameAction}. */
@RunWith(JUnit4.class)
public class DnsRefreshForHostRenameActionTest
extends MapreduceTestCase<DnsRefreshForHostRenameAction> {
@Rule
public final ExceptionRule thrown = new ExceptionRule();
@Rule
public InjectRule inject = new InjectRule();
private final DnsQueue dnsQueue = mock(DnsQueue.class);
@Before
public void setup() throws Exception {
inject.setStaticField(DnsRefreshForHostRenameAction.class, "dnsQueue", dnsQueue);
}
private void runMapreduce(String hostKeyString) throws Exception {
action = new DnsRefreshForHostRenameAction();
action.hostKeyString = hostKeyString;
action.mrRunner = makeDefaultRunner();
action.response = new FakeResponse();
action.run();
executeTasksUntilEmpty("mapreduce");
}
@Test
public void testSuccess_dnsUpdateEnqueued() throws Exception {
createTld("tld");
Key<HostResource> renamedHostKey = Key.create(persistActiveHost("ns1.example.tld"));
Key<HostResource> otherHostKey = Key.create(persistActiveHost("ns2.example.tld"));
persistResource(newDomainApplication("notadomain.tld").asBuilder()
.setNameservers(ImmutableSet.of(renamedHostKey))
.build());
persistResource(newDomainResource("example.tld").asBuilder()
.setNameservers(ImmutableSet.of(renamedHostKey))
.build());
persistResource(newDomainResource("otherexample.tld").asBuilder()
.setNameservers(ImmutableSet.of(renamedHostKey))
.build());
persistResource(newDomainResource("untouched.tld").asBuilder()
.setNameservers(ImmutableSet.of(otherHostKey))
.build());
runMapreduce(renamedHostKey.getString());
verify(dnsQueue).addDomainRefreshTask("example.tld");
verify(dnsQueue).addDomainRefreshTask("otherexample.tld");
verifyNoMoreInteractions(dnsQueue);
}
@Test
public void testSuccess_noDnsTasksForDeletedDomain() throws Exception {
createTld("tld");
Key<HostResource> renamedHostKey = Key.create(persistActiveHost("ns1.example.tld"));
persistResource(newDomainResource("example.tld").asBuilder()
.setNameservers(ImmutableSet.of(renamedHostKey))
.setDeletionTime(START_OF_TIME)
.build());
runMapreduce(renamedHostKey.getString());
verifyZeroInteractions(dnsQueue);
}
@Test
public void testFailure_badKeyPassed() throws Exception {
createTld("tld");
thrown.expect(BadRequestException.class, "Could not parse key string: a bad key");
runMapreduce("a bad key");
}
@Test
public void testFailure_hostDoesntExist() throws Exception {
createTld("tld");
HostResource notPersistedHost = newHostResource("ns1.example.tld");
thrown.expect(
BadRequestException.class,
"Could not load resource for key: Key<?>(HostResource(\"2-ROID\"))");
runMapreduce(Key.create(notPersistedHost).getString());
}
}