zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-15 17:07:15 +02:00
Code Issues Projects Releases Packages Wiki Activity

Improve logging/reduce shard count on async deletion []

Browse source 
-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=133280141
This commit is contained in:
mcilwain 2016-09-15 10:48:34 -07:00 • committed by Ben McIlwain
parent 949a360f46
commit c35de8144f
2 changed files with 16 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

1
java/google/registry/flows/BUILD
View file

@ -28,6 +28,7 @@ java_library(
"//java/com/google/common/base", "//java/com/google/common/base",
"//java/com/google/common/collect", "//java/com/google/common/collect",
"//java/com/google/common/io", "//java/com/google/common/io",
"//java/com/google/common/math",
"//java/com/google/common/net", "//java/com/google/common/net",
"//third_party/java/appengine:appengine-api", "//third_party/java/appengine:appengine-api",
"//third_party/java/appengine_mapreduce2:appengine_mapreduce", "//third_party/java/appengine_mapreduce2:appengine_mapreduce",

17
java/google/registry/flows/async/DeleteContactsAndHostsAction.java
View file

@ -18,6 +18,7 @@ import static com.google.appengine.api.taskqueue.QueueConstants.maxLeaseCount;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Preconditions.checkState;
import static com.google.common.math.IntMath.divide;
import static com.googlecode.objectify.Key.getKind; import static com.googlecode.objectify.Key.getKind;
import static google.registry.flows.ResourceFlowUtils.handlePendingTransferOnDelete; import static google.registry.flows.ResourceFlowUtils.handlePendingTransferOnDelete;
import static google.registry.flows.ResourceFlowUtils.prepareDeletedResourceAsBuilder; import static google.registry.flows.ResourceFlowUtils.prepareDeletedResourceAsBuilder;
@ -32,6 +33,7 @@ import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE;
import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE_FAILURE; import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE_FAILURE;
import static google.registry.util.FormattingLogger.getLoggerForCallerClass; import static google.registry.util.FormattingLogger.getLoggerForCallerClass;
import static google.registry.util.PipelineUtils.createJobPath; import static google.registry.util.PipelineUtils.createJobPath;
import static java.math.RoundingMode.CEILING;
import static java.util.concurrent.TimeUnit.DAYS; import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.MINUTES;
@ -42,9 +44,11 @@ import com.google.appengine.tools.mapreduce.Mapper;
import com.google.appengine.tools.mapreduce.Reducer; import com.google.appengine.tools.mapreduce.Reducer;
import com.google.appengine.tools.mapreduce.ReducerInput; import com.google.appengine.tools.mapreduce.ReducerInput;
import com.google.auto.value.AutoValue; import com.google.auto.value.AutoValue;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators; import com.google.common.collect.Iterators;
import com.google.common.collect.Multiset;
import com.googlecode.objectify.Key; import com.googlecode.objectify.Key;
import com.googlecode.objectify.Work; import com.googlecode.objectify.Work;
import google.registry.dns.DnsQueue; import google.registry.dns.DnsQueue;
@ -89,6 +93,8 @@ public class DeleteContactsAndHostsAction implements Runnable {
private static final long LEASE_MINUTES = 20; private static final long LEASE_MINUTES = 20;
private static final FormattingLogger logger = getLoggerForCallerClass(); private static final FormattingLogger logger = getLoggerForCallerClass();
private static final int MAX_REDUCE_SHARDS = 50;
private static final int DELETES_PER_SHARD = 5;
@Inject Clock clock; @Inject Clock clock;
@Inject MapreduceRunner mrRunner; @Inject MapreduceRunner mrRunner;
@ -104,10 +110,13 @@ public class DeleteContactsAndHostsAction implements Runnable {
if (tasks.isEmpty()) { if (tasks.isEmpty()) {
return; return;
} }
Multiset<String> kindCounts = HashMultiset.create(2);
ImmutableList.Builder<DeletionRequest> builder = new ImmutableList.Builder<>(); ImmutableList.Builder<DeletionRequest> builder = new ImmutableList.Builder<>();
for (TaskHandle task : tasks) { for (TaskHandle task : tasks) {
try { try {
builder.add(DeletionRequest.createFromTask(task, clock.nowUtc())); DeletionRequest deletionRequest = DeletionRequest.createFromTask(task, clock.nowUtc());
builder.add(deletionRequest);
kindCounts.add(deletionRequest.key().getKind());
} catch (Exception e) { } catch (Exception e) {
logger.severefmt( logger.severefmt(
e, "Could not parse async deletion request, delaying task for a day: %s", task); e, "Could not parse async deletion request, delaying task for a day: %s", task);
@ -118,15 +127,19 @@ public class DeleteContactsAndHostsAction implements Runnable {
} }
ImmutableList<DeletionRequest> deletionRequests = builder.build(); ImmutableList<DeletionRequest> deletionRequests = builder.build();
logger.infofmt( logger.infofmt(
"Processing asynchronous deletion of %d contacts and hosts.", deletionRequests.size()); "Processing asynchronous deletion of %d contacts and %d hosts.",
kindCounts.count(KIND_CONTACT), kindCounts.count(KIND_HOST));
runMapreduce(deletionRequests); runMapreduce(deletionRequests);
} }
private void runMapreduce(ImmutableList<DeletionRequest> deletionRequests) { private void runMapreduce(ImmutableList<DeletionRequest> deletionRequests) {
try { try {
int numReducers =
Math.min(MAX_REDUCE_SHARDS, divide(deletionRequests.size(), DELETES_PER_SHARD, CEILING));
response.sendJavaScriptRedirect(createJobPath(mrRunner response.sendJavaScriptRedirect(createJobPath(mrRunner
.setJobName("Check for EPP resource references and then delete") .setJobName("Check for EPP resource references and then delete")
.setModuleName("backend") .setModuleName("backend")
.setDefaultReduceShards(numReducers)
.runMapreduce( .runMapreduce(
new DeleteContactsAndHostsMapper(deletionRequests), new DeleteContactsAndHostsMapper(deletionRequests),
new DeleteEppResourceReducer(), new DeleteEppResourceReducer(),