mirror of
https://github.com/google/nomulus.git
synced 2025-06-27 23:03:34 +02:00
Reverse dependency between /flows/ and /batch/
Certain flows need to launch batched jobs. Logically this would mean that flows depend on batch. However, the current state of dependency was the other way around, and the reason for that was ResourceFlowUtils.java that had in it some utility functions that weren't used in the flows and were needed in the batch jobs. This CL removes these utility functions from the /flows/ directory, letting us reverse the dependency edge between flows/ and batch/ Part of this was moving the flows/async/ code into batch/ - which also makes sense because flows/async/ just "enqueued" tasks that would then be run by actions in batch/ It makes sense that the code that enqueues the tasks and the code that dequeues the tasks sit in the same library. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=228698761
This commit is contained in:
guyben
Ben McIlwain
parent
d0c73efac0
commit
898448b8a0
42 changed files with 419 additions and 423 deletions
169
java/google/registry/batch/AsyncTaskEnqueuer.java
Normal file
169
java/google/registry/batch/AsyncTaskEnqueuer.java
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.batch;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
|
||||
import com.google.appengine.api.taskqueue.Queue;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.appengine.api.taskqueue.TransientFailureException;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.ImmutableSortedSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.eppcommon.Trid;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.util.AppEngineServiceUtils;
|
||||
import google.registry.util.Retrier;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
/** Helper class to enqueue tasks for handling asynchronous operations in flows. */
|
||||
public final class AsyncTaskEnqueuer {
|
||||
|
||||
/** The HTTP parameter names used by async flows. */
|
||||
public static final String PARAM_RESOURCE_KEY = "resourceKey";
|
||||
public static final String PARAM_REQUESTING_CLIENT_ID = "requestingClientId";
|
||||
public static final String PARAM_CLIENT_TRANSACTION_ID = "clientTransactionId";
|
||||
public static final String PARAM_SERVER_TRANSACTION_ID = "serverTransactionId";
|
||||
public static final String PARAM_IS_SUPERUSER = "isSuperuser";
|
||||
public static final String PARAM_HOST_KEY = "hostKey";
|
||||
public static final String PARAM_REQUESTED_TIME = "requestedTime";
|
||||
public static final String PARAM_RESAVE_TIMES = "resaveTimes";
|
||||
|
||||
/** The task queue names used by async flows. */
|
||||
public static final String QUEUE_ASYNC_ACTIONS = "async-actions";
|
||||
public static final String QUEUE_ASYNC_DELETE = "async-delete-pull";
|
||||
public static final String QUEUE_ASYNC_HOST_RENAME = "async-host-rename-pull";
|
||||
|
||||
public static final String PATH_RESAVE_ENTITY = "/_dr/task/resaveEntity";
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final Duration MAX_ASYNC_ETA = Duration.standardDays(30);
|
||||
|
||||
private final Duration asyncDeleteDelay;
|
||||
private final Queue asyncActionsPushQueue;
|
||||
private final Queue asyncDeletePullQueue;
|
||||
private final Queue asyncDnsRefreshPullQueue;
|
||||
private final AppEngineServiceUtils appEngineServiceUtils;
|
||||
private final Retrier retrier;
|
||||
|
||||
@VisibleForTesting
|
||||
@Inject
|
||||
public AsyncTaskEnqueuer(
|
||||
@Named(QUEUE_ASYNC_ACTIONS) Queue asyncActionsPushQueue,
|
||||
@Named(QUEUE_ASYNC_DELETE) Queue asyncDeletePullQueue,
|
||||
@Named(QUEUE_ASYNC_HOST_RENAME) Queue asyncDnsRefreshPullQueue,
|
||||
@Config("asyncDeleteFlowMapreduceDelay") Duration asyncDeleteDelay,
|
||||
AppEngineServiceUtils appEngineServiceUtils,
|
||||
Retrier retrier) {
|
||||
this.asyncActionsPushQueue = asyncActionsPushQueue;
|
||||
this.asyncDeletePullQueue = asyncDeletePullQueue;
|
||||
this.asyncDnsRefreshPullQueue = asyncDnsRefreshPullQueue;
|
||||
this.asyncDeleteDelay = asyncDeleteDelay;
|
||||
this.appEngineServiceUtils = appEngineServiceUtils;
|
||||
this.retrier = retrier;
|
||||
}
|
||||
|
||||
/** Enqueues a task to asynchronously re-save an entity at some point in the future. */
|
||||
public void enqueueAsyncResave(
|
||||
ImmutableObject entityToResave, DateTime now, DateTime whenToResave) {
|
||||
enqueueAsyncResave(entityToResave, now, ImmutableSortedSet.of(whenToResave));
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueues a task to asynchronously re-save an entity at some point(s) in the future.
|
||||
*
|
||||
* <p>Multiple re-save times are chained one after the other, i.e. any given run will re-enqueue
|
||||
* itself to run at the next time if there are remaining re-saves scheduled.
|
||||
*/
|
||||
public void enqueueAsyncResave(
|
||||
ImmutableObject entityToResave, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
|
||||
DateTime firstResave = whenToResave.first();
|
||||
checkArgument(isBeforeOrAt(now, firstResave), "Can't enqueue a resave to run in the past");
|
||||
Key<ImmutableObject> entityKey = Key.create(entityToResave);
|
||||
Duration etaDuration = new Duration(now, firstResave);
|
||||
if (etaDuration.isLongerThan(MAX_ASYNC_ETA)) {
|
||||
logger.atInfo().log(
|
||||
"Ignoring async re-save of %s; %s is past the ETA threshold of %s.",
|
||||
entityKey, firstResave, MAX_ASYNC_ETA);
|
||||
return;
|
||||
}
|
||||
logger.atInfo().log("Enqueuing async re-save of %s to run at %s.", entityKey, whenToResave);
|
||||
String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
|
||||
TaskOptions task =
|
||||
TaskOptions.Builder.withUrl(PATH_RESAVE_ENTITY)
|
||||
.method(Method.POST)
|
||||
.header("Host", backendHostname)
|
||||
.countdownMillis(etaDuration.getMillis())
|
||||
.param(PARAM_RESOURCE_KEY, entityKey.getString())
|
||||
.param(PARAM_REQUESTED_TIME, now.toString());
|
||||
if (whenToResave.size() > 1) {
|
||||
task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
|
||||
}
|
||||
addTaskToQueueWithRetry(asyncActionsPushQueue, task);
|
||||
}
|
||||
|
||||
/** Enqueues a task to asynchronously delete a contact or host, by key. */
|
||||
public void enqueueAsyncDelete(
|
||||
EppResource resourceToDelete,
|
||||
DateTime now,
|
||||
String requestingClientId,
|
||||
Trid trid,
|
||||
boolean isSuperuser) {
|
||||
Key<EppResource> resourceKey = Key.create(resourceToDelete);
|
||||
logger.atInfo().log(
|
||||
"Enqueuing async deletion of %s on behalf of registrar %s.",
|
||||
resourceKey, requestingClientId);
|
||||
TaskOptions task =
|
||||
TaskOptions.Builder.withMethod(Method.PULL)
|
||||
.countdownMillis(asyncDeleteDelay.getMillis())
|
||||
.param(PARAM_RESOURCE_KEY, resourceKey.getString())
|
||||
.param(PARAM_REQUESTING_CLIENT_ID, requestingClientId)
|
||||
.param(PARAM_SERVER_TRANSACTION_ID, trid.getServerTransactionId())
|
||||
.param(PARAM_IS_SUPERUSER, Boolean.toString(isSuperuser))
|
||||
.param(PARAM_REQUESTED_TIME, now.toString());
|
||||
trid.getClientTransactionId()
|
||||
.ifPresent(clTrid -> task.param(PARAM_CLIENT_TRANSACTION_ID, clTrid));
|
||||
addTaskToQueueWithRetry(asyncDeletePullQueue, task);
|
||||
}
|
||||
|
||||
/** Enqueues a task to asynchronously refresh DNS for a renamed host. */
|
||||
public void enqueueAsyncDnsRefresh(HostResource host, DateTime now) {
|
||||
Key<HostResource> hostKey = Key.create(host);
|
||||
logger.atInfo().log("Enqueuing async DNS refresh for renamed host %s.", hostKey);
|
||||
addTaskToQueueWithRetry(
|
||||
asyncDnsRefreshPullQueue,
|
||||
TaskOptions.Builder.withMethod(Method.PULL)
|
||||
.param(PARAM_HOST_KEY, hostKey.getString())
|
||||
.param(PARAM_REQUESTED_TIME, now.toString()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a task to a queue with retrying, to avoid aborting the entire flow over a transient issue
|
||||
* enqueuing a task.
|
||||
*/
|
||||
private void addTaskToQueueWithRetry(final Queue queue, final TaskOptions task) {
|
||||
retrier.callWithRetry(() -> queue.add(task), TransientFailureException.class);
|
||||
}
|
||||
}
|
||||
165
java/google/registry/batch/AsyncTaskMetrics.java
Normal file
165
java/google/registry/batch/AsyncTaskMetrics.java
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.batch;
|
||||
|
||||
import static com.google.appengine.api.taskqueue.QueueConstants.maxLeaseCount;
|
||||
import static com.google.monitoring.metrics.EventMetric.DEFAULT_FITTER;
|
||||
import static google.registry.batch.AsyncTaskMetrics.OperationType.CONTACT_AND_HOST_DELETE;
|
||||
import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.monitoring.metrics.DistributionFitter;
|
||||
import com.google.monitoring.metrics.EventMetric;
|
||||
import com.google.monitoring.metrics.FibonacciFitter;
|
||||
import com.google.monitoring.metrics.IncrementableMetric;
|
||||
import com.google.monitoring.metrics.LabelDescriptor;
|
||||
import com.google.monitoring.metrics.MetricRegistryImpl;
|
||||
import google.registry.util.Clock;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
/**
|
||||
* Instrumentation for async flows (contact/host deletion and DNS refreshes).
|
||||
*
|
||||
* @see AsyncTaskEnqueuer
|
||||
*/
|
||||
public class AsyncTaskMetrics {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
private final Clock clock;
|
||||
|
||||
@Inject
|
||||
public AsyncTaskMetrics(Clock clock) {
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
/**
|
||||
* A Fibonacci fitter used for bucketing the batch count.
|
||||
*
|
||||
* <p>We use a Fibonacci filter because it provides better resolution at the low end than an
|
||||
* exponential fitter, which is important because most batch sizes are likely to be very low,
|
||||
* despite going up to 1,000 on the high end. Also, the precision is better, as batch size is
|
||||
* inherently an integer, whereas an exponential fitter with an exponent base less than 2 would
|
||||
* have unintuitive boundaries.
|
||||
*/
|
||||
private static final DistributionFitter FITTER_BATCH_SIZE =
|
||||
FibonacciFitter.create(maxLeaseCount());
|
||||
|
||||
private static final ImmutableSet<LabelDescriptor> LABEL_DESCRIPTORS =
|
||||
ImmutableSet.of(
|
||||
LabelDescriptor.create("operation_type", "The type of async flow operation."),
|
||||
LabelDescriptor.create("result", "The result of the async flow operation."));
|
||||
|
||||
@VisibleForTesting
|
||||
static final IncrementableMetric asyncFlowOperationCounts =
|
||||
MetricRegistryImpl.getDefault()
|
||||
.newIncrementableMetric(
|
||||
"/async_flows/operations",
|
||||
"Count of Async Flow Operations",
|
||||
"count",
|
||||
LABEL_DESCRIPTORS);
|
||||
|
||||
@VisibleForTesting
|
||||
static final EventMetric asyncFlowOperationProcessingTime =
|
||||
MetricRegistryImpl.getDefault()
|
||||
.newEventMetric(
|
||||
"/async_flows/processing_time",
|
||||
"Async Flow Processing Time",
|
||||
"milliseconds",
|
||||
LABEL_DESCRIPTORS,
|
||||
DEFAULT_FITTER);
|
||||
|
||||
@VisibleForTesting
|
||||
static final EventMetric asyncFlowBatchSize =
|
||||
MetricRegistryImpl.getDefault()
|
||||
.newEventMetric(
|
||||
"/async_flows/batch_size",
|
||||
"Async Operation Batch Size",
|
||||
"batch size",
|
||||
ImmutableSet.of(
|
||||
LabelDescriptor.create("operation_type", "The type of async flow operation.")),
|
||||
FITTER_BATCH_SIZE);
|
||||
|
||||
/** The type of asynchronous operation. */
|
||||
public enum OperationType {
|
||||
CONTACT_DELETE("contactDelete"),
|
||||
HOST_DELETE("hostDelete"),
|
||||
CONTACT_AND_HOST_DELETE("contactAndHostDelete"),
|
||||
DNS_REFRESH("dnsRefresh");
|
||||
|
||||
private final String metricLabelValue;
|
||||
|
||||
OperationType(String metricLabelValue) {
|
||||
this.metricLabelValue = metricLabelValue;
|
||||
}
|
||||
|
||||
String getMetricLabelValue() {
|
||||
return metricLabelValue;
|
||||
}
|
||||
}
|
||||
|
||||
/** The result of an asynchronous operation. */
|
||||
public enum OperationResult {
|
||||
/** The operation processed correctly and the result was success. */
|
||||
SUCCESS("success"),
|
||||
|
||||
/** The operation processed correctly and the result was failure. */
|
||||
FAILURE("failure"),
|
||||
|
||||
/** The operation did not process correctly due to some unexpected error. */
|
||||
ERROR("error"),
|
||||
|
||||
/** The operation was skipped because the request is now stale. */
|
||||
STALE("stale");
|
||||
|
||||
private final String metricLabelValue;
|
||||
|
||||
OperationResult(String metricLabelValue) {
|
||||
this.metricLabelValue = metricLabelValue;
|
||||
}
|
||||
|
||||
String getMetricLabelValue() {
|
||||
return metricLabelValue;
|
||||
}
|
||||
}
|
||||
|
||||
public void recordAsyncFlowResult(
|
||||
OperationType operationType, OperationResult operationResult, DateTime whenEnqueued) {
|
||||
asyncFlowOperationCounts.increment(
|
||||
operationType.getMetricLabelValue(), operationResult.getMetricLabelValue());
|
||||
long processingMillis = new Duration(whenEnqueued, clock.nowUtc()).getMillis();
|
||||
asyncFlowOperationProcessingTime.record(
|
||||
processingMillis,
|
||||
operationType.getMetricLabelValue(),
|
||||
operationResult.getMetricLabelValue());
|
||||
logger.atInfo().log(
|
||||
"Asynchronous %s operation took %d ms to process, yielding result: %s.",
|
||||
operationType.getMetricLabelValue(),
|
||||
processingMillis,
|
||||
operationResult.getMetricLabelValue());
|
||||
}
|
||||
|
||||
public void recordContactHostDeletionBatchSize(long batchSize) {
|
||||
asyncFlowBatchSize.record(batchSize, CONTACT_AND_HOST_DELETE.getMetricLabelValue());
|
||||
}
|
||||
|
||||
public void recordDnsRefreshBatchSize(long batchSize) {
|
||||
asyncFlowBatchSize.record(batchSize, DNS_REFRESH.getMetricLabelValue());
|
||||
}
|
||||
}
|
||||
|
|
@ -8,21 +8,16 @@ java_library(
|
|||
name = "batch",
|
||||
srcs = glob(["*.java"]),
|
||||
deps = [
|
||||
"//java/google/registry/bigquery",
|
||||
"//java/google/registry/config",
|
||||
"//java/google/registry/dns",
|
||||
"//java/google/registry/flows",
|
||||
"//java/google/registry/mapreduce",
|
||||
"//java/google/registry/mapreduce/inputs",
|
||||
"//java/google/registry/model",
|
||||
"//java/google/registry/monitoring/whitebox",
|
||||
"//java/google/registry/pricing",
|
||||
"//java/google/registry/request",
|
||||
"//java/google/registry/request:modules",
|
||||
"//java/google/registry/request/auth",
|
||||
"//java/google/registry/util",
|
||||
"//third_party/objectify:objectify-v4_1",
|
||||
"@com_google_apis_google_api_services_bigquery",
|
||||
"@com_google_appengine_api_1_0_sdk",
|
||||
"@com_google_appengine_tools_appengine_gcs_client",
|
||||
"@com_google_appengine_tools_appengine_mapreduce",
|
||||
|
|
@ -35,6 +30,7 @@ java_library(
|
|||
"@com_google_flogger_system_backend",
|
||||
"@com_google_guava",
|
||||
"@com_google_http_client",
|
||||
"@com_google_monitoring_client_metrics",
|
||||
"@javax_servlet_api",
|
||||
"@joda_time",
|
||||
"@org_joda_money",
|
||||
|
|
|
|||
|
|
@ -14,9 +14,13 @@
|
|||
|
||||
package google.registry.batch;
|
||||
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_REQUESTED_TIME;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESAVE_TIMES;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESOURCE_KEY;
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESAVE_TIMES;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_ACTIONS;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_DELETE;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
|
||||
import static google.registry.request.RequestParameters.extractOptionalBooleanParameter;
|
||||
import static google.registry.request.RequestParameters.extractOptionalIntParameter;
|
||||
import static google.registry.request.RequestParameters.extractOptionalParameter;
|
||||
|
|
@ -24,6 +28,7 @@ import static google.registry.request.RequestParameters.extractRequiredDatetimeP
|
|||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||
import static google.registry.request.RequestParameters.extractSetOfDatetimeParameters;
|
||||
|
||||
import com.google.appengine.api.taskqueue.Queue;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.googlecode.objectify.Key;
|
||||
import dagger.Module;
|
||||
|
|
@ -31,6 +36,7 @@ import dagger.Provides;
|
|||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.request.Parameter;
|
||||
import java.util.Optional;
|
||||
import javax.inject.Named;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
|
|
@ -87,4 +93,22 @@ public class BatchModule {
|
|||
static ImmutableSet<DateTime> provideResaveTimes(HttpServletRequest req) {
|
||||
return extractSetOfDatetimeParameters(req, PARAM_RESAVE_TIMES);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Named(QUEUE_ASYNC_ACTIONS)
|
||||
static Queue provideAsyncActionsPushQueue() {
|
||||
return getQueue(QUEUE_ASYNC_ACTIONS);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Named(QUEUE_ASYNC_DELETE)
|
||||
static Queue provideAsyncDeletePullQueue() {
|
||||
return getQueue(QUEUE_ASYNC_DELETE);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Named(QUEUE_ASYNC_HOST_RENAME)
|
||||
static Queue provideAsyncHostRenamePullQueue() {
|
||||
return getQueue(QUEUE_ASYNC_HOST_RENAME);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,18 +21,18 @@ import static com.google.common.base.Preconditions.checkState;
|
|||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.math.IntMath.divide;
|
||||
import static com.googlecode.objectify.Key.getKind;
|
||||
import static google.registry.flows.ResourceFlowUtils.denyPendingTransfer;
|
||||
import static google.registry.flows.ResourceFlowUtils.handlePendingTransferOnDelete;
|
||||
import static google.registry.flows.ResourceFlowUtils.updateForeignKeyIndexDeletionTime;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_CLIENT_TRANSACTION_ID;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_IS_SUPERUSER;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_REQUESTED_TIME;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_REQUESTING_CLIENT_ID;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESOURCE_KEY;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_SERVER_TRANSACTION_ID;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.QUEUE_ASYNC_DELETE;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_CLIENT_TRANSACTION_ID;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_IS_SUPERUSER;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTING_CLIENT_ID;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_SERVER_TRANSACTION_ID;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_DELETE;
|
||||
import static google.registry.model.EppResourceUtils.isActive;
|
||||
import static google.registry.model.EppResourceUtils.isDeleted;
|
||||
import static google.registry.model.ResourceTransferUtils.denyPendingTransfer;
|
||||
import static google.registry.model.ResourceTransferUtils.handlePendingTransferOnDelete;
|
||||
import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
|
||||
import static google.registry.model.eppcommon.StatusValue.PENDING_DELETE;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE;
|
||||
|
|
@ -63,11 +63,10 @@ import com.google.common.collect.Iterators;
|
|||
import com.google.common.collect.Multiset;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.batch.AsyncTaskMetrics.OperationResult;
|
||||
import google.registry.batch.AsyncTaskMetrics.OperationType;
|
||||
import google.registry.batch.DeleteContactsAndHostsAction.DeletionResult.Type;
|
||||
import google.registry.dns.DnsQueue;
|
||||
import google.registry.flows.async.AsyncFlowMetrics;
|
||||
import google.registry.flows.async.AsyncFlowMetrics.OperationResult;
|
||||
import google.registry.flows.async.AsyncFlowMetrics.OperationType;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.UnlockerOutput;
|
||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
||||
|
|
@ -124,7 +123,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
|
|||
private static final int MAX_REDUCE_SHARDS = 50;
|
||||
private static final int DELETES_PER_SHARD = 5;
|
||||
|
||||
@Inject AsyncFlowMetrics asyncFlowMetrics;
|
||||
@Inject AsyncTaskMetrics asyncTaskMetrics;
|
||||
@Inject Clock clock;
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject @Named(QUEUE_ASYNC_DELETE) Queue queue;
|
||||
|
|
@ -154,7 +153,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
|
|||
LeaseOptions.Builder.withCountLimit(maxLeaseCount())
|
||||
.leasePeriod(LEASE_LENGTH.getStandardSeconds(), SECONDS);
|
||||
List<TaskHandle> tasks = queue.leaseTasks(options);
|
||||
asyncFlowMetrics.recordContactHostDeletionBatchSize(tasks.size());
|
||||
asyncTaskMetrics.recordContactHostDeletionBatchSize(tasks.size());
|
||||
|
||||
// Check if there are no tasks to process, and if so, return early.
|
||||
if (tasks.isEmpty()) {
|
||||
|
|
@ -210,7 +209,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
|
|||
retrier.callWithRetry(() -> queue.deleteTask(tasks), TransientFailureException.class);
|
||||
deletionRequests.forEach(
|
||||
deletionRequest ->
|
||||
asyncFlowMetrics.recordAsyncFlowResult(
|
||||
asyncTaskMetrics.recordAsyncFlowResult(
|
||||
deletionRequest.getMetricOperationType(),
|
||||
OperationResult.STALE,
|
||||
deletionRequest.requestedTime()));
|
||||
|
|
@ -306,7 +305,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
|
|||
private static final DnsQueue dnsQueue = DnsQueue.create();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static AsyncFlowMetrics asyncFlowMetrics = new AsyncFlowMetrics(new SystemClock());
|
||||
private static AsyncTaskMetrics asyncTaskMetrics = new AsyncTaskMetrics(new SystemClock());
|
||||
|
||||
@Override
|
||||
public void reduce(final DeletionRequest deletionRequest, ReducerInput<Boolean> values) {
|
||||
|
|
@ -321,7 +320,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
|
|||
getQueue(QUEUE_ASYNC_DELETE).deleteTask(deletionRequest.task());
|
||||
return deletionResult;
|
||||
});
|
||||
asyncFlowMetrics.recordAsyncFlowResult(
|
||||
asyncTaskMetrics.recordAsyncFlowResult(
|
||||
deletionRequest.getMetricOperationType(),
|
||||
result.getMetricOperationResult(),
|
||||
deletionRequest.requestedTime());
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ package google.registry.batch;
|
|||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.flows.ResourceFlowUtils.updateForeignKeyIndexDeletionTime;
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.registry.Registries.getTldsOfType;
|
||||
import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_DELETE;
|
||||
|
|
|
|||
|
|
@ -18,10 +18,10 @@ import static com.google.appengine.api.taskqueue.QueueConstants.maxLeaseCount;
|
|||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_HOST_KEY;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_REQUESTED_TIME;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.QUEUE_ASYNC_HOST_RENAME;
|
||||
import static google.registry.flows.async.AsyncFlowMetrics.OperationType.DNS_REFRESH;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_HOST_KEY;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
|
||||
import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.EppResourceUtils.isActive;
|
||||
import static google.registry.model.EppResourceUtils.isDeleted;
|
||||
|
|
@ -46,9 +46,8 @@ import com.google.common.collect.ImmutableList;
|
|||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.batch.AsyncTaskMetrics.OperationResult;
|
||||
import google.registry.dns.DnsQueue;
|
||||
import google.registry.flows.async.AsyncFlowMetrics;
|
||||
import google.registry.flows.async.AsyncFlowMetrics.OperationResult;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.NullInput;
|
||||
import google.registry.model.domain.DomainResource;
|
||||
|
|
@ -83,7 +82,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
|
|||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final Duration LEASE_LENGTH = standardHours(4);
|
||||
|
||||
@Inject AsyncFlowMetrics asyncFlowMetrics;
|
||||
@Inject AsyncTaskMetrics asyncTaskMetrics;
|
||||
@Inject Clock clock;
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject @Named(QUEUE_ASYNC_HOST_RENAME) Queue pullQueue;
|
||||
|
|
@ -114,7 +113,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
|
|||
LeaseOptions.Builder.withCountLimit(maxLeaseCount())
|
||||
.leasePeriod(LEASE_LENGTH.getStandardSeconds(), SECONDS);
|
||||
List<TaskHandle> tasks = pullQueue.leaseTasks(options);
|
||||
asyncFlowMetrics.recordDnsRefreshBatchSize(tasks.size());
|
||||
asyncTaskMetrics.recordDnsRefreshBatchSize(tasks.size());
|
||||
|
||||
// Check if there are no tasks to process, and if so, return early.
|
||||
if (tasks.isEmpty()) {
|
||||
|
|
@ -146,7 +145,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
|
|||
}
|
||||
|
||||
deleteTasksWithRetry(
|
||||
requestsToDelete, pullQueue, asyncFlowMetrics, retrier, OperationResult.STALE);
|
||||
requestsToDelete, pullQueue, asyncTaskMetrics, retrier, OperationResult.STALE);
|
||||
ImmutableList<DnsRefreshRequest> refreshRequests = requestsBuilder.build();
|
||||
if (refreshRequests.isEmpty()) {
|
||||
logRespondAndUnlock(
|
||||
|
|
@ -245,7 +244,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
|
|||
private static final long serialVersionUID = 9077366205249562118L;
|
||||
|
||||
@NonFinalForTesting
|
||||
private static AsyncFlowMetrics asyncFlowMetrics = new AsyncFlowMetrics(new SystemClock());
|
||||
private static AsyncTaskMetrics asyncTaskMetrics = new AsyncTaskMetrics(new SystemClock());
|
||||
|
||||
private final Lock lock;
|
||||
private final Retrier retrier;
|
||||
|
|
@ -265,7 +264,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
|
|||
deleteTasksWithRetry(
|
||||
refreshRequests,
|
||||
getQueue(QUEUE_ASYNC_HOST_RENAME),
|
||||
asyncFlowMetrics,
|
||||
asyncTaskMetrics,
|
||||
retrier,
|
||||
OperationResult.SUCCESS);
|
||||
|
||||
|
|
@ -277,7 +276,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
|
|||
private static void deleteTasksWithRetry(
|
||||
final List<DnsRefreshRequest> refreshRequests,
|
||||
final Queue queue,
|
||||
AsyncFlowMetrics asyncFlowMetrics,
|
||||
AsyncTaskMetrics asyncTaskMetrics,
|
||||
Retrier retrier,
|
||||
OperationResult result) {
|
||||
if (refreshRequests.isEmpty()) {
|
||||
|
|
@ -287,7 +286,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
|
|||
refreshRequests.stream().map(DnsRefreshRequest::task).collect(toImmutableList());
|
||||
retrier.callWithRetry(() -> queue.deleteTask(tasks), TransientFailureException.class);
|
||||
refreshRequests.forEach(
|
||||
r -> asyncFlowMetrics.recordAsyncFlowResult(DNS_REFRESH, result, r.requestedTime()));
|
||||
r -> asyncTaskMetrics.recordAsyncFlowResult(DNS_REFRESH, result, r.requestedTime()));
|
||||
}
|
||||
|
||||
/** A class that encapsulates the values of a request to refresh DNS for a renamed host. */
|
||||
|
|
|
|||
|
|
@ -14,16 +14,15 @@
|
|||
|
||||
package google.registry.batch;
|
||||
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_REQUESTED_TIME;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESAVE_TIMES;
|
||||
import static google.registry.flows.async.AsyncFlowEnqueuer.PARAM_RESOURCE_KEY;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESAVE_TIMES;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSortedSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.flows.async.AsyncFlowEnqueuer;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.request.Action;
|
||||
|
|
@ -39,15 +38,17 @@ import org.joda.time.DateTime;
|
|||
*
|
||||
* <p>{@link EppResource}s will be projected forward to the current time.
|
||||
*/
|
||||
@Action(path = "/_dr/task/resaveEntity", auth = Auth.AUTH_INTERNAL_OR_ADMIN, method = Method.POST)
|
||||
@Action(path = ResaveEntityAction.PATH, auth = Auth.AUTH_INTERNAL_OR_ADMIN, method = Method.POST)
|
||||
public class ResaveEntityAction implements Runnable {
|
||||
|
||||
public static final String PATH = "/_dr/task/resaveEntity";
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
private final Key<ImmutableObject> resourceKey;
|
||||
private final DateTime requestedTime;
|
||||
private final ImmutableSortedSet<DateTime> resaveTimes;
|
||||
private final AsyncFlowEnqueuer asyncFlowEnqueuer;
|
||||
private final AsyncTaskEnqueuer asyncTaskEnqueuer;
|
||||
private final Response response;
|
||||
|
||||
@Inject
|
||||
|
|
@ -55,12 +56,12 @@ public class ResaveEntityAction implements Runnable {
|
|||
@Parameter(PARAM_RESOURCE_KEY) Key<ImmutableObject> resourceKey,
|
||||
@Parameter(PARAM_REQUESTED_TIME) DateTime requestedTime,
|
||||
@Parameter(PARAM_RESAVE_TIMES) ImmutableSet<DateTime> resaveTimes,
|
||||
AsyncFlowEnqueuer asyncFlowEnqueuer,
|
||||
AsyncTaskEnqueuer asyncTaskEnqueuer,
|
||||
Response response) {
|
||||
this.resourceKey = resourceKey;
|
||||
this.requestedTime = requestedTime;
|
||||
this.resaveTimes = ImmutableSortedSet.copyOf(resaveTimes);
|
||||
this.asyncFlowEnqueuer = asyncFlowEnqueuer;
|
||||
this.asyncTaskEnqueuer = asyncTaskEnqueuer;
|
||||
this.response = response;
|
||||
}
|
||||
|
||||
|
|
@ -75,7 +76,7 @@ public class ResaveEntityAction implements Runnable {
|
|||
? ((EppResource) entity).cloneProjectedAtTime(ofy().getTransactionTime()) : entity
|
||||
);
|
||||
if (!resaveTimes.isEmpty()) {
|
||||
asyncFlowEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
|
||||
asyncTaskEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
|
||||
}
|
||||
});
|
||||
response.setPayload("Entity re-saved.");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue