Add mapreduce cleanup action

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=149432516
This commit is contained in:
mountford 2017-03-07 10:13:07 -08:00 committed by Ben McIlwain
parent bd7db61606
commit 5d4287a375
11 changed files with 1171 additions and 2 deletions

View file

@ -25,6 +25,7 @@ java_library(
"@com_google_appengine_api_1_0_sdk",
"@com_google_appengine_tools_appengine_gcs_client",
"@com_google_appengine_tools_appengine_mapreduce",
"@com_google_appengine_tools_appengine_pipeline",
"@com_google_auto_factory",
"@com_google_auto_value",
"@com_google_code_findbugs_jsr305",

View file

@ -14,12 +14,19 @@
package google.registry.batch;
import static google.registry.request.RequestParameters.extractOptionalBooleanParameter;
import static google.registry.request.RequestParameters.extractOptionalIntParameter;
import static google.registry.request.RequestParameters.extractOptionalParameter;
import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import dagger.Module;
import dagger.Provides;
import dagger.multibindings.IntoMap;
import dagger.multibindings.StringKey;
import google.registry.request.Parameter;
import javax.servlet.http.HttpServletRequest;
/**
* Dagger module for injecting common settings for batch actions.
@ -33,4 +40,34 @@ public class BatchModule {
static ImmutableList<TableFieldSchema> provideEntityIntegrityAlertsSchema() {
return EntityIntegrityAlertsSchema.SCHEMA_FIELDS;
}
@Provides
@Parameter("jobName")
static Optional<String> provideJobName(HttpServletRequest req) {
return extractOptionalParameter(req, "jobName");
}
@Provides
@Parameter("jobId")
static Optional<String> provideJobId(HttpServletRequest req) {
return extractOptionalParameter(req, "jobId");
}
@Provides
@Parameter("numJobsToDelete")
static Optional<Integer> provideNumJobsToDelete(HttpServletRequest req) {
return extractOptionalIntParameter(req, "numJobsToDelete");
}
@Provides
@Parameter("daysOld")
static Optional<Integer> provideDaysOld(HttpServletRequest req) {
return extractOptionalIntParameter(req, "daysOld");
}
@Provides
@Parameter("force")
static Optional<Boolean> provideForce(HttpServletRequest req) {
return extractOptionalBooleanParameter(req, "force");
}
}

View file

@ -0,0 +1,236 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.batch;
import com.google.appengine.api.datastore.DatastoreService;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableSet;
import com.google.common.net.MediaType;
import google.registry.batch.MapreduceEntityCleanupUtil.EligibleJobResults;
import google.registry.mapreduce.MapreduceRunner;
import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.util.Clock;
import google.registry.util.FormattingLogger;
import java.util.Set;
import javax.inject.Inject;
import org.joda.time.DateTime;
/**
* Action to delete entities associated with the App Engine Mapreduce library.
*
* <p>To delete a specific job, set the jobId parameter. To delete all jobs with a specific job name
* which are older than the specified age, set the jobName parameter. Otherwise, all jobs older than
* the specified age are deleted. Examples:
*
* <ul>
* <li>jobId=12345: delete only the root pipeline job with ID 12345, and all descendant jobs
* <li>jobName=Generate+Important+Files: delete all root pipeline jobs with the display name
* "Generate Important Files" (subject to the limits imposed by the daysOld and numJobsToDelete
* parameters), and all descendant jobs
* <li>(neither specified): delete all jobs (subject to the limits imposed by the daysOld and
* numJobsToDelete parameters)
* </ul>
*
* <p>More about display names: The pipeline library assigns each root pipeline job a "display
* name". You can see the display name of each job using the pipeline Web interface, available at
* /_ah/pipeline/list, where the display name column is confusingly labeled "Class Path". Usually,
* the display name is set to a fixed value by the mapreduce code. For instance, when a pipeline job
* is created by the {@link MapreduceRunner} class, the display name is set by the
* {@link MapreduceRunner#setJobName} method. When formulating a URL to invoke {@link
* MapreduceEntityCleanupAction}, the display name must of course be URL-encoded -- spaces are
* replaced by the plus sign, and so forth. For more information, see <a
* href="https://en.wikipedia.org/wiki/Percent-encoding">the Wikipedia article on percent
* encoding.</a>
*
* <p>The daysOld parameter specifies the minimum allowable age of a job in days for it to be
* eligible for deletion. Jobs will not be deleted if they are newer than this threshold, unless
* specifically named using the jobId parameter.
*
* <p>The numJobsToDelete parameter specifies the maximum number of jobs to delete. If this is fewer
* than would ordinarily be deleted, the jobs to be deleted are chosen arbitrarily.
*
* <p>The force parameter, if present and true, indicates that jobs should be deleted even if they
* are not in FINALIZED or STOPPED state.
*/
@Action(path = "/_dr/task/mapreduceEntityCleanup")
public class MapreduceEntityCleanupAction implements Runnable {
private static final int DEFAULT_DAYS_OLD = 180;
private static final int DEFAULT_MAX_NUM_JOBS_TO_DELETE = 5;
private static final String ERROR_BOTH_JOB_ID_AND_NAME =
"Do not specify both a job ID and a job name";
private static final String ERROR_BOTH_JOB_ID_AND_NUMBER_OF_JOBS =
"Do not specify both a job ID and a number of jobs to delete";
private static final String ERROR_BOTH_JOB_ID_AND_DAYS_OLD =
"Do not specify both a job ID and a days old threshold";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
private final Optional<String> jobId;
private final Optional<String> jobName;
private final Optional<Integer> numJobsToDelete;
private final Optional<Integer> daysOld;
private final Optional<Boolean> force;
private final MapreduceEntityCleanupUtil mapreduceEntityCleanupUtil;
private final Clock clock;
private final DatastoreService datastore;
private final Response response;
@Inject
MapreduceEntityCleanupAction(
@Parameter("jobId") Optional<String> jobId,
@Parameter("jobName") Optional<String> jobName,
@Parameter("numJobsToDelete") Optional<Integer> numJobsToDelete,
@Parameter("daysOld") Optional<Integer> daysOld,
@Parameter("force") Optional<Boolean> force,
MapreduceEntityCleanupUtil mapreduceEntityCleanupUtil,
Clock clock,
DatastoreService datastore,
Response response) {
this.jobId = jobId;
this.jobName = jobName;
this.numJobsToDelete = numJobsToDelete;
this.daysOld = daysOld;
this.force = force;
this.mapreduceEntityCleanupUtil = mapreduceEntityCleanupUtil;
this.clock = clock;
this.datastore = datastore;
this.response = response;
}
@Override
public void run() {
response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
if (jobId.isPresent()) {
runWithJobId();
} else {
runWithoutJobId();
}
}
private void logSevereAndSetPayload(String message) {
logger.severe(message);
response.setPayload(message);
}
/** Delete the job with the specified job ID, checking for conflicting parameters. */
private void runWithJobId() {
if (jobName.isPresent()) {
logSevereAndSetPayload(ERROR_BOTH_JOB_ID_AND_NAME);
return;
}
if (numJobsToDelete.isPresent()) {
logSevereAndSetPayload(ERROR_BOTH_JOB_ID_AND_NUMBER_OF_JOBS);
return;
}
if (daysOld.isPresent()) {
logSevereAndSetPayload(ERROR_BOTH_JOB_ID_AND_DAYS_OLD);
return;
}
response.setPayload(requestDeletion(ImmutableSet.of(jobId.get()), true /* generatePayload */));
}
/**
* Delete jobs with a matching display name, or all jobs if no name is specified. Only pick jobs
* which are old enough.
*/
private void runWithoutJobId() {
int defaultedDaysOld = daysOld.or(DEFAULT_DAYS_OLD);
// Only generate the detailed response payload if there aren't too many jobs involved.
boolean generatePayload =
numJobsToDelete.isPresent() && (numJobsToDelete.get() <= DEFAULT_MAX_NUM_JOBS_TO_DELETE);
Optional<StringBuilder> payloadBuilder =
generatePayload ? Optional.of(new StringBuilder()) : Optional.<StringBuilder>absent();
String defaultPayload = "done";
// Since findEligibleJobsByJobName returns only a certain number of jobs, we must loop through
// until we find enough, requesting deletion as we go. We also stop if we don't find anything,
// or if there are no more jobs to be found (because no cursor is returned).
int numJobsDeletedSoFar = 0;
boolean isFirstTime = true;
Optional<String> cursor = Optional.<String>absent();
DateTime cutoffDate = clock.nowUtc().minusDays(defaultedDaysOld);
while ((isFirstTime || cursor.isPresent())
&& (!numJobsToDelete.isPresent() || (numJobsDeletedSoFar < numJobsToDelete.get()))) {
isFirstTime = false;
EligibleJobResults eligibleJobResults =
mapreduceEntityCleanupUtil.findEligibleJobsByJobName(
jobName.orNull(), cutoffDate, numJobsToDelete, force.or(false), cursor);
cursor = eligibleJobResults.cursor();
if (eligibleJobResults.eligibleJobs().isEmpty()) {
logger.infofmt(
"No eligible job with name '%s' older than %s days old.",
jobName.or("(null)"), defaultedDaysOld);
if (generatePayload) {
payloadBuilder.get().append("No eligible job.");
}
defaultPayload = "No eligible job.";
} else {
String payloadChunk = requestDeletion(eligibleJobResults.eligibleJobs(), generatePayload);
if (generatePayload) {
payloadBuilder.get().append(payloadChunk);
}
numJobsDeletedSoFar += eligibleJobResults.eligibleJobs().size();
}
}
logger.infofmt("A total of %s job(s) processed", numJobsDeletedSoFar);
if (generatePayload) {
payloadBuilder
.get()
.append(String.format("A total of %d job(s) processed\n", numJobsDeletedSoFar));
response.setPayload(payloadBuilder.get().toString());
} else {
response.setPayload(defaultPayload);
}
}
private String requestDeletion(Set<String> actualJobIds, boolean generatePayload) {
Optional<StringBuilder> payloadChunkBuilder =
generatePayload ? Optional.of(new StringBuilder()) : Optional.<StringBuilder>absent();
int errorCount = 0;
for (String actualJobId : actualJobIds) {
Optional<String> error =
mapreduceEntityCleanupUtil.deleteJobAsync(datastore, actualJobId, force.or(false));
if (error.isPresent()) {
errorCount++;
}
logger.infofmt("%s: %s", actualJobId, error.or("deletion requested"));
if (payloadChunkBuilder.isPresent()) {
payloadChunkBuilder
.get()
.append(String.format("%s: %s\n", actualJobId, error.or("deletion requested")));
}
}
logger.infofmt(
"successfully requested async deletion of %s job(s); errors received on %s",
actualJobIds.size() - errorCount,
errorCount);
if (payloadChunkBuilder.isPresent()) {
payloadChunkBuilder.get().append(String.format(
"successfully requested async deletion of %d job(s); errors received on %d\n",
actualJobIds.size() - errorCount,
errorCount));
return payloadChunkBuilder.get().toString();
} else {
return "";
}
}
}

View file

@ -0,0 +1,210 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.batch;
import com.google.appengine.api.datastore.BaseDatastoreService;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobServiceFactory;
import com.google.appengine.tools.pipeline.NoSuchObjectException;
import com.google.appengine.tools.pipeline.impl.PipelineManager;
import com.google.appengine.tools.pipeline.impl.model.JobRecord;
import com.google.appengine.tools.pipeline.util.Pair;
import com.google.auto.value.AutoValue;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableSet;
import java.util.HashSet;
import java.util.Set;
import javax.annotation.Nullable;
import javax.inject.Inject;
import org.joda.time.DateTime;
/** Utilities used in mapreduce datastore entity cleanup. */
class MapreduceEntityCleanupUtil {
/** Number of jobs to fetch at a time using PipelineManager.queryRootPipelines. */
protected static final int MAX_NUMBER_OF_JOBS_PER_SEARCH = 100;
private static final ImmutableSet<String> JOB_PREFIXES =
ImmutableSet.of("", "map-", "sort-", "merge-", "reduce-");
@Inject
MapreduceEntityCleanupUtil() {}
/** Return value from {@link #findEligibleJobsByJobName} */
@AutoValue
abstract static class EligibleJobResults {
static EligibleJobResults create(ImmutableSet<String> jobs, Optional<String> cursor) {
return new AutoValue_MapreduceEntityCleanupUtil_EligibleJobResults(jobs, cursor);
}
abstract ImmutableSet<String> eligibleJobs();
abstract Optional<String> cursor();
}
/**
* Returns the maximum number of jobs to return per search request.
*
* <p>This method is present to allow overriding by test subclasses.
*/
protected int getMaxNumberOfJobsPerSearch() {
return MAX_NUMBER_OF_JOBS_PER_SEARCH;
}
/**
* Finds the requested number of root pipeline jobs eligible for deletion.
*
* <p>Loops through the root jobs returned by the pipeline API, searching for those with a
* matching name in an appropriate state, and older than the specified cutoff date.
*
* <p>Regardless of the setting of maxJobs, a maximum of {@link
* #MAX_NUMBER_OF_JOBS_PER_SEARCH} will be returned. If there might be more jobs available to
* find, a cursor will be returned, which can be used in a subsequent call to {@link
* #findEligibleJobsByJobName} to continue the search.
*
* @param jobName the desired job name; if null, all jobs are considered to match
* @param cutoffDate eligible jobs must have both startTime and endTime before cutoffDate; if
* startTime and/or endTime are null, they are considered to be old enough -- this is because
* many jobs do lack at least one of these, and we don't want such jobs to stick around
* forever and not get deleted
* @param maxJobs the maximum number of jobs to return; if absent, return all eligible jobs (see
* note above about {@link #MAX_NUMBER_OF_JOBS_PER_SEARCH})
* @param ignoreState if true, jobs will be included regardless of the state
* @param cursor if present, a cursor returned from a previous call to the method; the search will
* be picked up where it left off
* @return job IDs of the eligible jobs
*/
EligibleJobResults findEligibleJobsByJobName(
@Nullable String jobName,
DateTime cutoffDate,
Optional<Integer> maxJobs,
boolean ignoreState,
Optional<String> cursor) {
if (maxJobs.isPresent() && (maxJobs.get() <= 0)) {
return EligibleJobResults.create(ImmutableSet.<String>of(), Optional.<String>absent());
}
Set<String> eligibleJobs = new HashSet<>();
Pair<? extends Iterable<JobRecord>, String> pair =
PipelineManager.queryRootPipelines(jobName, cursor.orNull(), getMaxNumberOfJobsPerSearch());
for (JobRecord jobRecord : pair.getFirst()) {
if (((jobRecord.getStartTime() == null)
|| jobRecord.getStartTime().before(cutoffDate.toDate()))
&& ((jobRecord.getEndTime() == null)
|| jobRecord.getEndTime().before(cutoffDate.toDate()))
&& (ignoreState
|| (jobRecord.getState() == JobRecord.State.FINALIZED)
|| (jobRecord.getState() == JobRecord.State.STOPPED))) {
eligibleJobs.add(jobRecord.getRootJobKey().getName());
if (maxJobs.isPresent() && (eligibleJobs.size() >= maxJobs.get())) {
return EligibleJobResults.create(
ImmutableSet.copyOf(eligibleJobs), Optional.<String>absent());
}
}
}
return EligibleJobResults.create(
ImmutableSet.copyOf(eligibleJobs), Optional.fromNullable(pair.getSecond()));
}
/**
* Requests asynchronous deletion of entities associated with the specified job ID.
*
* <p>The mapreduce API is used to delete the MR-* entities, and the pipeline API is used to
* delete the main job records. No attempt is made to check whether the deletion succeeds, only
* whether it appeared to be a valid deletion request up front.
*
* @param datastore The datastore service, which can be either synchronous or asynchronous, since
* the only interaction with the database is via prepared queries
* @param jobId the root pipeline job ID to be deleted; if the jobId does not exist, the deletion
* will be apparently successful, because the underlying library routines do not complain
* @param force passed to the pipeline API, indicating whether jobs should be forcibly deleted
* even if they are not in a completed state; however, there is no force flag on the mapreduce
* API call, meaning that running jobs cannot be deleted
* @return an error string, or absent if no error was detected
*/
Optional<String> deleteJobAsync(
BaseDatastoreService datastore, String jobId, boolean force) {
// Try to delete the MR-* entities. This is always done asynchronously. A return value of false
// indicates that the job is in RUNNING state, and nothing has been done.
// TODO(mountford) check the state of all sharded jobs before deleting any
for (String mrShardedJobId : getPossibleIdsForPipelineJob(datastore, jobId)) {
if (!ShardedJobServiceFactory.getShardedJobService().cleanupJob(mrShardedJobId)) {
return Optional.of(String.format("Skipping; job %s is in running state", mrShardedJobId));
}
}
// If we are successful (meaning, MR-* entity deletion has been kicked off asynchronously),
// delete the pipeline-* entities as well.
try {
PipelineManager.deletePipelineRecords(jobId, force, true /* async */);
return Optional.absent();
} catch (NoSuchObjectException ex) {
return Optional.of("No such pipeline job");
} catch (IllegalStateException ex) {
return Optional.of("Job is not in FINALIZED or STOPPED state");
}
}
/**
* Returns the possible MR-ShardedJob IDs associated with the specified pipeline job and any child
* jobs.
*
* @param datastore The datastore service, which can be either synchronous or asynchronous, since
* the only interaction with the database is via prepared queries
* @param jobId The pipeline job ID
* @return the IDs of MR-ShardedJob entities that the Mapreduce library might have created,
* depending on which steps of the mapreduce were used
*/
private ImmutableSet<String> getPossibleIdsForPipelineJob(
BaseDatastoreService datastore, String jobId) {
return getPossibleIdsForPipelineJobRecur(datastore, jobId, new HashSet<String>());
}
/**
* Called by getPossibleIdsForPipelineJob(), and by itself recursively.
*
* @param datastore The datastore service, which can be either synchronous or asynchronous, since
* the only interaction with the database is via prepared queries
* @param jobId The pipeline job ID
* @param handledJobIds The set of job IDs which have been handled so far; this is a sanity check
* to prevent an infinite loop if, for some crazy reason, the job dependency graph is cyclic
* @return the IDs of MR-ShardedJob entities that the Mapreduce library might have created,
* depending on which steps of the mapreduce were used
*/
private ImmutableSet<String> getPossibleIdsForPipelineJobRecur(
BaseDatastoreService datastore, String jobId, Set<String> handledJobIds) {
if (handledJobIds.contains(jobId)) {
return ImmutableSet.<String>of();
}
handledJobIds.add(jobId);
JobRecord jobRecord;
try {
jobRecord = PipelineManager.getJob(jobId);
} catch (NoSuchObjectException e) {
return ImmutableSet.<String>of();
}
ImmutableSet.Builder<String> idSetBuilder = new ImmutableSet.Builder<>();
for (String jobPrefix : JOB_PREFIXES) {
idSetBuilder.add("MR-ShardedJob", jobPrefix + jobId);
}
for (Key childKey : jobRecord.getChildKeys()) {
idSetBuilder
.addAll(getPossibleIdsForPipelineJobRecur(datastore, childKey.getName(), handledJobIds));
}
return idSetBuilder.build();
}
}

View file

@ -143,6 +143,12 @@
<url-pattern>/_dr/task/pollBigqueryJob</url-pattern>
</servlet-mapping>
<!-- Cleans up old mapreduce entities. -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>
<url-pattern>/_dr/task/mapreduceEntityCleanup</url-pattern>
</servlet-mapping>
<!-- Fans out a cron task over an adjustable range of TLDs. -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>

View file

@ -25,6 +25,7 @@ import google.registry.batch.BatchModule;
import google.registry.batch.DeleteContactsAndHostsAction;
import google.registry.batch.DeleteProberDataAction;
import google.registry.batch.ExpandRecurringBillingEventsAction;
import google.registry.batch.MapreduceEntityCleanupAction;
import google.registry.batch.RefreshDnsOnHostRenameAction;
import google.registry.batch.VerifyEntityIntegrityAction;
import google.registry.cron.CommitLogFanoutAction;
@ -110,6 +111,7 @@ interface BackendRequestComponent {
ExportReservedTermsAction exportReservedTermsAction();
ExportSnapshotAction exportSnapshotAction();
LoadSnapshotAction loadSnapshotAction();
MapreduceEntityCleanupAction mapreduceEntityCleanupAction();
MetricsExportAction metricsExportAction();
NordnUploadAction nordnUploadAction();
NordnVerifyAction nordnVerifyAction();

View file

@ -114,6 +114,19 @@ public final class RequestParameters {
}
}
/**
* Returns first GET or POST parameter associated with {@code name} as a boolean.
*
* @throws BadRequestException if request parameter is present but not a valid boolean
*/
public static Optional<Boolean> extractOptionalBooleanParameter(
HttpServletRequest req, String name) {
String stringParam = req.getParameter(name);
return isNullOrEmpty(stringParam)
? Optional.<Boolean>absent()
: Optional.of(Boolean.valueOf(stringParam));
}
/**
* Returns {@code true} if parameter is present and not empty and not {@code "false"}.
*