zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-07-24 03:30:46 +02:00
Code Issues Projects Releases Packages Wiki Activity

Batch NORDN pull queue task deletions

Browse source 
They were failing because the maximum App Engine task batch size is 1,000, and
we currently have more than 4,000 tasks in the pull queue. We keep re-uploading
those to NORDN because we're unable to delete the tasks after successful upload,
so the leases expire and they get processed again.

Also renames TaskEnqueuer to TaskQueueUtils to reflect its newly expanded role.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=197060903
This commit is contained in:
mcilwain 2018-05-17 15:22:34 -07:00 • committed by jianglai
parent 1248a7722b
commit c989911526
21 changed files with 129 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

31
java/google/registry/util/TaskEnqueuer.java → java/google/registry/util/TaskQueueUtils.java
View file

@ -1,4 +1,4 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -20,12 +20,14 @@ import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.appengine.api.taskqueue.TransientFailureException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import java.io.Serializable;
import java.util.List;
import javax.inject.Inject;
/** Utilities for dealing with App Engine task queues. */
public class TaskEnqueuer implements Serializable {
public class TaskQueueUtils implements Serializable {
private static final long serialVersionUID = 7893211200220508362L;
@ -34,10 +36,23 @@ public class TaskEnqueuer implements Serializable {
private final Retrier retrier;
@Inject
public TaskEnqueuer(Retrier retrier) {
public TaskQueueUtils(Retrier retrier) {
this.retrier = retrier;
}
@NonFinalForTesting
@VisibleForTesting
static int BATCH_SIZE = 1000;
/**
* The batch size to use for App Engine task queue operations.
*
* <p>Note that 1,000 is currently the maximum allowable batch size in App Engine.
*/
public static int getBatchSize() {
return BATCH_SIZE;
}
/**
* Adds a task to a App Engine task queue in a reliable manner.
*
@ -73,4 +88,14 @@ public class TaskEnqueuer implements Serializable {
},
TransientFailureException.class);
}
/** Deletes the specified tasks from the queue in batches, with retrying. */
public void deleteTasks(Queue queue, List<TaskHandle> tasks) {
Lists.partition(tasks, BATCH_SIZE)
.stream()
.forEach(
batch ->
retrier.callWithRetry(
() -> queue.deleteTask(batch), TransientFailureException.class));
}
}