mirror of
https://github.com/google/nomulus.git
synced 2025-05-30 17:24:03 +02:00
Chain ICANN report upload after staging
This converts the upload task from a cron job to a task chained after staging. This ensures the upload job only occurs when its dependencies are met, and provides a faster turnaround time to verify both the staging and upload jobs are complete. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=175045489
This commit is contained in:
larryruili
jianglai
parent
a6f0ab1429
commit
8dcc2d6833
3 changed files with 43 additions and 10 deletions
|
|
@ -256,18 +256,9 @@
|
|||
<description>
|
||||
Create ICANN activity and transaction reports for last month, storing them in
|
||||
gs://domain-registry-alpha-reporting/icann/monthly/yyyy-MM
|
||||
Upon success, enqueues the icannReportingUpload task to POST these files to ICANN.
|
||||
</description>
|
||||
<schedule>2 of month 09:00</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/icannReportingUpload&runInEmpty]]></url>
|
||||
<description>
|
||||
Upload ICANN activity and transaction reports for last month to ICANN, fetching them from
|
||||
gs://domain-registry-alpha-reporting/icann/monthly/yyyy-MM
|
||||
</description>
|
||||
<schedule>4 of month 09:00</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
</cronentries>
|
||||
|
|
|
|||
|
|
@ -18,17 +18,24 @@ import static google.registry.request.Action.Method.POST;
|
|||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.bigquery.BigqueryJobFailureException;
|
||||
import google.registry.reporting.IcannReportingModule.ReportType;
|
||||
import google.registry.reporting.IcannReportingModule.ReportingSubdir;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.FormattingLogger;
|
||||
import google.registry.util.Retrier;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.Duration;
|
||||
import org.joda.time.YearMonth;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
|
||||
/**
|
||||
* Action that generates monthly ICANN activity and transactions reports.
|
||||
|
|
@ -54,7 +61,10 @@ public final class IcannReportingStagingAction implements Runnable {
|
|||
static final String PATH = "/_dr/task/icannReportingStaging";
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
private static final String CRON_QUEUE = "retryable-cron-tasks";
|
||||
|
||||
@Inject YearMonth yearMonth;
|
||||
@Inject @ReportingSubdir String subdir;
|
||||
@Inject ImmutableList<ReportType> reportTypes;
|
||||
@Inject IcannReportingStager stager;
|
||||
@Inject Retrier retrier;
|
||||
|
|
@ -83,6 +93,16 @@ public final class IcannReportingStagingAction implements Runnable {
|
|||
response.setStatus(SC_OK);
|
||||
response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
|
||||
response.setPayload("Completed staging action.");
|
||||
|
||||
logger.infofmt("Enqueueing report upload :");
|
||||
TaskOptions uploadTask = TaskOptions.Builder.withUrl(IcannReportingUploadAction.PATH)
|
||||
.method(Method.POST)
|
||||
.countdownMillis(Duration.standardMinutes(2).getMillis())
|
||||
.param(
|
||||
IcannReportingModule.PARAM_YEAR_MONTH,
|
||||
DateTimeFormat.forPattern("yyyy-MM").print(yearMonth))
|
||||
.param(IcannReportingModule.PARAM_SUBDIR, subdir);
|
||||
QueueFactory.getQueue(CRON_QUEUE).add(uploadTask);
|
||||
return null;
|
||||
},
|
||||
new Retrier.FailureReporter() {
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ package google.registry.reporting;
|
|||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth.assertWithMessage;
|
||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
|
@ -28,7 +30,9 @@ import google.registry.testing.AppEngineRule;
|
|||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import google.registry.testing.FakeSleeper;
|
||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||
import google.registry.util.Retrier;
|
||||
import org.joda.time.YearMonth;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
|
|
@ -49,6 +53,7 @@ public class IcannReportingStagingActionTest {
|
|||
public final AppEngineRule appEngine = AppEngineRule.builder()
|
||||
.withDatastore()
|
||||
.withLocalModules()
|
||||
.withTaskQueue()
|
||||
.build();
|
||||
|
||||
@Before
|
||||
|
|
@ -57,8 +62,20 @@ public class IcannReportingStagingActionTest {
|
|||
when(stager.stageReports(ReportType.TRANSACTIONS)).thenReturn(ImmutableList.of("c", "d"));
|
||||
}
|
||||
|
||||
private static void assertUploadTaskEnqueued(String yearMonth, String subDir) throws Exception {
|
||||
TaskMatcher matcher =
|
||||
new TaskMatcher()
|
||||
.url("/_dr/task/icannReportingUpload")
|
||||
.method("POST")
|
||||
.param("yearMonth", yearMonth)
|
||||
.param("subdir", subDir);
|
||||
assertTasksEnqueued("retryable-cron-tasks", matcher);
|
||||
}
|
||||
|
||||
private IcannReportingStagingAction createAction(ImmutableList<ReportType> reportingMode) {
|
||||
IcannReportingStagingAction action = new IcannReportingStagingAction();
|
||||
action.yearMonth = new YearMonth(2017, 6);
|
||||
action.subdir = "default/dir";
|
||||
action.reportTypes = reportingMode;
|
||||
action.response = response;
|
||||
action.stager = stager;
|
||||
|
|
@ -77,6 +94,7 @@ public class IcannReportingStagingActionTest {
|
|||
.emailResults(
|
||||
"ICANN Monthly report staging summary [SUCCESS]",
|
||||
"Completed staging the following 2 ICANN reports:\na\nb");
|
||||
assertUploadTaskEnqueued("2017-06", "default/dir");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -91,6 +109,7 @@ public class IcannReportingStagingActionTest {
|
|||
.emailResults(
|
||||
"ICANN Monthly report staging summary [SUCCESS]",
|
||||
"Completed staging the following 4 ICANN reports:\na\nb\nc\nd");
|
||||
assertUploadTaskEnqueued("2017-06", "default/dir");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -108,6 +127,7 @@ public class IcannReportingStagingActionTest {
|
|||
.emailResults(
|
||||
"ICANN Monthly report staging summary [SUCCESS]",
|
||||
"Completed staging the following 4 ICANN reports:\na\nb\nc\nd");
|
||||
assertUploadTaskEnqueued("2017-06", "default/dir");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -129,5 +149,7 @@ public class IcannReportingStagingActionTest {
|
|||
"ICANN Monthly report staging summary [FAILURE]",
|
||||
"Staging failed due to BigqueryJobFailureException: Expected failure,"
|
||||
+ " check logs for more details.");
|
||||
// Assert no upload task enqueued
|
||||
assertNoTasksEnqueued("retryable-cron-tasks");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue