Fix IcannReportingUploadAction to use correct year in January (#452)

* Fix IcannReportingUploadAction to use correct year in January

* small fixes
This commit is contained in:
sarahcaseybot 2020-01-27 17:38:59 -05:00 committed by GitHub
parent 43682aa7f4
commit 1912453f4e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 58 additions and 23 deletions

View file

@ -122,8 +122,7 @@ public final class IcannReportingStagingAction implements Runnable {
TaskOptions uploadTask = TaskOptions uploadTask =
TaskOptions.Builder.withUrl(IcannReportingUploadAction.PATH) TaskOptions.Builder.withUrl(IcannReportingUploadAction.PATH)
.method(Method.POST) .method(Method.POST)
.countdownMillis(Duration.standardMinutes(2).getMillis()) .countdownMillis(Duration.standardMinutes(2).getMillis());
.param(PARAM_SUBDIR, subdir);
QueueFactory.getQueue(CRON_QUEUE).add(uploadTask); QueueFactory.getQueue(CRON_QUEUE).add(uploadTask);
return null; return null;
}, },

View file

@ -20,7 +20,6 @@ import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime; import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.reporting.icann.IcannReportingModule.PARAM_SUBDIR;
import static google.registry.request.Action.Method.POST; import static google.registry.request.Action.Method.POST;
import static javax.servlet.http.HttpServletResponse.SC_OK; import static javax.servlet.http.HttpServletResponse.SC_OK;
@ -40,7 +39,6 @@ import google.registry.model.registry.Registry;
import google.registry.model.registry.Registry.TldType; import google.registry.model.registry.Registry.TldType;
import google.registry.request.Action; import google.registry.request.Action;
import google.registry.request.HttpException.ServiceUnavailableException; import google.registry.request.HttpException.ServiceUnavailableException;
import google.registry.request.Parameter;
import google.registry.request.Response; import google.registry.request.Response;
import google.registry.request.auth.Auth; import google.registry.request.auth.Auth;
import google.registry.request.lock.LockHandler; import google.registry.request.lock.LockHandler;
@ -91,8 +89,6 @@ public final class IcannReportingUploadAction implements Runnable {
@Config("reportingBucket") @Config("reportingBucket")
String reportingBucket; String reportingBucket;
@Inject @Parameter(PARAM_SUBDIR) String subdir;
@Inject GcsUtils gcsUtils; @Inject GcsUtils gcsUtils;
@Inject IcannHttpReporter icannReporter; @Inject IcannHttpReporter icannReporter;
@Inject Retrier retrier; @Inject Retrier retrier;
@ -145,7 +141,12 @@ public final class IcannReportingUploadAction implements Runnable {
CursorType cursorType, CursorType cursorType,
String tldStr, String tldStr,
ImmutableMap.Builder<String, Boolean> reportSummaryBuilder) { ImmutableMap.Builder<String, Boolean> reportSummaryBuilder) {
String reportBucketname = String.format("%s/%s", reportingBucket, subdir); DateTime cursorTimeMinusMonth = cursorTime.withDayOfMonth(1).minusMonths(1);
String reportSubdir =
String.format(
"icann/monthly/%d-%02d",
cursorTimeMinusMonth.getYear(), cursorTimeMinusMonth.getMonthOfYear());
String reportBucketname = String.format("%s/%s", reportingBucket, reportSubdir);
String filename = getFileName(cursorType, cursorTime, tldStr); String filename = getFileName(cursorType, cursorTime, tldStr);
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, filename); final GcsFilename gcsFilename = new GcsFilename(reportBucketname, filename);
logger.atInfo().log("Reading ICANN report %s from bucket %s", filename, reportBucketname); logger.atInfo().log("Reading ICANN report %s from bucket %s", filename, reportBucketname);
@ -195,12 +196,13 @@ public final class IcannReportingUploadAction implements Runnable {
} }
private String getFileName(CursorType cursorType, DateTime cursorTime, String tld) { private String getFileName(CursorType cursorType, DateTime cursorTime, String tld) {
DateTime cursorTimeMinusMonth = cursorTime.withDayOfMonth(1).minusMonths(1);
return String.format( return String.format(
"%s%s%d%02d.csv", "%s%s%d%02d.csv",
tld, tld,
(cursorType.equals(CursorType.ICANN_UPLOAD_ACTIVITY) ? "-activity-" : "-transactions-"), (cursorType.equals(CursorType.ICANN_UPLOAD_ACTIVITY) ? "-activity-" : "-transactions-"),
cursorTime.year().get(), cursorTimeMinusMonth.year().get(),
cursorTime.withDayOfMonth(1).minusMonths(1).monthOfYear().get()); cursorTimeMinusMonth.monthOfYear().get());
} }
/** Returns a map of each cursor to the CursorType and tld. */ /** Returns a map of each cursor to the CursorType and tld. */

View file

@ -81,12 +81,8 @@ public class IcannReportingStagingActionTest {
.thenReturn(ImmutableList.of("c", "d")); .thenReturn(ImmutableList.of("c", "d"));
} }
private static void assertUploadTaskEnqueued(String subdir) { private static void assertUploadTaskEnqueued() {
TaskMatcher matcher = TaskMatcher matcher = new TaskMatcher().url("/_dr/task/icannReportingUpload").method("POST");
new TaskMatcher()
.url("/_dr/task/icannReportingUpload")
.method("POST")
.param("subdir", subdir);
assertTasksEnqueued("retryable-cron-tasks", matcher); assertTasksEnqueued("retryable-cron-tasks", matcher);
} }
@ -103,7 +99,7 @@ public class IcannReportingStagingActionTest {
"Completed staging the following 2 ICANN reports:\na\nb", "Completed staging the following 2 ICANN reports:\na\nb",
new InternetAddress("recipient@example.com"), new InternetAddress("recipient@example.com"),
new InternetAddress("sender@example.com"))); new InternetAddress("sender@example.com")));
assertUploadTaskEnqueued("default/dir"); assertUploadTaskEnqueued();
} }
@Test @Test
@ -119,7 +115,7 @@ public class IcannReportingStagingActionTest {
"Completed staging the following 4 ICANN reports:\na\nb\nc\nd", "Completed staging the following 4 ICANN reports:\na\nb\nc\nd",
new InternetAddress("recipient@example.com"), new InternetAddress("recipient@example.com"),
new InternetAddress("sender@example.com"))); new InternetAddress("sender@example.com")));
assertUploadTaskEnqueued("default/dir"); assertUploadTaskEnqueued();
} }
@Test @Test
@ -138,7 +134,7 @@ public class IcannReportingStagingActionTest {
"Completed staging the following 4 ICANN reports:\na\nb\nc\nd", "Completed staging the following 4 ICANN reports:\na\nb\nc\nd",
new InternetAddress("recipient@example.com"), new InternetAddress("recipient@example.com"),
new InternetAddress("sender@example.com"))); new InternetAddress("sender@example.com")));
assertUploadTaskEnqueued("default/dir"); assertUploadTaskEnqueued();
} }
@Test @Test

View file

@ -78,7 +78,6 @@ public class IcannReportingUploadActionTest {
action.icannReporter = mockReporter; action.icannReporter = mockReporter;
action.gcsUtils = new GcsUtils(gcsService, 1024); action.gcsUtils = new GcsUtils(gcsService, 1024);
action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
action.subdir = "icann/monthly/2006-06";
action.reportingBucket = "basin"; action.reportingBucket = "basin";
action.emailService = emailService; action.emailService = emailService;
action.sender = new InternetAddress("sender@example.com"); action.sender = new InternetAddress("sender@example.com");
@ -151,6 +150,43 @@ public class IcannReportingUploadActionTest {
new InternetAddress("sender@example.com"))); new InternetAddress("sender@example.com")));
} }
@Test
public void testSuccess_january() throws Exception {
clock.setTo(DateTime.parse("2006-01-22T00:30:00Z"));
persistResource(
Cursor.create(
CursorType.ICANN_UPLOAD_ACTIVITY, DateTime.parse("2006-01-01TZ"), Registry.get("tld")));
persistResource(
Cursor.create(
CursorType.ICANN_UPLOAD_TX, DateTime.parse("2006-01-01TZ"), Registry.get("tld")));
writeGcsFile(
gcsService,
new GcsFilename("basin/icann/monthly/2005-12", "tld-transactions-200512.csv"),
PAYLOAD_SUCCESS);
writeGcsFile(
gcsService,
new GcsFilename("basin/icann/monthly/2005-12", "tld-activity-200512.csv"),
PAYLOAD_SUCCESS);
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200512.csv")).thenReturn(true);
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200512.csv")).thenReturn(true);
IcannReportingUploadAction action = createAction();
action.run();
verify(mockReporter).send(PAYLOAD_SUCCESS, "tld-activity-200512.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "tld-transactions-200512.csv");
verifyNoMoreInteractions(mockReporter);
verify(emailService)
.sendEmail(
EmailMessage.create(
"ICANN Monthly report upload summary: 2/2 succeeded",
"Report Filename - Upload status:\n"
+ "tld-activity-200512.csv - SUCCESS\n"
+ "tld-transactions-200512.csv - SUCCESS",
new InternetAddress("recipient@example.com"),
new InternetAddress("sender@example.com")));
}
@Test @Test
public void testSuccess_advancesCursor() throws Exception { public void testSuccess_advancesCursor() throws Exception {
writeGcsFile( writeGcsFile(
@ -280,15 +316,18 @@ public class IcannReportingUploadActionTest {
@Test @Test
public void testFail_fileNotFound() throws Exception { public void testFail_fileNotFound() throws Exception {
clock.setTo(DateTime.parse("2006-01-22T00:30:00Z"));
persistResource(
Cursor.create(
CursorType.ICANN_UPLOAD_ACTIVITY, DateTime.parse("2006-01-01TZ"), Registry.get("tld")));
IcannReportingUploadAction action = createAction(); IcannReportingUploadAction action = createAction();
action.subdir = "somewhere/else";
action.run(); action.run();
assertAboutLogs() assertAboutLogs()
.that(logHandler) .that(logHandler)
.hasLogAtLevelWithMessage( .hasLogAtLevelWithMessage(
Level.SEVERE, Level.SEVERE,
"Could not upload ICANN_UPLOAD_ACTIVITY report for foo because file" "Could not upload ICANN_UPLOAD_ACTIVITY report for tld because file"
+ " foo-activity-200606.csv did not exist"); + " tld-activity-200512.csv did not exist");
} }
@Test @Test
@ -298,7 +337,6 @@ public class IcannReportingUploadActionTest {
CursorType.ICANN_UPLOAD_ACTIVITY, DateTime.parse("2006-08-01TZ"), Registry.get("foo"))); CursorType.ICANN_UPLOAD_ACTIVITY, DateTime.parse("2006-08-01TZ"), Registry.get("foo")));
clock.setTo(DateTime.parse("2006-08-01T00:30:00Z")); clock.setTo(DateTime.parse("2006-08-01T00:30:00Z"));
IcannReportingUploadAction action = createAction(); IcannReportingUploadAction action = createAction();
action.subdir = "icann/monthly/2006-07";
action.run(); action.run();
assertAboutLogs() assertAboutLogs()
.that(logHandler) .that(logHandler)