From e318f47fc6bc8a8714bafb17c83d839a8e6fb081 Mon Sep 17 00:00:00 2001 From: sarahcaseybot Date: Fri, 22 Nov 2019 17:40:31 -0500 Subject: [PATCH] Add a cursor for tracking monthly uploads of ICANN report (#343) * Add a cursor for tracking monthly uploads of the transaction report to ICANN * Add cursors to track activity, transaction, and manifest report uploads. * Address comments * Add @Nullable annotation to manifestCursor * Add lock and batch load cursors. * Add string formatting, autovalue CursorInfo object, and handling for null cursors * Add some helper functions for loadCursors and restructure to require less round trips to the database * Switch new cursors to be created with cursorTime at first of next month --- .../icann/IcannReportingUploadAction.java | 234 +++++++++++--- .../icann/IcannReportingUploadActionTest.java | 306 ++++++++++++++---- 2 files changed, 441 insertions(+), 99 deletions(-) diff --git a/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java b/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java index 323fd0ef7..816886a5e 100644 --- a/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java +++ b/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java @@ -15,34 +15,51 @@ package google.registry.reporting.icann; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; +import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.model.transaction.TransactionManagerFactory.tm; import static google.registry.reporting.icann.IcannReportingModule.MANIFEST_FILE_NAME; import static google.registry.reporting.icann.IcannReportingModule.PARAM_SUBDIR; import static google.registry.request.Action.Method.POST; -import static java.nio.charset.StandardCharsets.UTF_8; import static javax.servlet.http.HttpServletResponse.SC_OK; import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; +import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.flogger.FluentLogger; import com.google.common.io.ByteStreams; +import com.googlecode.objectify.Key; import google.registry.config.RegistryConfig.Config; import google.registry.gcs.GcsUtils; +import google.registry.model.common.Cursor; +import google.registry.model.common.Cursor.CursorType; +import google.registry.model.registry.Registries; +import google.registry.model.registry.Registry; +import google.registry.model.registry.Registry.TldType; import google.registry.request.Action; +import google.registry.request.HttpException.ServiceUnavailableException; import google.registry.request.Parameter; import google.registry.request.Response; import google.registry.request.auth.Auth; +import google.registry.request.lock.LockHandler; +import google.registry.util.Clock; import google.registry.util.EmailMessage; import google.registry.util.Retrier; import google.registry.util.SendEmailService; import java.io.IOException; import java.io.InputStream; +import java.util.Map; +import java.util.concurrent.Callable; import java.util.regex.Pattern; import java.util.stream.Collectors; +import javax.annotation.Nullable; import javax.inject.Inject; import javax.mail.internet.InternetAddress; +import org.joda.time.DateTime; +import org.joda.time.Duration; /** * Action that uploads the monthly activity/transactions reports from GCS to ICANN via an HTTP PUT. @@ -81,44 +98,175 @@ public final class IcannReportingUploadAction implements Runnable { @Inject @Config("gSuiteOutgoingEmailAddress") InternetAddress sender; @Inject @Config("alertRecipientEmailAddress") InternetAddress recipient; @Inject SendEmailService emailService; + @Inject Clock clock; + @Inject LockHandler lockHandler; + @Inject IcannReportingUploadAction() {} @Override public void run() { - String reportBucketname = String.format("%s/%s", reportingBucket, subdir); - ImmutableList manifestedFiles = getManifestedFiles(reportBucketname); - ImmutableMap.Builder reportSummaryBuilder = new ImmutableMap.Builder<>(); - // Report on all manifested files - for (String reportFilename : manifestedFiles) { - logger.atInfo().log( - "Reading ICANN report %s from bucket %s", reportFilename, reportBucketname); - final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename); - verifyFileExists(gcsFilename); - boolean success = false; - try { - success = - retrier.callWithRetry( - () -> { - final byte[] payload = readBytesFromGcs(gcsFilename); - return icannReporter.send(payload, reportFilename); - }, - IcannReportingUploadAction::isUploadFailureRetryable); - } catch (RuntimeException e) { - logger.atWarning().withCause(e).log("Upload to %s failed.", gcsFilename); - } - reportSummaryBuilder.put(reportFilename, success); + Callable lockRunner = + () -> { + ImmutableMap.Builder reportSummaryBuilder = new ImmutableMap.Builder<>(); + + ImmutableMap cursors = loadCursors(); + + // If cursor time is before now, upload the corresponding report + cursors.entrySet().stream() + .filter(entry -> getCursorTimeOrStartOfTime(entry.getKey()).isBefore(clock.nowUtc())) + .forEach( + entry -> { + DateTime cursorTime = getCursorTimeOrStartOfTime(entry.getKey()); + uploadReport( + cursorTime, + entry.getValue().getType(), + entry.getValue().getTld(), + reportSummaryBuilder); + }); + // Send email of which reports were uploaded + emailUploadResults(reportSummaryBuilder.build()); + response.setStatus(SC_OK); + response.setContentType(PLAIN_TEXT_UTF_8); + return null; + }; + + String lockname = "IcannReportingUploadAction"; + if (!lockHandler.executeWithLocks(lockRunner, null, Duration.standardHours(2), lockname)) { + throw new ServiceUnavailableException("Lock for IcannReportingUploadAction already in use"); } - emailUploadResults(reportSummaryBuilder.build()); - response.setStatus(SC_OK); - response.setContentType(PLAIN_TEXT_UTF_8); - response.setPayload( - String.format("OK, attempted uploading %d reports", manifestedFiles.size())); + } + + /** Uploads the report and rolls forward the cursor for that report. */ + private void uploadReport( + DateTime cursorTime, + CursorType cursorType, + String tldStr, + ImmutableMap.Builder reportSummaryBuilder) { + String reportBucketname = String.format("%s/%s", reportingBucket, subdir); + String filename = getFileName(cursorType, cursorTime, tldStr); + final GcsFilename gcsFilename = new GcsFilename(reportBucketname, filename); + logger.atInfo().log("Reading ICANN report %s from bucket %s", filename, reportBucketname); + // Check that the report exists + try { + verifyFileExists(gcsFilename); + } catch (IllegalArgumentException e) { + String logMessage = + String.format( + "Could not upload %s report for %s because file %s did not exist.", + cursorType, tldStr, filename); + if (clock.nowUtc().dayOfMonth().get() == 1) { + logger.atInfo().withCause(e).log(logMessage + " This report may not have been staged yet."); + } else { + logger.atSevere().withCause(e).log(logMessage); + } + reportSummaryBuilder.put(filename, false); + return; + } + + // Upload the report + boolean success = false; + try { + success = + retrier.callWithRetry( + () -> { + final byte[] payload = readBytesFromGcs(gcsFilename); + return icannReporter.send(payload, filename); + }, + IcannReportingUploadAction::isUploadFailureRetryable); + } catch (RuntimeException e) { + logger.atWarning().withCause(e).log("Upload to %s failed", gcsFilename); + } + reportSummaryBuilder.put(filename, success); + + // Set cursor to first day of next month if the upload succeeded + if (success) { + Cursor newCursor; + if (cursorType.equals(CursorType.ICANN_UPLOAD_MANIFEST)) { + newCursor = + Cursor.createGlobal( + cursorType, cursorTime.withTimeAtStartOfDay().withDayOfMonth(1).plusMonths(1)); + } else { + newCursor = + Cursor.create( + cursorType, + cursorTime.withTimeAtStartOfDay().withDayOfMonth(1).plusMonths(1), + Registry.get(tldStr)); + } + tm().transact(() -> ofy().save().entity(newCursor)); + } + } + + private String getFileName(CursorType cursorType, DateTime cursorTime, String tld) { + if (cursorType.equals(CursorType.ICANN_UPLOAD_MANIFEST)) { + return MANIFEST_FILE_NAME; + } + return String.format( + "%s%s%d%02d.csv", + tld, + (cursorType.equals(CursorType.ICANN_UPLOAD_ACTIVITY) ? "-activity-" : "-transactions-"), + cursorTime.year().get(), + cursorTime.monthOfYear().get()); + } + + /** Returns a map of each cursor to the CursorType and tld. */ + private ImmutableMap loadCursors() { + + ImmutableSet registries = Registries.getTldEntitiesOfType(TldType.REAL); + + Map, Registry> activityKeyMap = + loadKeyMap(registries, CursorType.ICANN_UPLOAD_ACTIVITY); + Map, Registry> transactionKeyMap = + loadKeyMap(registries, CursorType.ICANN_UPLOAD_TX); + + ImmutableSet.Builder> keys = new ImmutableSet.Builder<>(); + keys.addAll(activityKeyMap.keySet()); + keys.addAll(transactionKeyMap.keySet()); + keys.add(Cursor.createGlobalKey(CursorType.ICANN_UPLOAD_MANIFEST)); + + Map, Cursor> cursorMap = ofy().load().keys(keys.build()); + ImmutableMap.Builder cursors = new ImmutableMap.Builder<>(); + defaultNullCursorsToNextMonthAndAddToMap( + activityKeyMap, CursorType.ICANN_UPLOAD_ACTIVITY, cursorMap, cursors); + defaultNullCursorsToNextMonthAndAddToMap( + transactionKeyMap, CursorType.ICANN_UPLOAD_TX, cursorMap, cursors); + Cursor manifestCursor = + cursorMap.getOrDefault( + Cursor.createGlobalKey(CursorType.ICANN_UPLOAD_MANIFEST), + Cursor.createGlobal(CursorType.ICANN_UPLOAD_MANIFEST, clock.nowUtc().minusDays(1))); + cursors.put(manifestCursor, CursorInfo.create(CursorType.ICANN_UPLOAD_MANIFEST, null)); + return cursors.build(); + } + + private Map, Registry> loadKeyMap( + ImmutableSet registries, CursorType type) { + return registries.stream().collect(toImmutableMap(r -> Cursor.createKey(type, r), r -> r)); + } + + /** + * Populate the cursors map with the Cursor and CursorInfo for each key in the keyMap. If the key + * from the keyMap does not have an existing cursor, create a new cursor with a default cursorTime + * of the first of next month. + */ + private void defaultNullCursorsToNextMonthAndAddToMap( + Map, Registry> keyMap, + CursorType type, + Map, Cursor> cursorMap, + ImmutableMap.Builder cursors) { + keyMap.forEach( + (key, registry) -> { + // Cursor time is defaulted to the first of next month since a new tld will not yet have a + // report staged for upload. + Cursor cursor = + cursorMap.getOrDefault( + key, Cursor.create(type, clock.nowUtc().minusDays(1), registry)); + cursors.put(cursor, CursorInfo.create(type, registry.getTldStr())); + }); } /** Don't retry when reports are already uploaded or can't be uploaded. */ private static final String ICANN_UPLOAD_PERMANENT_ERROR_MESSAGE = - "A report for that month already exists, the cut-off date already passed."; + "A report for that month already exists, the cut-off date already passed"; /** Don't retry when the IP address isn't whitelisted, as retries go through the same IP. */ private static final Pattern ICANN_UPLOAD_WHITELIST_ERROR = @@ -146,18 +294,6 @@ public final class IcannReportingUploadAction implements Runnable { emailService.sendEmail(EmailMessage.create(subject, body, recipient, sender)); } - private ImmutableList getManifestedFiles(String reportBucketname) { - GcsFilename manifestFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME); - verifyFileExists(manifestFilename); - return retrier.callWithRetry( - () -> - ImmutableList.copyOf( - Splitter.on('\n') - .omitEmptyStrings() - .split(new String(readBytesFromGcs(manifestFilename), UTF_8))), - IOException.class); - } - private byte[] readBytesFromGcs(GcsFilename reportFilename) throws IOException { try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) { return ByteStreams.toByteArray(gcsInput); @@ -171,4 +307,16 @@ public final class IcannReportingUploadAction implements Runnable { gcsFilename.getObjectName(), gcsFilename.getBucketName()); } + + @AutoValue + abstract static class CursorInfo { + static CursorInfo create(CursorType type, @Nullable String tld) { + return new AutoValue_IcannReportingUploadAction_CursorInfo(type, tld); + } + + public abstract CursorType getType(); + + @Nullable + abstract String getTld(); + } } diff --git a/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java b/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java index aa1109d2d..2f9ebbfd9 100644 --- a/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java +++ b/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java @@ -15,8 +15,12 @@ package google.registry.reporting.icann; import static com.google.common.truth.Truth.assertThat; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.testing.DatastoreHelper.createTlds; +import static google.registry.testing.DatastoreHelper.persistResource; import static google.registry.testing.GcsTestingUtils.writeGcsFile; import static google.registry.testing.JUnitBackports.assertThrows; +import static google.registry.testing.LogsSubject.assertAboutLogs; import static java.nio.charset.StandardCharsets.UTF_8; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -27,16 +31,25 @@ import static org.mockito.Mockito.when; import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.appengine.tools.cloudstorage.GcsService; import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.common.testing.TestLogHandler; import google.registry.gcs.GcsUtils; +import google.registry.model.common.Cursor; +import google.registry.model.common.Cursor.CursorType; +import google.registry.model.registry.Registry; +import google.registry.request.HttpException.ServiceUnavailableException; import google.registry.testing.AppEngineRule; import google.registry.testing.FakeClock; +import google.registry.testing.FakeLockHandler; import google.registry.testing.FakeResponse; import google.registry.testing.FakeSleeper; import google.registry.util.EmailMessage; import google.registry.util.Retrier; import google.registry.util.SendEmailService; import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.mail.internet.InternetAddress; +import org.joda.time.DateTime; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -52,105 +65,175 @@ public class IcannReportingUploadActionTest { private static final byte[] PAYLOAD_SUCCESS = "test,csv\n13,37".getBytes(UTF_8); private static final byte[] PAYLOAD_FAIL = "ahah,csv\n12,34".getBytes(UTF_8); private static final byte[] MANIFEST_PAYLOAD = - "test-transactions-201706.csv\na-activity-201706.csv\n".getBytes(UTF_8); + "tld-transactions-200606.csv\ntld-activity-200606.csv\nfoo-transactions-200606.csv\nfoo-activity-200606.csv\n" + .getBytes(UTF_8); private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class); private final SendEmailService emailService = mock(SendEmailService.class); private final FakeResponse response = new FakeResponse(); private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final TestLogHandler logHandler = new TestLogHandler(); + private final Logger loggerToIntercept = + Logger.getLogger(IcannReportingUploadAction.class.getCanonicalName()); + private final FakeClock clock = new FakeClock(DateTime.parse("2000-01-01TZ")); private IcannReportingUploadAction createAction() throws Exception { IcannReportingUploadAction action = new IcannReportingUploadAction(); action.icannReporter = mockReporter; action.gcsUtils = new GcsUtils(gcsService, 1024); action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); - action.subdir = "icann/monthly/2017-06"; + action.subdir = "icann/monthly/2006-06"; action.reportingBucket = "basin"; action.emailService = emailService; action.sender = new InternetAddress("sender@example.com"); action.recipient = new InternetAddress("recipient@example.com"); action.response = response; + action.clock = clock; + action.lockHandler = new FakeLockHandler(true); return action; } @Before public void before() throws Exception { + createTlds("tld", "foo"); writeGcsFile( gcsService, - new GcsFilename("basin/icann/monthly/2017-06", "test-transactions-201706.csv"), + new GcsFilename("basin/icann/monthly/2006-06", "tld-transactions-200606.csv"), PAYLOAD_SUCCESS); writeGcsFile( gcsService, - new GcsFilename("basin/icann/monthly/2017-06", "a-activity-201706.csv"), + new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), PAYLOAD_FAIL); writeGcsFile( gcsService, - new GcsFilename("basin/icann/monthly/2017-06", "MANIFEST.txt"), + new GcsFilename("basin/icann/monthly/2006-06", "foo-transactions-200606.csv"), + PAYLOAD_SUCCESS); + writeGcsFile( + gcsService, + new GcsFilename("basin/icann/monthly/2006-06", "foo-activity-200606.csv"), + PAYLOAD_SUCCESS); + writeGcsFile( + gcsService, + new GcsFilename("basin/icann/monthly/2006-06", "MANIFEST.txt"), MANIFEST_PAYLOAD); - when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv")).thenReturn(true); - when(mockReporter.send(PAYLOAD_FAIL, "a-activity-201706.csv")).thenReturn(false); + when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv")).thenReturn(true); + when(mockReporter.send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv")).thenReturn(true); + when(mockReporter.send(PAYLOAD_FAIL, "tld-activity-200606.csv")).thenReturn(false); + when(mockReporter.send(PAYLOAD_SUCCESS, "foo-activity-200606.csv")).thenReturn(true); + when(mockReporter.send(MANIFEST_PAYLOAD, "MANIFEST.txt")).thenReturn(true); + clock.setTo(DateTime.parse("2006-06-06T00:30:00Z")); + persistResource( + Cursor.create( + CursorType.ICANN_UPLOAD_ACTIVITY, DateTime.parse("2006-06-06TZ"), Registry.get("tld"))); + persistResource( + Cursor.create( + CursorType.ICANN_UPLOAD_TX, DateTime.parse("2006-06-06TZ"), Registry.get("tld"))); + persistResource( + Cursor.createGlobal(CursorType.ICANN_UPLOAD_MANIFEST, DateTime.parse("2006-07-06TZ"))); + persistResource( + Cursor.create( + CursorType.ICANN_UPLOAD_ACTIVITY, DateTime.parse("2006-06-06TZ"), Registry.get("foo"))); + persistResource( + Cursor.create( + CursorType.ICANN_UPLOAD_TX, DateTime.parse("2006-06-06TZ"), Registry.get("foo"))); + loggerToIntercept.addHandler(logHandler); } @Test public void testSuccess() throws Exception { IcannReportingUploadAction action = createAction(); action.run(); - verify(mockReporter).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv"); - verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-activity-200606.csv"); + verify(mockReporter).send(PAYLOAD_FAIL, "tld-activity-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv"); + verifyNoMoreInteractions(mockReporter); - assertThat(((FakeResponse) action.response).getPayload()) - .isEqualTo("OK, attempted uploading 2 reports"); verify(emailService) .sendEmail( EmailMessage.create( - "ICANN Monthly report upload summary: 1/2 succeeded", + "ICANN Monthly report upload summary: 3/4 succeeded", "Report Filename - Upload status:\n" - + "test-transactions-201706.csv - SUCCESS\n" - + "a-activity-201706.csv - FAILURE", + + "foo-activity-200606.csv - SUCCESS\n" + + "tld-activity-200606.csv - FAILURE\n" + + "foo-transactions-200606.csv - SUCCESS\n" + + "tld-transactions-200606.csv - SUCCESS", new InternetAddress("recipient@example.com"), new InternetAddress("sender@example.com"))); } @Test - public void testSuccess_WithRetry() throws Exception { + public void testSuccess_advancesCursor() throws Exception { + writeGcsFile( + gcsService, + new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), + PAYLOAD_SUCCESS); + when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200606.csv")).thenReturn(true); IcannReportingUploadAction action = createAction(); - when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv")) + action.run(); + ofy().clearSessionCache(); + Cursor cursor = + ofy() + .load() + .key(Cursor.createKey(CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("tld"))) + .now(); + assertThat(cursor.getCursorTime()).isEqualTo(DateTime.parse("2006-07-01TZ")); + } + + @Test + public void testSuccess_noUploadsNeeded() throws Exception { + clock.setTo(DateTime.parse("2006-5-01T00:30:00Z")); + IcannReportingUploadAction action = createAction(); + action.run(); + ofy().clearSessionCache(); + verifyNoMoreInteractions(mockReporter); + verify(emailService) + .sendEmail( + EmailMessage.create( + "ICANN Monthly report upload summary: 0/0 succeeded", + "Report Filename - Upload status:\n", + new InternetAddress("recipient@example.com"), + new InternetAddress("sender@example.com"))); + } + + @Test + public void testSuccess_uploadManifest() throws Exception { + persistResource( + Cursor.createGlobal(CursorType.ICANN_UPLOAD_MANIFEST, DateTime.parse("2006-06-06TZ"))); + IcannReportingUploadAction action = createAction(); + action.run(); + ofy().clearSessionCache(); + Cursor cursor = + ofy().load().key(Cursor.createGlobalKey(CursorType.ICANN_UPLOAD_MANIFEST)).now(); + assertThat(cursor.getCursorTime()).isEqualTo(DateTime.parse("2006-07-01TZ")); + verify(mockReporter).send(PAYLOAD_FAIL, "tld-activity-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-activity-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv"); + verify(mockReporter).send(MANIFEST_PAYLOAD, "MANIFEST.txt"); + verifyNoMoreInteractions(mockReporter); + } + + @Test + public void testSuccess_withRetry() throws Exception { + IcannReportingUploadAction action = createAction(); + when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv")) .thenThrow(new IOException("Expected exception.")) .thenReturn(true); action.run(); - verify(mockReporter, times(2)).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv"); - verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-activity-200606.csv"); + verify(mockReporter).send(PAYLOAD_FAIL, "tld-activity-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv"); + verify(mockReporter, times(2)).send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv"); verifyNoMoreInteractions(mockReporter); - assertThat(((FakeResponse) action.response).getPayload()) - .isEqualTo("OK, attempted uploading 2 reports"); verify(emailService) .sendEmail( EmailMessage.create( - "ICANN Monthly report upload summary: 1/2 succeeded", + "ICANN Monthly report upload summary: 3/4 succeeded", "Report Filename - Upload status:\n" - + "test-transactions-201706.csv - SUCCESS\n" - + "a-activity-201706.csv - FAILURE", - new InternetAddress("recipient@example.com"), - new InternetAddress("sender@example.com"))); - } - - @Test - public void testFailure_firstUnrecoverable_stillAttemptsUploadingBoth() throws Exception { - IcannReportingUploadAction action = createAction(); - when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv")) - .thenThrow(new IOException("Expected exception")); - action.run(); - verify(mockReporter, times(3)).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv"); - verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv"); - verifyNoMoreInteractions(mockReporter); - assertThat(((FakeResponse) action.response).getPayload()) - .isEqualTo("OK, attempted uploading 2 reports"); - verify(emailService) - .sendEmail( - EmailMessage.create( - "ICANN Monthly report upload summary: 0/2 succeeded", - "Report Filename - Upload status:\n" - + "test-transactions-201706.csv - FAILURE\n" - + "a-activity-201706.csv - FAILURE", + + "foo-activity-200606.csv - SUCCESS\n" + + "tld-activity-200606.csv - FAILURE\n" + + "foo-transactions-200606.csv - SUCCESS\n" + + "tld-transactions-200606.csv - SUCCESS", new InternetAddress("recipient@example.com"), new InternetAddress("sender@example.com"))); } @@ -169,38 +252,149 @@ public class IcannReportingUploadActionTest { new IOException("Your IP address 25.147.130.158 is not allowed to connect")); } + @Test + public void testFailure_cursorIsNotAdvancedForward() throws Exception { + runTest_nonRetryableException( + new IOException("Your IP address 25.147.130.158 is not allowed to connect")); + ofy().clearSessionCache(); + Cursor cursor = + ofy() + .load() + .key(Cursor.createKey(CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("tld"))) + .now(); + assertThat(cursor.getCursorTime()).isEqualTo(DateTime.parse("2006-06-06TZ")); + } + + @Test + public void testNotRunIfCursorDateIsAfterToday() throws Exception { + clock.setTo(DateTime.parse("2006-05-01T00:30:00Z")); + IcannReportingUploadAction action = createAction(); + action.run(); + ofy().clearSessionCache(); + Cursor cursor = + ofy() + .load() + .key(Cursor.createKey(CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("foo"))) + .now(); + assertThat(cursor.getCursorTime()).isEqualTo(DateTime.parse("2006-06-06TZ")); + verifyNoMoreInteractions(mockReporter); + } + private void runTest_nonRetryableException(Exception nonRetryableException) throws Exception { IcannReportingUploadAction action = createAction(); - when(mockReporter.send(PAYLOAD_FAIL, "a-activity-201706.csv")) + when(mockReporter.send(PAYLOAD_FAIL, "tld-activity-200606.csv")) .thenThrow(nonRetryableException) .thenThrow( new AssertionError( "This should never be thrown because the previous exception isn't retryable")); action.run(); - verify(mockReporter, times(1)).send(PAYLOAD_FAIL, "a-activity-201706.csv"); - verify(mockReporter).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-activity-200606.csv"); + verify(mockReporter, times(1)).send(PAYLOAD_FAIL, "tld-activity-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv"); verifyNoMoreInteractions(mockReporter); - assertThat(((FakeResponse) action.response).getPayload()) - .isEqualTo("OK, attempted uploading 2 reports"); verify(emailService) .sendEmail( EmailMessage.create( - "ICANN Monthly report upload summary: 1/2 succeeded", + "ICANN Monthly report upload summary: 3/4 succeeded", "Report Filename - Upload status:\n" - + "test-transactions-201706.csv - SUCCESS\n" - + "a-activity-201706.csv - FAILURE", + + "foo-activity-200606.csv - SUCCESS\n" + + "tld-activity-200606.csv - FAILURE\n" + + "foo-transactions-200606.csv - SUCCESS\n" + + "tld-transactions-200606.csv - SUCCESS", new InternetAddress("recipient@example.com"), new InternetAddress("sender@example.com"))); } @Test - public void testFail_FileNotFound() throws Exception { + public void testFail_fileNotFound() throws Exception { IcannReportingUploadAction action = createAction(); action.subdir = "somewhere/else"; - IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, action::run); + action.run(); + assertAboutLogs() + .that(logHandler) + .hasLogAtLevelWithMessage( + Level.SEVERE, + "Could not upload ICANN_UPLOAD_ACTIVITY report for foo because file" + + " foo-activity-200606.csv did not exist"); + } + + @Test + public void testWarning_fileNotStagedYet() throws Exception { + persistResource( + Cursor.create( + CursorType.ICANN_UPLOAD_ACTIVITY, DateTime.parse("2006-07-01TZ"), Registry.get("foo"))); + clock.setTo(DateTime.parse("2006-07-01T00:30:00Z")); + IcannReportingUploadAction action = createAction(); + action.subdir = "icann/monthly/2006-07"; + action.run(); + assertAboutLogs() + .that(logHandler) + .hasLogAtLevelWithMessage( + Level.INFO, + "Could not upload ICANN_UPLOAD_ACTIVITY report for foo because file" + + " foo-activity-200607.csv did not exist. This report may not have been staged" + + " yet."); + } + + @Test + public void testFailure_lockIsntAvailable() throws Exception { + IcannReportingUploadAction action = createAction(); + action.lockHandler = new FakeLockHandler(false); + ServiceUnavailableException thrown = + assertThrows(ServiceUnavailableException.class, () -> action.run()); assertThat(thrown) .hasMessageThat() - .isEqualTo("Object MANIFEST.txt in bucket basin/somewhere/else not found"); + .contains("Lock for IcannReportingUploadAction already in use"); + } + + @Test + public void testSuccess_nullCursors() throws Exception { + createTlds("new"); + writeGcsFile( + gcsService, + new GcsFilename("basin/icann/monthly/2006-06", "new-transactions-200606.csv"), + PAYLOAD_SUCCESS); + writeGcsFile( + gcsService, + new GcsFilename("basin/icann/monthly/2006-06", "new-activity-200606.csv"), + PAYLOAD_SUCCESS); + when(mockReporter.send(PAYLOAD_SUCCESS, "new-transactions-200606.csv")).thenReturn(true); + when(mockReporter.send(PAYLOAD_SUCCESS, "new-activity-200606.csv")).thenReturn(true); + + IcannReportingUploadAction action = createAction(); + action.run(); + verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-activity-200606.csv"); + verify(mockReporter).send(PAYLOAD_FAIL, "tld-activity-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "new-activity-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv"); + verify(mockReporter).send(PAYLOAD_SUCCESS, "new-transactions-200606.csv"); + verifyNoMoreInteractions(mockReporter); + + verify(emailService) + .sendEmail( + EmailMessage.create( + "ICANN Monthly report upload summary: 5/6 succeeded", + "Report Filename - Upload status:\n" + + "foo-activity-200606.csv - SUCCESS\n" + + "new-activity-200606.csv - SUCCESS\n" + + "tld-activity-200606.csv - FAILURE\n" + + "foo-transactions-200606.csv - SUCCESS\n" + + "new-transactions-200606.csv - SUCCESS\n" + + "tld-transactions-200606.csv - SUCCESS", + new InternetAddress("recipient@example.com"), + new InternetAddress("sender@example.com"))); + + Cursor newActivityCursor = + ofy() + .load() + .key(Cursor.createKey(CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("new"))) + .now(); + assertThat(newActivityCursor.getCursorTime()).isEqualTo(DateTime.parse("2006-07-01TZ")); + Cursor newTransactionCursor = + ofy().load().key(Cursor.createKey(CursorType.ICANN_UPLOAD_TX, Registry.get("new"))).now(); + assertThat(newTransactionCursor.getCursorTime()).isEqualTo(DateTime.parse("2006-07-01TZ")); } }