zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-04-30 03:57:51 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add a cursor for tracking monthly uploads of ICANN report (#343)

Browse source 
* Add a cursor for tracking monthly uploads of the transaction report to ICANN

* Add cursors to track activity, transaction, and manifest report uploads.

* Address comments

* Add @Nullable annotation to manifestCursor

* Add lock and batch load cursors.

* Add string formatting, autovalue CursorInfo object, and handling for null cursors

* Add some helper functions for loadCursors and restructure to require less round trips to the database

* Switch new cursors to be created with cursorTime at first of next month
This commit is contained in:
sarahcaseybot 2019-11-22 17:40:31 -05:00 • committed by GitHub
parent 3a6e55f2da
commit 2eacaf4cdc
2 changed files with 441 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

234
core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java
View file

@ -15,34 +15,51 @@
package google.registry.reporting.icann;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.transaction.TransactionManagerFactory.tm;
import static google.registry.reporting.icann.IcannReportingModule.MANIFEST_FILE_NAME;
import static google.registry.reporting.icann.IcannReportingModule.PARAM_SUBDIR;
import static google.registry.request.Action.Method.POST;
import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.flogger.FluentLogger;
import com.google.common.io.ByteStreams;
import com.googlecode.objectify.Key;
import google.registry.config.RegistryConfig.Config;
import google.registry.gcs.GcsUtils;
import google.registry.model.common.Cursor;
import google.registry.model.common.Cursor.CursorType;
import google.registry.model.registry.Registries;
import google.registry.model.registry.Registry;
import google.registry.model.registry.Registry.TldType;
import google.registry.request.Action;
import google.registry.request.HttpException.ServiceUnavailableException;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.request.lock.LockHandler;
import google.registry.util.Clock;
import google.registry.util.EmailMessage;
import google.registry.util.Retrier;
import google.registry.util.SendEmailService;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.mail.internet.InternetAddress;
import org.joda.time.DateTime;
import org.joda.time.Duration;
/**
* Action that uploads the monthly activity/transactions reports from GCS to ICANN via an HTTP PUT.
@ -81,44 +98,175 @@ public final class IcannReportingUploadAction implements Runnable {
@Inject @Config("gSuiteOutgoingEmailAddress") InternetAddress sender;
@Inject @Config("alertRecipientEmailAddress") InternetAddress recipient;
@Inject SendEmailService emailService;
@Inject Clock clock;
@Inject LockHandler lockHandler;
@Inject
IcannReportingUploadAction() {}
@Override
public void run() {
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
ImmutableList<String> manifestedFiles = getManifestedFiles(reportBucketname);
ImmutableMap.Builder<String, Boolean> reportSummaryBuilder = new ImmutableMap.Builder<>();
// Report on all manifested files
for (String reportFilename : manifestedFiles) {
logger.atInfo().log(
"Reading ICANN report %s from bucket %s", reportFilename, reportBucketname);
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
verifyFileExists(gcsFilename);
boolean success = false;
try {
success =
retrier.callWithRetry(
() -> {
final byte[] payload = readBytesFromGcs(gcsFilename);
return icannReporter.send(payload, reportFilename);
},
IcannReportingUploadAction::isUploadFailureRetryable);
} catch (RuntimeException e) {
logger.atWarning().withCause(e).log("Upload to %s failed.", gcsFilename);
}
reportSummaryBuilder.put(reportFilename, success);
Callable<Void> lockRunner =
() -> {
ImmutableMap.Builder<String, Boolean> reportSummaryBuilder = new ImmutableMap.Builder<>();
ImmutableMap<Cursor, CursorInfo> cursors = loadCursors();
// If cursor time is before now, upload the corresponding report
cursors.entrySet().stream()
.filter(entry -> getCursorTimeOrStartOfTime(entry.getKey()).isBefore(clock.nowUtc()))
.forEach(
entry -> {
DateTime cursorTime = getCursorTimeOrStartOfTime(entry.getKey());
uploadReport(
cursorTime,
entry.getValue().getType(),
entry.getValue().getTld(),
reportSummaryBuilder);
});
// Send email of which reports were uploaded
emailUploadResults(reportSummaryBuilder.build());
response.setStatus(SC_OK);
response.setContentType(PLAIN_TEXT_UTF_8);
return null;
};
String lockname = "IcannReportingUploadAction";
if (!lockHandler.executeWithLocks(lockRunner, null, Duration.standardHours(2), lockname)) {
throw new ServiceUnavailableException("Lock for IcannReportingUploadAction already in use");
}
emailUploadResults(reportSummaryBuilder.build());
response.setStatus(SC_OK);
response.setContentType(PLAIN_TEXT_UTF_8);
response.setPayload(
String.format("OK, attempted uploading %d reports", manifestedFiles.size()));
}
/** Uploads the report and rolls forward the cursor for that report. */
private void uploadReport(
DateTime cursorTime,
CursorType cursorType,
String tldStr,
ImmutableMap.Builder<String, Boolean> reportSummaryBuilder) {
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
String filename = getFileName(cursorType, cursorTime, tldStr);
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, filename);
logger.atInfo().log("Reading ICANN report %s from bucket %s", filename, reportBucketname);
// Check that the report exists
try {
verifyFileExists(gcsFilename);
} catch (IllegalArgumentException e) {
String logMessage =
String.format(
"Could not upload %s report for %s because file %s did not exist.",
cursorType, tldStr, filename);
if (clock.nowUtc().dayOfMonth().get() == 1) {
logger.atInfo().withCause(e).log(logMessage + " This report may not have been staged yet.");
} else {
logger.atSevere().withCause(e).log(logMessage);
}
reportSummaryBuilder.put(filename, false);
return;
}
// Upload the report
boolean success = false;
try {
success =
retrier.callWithRetry(
() -> {
final byte[] payload = readBytesFromGcs(gcsFilename);
return icannReporter.send(payload, filename);
},
IcannReportingUploadAction::isUploadFailureRetryable);
} catch (RuntimeException e) {
logger.atWarning().withCause(e).log("Upload to %s failed", gcsFilename);
}
reportSummaryBuilder.put(filename, success);
// Set cursor to first day of next month if the upload succeeded
if (success) {
Cursor newCursor;
if (cursorType.equals(CursorType.ICANN_UPLOAD_MANIFEST)) {
newCursor =
Cursor.createGlobal(
cursorType, cursorTime.withTimeAtStartOfDay().withDayOfMonth(1).plusMonths(1));
} else {
newCursor =
Cursor.create(
cursorType,
cursorTime.withTimeAtStartOfDay().withDayOfMonth(1).plusMonths(1),
Registry.get(tldStr));
}
tm().transact(() -> ofy().save().entity(newCursor));
}
}
private String getFileName(CursorType cursorType, DateTime cursorTime, String tld) {
if (cursorType.equals(CursorType.ICANN_UPLOAD_MANIFEST)) {
return MANIFEST_FILE_NAME;
}
return String.format(
"%s%s%d%02d.csv",
tld,
(cursorType.equals(CursorType.ICANN_UPLOAD_ACTIVITY) ? "-activity-" : "-transactions-"),
cursorTime.year().get(),
cursorTime.monthOfYear().get());
}
/** Returns a map of each cursor to the CursorType and tld. */
private ImmutableMap<Cursor, CursorInfo> loadCursors() {
ImmutableSet<Registry> registries = Registries.getTldEntitiesOfType(TldType.REAL);
Map<Key<Cursor>, Registry> activityKeyMap =
loadKeyMap(registries, CursorType.ICANN_UPLOAD_ACTIVITY);
Map<Key<Cursor>, Registry> transactionKeyMap =
loadKeyMap(registries, CursorType.ICANN_UPLOAD_TX);
ImmutableSet.Builder<Key<Cursor>> keys = new ImmutableSet.Builder<>();
keys.addAll(activityKeyMap.keySet());
keys.addAll(transactionKeyMap.keySet());
keys.add(Cursor.createGlobalKey(CursorType.ICANN_UPLOAD_MANIFEST));
Map<Key<Cursor>, Cursor> cursorMap = ofy().load().keys(keys.build());
ImmutableMap.Builder<Cursor, CursorInfo> cursors = new ImmutableMap.Builder<>();
defaultNullCursorsToNextMonthAndAddToMap(
activityKeyMap, CursorType.ICANN_UPLOAD_ACTIVITY, cursorMap, cursors);
defaultNullCursorsToNextMonthAndAddToMap(
transactionKeyMap, CursorType.ICANN_UPLOAD_TX, cursorMap, cursors);
Cursor manifestCursor =
cursorMap.getOrDefault(
Cursor.createGlobalKey(CursorType.ICANN_UPLOAD_MANIFEST),
Cursor.createGlobal(CursorType.ICANN_UPLOAD_MANIFEST, clock.nowUtc().minusDays(1)));
cursors.put(manifestCursor, CursorInfo.create(CursorType.ICANN_UPLOAD_MANIFEST, null));
return cursors.build();
}
private Map<Key<Cursor>, Registry> loadKeyMap(
ImmutableSet<Registry> registries, CursorType type) {
return registries.stream().collect(toImmutableMap(r -> Cursor.createKey(type, r), r -> r));
}
/**
* Populate the cursors map with the Cursor and CursorInfo for each key in the keyMap. If the key
* from the keyMap does not have an existing cursor, create a new cursor with a default cursorTime
* of the first of next month.
*/
private void defaultNullCursorsToNextMonthAndAddToMap(
Map<Key<Cursor>, Registry> keyMap,
CursorType type,
Map<Key<Cursor>, Cursor> cursorMap,
ImmutableMap.Builder<Cursor, CursorInfo> cursors) {
keyMap.forEach(
(key, registry) -> {
// Cursor time is defaulted to the first of next month since a new tld will not yet have a
// report staged for upload.
Cursor cursor =
cursorMap.getOrDefault(
key, Cursor.create(type, clock.nowUtc().minusDays(1), registry));
cursors.put(cursor, CursorInfo.create(type, registry.getTldStr()));
});
}
/** Don't retry when reports are already uploaded or can't be uploaded. */
private static final String ICANN_UPLOAD_PERMANENT_ERROR_MESSAGE =
"A report for that month already exists, the cut-off date already passed.";
"A report for that month already exists, the cut-off date already passed";
/** Don't retry when the IP address isn't whitelisted, as retries go through the same IP. */
private static final Pattern ICANN_UPLOAD_WHITELIST_ERROR =
@ -146,18 +294,6 @@ public final class IcannReportingUploadAction implements Runnable {
emailService.sendEmail(EmailMessage.create(subject, body, recipient, sender));
}
private ImmutableList<String> getManifestedFiles(String reportBucketname) {
GcsFilename manifestFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
verifyFileExists(manifestFilename);
return retrier.callWithRetry(
() ->
ImmutableList.copyOf(
Splitter.on('\n')
.omitEmptyStrings()
.split(new String(readBytesFromGcs(manifestFilename), UTF_8))),
IOException.class);
}
private byte[] readBytesFromGcs(GcsFilename reportFilename) throws IOException {
try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) {
return ByteStreams.toByteArray(gcsInput);
@ -171,4 +307,16 @@ public final class IcannReportingUploadAction implements Runnable {
gcsFilename.getObjectName(),
gcsFilename.getBucketName());
}
@AutoValue
abstract static class CursorInfo {
static CursorInfo create(CursorType type, @Nullable String tld) {
return new AutoValue_IcannReportingUploadAction_CursorInfo(type, tld);
}
public abstract CursorType getType();
@Nullable
abstract String getTld();
}
}

306
core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java
View file

@ -15,8 +15,12 @@
package google.registry.reporting.icann;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.testing.DatastoreHelper.createTlds;
import static google.registry.testing.DatastoreHelper.persistResource;
import static google.registry.testing.GcsTestingUtils.writeGcsFile;
import static google.registry.testing.JUnitBackports.assertThrows;
import static google.registry.testing.LogsSubject.assertAboutLogs;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@ -27,16 +31,25 @@ import static org.mockito.Mockito.when;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.common.testing.TestLogHandler;
import google.registry.gcs.GcsUtils;
import google.registry.model.common.Cursor;
import google.registry.model.common.Cursor.CursorType;
import google.registry.model.registry.Registry;
import google.registry.request.HttpException.ServiceUnavailableException;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeLockHandler;
import google.registry.testing.FakeResponse;
import google.registry.testing.FakeSleeper;
import google.registry.util.EmailMessage;
import google.registry.util.Retrier;
import google.registry.util.SendEmailService;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.mail.internet.InternetAddress;
import org.joda.time.DateTime;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@ -52,105 +65,175 @@ public class IcannReportingUploadActionTest {
private static final byte[] PAYLOAD_SUCCESS = "test,csv\n13,37".getBytes(UTF_8);
private static final byte[] PAYLOAD_FAIL = "ahah,csv\n12,34".getBytes(UTF_8);
private static final byte[] MANIFEST_PAYLOAD =
"test-transactions-201706.csv\na-activity-201706.csv\n".getBytes(UTF_8);
"tld-transactions-200606.csv\ntld-activity-200606.csv\nfoo-transactions-200606.csv\nfoo-activity-200606.csv\n"
.getBytes(UTF_8);
private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class);
private final SendEmailService emailService = mock(SendEmailService.class);
private final FakeResponse response = new FakeResponse();
private final GcsService gcsService = GcsServiceFactory.createGcsService();
private final TestLogHandler logHandler = new TestLogHandler();
private final Logger loggerToIntercept =
Logger.getLogger(IcannReportingUploadAction.class.getCanonicalName());
private final FakeClock clock = new FakeClock(DateTime.parse("2000-01-01TZ"));
private IcannReportingUploadAction createAction() throws Exception {
IcannReportingUploadAction action = new IcannReportingUploadAction();
action.icannReporter = mockReporter;
action.gcsUtils = new GcsUtils(gcsService, 1024);
action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
action.subdir = "icann/monthly/2017-06";
action.subdir = "icann/monthly/2006-06";
action.reportingBucket = "basin";
action.emailService = emailService;
action.sender = new InternetAddress("sender@example.com");
action.recipient = new InternetAddress("recipient@example.com");
action.response = response;
action.clock = clock;
action.lockHandler = new FakeLockHandler(true);
return action;
}
@Before
public void before() throws Exception {
createTlds("tld", "foo");
writeGcsFile(
gcsService,
new GcsFilename("basin/icann/monthly/2017-06", "test-transactions-201706.csv"),
new GcsFilename("basin/icann/monthly/2006-06", "tld-transactions-200606.csv"),
PAYLOAD_SUCCESS);
writeGcsFile(
gcsService,
new GcsFilename("basin/icann/monthly/2017-06", "a-activity-201706.csv"),
new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"),
PAYLOAD_FAIL);
writeGcsFile(
gcsService,
new GcsFilename("basin/icann/monthly/2017-06", "MANIFEST.txt"),
new GcsFilename("basin/icann/monthly/2006-06", "foo-transactions-200606.csv"),
PAYLOAD_SUCCESS);
writeGcsFile(
gcsService,
new GcsFilename("basin/icann/monthly/2006-06", "foo-activity-200606.csv"),
PAYLOAD_SUCCESS);
writeGcsFile(
gcsService,
new GcsFilename("basin/icann/monthly/2006-06", "MANIFEST.txt"),
MANIFEST_PAYLOAD);
when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv")).thenReturn(true);
when(mockReporter.send(PAYLOAD_FAIL, "a-activity-201706.csv")).thenReturn(false);
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv")).thenReturn(true);
when(mockReporter.send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv")).thenReturn(true);
when(mockReporter.send(PAYLOAD_FAIL, "tld-activity-200606.csv")).thenReturn(false);
when(mockReporter.send(PAYLOAD_SUCCESS, "foo-activity-200606.csv")).thenReturn(true);
when(mockReporter.send(MANIFEST_PAYLOAD, "MANIFEST.txt")).thenReturn(true);
clock.setTo(DateTime.parse("2006-06-06T00:30:00Z"));
persistResource(
Cursor.create(
CursorType.ICANN_UPLOAD_ACTIVITY, DateTime.parse("2006-06-06TZ"), Registry.get("tld")));
persistResource(
Cursor.create(
CursorType.ICANN_UPLOAD_TX, DateTime.parse("2006-06-06TZ"), Registry.get("tld")));
persistResource(
Cursor.createGlobal(CursorType.ICANN_UPLOAD_MANIFEST, DateTime.parse("2006-07-06TZ")));
persistResource(
Cursor.create(
CursorType.ICANN_UPLOAD_ACTIVITY, DateTime.parse("2006-06-06TZ"), Registry.get("foo")));
persistResource(
Cursor.create(
CursorType.ICANN_UPLOAD_TX, DateTime.parse("2006-06-06TZ"), Registry.get("foo")));
loggerToIntercept.addHandler(logHandler);
}
@Test
public void testSuccess() throws Exception {
IcannReportingUploadAction action = createAction();
action.run();
verify(mockReporter).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv");
verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-activity-200606.csv");
verify(mockReporter).send(PAYLOAD_FAIL, "tld-activity-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv");
verifyNoMoreInteractions(mockReporter);
assertThat(((FakeResponse) action.response).getPayload())
.isEqualTo("OK, attempted uploading 2 reports");
verify(emailService)
.sendEmail(
EmailMessage.create(
"ICANN Monthly report upload summary: 1/2 succeeded",
"ICANN Monthly report upload summary: 3/4 succeeded",
"Report Filename - Upload status:\n"
+ "test-transactions-201706.csv - SUCCESS\n"
+ "a-activity-201706.csv - FAILURE",
+ "foo-activity-200606.csv - SUCCESS\n"
+ "tld-activity-200606.csv - FAILURE\n"
+ "foo-transactions-200606.csv - SUCCESS\n"
+ "tld-transactions-200606.csv - SUCCESS",
new InternetAddress("recipient@example.com"),
new InternetAddress("sender@example.com")));
}
@Test
public void testSuccess_WithRetry() throws Exception {
public void testSuccess_advancesCursor() throws Exception {
writeGcsFile(
gcsService,
new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"),
PAYLOAD_SUCCESS);
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200606.csv")).thenReturn(true);
IcannReportingUploadAction action = createAction();
when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv"))
action.run();
ofy().clearSessionCache();
Cursor cursor =
ofy()
.load()
.key(Cursor.createKey(CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("tld")))
.now();
assertThat(cursor.getCursorTime()).isEqualTo(DateTime.parse("2006-07-01TZ"));
}
@Test
public void testSuccess_noUploadsNeeded() throws Exception {
clock.setTo(DateTime.parse("2006-5-01T00:30:00Z"));
IcannReportingUploadAction action = createAction();
action.run();
ofy().clearSessionCache();
verifyNoMoreInteractions(mockReporter);
verify(emailService)
.sendEmail(
EmailMessage.create(
"ICANN Monthly report upload summary: 0/0 succeeded",
"Report Filename - Upload status:\n",
new InternetAddress("recipient@example.com"),
new InternetAddress("sender@example.com")));
}
@Test
public void testSuccess_uploadManifest() throws Exception {
persistResource(
Cursor.createGlobal(CursorType.ICANN_UPLOAD_MANIFEST, DateTime.parse("2006-06-06TZ")));
IcannReportingUploadAction action = createAction();
action.run();
ofy().clearSessionCache();
Cursor cursor =
ofy().load().key(Cursor.createGlobalKey(CursorType.ICANN_UPLOAD_MANIFEST)).now();
assertThat(cursor.getCursorTime()).isEqualTo(DateTime.parse("2006-07-01TZ"));
verify(mockReporter).send(PAYLOAD_FAIL, "tld-activity-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-activity-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv");
verify(mockReporter).send(MANIFEST_PAYLOAD, "MANIFEST.txt");
verifyNoMoreInteractions(mockReporter);
}
@Test
public void testSuccess_withRetry() throws Exception {
IcannReportingUploadAction action = createAction();
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv"))
.thenThrow(new IOException("Expected exception."))
.thenReturn(true);
action.run();
verify(mockReporter, times(2)).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv");
verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-activity-200606.csv");
verify(mockReporter).send(PAYLOAD_FAIL, "tld-activity-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv");
verify(mockReporter, times(2)).send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv");
verifyNoMoreInteractions(mockReporter);
assertThat(((FakeResponse) action.response).getPayload())
.isEqualTo("OK, attempted uploading 2 reports");
verify(emailService)
.sendEmail(
EmailMessage.create(
"ICANN Monthly report upload summary: 1/2 succeeded",
"ICANN Monthly report upload summary: 3/4 succeeded",
"Report Filename - Upload status:\n"
+ "test-transactions-201706.csv - SUCCESS\n"
+ "a-activity-201706.csv - FAILURE",
new InternetAddress("recipient@example.com"),
new InternetAddress("sender@example.com")));
}
@Test
public void testFailure_firstUnrecoverable_stillAttemptsUploadingBoth() throws Exception {
IcannReportingUploadAction action = createAction();
when(mockReporter.send(PAYLOAD_SUCCESS, "test-transactions-201706.csv"))
.thenThrow(new IOException("Expected exception"));
action.run();
verify(mockReporter, times(3)).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv");
verify(mockReporter).send(PAYLOAD_FAIL, "a-activity-201706.csv");
verifyNoMoreInteractions(mockReporter);
assertThat(((FakeResponse) action.response).getPayload())
.isEqualTo("OK, attempted uploading 2 reports");
verify(emailService)
.sendEmail(
EmailMessage.create(
"ICANN Monthly report upload summary: 0/2 succeeded",
"Report Filename - Upload status:\n"
+ "test-transactions-201706.csv - FAILURE\n"
+ "a-activity-201706.csv - FAILURE",
+ "foo-activity-200606.csv - SUCCESS\n"
+ "tld-activity-200606.csv - FAILURE\n"
+ "foo-transactions-200606.csv - SUCCESS\n"
+ "tld-transactions-200606.csv - SUCCESS",
new InternetAddress("recipient@example.com"),
new InternetAddress("sender@example.com")));
}
@ -169,38 +252,149 @@ public class IcannReportingUploadActionTest {
new IOException("Your IP address 25.147.130.158 is not allowed to connect"));
}
@Test
public void testFailure_cursorIsNotAdvancedForward() throws Exception {
runTest_nonRetryableException(
new IOException("Your IP address 25.147.130.158 is not allowed to connect"));
ofy().clearSessionCache();
Cursor cursor =
ofy()
.load()
.key(Cursor.createKey(CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("tld")))
.now();
assertThat(cursor.getCursorTime()).isEqualTo(DateTime.parse("2006-06-06TZ"));
}
@Test
public void testNotRunIfCursorDateIsAfterToday() throws Exception {
clock.setTo(DateTime.parse("2006-05-01T00:30:00Z"));
IcannReportingUploadAction action = createAction();
action.run();
ofy().clearSessionCache();
Cursor cursor =
ofy()
.load()
.key(Cursor.createKey(CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("foo")))
.now();
assertThat(cursor.getCursorTime()).isEqualTo(DateTime.parse("2006-06-06TZ"));
verifyNoMoreInteractions(mockReporter);
}
private void runTest_nonRetryableException(Exception nonRetryableException) throws Exception {
IcannReportingUploadAction action = createAction();
when(mockReporter.send(PAYLOAD_FAIL, "a-activity-201706.csv"))
when(mockReporter.send(PAYLOAD_FAIL, "tld-activity-200606.csv"))
.thenThrow(nonRetryableException)
.thenThrow(
new AssertionError(
"This should never be thrown because the previous exception isn't retryable"));
action.run();
verify(mockReporter, times(1)).send(PAYLOAD_FAIL, "a-activity-201706.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "test-transactions-201706.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-activity-200606.csv");
verify(mockReporter, times(1)).send(PAYLOAD_FAIL, "tld-activity-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv");
verifyNoMoreInteractions(mockReporter);
assertThat(((FakeResponse) action.response).getPayload())
.isEqualTo("OK, attempted uploading 2 reports");
verify(emailService)
.sendEmail(
EmailMessage.create(
"ICANN Monthly report upload summary: 1/2 succeeded",
"ICANN Monthly report upload summary: 3/4 succeeded",
"Report Filename - Upload status:\n"
+ "test-transactions-201706.csv - SUCCESS\n"
+ "a-activity-201706.csv - FAILURE",
+ "foo-activity-200606.csv - SUCCESS\n"
+ "tld-activity-200606.csv - FAILURE\n"
+ "foo-transactions-200606.csv - SUCCESS\n"
+ "tld-transactions-200606.csv - SUCCESS",
new InternetAddress("recipient@example.com"),
new InternetAddress("sender@example.com")));
}
@Test
public void testFail_FileNotFound() throws Exception {
public void testFail_fileNotFound() throws Exception {
IcannReportingUploadAction action = createAction();
action.subdir = "somewhere/else";
IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, action::run);
action.run();
assertAboutLogs()
.that(logHandler)
.hasLogAtLevelWithMessage(
Level.SEVERE,
"Could not upload ICANN_UPLOAD_ACTIVITY report for foo because file"
+ " foo-activity-200606.csv did not exist");
}
@Test
public void testWarning_fileNotStagedYet() throws Exception {
persistResource(
Cursor.create(
CursorType.ICANN_UPLOAD_ACTIVITY, DateTime.parse("2006-07-01TZ"), Registry.get("foo")));
clock.setTo(DateTime.parse("2006-07-01T00:30:00Z"));
IcannReportingUploadAction action = createAction();
action.subdir = "icann/monthly/2006-07";
action.run();
assertAboutLogs()
.that(logHandler)
.hasLogAtLevelWithMessage(
Level.INFO,
"Could not upload ICANN_UPLOAD_ACTIVITY report for foo because file"
+ " foo-activity-200607.csv did not exist. This report may not have been staged"
+ " yet.");
}
@Test
public void testFailure_lockIsntAvailable() throws Exception {
IcannReportingUploadAction action = createAction();
action.lockHandler = new FakeLockHandler(false);
ServiceUnavailableException thrown =
assertThrows(ServiceUnavailableException.class, () -> action.run());
assertThat(thrown)
.hasMessageThat()
.isEqualTo("Object MANIFEST.txt in bucket basin/somewhere/else not found");
.contains("Lock for IcannReportingUploadAction already in use");
}
@Test
public void testSuccess_nullCursors() throws Exception {
createTlds("new");
writeGcsFile(
gcsService,
new GcsFilename("basin/icann/monthly/2006-06", "new-transactions-200606.csv"),
PAYLOAD_SUCCESS);
writeGcsFile(
gcsService,
new GcsFilename("basin/icann/monthly/2006-06", "new-activity-200606.csv"),
PAYLOAD_SUCCESS);
when(mockReporter.send(PAYLOAD_SUCCESS, "new-transactions-200606.csv")).thenReturn(true);
when(mockReporter.send(PAYLOAD_SUCCESS, "new-activity-200606.csv")).thenReturn(true);
IcannReportingUploadAction action = createAction();
action.run();
verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-activity-200606.csv");
verify(mockReporter).send(PAYLOAD_FAIL, "tld-activity-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "new-activity-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv");
verify(mockReporter).send(PAYLOAD_SUCCESS, "new-transactions-200606.csv");
verifyNoMoreInteractions(mockReporter);
verify(emailService)
.sendEmail(
EmailMessage.create(
"ICANN Monthly report upload summary: 5/6 succeeded",
"Report Filename - Upload status:\n"
+ "foo-activity-200606.csv - SUCCESS\n"
+ "new-activity-200606.csv - SUCCESS\n"
+ "tld-activity-200606.csv - FAILURE\n"
+ "foo-transactions-200606.csv - SUCCESS\n"
+ "new-transactions-200606.csv - SUCCESS\n"
+ "tld-transactions-200606.csv - SUCCESS",
new InternetAddress("recipient@example.com"),
new InternetAddress("sender@example.com")));
Cursor newActivityCursor =
ofy()
.load()
.key(Cursor.createKey(CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("new")))
.now();
assertThat(newActivityCursor.getCursorTime()).isEqualTo(DateTime.parse("2006-07-01TZ"));
Cursor newTransactionCursor =
ofy().load().key(Cursor.createKey(CursorType.ICANN_UPLOAD_TX, Registry.get("new"))).now();
assertThat(newTransactionCursor.getCursorTime()).isEqualTo(DateTime.parse("2006-07-01TZ"));
}
}