mirror of
https://github.com/google/nomulus.git
synced 2025-08-03 08:22:13 +02:00
Allow RdeStagingAction to be invoked manually
RdeStagingAction always processed all RDE and BRDA deposits currently outstanding, updating the cursors appropriately and kicking off the upload job. Sometimes we don't want all that. We just want to create a specific deposit by hand, without modifying the cursors or uploading. This CL adds parameters to support that. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=152415959
This commit is contained in:
mountford
Ben McIlwain
parent
08009e755f
commit
4f94464eaf
8 changed files with 499 additions and 25 deletions
|
|
@ -36,6 +36,13 @@ import static java.util.Arrays.asList;
|
|||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.ListItem;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.base.Optional;
|
||||
import com.google.common.collect.FluentIterable;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.net.InetAddresses;
|
||||
|
|
@ -47,7 +54,9 @@ import google.registry.model.common.Cursor.CursorType;
|
|||
import google.registry.model.host.HostResource;
|
||||
import google.registry.model.ofy.Ofy;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.RequestParameters;
|
||||
import google.registry.testing.ExceptionRule;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeKeyringModule;
|
||||
import google.registry.testing.FakeResponse;
|
||||
|
|
@ -101,6 +110,9 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
@Rule
|
||||
public final InjectRule inject = new InjectRule();
|
||||
|
||||
@Rule
|
||||
public final ExceptionRule thrown = new ExceptionRule();
|
||||
|
||||
private final FakeClock clock = new FakeClock();
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
|
|
@ -136,6 +148,47 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
action.pendingDepositChecker.rdeInterval = Duration.standardDays(1);
|
||||
action.response = response;
|
||||
action.transactionCooldown = Duration.ZERO;
|
||||
action.directory = Optional.<String>absent();
|
||||
action.modeStrings = ImmutableSet.<String>of();
|
||||
action.tlds = ImmutableSet.<String>of();
|
||||
action.watermarks = ImmutableSet.<DateTime>of();
|
||||
action.revision = Optional.<Integer>absent();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_modeInNonManualMode_throwsException() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
clock.setTo(DateTime.parse("2000-01-01TZ"));
|
||||
action.modeStrings = ImmutableSet.of("full");
|
||||
thrown.expect(BadRequestException.class);
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_tldInNonManualMode_throwsException() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
clock.setTo(DateTime.parse("2000-01-01TZ"));
|
||||
action.tlds = ImmutableSet.of("tld");
|
||||
thrown.expect(BadRequestException.class);
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_watermarkInNonManualMode_throwsException() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
clock.setTo(DateTime.parse("2000-01-01TZ"));
|
||||
action.watermarks = ImmutableSet.of(clock.nowUtc());
|
||||
thrown.expect(BadRequestException.class);
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_revisionInNonManualMode_throwsException() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
clock.setTo(DateTime.parse("2000-01-01TZ"));
|
||||
action.revision = Optional.of(42);
|
||||
thrown.expect(BadRequestException.class);
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -184,6 +237,86 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
assertAtLeastOneTaskIsEnqueued("mapreduce");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testManualRun_emptyMode_throwsException() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
clock.setTo(DateTime.parse("2000-01-01TZ"));
|
||||
action.manual = true;
|
||||
action.directory = Optional.of("test/");
|
||||
action.modeStrings = ImmutableSet.<String>of();
|
||||
action.tlds = ImmutableSet.of("lol");
|
||||
action.watermarks = ImmutableSet.of(clock.nowUtc());
|
||||
thrown.expect(BadRequestException.class);
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testManualRun_invalidMode_throwsException() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
clock.setTo(DateTime.parse("2000-01-01TZ"));
|
||||
action.manual = true;
|
||||
action.directory = Optional.of("test/");
|
||||
action.modeStrings = ImmutableSet.of("full", "thing");
|
||||
action.tlds = ImmutableSet.of("lol");
|
||||
action.watermarks = ImmutableSet.of(clock.nowUtc());
|
||||
thrown.expect(BadRequestException.class);
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testManualRun_emptyTld_throwsException() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
clock.setTo(DateTime.parse("2000-01-01TZ"));
|
||||
action.manual = true;
|
||||
action.directory = Optional.of("test/");
|
||||
action.modeStrings = ImmutableSet.of("full");
|
||||
action.tlds = ImmutableSet.<String>of();
|
||||
action.watermarks = ImmutableSet.of(clock.nowUtc());
|
||||
thrown.expect(BadRequestException.class);
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testManualRun_emptyWatermark_throwsException() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
clock.setTo(DateTime.parse("2000-01-01TZ"));
|
||||
action.manual = true;
|
||||
action.directory = Optional.of("test/");
|
||||
action.modeStrings = ImmutableSet.of("full");
|
||||
action.tlds = ImmutableSet.of("lol");
|
||||
action.watermarks = ImmutableSet.of();
|
||||
thrown.expect(BadRequestException.class);
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testManualRun_nonDayStartWatermark_throwsException() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
clock.setTo(DateTime.parse("2000-01-01TZ"));
|
||||
action.manual = true;
|
||||
action.directory = Optional.of("test/");
|
||||
action.modeStrings = ImmutableSet.of("full");
|
||||
action.tlds = ImmutableSet.of("lol");
|
||||
action.watermarks = ImmutableSet.of(DateTime.parse("2001-01-01T01:36:45Z"));
|
||||
thrown.expect(BadRequestException.class);
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testManualRun_validParameters_runsMapReduce() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
clock.setTo(DateTime.parse("2000-01-01TZ"));
|
||||
action.manual = true;
|
||||
action.directory = Optional.of("test/");
|
||||
action.modeStrings = ImmutableSet.of("full");
|
||||
action.tlds = ImmutableSet.of("lol");
|
||||
action.watermarks = ImmutableSet.of(DateTime.parse("2001-01-01TZ"));
|
||||
action.run();
|
||||
assertThat(response.getStatus()).isEqualTo(200);
|
||||
assertThat(response.getPayload()).contains("_ah/pipeline/status.html?root=");
|
||||
assertAtLeastOneTaskIsEnqueued("mapreduce");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMapReduce_bunchOfResources_headerHasCorrectCounts() throws Exception {
|
||||
clock.setTo(DateTime.parse("1999-12-31TZ"));
|
||||
|
|
@ -610,6 +743,87 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
.isEqualTo(DateTime.parse("1984-12-21TZ"));
|
||||
}
|
||||
|
||||
private void doManualModeMapReduceTest(int revision, ImmutableSet<String> tlds) throws Exception {
|
||||
clock.setTo(DateTime.parse("1999-12-31TZ"));
|
||||
for (String tld : tlds) {
|
||||
createTldWithEscrowEnabled(tld);
|
||||
makeDomainResource(clock, tld);
|
||||
setCursor(Registry.get(tld), RDE_STAGING, DateTime.parse("1999-01-01TZ"));
|
||||
setCursor(Registry.get(tld), BRDA, DateTime.parse("2001-01-01TZ"));
|
||||
}
|
||||
|
||||
action.manual = true;
|
||||
action.directory = Optional.of("test/");
|
||||
action.modeStrings = ImmutableSet.of("full", "thin");
|
||||
action.tlds = tlds;
|
||||
action.watermarks =
|
||||
ImmutableSet.of(DateTime.parse("2000-01-01TZ"), DateTime.parse("2000-01-02TZ"));
|
||||
action.revision = Optional.of(revision);
|
||||
|
||||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce", clock);
|
||||
|
||||
ListResult listResult =
|
||||
gcsService.list("rde-bucket", new ListOptions.Builder().setPrefix("manual/test").build());
|
||||
ImmutableSet<String> filenames =
|
||||
FluentIterable.from(ImmutableList.copyOf(listResult))
|
||||
.transform(
|
||||
new Function<ListItem, String>() {
|
||||
@Override
|
||||
public String apply(ListItem listItem) {
|
||||
return listItem.getName();
|
||||
}
|
||||
})
|
||||
.toSet();
|
||||
for (String tld : tlds) {
|
||||
assertThat(filenames)
|
||||
.containsAllOf(
|
||||
"manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + "-report.xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.length",
|
||||
"manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.length",
|
||||
"manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + "-report.xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.length",
|
||||
"manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.length");
|
||||
|
||||
assertThat(
|
||||
ofy()
|
||||
.load()
|
||||
.key(Cursor.createKey(RDE_STAGING, Registry.get(tld)))
|
||||
.now()
|
||||
.getCursorTime())
|
||||
.isEqualTo(DateTime.parse("1999-01-01TZ"));
|
||||
assertThat(ofy().load().key(Cursor.createKey(BRDA, Registry.get(tld))).now().getCursorTime())
|
||||
.isEqualTo(DateTime.parse("2001-01-01TZ"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMapReduce_manualMode_generatesCorrectDepositsWithoutAdvancingCursors()
|
||||
throws Exception {
|
||||
doManualModeMapReduceTest(0, ImmutableSet.of("lol"));
|
||||
XmlTestUtils.assertXmlEquals(
|
||||
readResourceUtf8(getClass(), "testdata/testMapReduce_withDomain_producesExpectedXml.xml"),
|
||||
readXml("manual/test/lol_2000-01-01_full_S1_R0.xml.ghostryde"),
|
||||
"deposit.contents.registrar.crDate",
|
||||
"deposit.contents.registrar.upDate");
|
||||
XmlTestUtils.assertXmlEquals(
|
||||
readResourceUtf8(getClass(), "testdata/testMapReduce_withDomain_producesReportXml.xml"),
|
||||
readXml(
|
||||
"manual/test/lol_2000-01-01_full_S1_R0-report.xml.ghostryde"),
|
||||
"deposit.contents.registrar.crDate",
|
||||
"deposit.contents.registrar.upDate");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMapReduce_manualMode_nonZeroRevisionAndMultipleTlds()
|
||||
throws Exception {
|
||||
doManualModeMapReduceTest(42, ImmutableSet.of("lol", "slug"));
|
||||
}
|
||||
|
||||
private String readXml(String objectName) throws IOException, PGPException {
|
||||
GcsFilename file = new GcsFilename("rde-bucket", objectName);
|
||||
return new String(Ghostryde.decode(readGcsFile(gcsService, file), decryptKey).getData(), UTF_8);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue