mirror of
https://github.com/google/nomulus.git
synced 2025-05-13 16:07:15 +02:00
Change GenerateEscrowDepositCommand to trigger back end deposit generation
Previously, GenerateEscrowDepositCommand generated the deposit itself. Channeling it through the existing deposit generation code make things more maintainable. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=152847950
This commit is contained in:
mountford
Ben McIlwain
parent
8653d2f204
commit
dea386d08a
7 changed files with 224 additions and 367 deletions
|
|
@ -40,11 +40,11 @@ import org.joda.time.DateTime;
|
|||
@Module
|
||||
public final class RdeModule {
|
||||
|
||||
static final String PARAM_WATERMARK = "watermark";
|
||||
static final String PARAM_MANUAL = "manual";
|
||||
static final String PARAM_DIRECTORY = "directory";
|
||||
static final String PARAM_MODE = "mode";
|
||||
static final String PARAM_REVISION = "revision";
|
||||
public static final String PARAM_WATERMARK = "watermark";
|
||||
public static final String PARAM_MANUAL = "manual";
|
||||
public static final String PARAM_DIRECTORY = "directory";
|
||||
public static final String PARAM_MODE = "mode";
|
||||
public static final String PARAM_REVISION = "revision";
|
||||
|
||||
@Provides
|
||||
@Parameter(PARAM_WATERMARK)
|
||||
|
|
|
|||
|
|
@ -181,9 +181,11 @@ import org.joda.time.Duration;
|
|||
* @see <a href="https://tools.ietf.org/html/draft-arias-noguchi-registry-data-escrow-06">Registry Data Escrow Specification</a>
|
||||
* @see <a href="https://tools.ietf.org/html/draft-arias-noguchi-dnrd-objects-mapping-05">Domain Name Registration Data Objects Mapping</a>
|
||||
*/
|
||||
@Action(path = "/_dr/task/rdeStaging")
|
||||
@Action(path = RdeStagingAction.PATH)
|
||||
public final class RdeStagingAction implements Runnable {
|
||||
|
||||
public static final String PATH = "/_dr/task/rdeStaging";
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@Inject Clock clock;
|
||||
|
|
@ -302,6 +304,10 @@ public final class RdeStagingAction implements Runnable {
|
|||
}
|
||||
}
|
||||
|
||||
if (revision.isPresent() && (revision.get() < 0)) {
|
||||
throw new BadRequestException("Revision must be greater than or equal to zero");
|
||||
}
|
||||
|
||||
ImmutableSetMultimap.Builder<String, PendingDeposit> pendingsBuilder =
|
||||
new ImmutableSetMultimap.Builder<>();
|
||||
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ java_library(
|
|||
"//java/google/registry/model",
|
||||
"//java/google/registry/pricing",
|
||||
"//java/google/registry/rde",
|
||||
"//java/google/registry/request",
|
||||
"//java/google/registry/request:modules",
|
||||
"//java/google/registry/security",
|
||||
"//java/google/registry/tldconfig/idn",
|
||||
|
|
|
|||
|
|
@ -14,234 +14,102 @@
|
|||
|
||||
package google.registry.tools;
|
||||
|
||||
import static google.registry.model.EppResourceUtils.loadAtPointInTime;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.registry.Registries.assertTldExists;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
||||
import static google.registry.model.registry.Registries.assertTldsExist;
|
||||
|
||||
import com.beust.jcommander.Parameter;
|
||||
import com.beust.jcommander.ParameterException;
|
||||
import com.beust.jcommander.Parameters;
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.base.Predicates;
|
||||
import com.google.common.collect.FluentIterable;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.Result;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.EppResourceUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
import google.registry.model.domain.DomainResource;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.EppResourceIndexBucket;
|
||||
import com.google.appengine.api.modules.ModulesService;
|
||||
import com.google.appengine.api.taskqueue.Queue;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.common.base.Joiner;
|
||||
import google.registry.model.rde.RdeMode;
|
||||
import google.registry.model.rde.RdeNamingUtils;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.rde.DepositFragment;
|
||||
import google.registry.rde.RdeCounter;
|
||||
import google.registry.rde.RdeMarshaller;
|
||||
import google.registry.rde.RdeResourceType;
|
||||
import google.registry.rde.RdeUtil;
|
||||
import google.registry.tldconfig.idn.IdnTableEnum;
|
||||
import google.registry.rde.RdeModule;
|
||||
import google.registry.rde.RdeStagingAction;
|
||||
import google.registry.request.RequestParameters;
|
||||
import google.registry.tools.Command.RemoteApiCommand;
|
||||
import google.registry.tools.params.DateTimeParameter;
|
||||
import google.registry.tools.params.PathParameter;
|
||||
import google.registry.xjc.rdeheader.XjcRdeHeader;
|
||||
import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** Command to generate an XML RDE escrow deposit (with relevant files) in current directory. */
|
||||
/**
|
||||
* Command to kick off the server-side generation of an XML RDE or BRDA escrow deposit, which will
|
||||
* be stored in the specified manual subdirectory of the GCS RDE bucket.
|
||||
*/
|
||||
@Parameters(separators = " =", commandDescription = "Generate an XML escrow deposit.")
|
||||
final class GenerateEscrowDepositCommand implements RemoteApiCommand {
|
||||
|
||||
@Parameter(
|
||||
names = {"-t", "--tld"},
|
||||
description = "Top level domain for which deposit should be generated.",
|
||||
description = "Top level domain(s) for which deposit should be generated.",
|
||||
required = true)
|
||||
private String tld;
|
||||
private List<String> tlds;
|
||||
|
||||
@Parameter(
|
||||
names = {"-w", "--watermark"},
|
||||
description = "Point-in-time timestamp for snapshotting Datastore.",
|
||||
validateWith = DateTimeParameter.class)
|
||||
private DateTime watermark = DateTime.now(UTC);
|
||||
description = "Point-in-time timestamp(s) for snapshotting Datastore.",
|
||||
required = true,
|
||||
converter = DateTimeParameter.class)
|
||||
private List<DateTime> watermarks;
|
||||
|
||||
@Parameter(
|
||||
names = {"-m", "--mode"},
|
||||
description = "FULL/THIN mode of operation.")
|
||||
description = "Mode of operation: FULL for RDE deposits, THIN for BRDA deposits.")
|
||||
private RdeMode mode = RdeMode.FULL;
|
||||
|
||||
@Parameter(
|
||||
names = {"-r", "--revision"},
|
||||
description = "Revision number. Use >0 for resends.")
|
||||
private int revision = 0;
|
||||
private Integer revision;
|
||||
|
||||
@Parameter(
|
||||
names = {"-o", "--outdir"},
|
||||
description = "Specify output directory. Default is current directory.",
|
||||
validateWith = PathParameter.OutputDirectory.class)
|
||||
private Path outdir = Paths.get(".");
|
||||
description = "Specify output subdirectory (under GCS RDE bucket, directory manual).",
|
||||
required = true)
|
||||
private String outdir;
|
||||
|
||||
@Inject
|
||||
EscrowDepositEncryptor encryptor;
|
||||
|
||||
@Inject
|
||||
RdeCounter counter;
|
||||
|
||||
@Inject
|
||||
@Config("eppResourceIndexBucketCount")
|
||||
int eppResourceIndexBucketCount;
|
||||
@Inject ModulesService modulesService;
|
||||
@Inject @Named("rde-report") Queue queue;
|
||||
|
||||
@Override
|
||||
public void run() throws Exception {
|
||||
RdeMarshaller marshaller = new RdeMarshaller();
|
||||
assertTldExists(tld);
|
||||
String suffix = String.format("-%s-%s.tmp.xml", tld, watermark);
|
||||
Path xmlPath = outdir.resolve("deposit" + suffix);
|
||||
Path reportPath = outdir.resolve("report" + suffix);
|
||||
try {
|
||||
String id = RdeUtil.timestampToId(watermark);
|
||||
XjcRdeHeader header;
|
||||
try (OutputStream xmlOutputBytes = Files.newOutputStream(xmlPath);
|
||||
Writer xmlOutput = new OutputStreamWriter(xmlOutputBytes, UTF_8)) {
|
||||
xmlOutput.write(
|
||||
marshaller.makeHeader(id, watermark, RdeResourceType.getUris(mode), revision));
|
||||
for (ImmutableObject resource
|
||||
: Iterables.concat(Registrar.loadAll(), load(scan()))) {
|
||||
DepositFragment frag;
|
||||
if (resource instanceof Registrar) {
|
||||
frag = marshaller.marshalRegistrar((Registrar) resource);
|
||||
} else if (resource instanceof ContactResource) {
|
||||
frag = marshaller.marshalContact((ContactResource) resource);
|
||||
} else if (resource instanceof DomainResource) {
|
||||
DomainResource domain = (DomainResource) resource;
|
||||
if (!domain.getTld().equals(tld)) {
|
||||
continue;
|
||||
|
||||
if (tlds.isEmpty()) {
|
||||
throw new ParameterException("At least one TLD must be specified");
|
||||
}
|
||||
frag = marshaller.marshalDomain(domain, mode);
|
||||
} else if (resource instanceof HostResource) {
|
||||
HostResource host = (HostResource) resource;
|
||||
frag = host.isSubordinate()
|
||||
? marshaller.marshalSubordinateHost(
|
||||
host,
|
||||
// Note that loadAtPointInTime() does cloneProjectedAtTime(watermark) for us.
|
||||
loadAtPointInTime(
|
||||
ofy().load().key(host.getSuperordinateDomain()).now(), watermark).now())
|
||||
: marshaller.marshalExternalHost(host);
|
||||
} else {
|
||||
continue; // Surprise polymorphic entities, e.g. DomainApplication.
|
||||
}
|
||||
if (!frag.xml().isEmpty()) {
|
||||
xmlOutput.write(frag.xml());
|
||||
counter.increment(frag.type());
|
||||
}
|
||||
if (!frag.error().isEmpty()) {
|
||||
System.err.print(frag.error());
|
||||
}
|
||||
}
|
||||
for (IdnTableEnum idn : IdnTableEnum.values()) {
|
||||
xmlOutput.write(marshaller.marshalIdn(idn.getTable()));
|
||||
counter.increment(RdeResourceType.IDN);
|
||||
}
|
||||
header = counter.makeHeader(tld, mode);
|
||||
xmlOutput.write(marshaller.marshalStrictlyOrDie(new XjcRdeHeaderElement(header)));
|
||||
xmlOutput.write(marshaller.makeFooter());
|
||||
}
|
||||
try (OutputStream reportOutputBytes = Files.newOutputStream(reportPath)) {
|
||||
counter.makeReport(id, watermark, header, revision).marshal(reportOutputBytes, UTF_8);
|
||||
}
|
||||
String name = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
|
||||
encryptor.encrypt(tld, xmlPath, outdir);
|
||||
Files.move(xmlPath, outdir.resolve(name + ".xml"), REPLACE_EXISTING);
|
||||
Files.move(reportPath, outdir.resolve(name + "-report.xml"), REPLACE_EXISTING);
|
||||
} finally {
|
||||
Files.deleteIfExists(xmlPath);
|
||||
Files.deleteIfExists(reportPath);
|
||||
assertTldsExist(tlds);
|
||||
|
||||
for (DateTime watermark : watermarks) {
|
||||
if (!watermark.withTimeAtStartOfDay().equals(watermark)) {
|
||||
throw new ParameterException("Each watermark date must be the start of a day");
|
||||
}
|
||||
}
|
||||
|
||||
private Iterable<EppResource> scan() {
|
||||
return Iterables.concat(
|
||||
Iterables.transform(
|
||||
getEppResourceIndexBuckets(),
|
||||
new Function<Key<EppResourceIndexBucket>, Iterable<EppResource>>() {
|
||||
@Override
|
||||
public Iterable<EppResource> apply(Key<EppResourceIndexBucket> bucket) {
|
||||
System.err.printf("Scanning EppResourceIndexBucket %d of %d...\n",
|
||||
bucket.getId(), eppResourceIndexBucketCount);
|
||||
return scanBucket(bucket);
|
||||
}}));
|
||||
if ((revision != null) && (revision < 0)) {
|
||||
throw new ParameterException("Revision must be greater than or equal to zero");
|
||||
}
|
||||
|
||||
private Iterable<EppResource> scanBucket(final Key<EppResourceIndexBucket> bucket) {
|
||||
return ofy().load()
|
||||
.keys(FluentIterable
|
||||
.from(mode == RdeMode.FULL
|
||||
? Arrays.asList(
|
||||
Key.getKind(ContactResource.class),
|
||||
Key.getKind(DomainResource.class),
|
||||
Key.getKind(HostResource.class))
|
||||
: Arrays.asList(
|
||||
Key.getKind(DomainResource.class)))
|
||||
.transformAndConcat(new Function<String, Iterable<EppResourceIndex>>() {
|
||||
@Override
|
||||
public Iterable<EppResourceIndex> apply(String kind) {
|
||||
return ofy().load()
|
||||
.type(EppResourceIndex.class)
|
||||
.ancestor(bucket)
|
||||
.filter("kind", kind)
|
||||
.iterable();
|
||||
}})
|
||||
.transform(new Function<EppResourceIndex, Key<EppResource>>() {
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public Key<EppResource> apply(EppResourceIndex index) {
|
||||
return (Key<EppResource>) index.getKey();
|
||||
}}))
|
||||
.values();
|
||||
if (outdir.isEmpty()) {
|
||||
throw new ParameterException("Output subdirectory must not be empty");
|
||||
}
|
||||
|
||||
private <T extends EppResource> Iterable<T> load(final Iterable<T> resources) {
|
||||
return FluentIterable
|
||||
.from(Iterables.partition(
|
||||
Iterables.transform(resources,
|
||||
new Function<T, Result<T>>() {
|
||||
@Override
|
||||
public Result<T> apply(T resource) {
|
||||
return EppResourceUtils.loadAtPointInTime(resource, watermark);
|
||||
}}),
|
||||
1000))
|
||||
.transformAndConcat(new Function<Iterable<Result<T>>, Iterable<T>>() {
|
||||
@Override
|
||||
public Iterable<T> apply(Iterable<Result<T>> results) {
|
||||
return Iterables.transform(results,
|
||||
new Function<Result<T>, T>() {
|
||||
@Override
|
||||
public T apply(Result<T> result) {
|
||||
return result.now();
|
||||
}});
|
||||
}})
|
||||
.filter(Predicates.notNull());
|
||||
// Unlike many tool commands, this command is actually invoking an action on the backend module
|
||||
// (because it's a mapreduce). So we invoke it in a different way.
|
||||
String hostname = modulesService.getVersionHostname("backend", null);
|
||||
TaskOptions opts =
|
||||
withUrl(RdeStagingAction.PATH)
|
||||
.header("Host", hostname)
|
||||
.param(RdeModule.PARAM_MANUAL, String.valueOf(true))
|
||||
.param(RequestParameters.PARAM_TLD, Joiner.on(',').join(tlds))
|
||||
.param(RdeModule.PARAM_WATERMARK, Joiner.on(',').join(watermarks))
|
||||
.param(RdeModule.PARAM_MODE, mode.toString())
|
||||
.param(RdeModule.PARAM_DIRECTORY, outdir);
|
||||
if (revision != null) {
|
||||
opts = opts.param(RdeModule.PARAM_REVISION, String.valueOf(revision));
|
||||
}
|
||||
|
||||
private ImmutableList<Key<EppResourceIndexBucket>> getEppResourceIndexBuckets() {
|
||||
ImmutableList.Builder<Key<EppResourceIndexBucket>> builder = new ImmutableList.Builder<>();
|
||||
for (int i = 1; i <= eppResourceIndexBucketCount; i++) {
|
||||
builder.add(Key.create(EppResourceIndexBucket.class, i));
|
||||
}
|
||||
return builder.build();
|
||||
queue.add(opts);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,9 +22,11 @@ import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule;
|
|||
import google.registry.keyring.api.DummyKeyringModule;
|
||||
import google.registry.keyring.api.KeyModule;
|
||||
import google.registry.keyring.kms.KmsModule;
|
||||
import google.registry.rde.RdeModule;
|
||||
import google.registry.request.Modules.AppIdentityCredentialModule;
|
||||
import google.registry.request.Modules.DatastoreServiceModule;
|
||||
import google.registry.request.Modules.Jackson2Module;
|
||||
import google.registry.request.Modules.ModulesServiceModule;
|
||||
import google.registry.request.Modules.URLFetchServiceModule;
|
||||
import google.registry.request.Modules.UrlFetchTransportModule;
|
||||
import google.registry.request.Modules.UseAppIdentityCredentialForGoogleApisModule;
|
||||
|
|
@ -57,6 +59,8 @@ import javax.inject.Singleton;
|
|||
Jackson2Module.class,
|
||||
KeyModule.class,
|
||||
KmsModule.class,
|
||||
ModulesServiceModule.class,
|
||||
RdeModule.class,
|
||||
RegistryToolModule.class,
|
||||
SystemClockModule.class,
|
||||
SystemSleeperModule.class,
|
||||
|
|
|
|||
|
|
@ -302,6 +302,20 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testManualRun_invalidRevision_throwsException() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
clock.setTo(DateTime.parse("2000-01-01TZ"));
|
||||
action.manual = true;
|
||||
action.directory = Optional.of("test/");
|
||||
action.modeStrings = ImmutableSet.of("full");
|
||||
action.tlds = ImmutableSet.of("lol");
|
||||
action.watermarks = ImmutableSet.of(DateTime.parse("2001-01-01T00:00:00Z"));
|
||||
action.revision = Optional.of(-1);
|
||||
thrown.expect(BadRequestException.class);
|
||||
action.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testManualRun_validParameters_runsMapReduce() throws Exception {
|
||||
createTldWithEscrowEnabled("lol");
|
||||
|
|
|
|||
|
|
@ -14,50 +14,20 @@
|
|||
|
||||
package google.registry.tools;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static google.registry.testing.DatastoreHelper.createTld;
|
||||
import static google.registry.testing.DatastoreHelper.createTlds;
|
||||
import static google.registry.testing.DatastoreHelper.generateNewContactHostRoid;
|
||||
import static google.registry.testing.DatastoreHelper.persistResourceWithCommitLog;
|
||||
import static google.registry.util.ResourceUtils.readResourceUtf8;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static java.util.Arrays.asList;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.beust.jcommander.ParameterException;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.net.InetAddresses;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.model.eppcommon.StatusValue;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.model.ofy.Ofy;
|
||||
import google.registry.rde.RdeCounter;
|
||||
import google.registry.rde.RdeResourceType;
|
||||
import google.registry.rde.RdeUtil;
|
||||
import google.registry.testing.BouncyCastleProviderRule;
|
||||
import google.registry.testing.FakeClock;
|
||||
import com.google.appengine.api.modules.ModulesService;
|
||||
import google.registry.testing.InjectRule;
|
||||
import google.registry.util.Idn;
|
||||
import google.registry.xjc.XjcXmlTransformer;
|
||||
import google.registry.xjc.rde.XjcRdeContentType;
|
||||
import google.registry.xjc.rde.XjcRdeDeposit;
|
||||
import google.registry.xjc.rde.XjcRdeDepositTypeType;
|
||||
import google.registry.xjc.rdeheader.XjcRdeHeader;
|
||||
import google.registry.xjc.rdeheader.XjcRdeHeaderCount;
|
||||
import google.registry.xjc.rdehost.XjcRdeHost;
|
||||
import google.registry.xjc.rderegistrar.XjcRdeRegistrar;
|
||||
import google.registry.xml.XmlException;
|
||||
import google.registry.xml.XmlTestUtils;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import javax.xml.bind.JAXBElement;
|
||||
import org.joda.time.DateTime;
|
||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Matchers;
|
||||
import org.mockito.Mock;
|
||||
|
||||
/** Unit tests for {@link GenerateEscrowDepositCommand}. */
|
||||
public class GenerateEscrowDepositCommandTest
|
||||
|
|
@ -66,169 +36,163 @@ public class GenerateEscrowDepositCommandTest
|
|||
@Rule
|
||||
public final InjectRule inject = new InjectRule();
|
||||
|
||||
@Rule
|
||||
public final BouncyCastleProviderRule bouncy = new BouncyCastleProviderRule();
|
||||
|
||||
private final FakeClock clock = new FakeClock(DateTime.parse("2010-10-17T04:20:00Z"));
|
||||
private final List<? super XjcRdeContentType> alreadyExtracted = new ArrayList<>();
|
||||
@Mock ModulesService modulesService;
|
||||
|
||||
@Before
|
||||
public void before() throws Exception {
|
||||
inject.setStaticField(Ofy.class, "clock", clock);
|
||||
command.encryptor = EncryptEscrowDepositCommandTest.createEncryptor();
|
||||
command.counter = new RdeCounter();
|
||||
command.eppResourceIndexBucketCount = RegistryConfig.getEppResourceIndexBucketCount();
|
||||
createTld("tld");
|
||||
createTld("anothertld");
|
||||
command = new GenerateEscrowDepositCommand();
|
||||
command.modulesService = modulesService;
|
||||
command.queue = getQueue("rde-report");
|
||||
when(modulesService.getVersionHostname(Matchers.anyString(), Matchers.anyString()))
|
||||
.thenReturn("1.backend.test.localhost");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_randomDomain_generatesXmlAndEncryptsItToo() throws Exception {
|
||||
createTld("xn--q9jyb4c");
|
||||
runCommand(
|
||||
"--outdir=" + tmpDir.getRoot(),
|
||||
"--tld=xn--q9jyb4c",
|
||||
"--watermark=" + clock.nowUtc().withTimeAtStartOfDay());
|
||||
assertThat(tmpDir.getRoot().list()).asList().containsExactly(
|
||||
"xn--q9jyb4c_2010-10-17_full_S1_R0.xml",
|
||||
"xn--q9jyb4c_2010-10-17_full_S1_R0-report.xml",
|
||||
"xn--q9jyb4c_2010-10-17_full_S1_R0.ryde",
|
||||
"xn--q9jyb4c_2010-10-17_full_S1_R0.sig",
|
||||
"xn--q9jyb4c.pub");
|
||||
public void testCommand_missingTld() throws Exception {
|
||||
thrown.expect(ParameterException.class, "The following option is required: -t, --tld");
|
||||
runCommand("--watermark=2017-01-01T00:00:00Z", "--mode=thin", "-r 42", "-o test");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_missingTldName_fails() throws Exception {
|
||||
thrown.expect(ParameterException.class);
|
||||
runCommand(
|
||||
"--outdir=" + tmpDir.getRoot(),
|
||||
"--watermark=" + clock.nowUtc());
|
||||
public void testCommand_emptyTld() throws Exception {
|
||||
thrown.expect(IllegalArgumentException.class, "Null or empty TLD specified");
|
||||
runCommand("--tld=", "--watermark=2017-01-01T00:00:00Z", "--mode=thin", "-r 42", "-o test");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_nonexistentTld_fails() throws Exception {
|
||||
thrown.expect(IllegalArgumentException.class);
|
||||
public void testCommand_invalidTld() throws Exception {
|
||||
thrown.expect(IllegalArgumentException.class, "TLDs do not exist: invalid");
|
||||
runCommand(
|
||||
"--outdir=" + tmpDir.getRoot(),
|
||||
"--tld=xn--q9jyb4c",
|
||||
"--watermark=" + clock.nowUtc());
|
||||
"--tld=invalid", "--watermark=2017-01-01T00:00:00Z", "--mode=thin", "-r 42", "-o test");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_oneHostNotDeletedOrFuture_producesValidDepositXml() throws Exception {
|
||||
createTlds("lol", "ussr", "xn--q9jyb4c");
|
||||
clock.setTo(DateTime.parse("1980-01-01TZ"));
|
||||
persistResourceWithCommitLog(
|
||||
newHostResource("communism.ussr", "dead:beef::cafe").asBuilder()
|
||||
.setDeletionTime(DateTime.parse("1991-12-25TZ")) // nope you're deleted
|
||||
.build());
|
||||
clock.setTo(DateTime.parse("1999-12-31TZ"));
|
||||
saveHostResource("ns1.cat.lol", "feed::a:bee"); // different tld doesn't matter
|
||||
clock.setTo(DateTime.parse("2020-12-31TZ"));
|
||||
saveHostResource("ns2.cat.lol", "a:fed::acab"); // nope you're from the future
|
||||
clock.setTo(DateTime.parse("2010-10-17TZ"));
|
||||
runCommand(
|
||||
"--outdir=" + tmpDir.getRoot(),
|
||||
"--tld=xn--q9jyb4c",
|
||||
"--watermark=" + clock.nowUtc());
|
||||
XmlTestUtils.assertXmlEquals(
|
||||
readResourceUtf8(getClass(), "testdata/xn--q9jyb4c_2010-10-17_full_S1_R0.xml"),
|
||||
new String(
|
||||
Files.readAllBytes(
|
||||
Paths.get(tmpDir.getRoot().toString(), "xn--q9jyb4c_2010-10-17_full_S1_R0.xml")),
|
||||
UTF_8),
|
||||
"deposit.contents.registrar.crDate",
|
||||
"deposit.contents.registrar.upDate");
|
||||
public void testCommand_missingWatermark() throws Exception {
|
||||
thrown.expect(ParameterException.class, "The following option is required: -w, --watermark");
|
||||
runCommand("--tld=tld", "--mode=full", "-r 42", "-o test");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_generatedXml_isSchemaValidAndHasStuffInIt() throws Exception {
|
||||
clock.setTo(DateTime.parse("1984-12-17TZ"));
|
||||
createTld("xn--q9jyb4c");
|
||||
saveHostResource("ns1.cat.lol", "feed::a:bee");
|
||||
clock.setTo(DateTime.parse("1984-12-18TZ"));
|
||||
runCommand(
|
||||
"--outdir=" + tmpDir.getRoot(),
|
||||
"--tld=xn--q9jyb4c",
|
||||
"--watermark=" + clock.nowUtc());
|
||||
XjcRdeDeposit deposit = (XjcRdeDeposit)
|
||||
unmarshal(Files.readAllBytes(
|
||||
Paths.get(tmpDir.getRoot().toString(), "xn--q9jyb4c_1984-12-18_full_S1_R0.xml")));
|
||||
assertThat(deposit.getType()).isEqualTo(XjcRdeDepositTypeType.FULL);
|
||||
assertThat(deposit.getId()).isEqualTo(RdeUtil.timestampToId(DateTime.parse("1984-12-18TZ")));
|
||||
assertThat(deposit.getWatermark()).isEqualTo(DateTime.parse("1984-12-18TZ"));
|
||||
XjcRdeRegistrar registrar1 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit);
|
||||
XjcRdeRegistrar registrar2 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit);
|
||||
XjcRdeHost host = extractAndRemoveContentWithType(XjcRdeHost.class, deposit);
|
||||
XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);
|
||||
assertThat(host.getName()).isEqualTo("ns1.cat.lol");
|
||||
assertThat(asList(registrar1.getName(), registrar2.getName()))
|
||||
.containsExactly("New Registrar", "The Registrar");
|
||||
assertThat(mapifyCounts(header)).containsEntry(RdeResourceType.HOST.getUri(), 1L);
|
||||
assertThat(mapifyCounts(header)).containsEntry(RdeResourceType.REGISTRAR.getUri(), 2L);
|
||||
public void testCommand_emptyWatermark() throws Exception {
|
||||
thrown.expect(IllegalArgumentException.class, "Invalid format: \"\"");
|
||||
runCommand("--tld=tld", "--watermark=", "--mode=full", "-r 42", "-o test");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRun_thinBrdaDeposit_hostsGetExcluded() throws Exception {
|
||||
clock.setTo(DateTime.parse("1984-12-17TZ"));
|
||||
createTld("xn--q9jyb4c");
|
||||
saveHostResource("ns1.cat.lol", "feed::a:bee");
|
||||
clock.setTo(DateTime.parse("1984-12-18TZ"));
|
||||
public void testCommand_missingOutdir() throws Exception {
|
||||
thrown.expect(ParameterException.class, "The following option is required: -o, --outdir");
|
||||
runCommand("--tld=tld", "--watermark=2017-01-01T00:00:00Z", "--mode=thin", "-r 42");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCommand_emptyOutdir() throws Exception {
|
||||
thrown.expect(ParameterException.class, "Output subdirectory must not be empty");
|
||||
runCommand(
|
||||
"--outdir=" + tmpDir.getRoot(),
|
||||
"--tld=xn--q9jyb4c",
|
||||
"--watermark=" + clock.nowUtc(),
|
||||
"--mode=THIN");
|
||||
XjcRdeDeposit deposit = (XjcRdeDeposit)
|
||||
unmarshal(Files.readAllBytes(
|
||||
Paths.get(tmpDir.getRoot().toString(), "xn--q9jyb4c_1984-12-18_thin_S1_R0.xml")));
|
||||
assertThat(deposit.getType()).isEqualTo(XjcRdeDepositTypeType.FULL);
|
||||
assertThat(deposit.getId()).isEqualTo(RdeUtil.timestampToId(DateTime.parse("1984-12-18TZ")));
|
||||
assertThat(deposit.getWatermark()).isEqualTo(DateTime.parse("1984-12-18TZ"));
|
||||
XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);
|
||||
assertThat(mapifyCounts(header)).doesNotContainKey(RdeResourceType.HOST.getUri());
|
||||
assertThat(mapifyCounts(header)).containsEntry(RdeResourceType.REGISTRAR.getUri(), 2L);
|
||||
"--tld=tld", "--watermark=2017-01-01T00:00:00Z", "--mode=thin", "--outdir=", "-r 42");
|
||||
}
|
||||
|
||||
private HostResource saveHostResource(String fqdn, String ip) {
|
||||
clock.advanceOneMilli();
|
||||
return persistResourceWithCommitLog(newHostResource(fqdn, ip));
|
||||
@Test
|
||||
public void testCommand_invalidWatermark() throws Exception {
|
||||
thrown.expect(ParameterException.class, "Each watermark date must be the start of a day");
|
||||
runCommand(
|
||||
"--tld=tld",
|
||||
"--watermark=2017-01-01T10:00:00Z,2017-01-02T00:00:00Z",
|
||||
"--mode=thin",
|
||||
"-r 42",
|
||||
"-o test");
|
||||
}
|
||||
|
||||
private HostResource newHostResource(String fqdn, String ip) {
|
||||
return new HostResource.Builder()
|
||||
.setRepoId(generateNewContactHostRoid())
|
||||
.setCreationClientId("LawyerCat")
|
||||
.setCreationTimeForTest(clock.nowUtc())
|
||||
.setPersistedCurrentSponsorClientId("BusinessCat")
|
||||
.setFullyQualifiedHostName(Idn.toASCII(fqdn))
|
||||
.setInetAddresses(ImmutableSet.of(InetAddresses.forString(ip)))
|
||||
.setLastTransferTime(DateTime.parse("1910-01-01T00:00:00Z"))
|
||||
.setLastEppUpdateClientId("CeilingCat")
|
||||
.setLastEppUpdateTime(clock.nowUtc())
|
||||
.setStatusValues(ImmutableSet.of(StatusValue.OK))
|
||||
.build();
|
||||
@Test
|
||||
public void testCommand_invalidMode() throws Exception {
|
||||
thrown.expect(
|
||||
ParameterException.class, "Invalid value for -m parameter. Allowed values:[FULL, THIN]");
|
||||
runCommand(
|
||||
"--tld=tld",
|
||||
"--watermark=2017-01-01T00:00:00Z",
|
||||
"--mode=thing",
|
||||
"-r 42",
|
||||
"-o test");
|
||||
}
|
||||
|
||||
public static Object unmarshal(byte[] xml) throws XmlException {
|
||||
return XjcXmlTransformer.unmarshal(Object.class, new ByteArrayInputStream(xml));
|
||||
@Test
|
||||
public void testCommand_invalidRevision() throws Exception {
|
||||
thrown.expect(
|
||||
ParameterException.class, "Revision must be greater than or equal to zero");
|
||||
runCommand(
|
||||
"--tld=tld",
|
||||
"--watermark=2017-01-01T00:00:00Z",
|
||||
"--mode=thin",
|
||||
"-r -1",
|
||||
"-o test");
|
||||
}
|
||||
|
||||
private static ImmutableMap<String, Long> mapifyCounts(XjcRdeHeader header) {
|
||||
ImmutableMap.Builder<String, Long> builder = new ImmutableMap.Builder<>();
|
||||
for (XjcRdeHeaderCount count : header.getCounts()) {
|
||||
builder.put(count.getUri(), count.getValue());
|
||||
}
|
||||
return builder.build();
|
||||
@Test
|
||||
public void testCommand_success() throws Exception {
|
||||
runCommand("--tld=tld", "--watermark=2017-01-01T00:00:00Z", "--mode=thin", "-r 42", "-o test");
|
||||
|
||||
assertTasksEnqueued("rde-report",
|
||||
new TaskMatcher()
|
||||
.url("/_dr/task/rdeStaging")
|
||||
.header("Host", "1.backend.test.localhost")
|
||||
.param("mode", "THIN")
|
||||
.param("watermark", "2017-01-01T00:00:00.000Z")
|
||||
.param("tld", "tld")
|
||||
.param("directory", "test")
|
||||
.param("manual", "true")
|
||||
.param("revision", "42"));
|
||||
}
|
||||
|
||||
private <T extends XjcRdeContentType>
|
||||
T extractAndRemoveContentWithType(Class<T> type, XjcRdeDeposit deposit) {
|
||||
for (JAXBElement<? extends XjcRdeContentType> content : deposit.getContents().getContents()) {
|
||||
XjcRdeContentType piece = content.getValue();
|
||||
if (type.isInstance(piece) && !alreadyExtracted.contains(piece)) {
|
||||
alreadyExtracted.add(piece);
|
||||
return type.cast(piece);
|
||||
}
|
||||
}
|
||||
throw new AssertionError("Expected deposit to contain another " + type.getSimpleName());
|
||||
@Test
|
||||
public void testCommand_successWithDefaultRevision() throws Exception {
|
||||
runCommand("--tld=tld", "--watermark=2017-01-01T00:00:00Z", "--mode=thin", "-o test");
|
||||
|
||||
assertTasksEnqueued("rde-report",
|
||||
new TaskMatcher()
|
||||
.url("/_dr/task/rdeStaging")
|
||||
.header("Host", "1.backend.test.localhost")
|
||||
.param("mode", "THIN")
|
||||
.param("watermark", "2017-01-01T00:00:00.000Z")
|
||||
.param("tld", "tld")
|
||||
.param("directory", "test")
|
||||
.param("manual", "true"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCommand_successWithDefaultMode() throws Exception {
|
||||
runCommand("--tld=tld", "--watermark=2017-01-01T00:00:00Z", "-r=42", "-o test");
|
||||
|
||||
assertTasksEnqueued("rde-report",
|
||||
new TaskMatcher()
|
||||
.url("/_dr/task/rdeStaging")
|
||||
.header("Host", "1.backend.test.localhost")
|
||||
.param("mode", "FULL")
|
||||
.param("watermark", "2017-01-01T00:00:00.000Z")
|
||||
.param("tld", "tld")
|
||||
.param("directory", "test")
|
||||
.param("manual", "true")
|
||||
.param("revision", "42"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCommand_successWithMultipleArgumentValues() throws Exception {
|
||||
runCommand(
|
||||
"--tld=tld,anothertld",
|
||||
"--watermark=2017-01-01T00:00:00Z,2017-01-02T00:00:00Z",
|
||||
"--mode=thin",
|
||||
"-r 42",
|
||||
"-o test");
|
||||
|
||||
assertTasksEnqueued("rde-report",
|
||||
new TaskMatcher()
|
||||
.url("/_dr/task/rdeStaging")
|
||||
.header("Host", "1.backend.test.localhost")
|
||||
.param("mode", "THIN")
|
||||
.param("watermark", "2017-01-01T00:00:00.000Z,2017-01-02T00:00:00.000Z")
|
||||
.param("tld", "tld,anothertld")
|
||||
.param("directory", "test")
|
||||
.param("manual", "true")
|
||||
.param("revision", "42"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue