zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-19 02:39:34 +02:00
Code Issues Projects Releases Packages Wiki Activity

mv com/google/domain/registry google/registry

Browse source 
This change renames directories in preparation for the great package
rename. The repository is now in a broken state because the code
itself hasn't been updated. However this should ensure that git
correctly preserves history for each file.
This commit is contained in:
Justine Tunney 2016-05-13 18:55:08 -04:00
parent a41677aea1
commit 5012893c1d
2396 changed files with 0 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

42
java/google/registry/rde/BUILD Normal file
View file

@ -0,0 +1,42 @@
package(
default_visibility = ["//java/com/google/domain/registry:registry_project"],
)
java_library(
name = "rde",
srcs = glob(["*.java"]),
deps = [
"//java/com/google/common/annotations",
"//java/com/google/common/base",
"//java/com/google/common/collect",
"//java/com/google/common/html",
"//java/com/google/common/io",
"//java/com/google/common/net",
"//java/com/google/domain/registry/config",
"//java/com/google/domain/registry/gcs",
"//java/com/google/domain/registry/keyring/api",
"//java/com/google/domain/registry/mapreduce",
"//java/com/google/domain/registry/mapreduce/inputs",
"//java/com/google/domain/registry/model",
"//java/com/google/domain/registry/request",
"//java/com/google/domain/registry/tldconfig/idn",
"//java/com/google/domain/registry/util",
"//java/com/google/domain/registry/xjc",
"//java/com/google/domain/registry/xml",
"//third_party/java/appengine:appengine-api",
"//third_party/java/appengine_gcs_client",
"//third_party/java/appengine_mapreduce2:appengine_mapreduce",
"//third_party/java/auto:auto_factory",
"//third_party/java/auto:auto_value",
"//third_party/java/bouncycastle",
"//third_party/java/bouncycastle_bcpg",
"//third_party/java/dagger",
"//third_party/java/joda_time",
"//third_party/java/jsch",
"//third_party/java/jsr305_annotations",
"//third_party/java/jsr330_inject",
"//third_party/java/objectify:objectify-v4_1",
"//third_party/java/servlet/servlet_api",
],
)

130
java/google/registry/rde/BrdaCopyAction.java Normal file
View file

@ -0,0 +1,130 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.domain.registry.model.rde.RdeMode.THIN;
import static com.google.domain.registry.request.Action.Method.POST;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.io.ByteStreams;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.gcs.GcsUtils;
import com.google.domain.registry.keyring.api.KeyModule.Key;
import com.google.domain.registry.model.rde.RdeNamingUtils;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.request.RequestParameters;
import com.google.domain.registry.util.FormattingLogger;
import org.bouncycastle.openpgp.PGPException;
import org.bouncycastle.openpgp.PGPKeyPair;
import org.bouncycastle.openpgp.PGPPrivateKey;
import org.bouncycastle.openpgp.PGPPublicKey;
import org.joda.time.DateTime;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import javax.inject.Inject;
/**
* Action that re-encrypts a BRDA escrow deposit and puts it into the upload bucket.
*
* <p>This action is run by the mapreduce for each BRDA staging file it generates. The staging file
* is encrypted with our internal {@link Ghostryde} encryption. We then re-encrypt it as a RyDE
* file, which is what the third-party escrow provider understands.
*
* <p>Then we put the RyDE file (along with our digital signature) into the configured BRDA bucket.
* This bucket is special because a separate script will rsync it to the third party escrow provider
* SFTP server. This is why the internal staging files are stored in the separate RDE bucket.
*
* @see "http://newgtlds.icann.org/en/applicants/agb/agreement-approved-09jan14-en.htm"
*/
@Action(path = BrdaCopyAction.PATH, method = POST, automaticallyPrintOk = true)
public final class BrdaCopyAction implements Runnable {
static final String PATH = "/_dr/task/brdaCopy";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject GcsUtils gcsUtils;
@Inject Ghostryde ghostryde;
@Inject RydePgpCompressionOutputStreamFactory pgpCompressionFactory;
@Inject RydePgpFileOutputStreamFactory pgpFileFactory;
@Inject RydePgpEncryptionOutputStreamFactory pgpEncryptionFactory;
@Inject RydePgpSigningOutputStreamFactory pgpSigningFactory;
@Inject RydeTarOutputStreamFactory tarFactory;
@Inject @Config("brdaBucket") String brdaBucket;
@Inject @Config("rdeBucket") String stagingBucket;
@Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@Inject @Parameter(RdeModule.PARAM_WATERMARK) DateTime watermark;
@Inject @Key("brdaReceiverKey") PGPPublicKey receiverKey;
@Inject @Key("brdaSigningKey") PGPKeyPair signingKey;
@Inject @Key("rdeStagingDecryptionKey") PGPPrivateKey stagingDecryptionKey;
@Inject BrdaCopyAction() {}
@Override
public void run() {
try {
copyAsRyde();
} catch (IOException | PGPException e) {
throw new RuntimeException(e);
}
}
private void copyAsRyde() throws IOException, PGPException {
String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, 0);
GcsFilename xmlFilename = new GcsFilename(stagingBucket, prefix + ".xml.ghostryde");
GcsFilename xmlLengthFilename = new GcsFilename(stagingBucket, prefix + ".xml.length");
GcsFilename rydeFile = new GcsFilename(brdaBucket, prefix + ".ryde");
GcsFilename sigFile = new GcsFilename(brdaBucket, prefix + ".sig");
long xmlLength = readXmlLength(xmlLengthFilename);
logger.infofmt("Writing %s", rydeFile);
byte[] signature;
try (InputStream gcsInput = gcsUtils.openInputStream(xmlFilename);
Ghostryde.Decryptor decryptor = ghostryde.openDecryptor(gcsInput, stagingDecryptionKey);
Ghostryde.Decompressor decompressor = ghostryde.openDecompressor(decryptor);
Ghostryde.Input ghostInput = ghostryde.openInput(decompressor);
BufferedInputStream xmlInput = new BufferedInputStream(ghostInput);
OutputStream gcsOutput = gcsUtils.openOutputStream(rydeFile);
RydePgpSigningOutputStream signLayer = pgpSigningFactory.create(gcsOutput, signingKey)) {
try (OutputStream encryptLayer = pgpEncryptionFactory.create(signLayer, receiverKey);
OutputStream compressLayer = pgpCompressionFactory.create(encryptLayer);
OutputStream fileLayer = pgpFileFactory.create(compressLayer, watermark, prefix + ".tar");
OutputStream tarLayer =
tarFactory.create(fileLayer, xmlLength, watermark, prefix + ".xml")) {
ByteStreams.copy(xmlInput, tarLayer);
}
signature = signLayer.getSignature();
}
logger.infofmt("Writing %s", sigFile);
try (OutputStream gcsOutput = gcsUtils.openOutputStream(sigFile)) {
gcsOutput.write(signature);
}
}
/** Reads the contents of a file from Cloud Storage that contains nothing but an integer. */
private long readXmlLength(GcsFilename xmlLengthFilename) throws IOException {
try (InputStream input = gcsUtils.openInputStream(xmlLengthFilename)) {
return Long.parseLong(new String(ByteStreams.toByteArray(input), UTF_8).trim());
}
}
}

193
java/google/registry/rde/ContactResourceToXjcConverter.java Normal file
View file

@ -0,0 +1,193 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.domain.registry.util.XmlEnumUtils.enumToXml;
import com.google.domain.registry.model.contact.ContactAddress;
import com.google.domain.registry.model.contact.ContactPhoneNumber;
import com.google.domain.registry.model.contact.ContactResource;
import com.google.domain.registry.model.contact.Disclose;
import com.google.domain.registry.model.contact.Disclose.PostalInfoChoice;
import com.google.domain.registry.model.contact.PostalInfo;
import com.google.domain.registry.model.eppcommon.StatusValue;
import com.google.domain.registry.model.transfer.TransferData;
import com.google.domain.registry.xjc.contact.XjcContactAddrType;
import com.google.domain.registry.xjc.contact.XjcContactDiscloseType;
import com.google.domain.registry.xjc.contact.XjcContactE164Type;
import com.google.domain.registry.xjc.contact.XjcContactIntLocType;
import com.google.domain.registry.xjc.contact.XjcContactPostalInfoEnumType;
import com.google.domain.registry.xjc.contact.XjcContactPostalInfoType;
import com.google.domain.registry.xjc.contact.XjcContactStatusType;
import com.google.domain.registry.xjc.contact.XjcContactStatusValueType;
import com.google.domain.registry.xjc.eppcom.XjcEppcomTrStatusType;
import com.google.domain.registry.xjc.rdecontact.XjcRdeContact;
import com.google.domain.registry.xjc.rdecontact.XjcRdeContactElement;
import com.google.domain.registry.xjc.rdecontact.XjcRdeContactTransferDataType;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
/** Utility class that turns {@link ContactResource} as {@link XjcRdeContactElement}. */
final class ContactResourceToXjcConverter {
/** Converts {@link ContactResource} to {@link XjcRdeContactElement}. */
static XjcRdeContactElement convert(ContactResource host) {
return new XjcRdeContactElement(convertContact(host));
}
/** Converts {@link ContactResource} to {@link XjcRdeContact}. */
static XjcRdeContact convertContact(ContactResource model) {
XjcRdeContact bean = new XjcRdeContact();
bean.setRoid(model.getRepoId());
for (StatusValue status : model.getStatusValues()) {
bean.getStatuses().add(convertStatusValue(status));
}
PostalInfo localizedPostalInfo = model.getLocalizedPostalInfo();
if (localizedPostalInfo != null) {
bean.getPostalInfos().add(convertPostalInfo(localizedPostalInfo));
}
PostalInfo internationalizedPostalInfo = model.getInternationalizedPostalInfo();
if (internationalizedPostalInfo != null) {
bean.getPostalInfos().add(convertPostalInfo(internationalizedPostalInfo));
}
bean.setId(model.getContactId());
bean.setClID(model.getCurrentSponsorClientId());
bean.setCrRr(RdeAdapter.convertRr(model.getCreationClientId(), null));
bean.setUpRr(RdeAdapter.convertRr(model.getLastEppUpdateClientId(), null));
bean.setCrDate(model.getCreationTime());
bean.setUpDate(model.getLastEppUpdateTime());
bean.setTrDate(model.getLastTransferTime());
bean.setVoice(convertPhoneNumber(model.getVoiceNumber()));
bean.setFax(convertPhoneNumber(model.getFaxNumber()));
bean.setEmail(model.getEmailAddress());
bean.setDisclose(convertDisclose(model.getDisclose()));
// o An OPTIONAL <trnData> element that contains the following child
// elements related to the last transfer request of the contact
// object:
//
// * A <trStatus> element that contains the state of the most recent
// transfer request.
//
// * A <reRr> element that contains the identifier of the registrar
// that requested the domain name object transfer. An OPTIONAL
// client attribute is used to specify the client that performed
// the operation.
//
// * An <acRr> element that contains the identifier of the registrar
// that SHOULD act upon a PENDING transfer request. For all other
// status types, the value identifies the registrar that took the
// indicated action. An OPTIONAL client attribute is used to
// specify the client that performed the operation.
//
// * A <reDate> element that contains the date and time that the
// transfer was requested.
//
// * An <acDate> element that contains the date and time of a
// required or completed response. For a PENDING request, the
// value identifies the date and time by which a response is
// required before an automated response action will be taken by
// the registry. For all other status types, the value identifies
// the date and time when the request was completed.
if (model.getTransferData() != TransferData.EMPTY) {
bean.setTrnData(convertTransferData(model.getTransferData()));
}
return bean;
}
/** Converts {@link TransferData} to {@link XjcRdeContactTransferDataType}. */
private static XjcRdeContactTransferDataType convertTransferData(TransferData model) {
XjcRdeContactTransferDataType bean = new XjcRdeContactTransferDataType();
bean.setTrStatus(XjcEppcomTrStatusType.fromValue(model.getTransferStatus().getXmlName()));
bean.setReRr(RdeUtil.makeXjcRdeRrType(model.getGainingClientId()));
bean.setAcRr(RdeUtil.makeXjcRdeRrType(model.getLosingClientId()));
bean.setReDate(model.getTransferRequestTime());
bean.setAcDate(model.getPendingTransferExpirationTime());
return bean;
}
/** Converts {@link ContactAddress} to {@link XjcContactAddrType}. */
private static XjcContactAddrType convertAddress(ContactAddress model) {
XjcContactAddrType bean = new XjcContactAddrType();
bean.getStreets().addAll(model.getStreet());
bean.setCity(model.getCity());
bean.setSp(model.getState());
bean.setPc(model.getZip());
bean.setCc(model.getCountryCode());
return bean;
}
/** Converts {@link Disclose} to {@link XjcContactDiscloseType}. */
@Nullable
@CheckForNull
static XjcContactDiscloseType convertDisclose(@Nullable Disclose model) {
if (model == null) {
return null;
}
XjcContactDiscloseType bean = new XjcContactDiscloseType();
bean.setFlag(model.getFlag());
for (PostalInfoChoice loc : model.getNames()) {
bean.getNames().add(convertPostalInfoChoice(loc));
}
for (PostalInfoChoice loc : model.getOrgs()) {
bean.getOrgs().add(convertPostalInfoChoice(loc));
}
for (PostalInfoChoice loc : model.getAddrs()) {
bean.getAddrs().add(convertPostalInfoChoice(loc));
}
return bean;
}
/** Converts {@link ContactPhoneNumber} to {@link XjcContactE164Type}. */
@Nullable
@CheckForNull
private static XjcContactE164Type convertPhoneNumber(@Nullable ContactPhoneNumber model) {
if (model == null) {
return null;
}
XjcContactE164Type bean = new XjcContactE164Type();
bean.setValue(model.getPhoneNumber());
bean.setX(model.getExtension());
return bean;
}
/** Converts {@link PostalInfoChoice} to {@link XjcContactIntLocType}. */
private static XjcContactIntLocType convertPostalInfoChoice(PostalInfoChoice model) {
XjcContactIntLocType bean = new XjcContactIntLocType();
bean.setType(XjcContactPostalInfoEnumType.fromValue(enumToXml(model.getType())));
return bean;
}
/** Converts {@link PostalInfo} to {@link XjcContactPostalInfoType}. */
private static XjcContactPostalInfoType convertPostalInfo(PostalInfo model) {
XjcContactPostalInfoType bean = new XjcContactPostalInfoType();
bean.setName(model.getName());
bean.setOrg(model.getOrg());
bean.setAddr(convertAddress(model.getAddress()));
bean.setType(XjcContactPostalInfoEnumType.fromValue(enumToXml(model.getType())));
return bean;
}
/** Converts {@link StatusValue} to {@link XjcContactStatusType}. */
private static XjcContactStatusType convertStatusValue(StatusValue model) {
XjcContactStatusType bean = new XjcContactStatusType();
bean.setS(XjcContactStatusValueType.fromValue(model.getXmlName()));
return bean;
}
private ContactResourceToXjcConverter() {}
}

36
java/google/registry/rde/DepositFragment.java Normal file
View file

@ -0,0 +1,36 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import com.google.auto.value.AutoValue;
import java.io.Serializable;
/** Container of datastore resource marshalled by {@link RdeMarshaller}. */
@AutoValue
public abstract class DepositFragment implements Serializable {
private static final long serialVersionUID = -5241410684255467454L;
public abstract RdeResourceType type();
public abstract String xml();
public abstract String error();
public static DepositFragment create(RdeResourceType type, String xml, String error) {
return new AutoValue_DepositFragment(type, xml, error);
}
DepositFragment() {}
}

291
java/google/registry/rde/DomainResourceToXjcConverter.java Normal file
View file

@ -0,0 +1,291 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.model.contact.ContactResource;
import com.google.domain.registry.model.domain.DesignatedContact;
import com.google.domain.registry.model.domain.DomainResource;
import com.google.domain.registry.model.domain.ReferenceUnion;
import com.google.domain.registry.model.domain.rgp.GracePeriodStatus;
import com.google.domain.registry.model.domain.secdns.DelegationSignerData;
import com.google.domain.registry.model.eppcommon.StatusValue;
import com.google.domain.registry.model.host.HostResource;
import com.google.domain.registry.model.rde.RdeMode;
import com.google.domain.registry.model.transfer.TransferData;
import com.google.domain.registry.model.transfer.TransferStatus;
import com.google.domain.registry.util.Idn;
import com.google.domain.registry.xjc.domain.XjcDomainContactAttrType;
import com.google.domain.registry.xjc.domain.XjcDomainContactType;
import com.google.domain.registry.xjc.domain.XjcDomainNsType;
import com.google.domain.registry.xjc.domain.XjcDomainStatusType;
import com.google.domain.registry.xjc.domain.XjcDomainStatusValueType;
import com.google.domain.registry.xjc.eppcom.XjcEppcomTrStatusType;
import com.google.domain.registry.xjc.rdedomain.XjcRdeDomain;
import com.google.domain.registry.xjc.rdedomain.XjcRdeDomainElement;
import com.google.domain.registry.xjc.rdedomain.XjcRdeDomainTransferDataType;
import com.google.domain.registry.xjc.rgp.XjcRgpStatusType;
import com.google.domain.registry.xjc.rgp.XjcRgpStatusValueType;
import com.google.domain.registry.xjc.secdns.XjcSecdnsDsDataType;
import com.google.domain.registry.xjc.secdns.XjcSecdnsDsOrKeyType;
import org.joda.time.DateTime;
/** Utility class that turns {@link DomainResource} as {@link XjcRdeDomainElement}. */
final class DomainResourceToXjcConverter {
/** Converts {@link DomainResource} to {@link XjcRdeDomainElement}. */
static XjcRdeDomainElement convert(DomainResource domain, RdeMode mode) {
return new XjcRdeDomainElement(convertDomain(domain, mode));
}
/** Converts {@link DomainResource} to {@link XjcRdeDomain}. */
static XjcRdeDomain convertDomain(DomainResource model, RdeMode mode) {
XjcRdeDomain bean = new XjcRdeDomain();
// o A <name> element that contains the fully qualified name of the
// domain name object.
bean.setName(model.getFullyQualifiedDomainName());
// o A <roid> element that contains the repository object identifier
// assigned to the domain name object when it was created.
bean.setRoid(model.getRepoId());
// o An OPTIONAL <uName> element that contains the name of the domain
// name in Unicode character set. It MUST be provided if available.
bean.setUName(Idn.toUnicode(model.getFullyQualifiedDomainName()));
// o An OPTIONAL <idnTableId> element that references the IDN Table
// used for the IDN. This corresponds to the "id" attribute of the
// <idnTableRef> element. This element MUST be present if the domain
// name is an IDN.
// ✭ We have to add some code to determine the IDN table id at creation
// time, then either save it somewhere, or else re-derive it here.
bean.setIdnTableId(model.getIdnTableName());
// o An OPTIONAL <originalName> element is used to indicate that the
// domain name is an IDN variant. This element contains the domain
// name used to generate the IDN variant.
// ☠ Not relevant for now. We may do some bundling of variants in the
// future, but right now we're going to be doing blocking - which
// means we won't canonicalize the IDN name at the present time.
// bean.setOriginalName(...);
// o A <clID> element that contains the identifier of the sponsoring
// registrar.
bean.setClID(model.getCurrentSponsorClientId());
// o A <crRr> element that contains the identifier of the registrar
// that created the domain name object. An OPTIONAL client attribute
// is used to specify the client that performed the operation.
bean.setCrRr(RdeAdapter.convertRr(model.getCreationClientId(), null));
// o An OPTIONAL <crDate> element that contains the date and time of
// the domain name object creation. This element MUST be present if
// the domain name has been allocated.
bean.setCrDate(model.getCreationTime());
// o An OPTIONAL <exDate> element that contains the date and time
// identifying the end (expiration) of the domain name object's
// registration period. This element MUST be present if the domain
// name has been allocated.
bean.setExDate(model.getRegistrationExpirationTime());
// o An OPTIONAL <upDate> element that contains the date and time of
// the most recent domain-name-object modification. This element
// MUST NOT be present if the domain name object has never been
// modified.
bean.setUpDate(model.getLastEppUpdateTime());
// o An OPTIONAL <upRr> element that contains the identifier of the
// registrar that last updated the domain name object. This element
// MUST NOT be present if the domain has never been modified. An
// OPTIONAL client attribute is used to specify the client that
// performed the operation.
bean.setUpRr(RdeAdapter.convertRr(model.getLastEppUpdateClientId(), null));
// o An OPTIONAL <trDate> element that contains the date and time of
// the most recent domain object successful transfer. This element
// MUST NOT be present if the domain name object has never been
// transfered.
bean.setTrDate(model.getLastTransferTime());
// o One or more <status> elements that contain the current status
// descriptors associated with the domain name.
for (StatusValue status : model.getStatusValues()) {
bean.getStatuses().add(convertStatusValue(status));
}
// o An OPTIONAL <ns> element that contains the fully qualified names
// of the delegated host objects or host attributes (name servers)
// associated with the domain name object. See Section 1.1 of
// [RFC5731] for a description of the elements used to specify host
// objects or host attributes.
// ✭ We don't support host attributes, only host objects. The RFC says
// you have to support one or the other, but not both. The gist of
// it is that with host attributes, you inline the nameserver data
// on each domain; with host objects, you normalize the nameserver
// data to a separate EPP object.
ImmutableSet<HostResource> linkedNameservers = model.loadNameservers();
if (!linkedNameservers.isEmpty()) {
XjcDomainNsType nameservers = new XjcDomainNsType();
for (HostResource host : linkedNameservers) {
nameservers.getHostObjs().add(host.getFullyQualifiedHostName());
}
bean.setNs(nameservers);
}
switch (mode) {
case FULL:
// o Zero or more OPTIONAL <rgpStatus> element to represent
// "pendingDelete" sub-statuses, including "redemptionPeriod",
// "pendingRestore", and "pendingDelete", that a domain name can be
// in as a result of grace period processing as specified in
// [RFC3915].
for (GracePeriodStatus status : model.getGracePeriodStatuses()) {
bean.getRgpStatuses().add(convertGracePeriodStatus(status));
}
// o An OPTIONAL <registrant> element that contain the identifier for
// the human or organizational social information object associated
// as the holder of the domain name object.
ReferenceUnion<ContactResource> registrant = model.getRegistrant();
if (registrant != null) {
bean.setRegistrant(registrant.getLinked().get().getContactId());
}
// o Zero or more OPTIONAL <contact> elements that contain identifiers
// for the human or organizational social information objects
// associated with the domain name object.
for (DesignatedContact contact : model.getContacts()) {
bean.getContacts().add(convertDesignatedContact(contact));
}
// o An OPTIONAL <secDNS> element that contains the public key
// information associated with Domain Name System security (DNSSEC)
// extensions for the domain name as specified in [RFC5910].
// ☠ We don't set keyData because we use dsData. The RFCs offer us a
// choice between the two, similar to hostAttr vs. hostObj above.
// ☠ We're not going to support maxSigLife since it seems to be
// completely useless.
if (!model.getDsData().isEmpty()) {
XjcSecdnsDsOrKeyType secdns = new XjcSecdnsDsOrKeyType();
for (DelegationSignerData ds : model.getDsData()) {
secdns.getDsDatas().add(convertDelegationSignerData(ds));
}
bean.setSecDNS(secdns);
}
// o An OPTIONAL <trnData> element that contains the following child
// elements related to the last transfer request of the domain name
// object. This element MUST NOT be present if a transfer request
// for the domain name has never been created.
//
// * A <trStatus> element that contains the state of the most recent
// transfer request.
//
// * A <reRr> element that contains the identifier of the registrar
// that requested the domain name object transfer. An OPTIONAL
// client attribute is used to specify the client that performed
// the operation.
//
// * A <reDate> element that contains the date and time that the
// transfer was requested.
//
// * An <acRr> element that contains the identifier of the registrar
// that SHOULD act upon a PENDING transfer request. For all other
// status types, the value identifies the registrar that took the
// indicated action. An OPTIONAL client attribute is used to
// specify the client that performed the operation.
//
// * An <acDate> element that contains the date and time of a
// required or completed response. For a PENDING request, the
// value identifies the date and time by which a response is
// required before an automated response action will be taken by
// the registry. For all other status types, the value identifies
// the date and time when the request was completed.
//
// * An OPTIONAL <exDate> element that contains the end of the
// domain name object's validity period (expiry date) if the
// transfer caused or causes a change in the validity period.
if (model.getTransferData() != TransferData.EMPTY) {
bean.setTrnData(
convertTransferData(model.getTransferData(), model.getRegistrationExpirationTime()));
}
break;
case THIN:
break;
}
return bean;
}
/** Converts {@link TransferData} to {@link XjcRdeDomainTransferDataType}. */
private static XjcRdeDomainTransferDataType convertTransferData(
TransferData model, DateTime domainExpires) {
XjcRdeDomainTransferDataType bean = new XjcRdeDomainTransferDataType();
bean.setTrStatus(
XjcEppcomTrStatusType.fromValue(model.getTransferStatus().getXmlName()));
bean.setReRr(RdeUtil.makeXjcRdeRrType(model.getGainingClientId()));
bean.setAcRr(RdeUtil.makeXjcRdeRrType(model.getLosingClientId()));
bean.setReDate(model.getTransferRequestTime());
bean.setAcDate(model.getPendingTransferExpirationTime());
if (model.getTransferStatus() == TransferStatus.PENDING) {
int years = Optional.fromNullable(model.getExtendedRegistrationYears()).or(0);
bean.setExDate(domainExpires.plusYears(years));
} else {
bean.setExDate(domainExpires);
}
return bean;
}
/** Converts {@link GracePeriodStatus} to {@link XjcRgpStatusType}. */
private static XjcRgpStatusType convertGracePeriodStatus(GracePeriodStatus model) {
XjcRgpStatusType bean = new XjcRgpStatusType();
bean.setS(XjcRgpStatusValueType.fromValue(model.getXmlName()));
return bean;
}
/** Converts {@link StatusValue} to {@link XjcDomainStatusType}. */
private static XjcDomainStatusType convertStatusValue(StatusValue model) {
XjcDomainStatusType bean = new XjcDomainStatusType();
bean.setS(XjcDomainStatusValueType.fromValue(model.getXmlName()));
return bean;
}
/** Converts {@link DelegationSignerData} to {@link XjcSecdnsDsDataType}. */
private static XjcSecdnsDsDataType convertDelegationSignerData(DelegationSignerData model) {
XjcSecdnsDsDataType bean = new XjcSecdnsDsDataType();
bean.setKeyTag(model.getKeyTag());
bean.setAlg((short) model.getAlgorithm());
bean.setDigestType((short) model.getDigestType());
bean.setDigest(model.getDigest());
bean.setKeyData(null);
return bean;
}
/** Converts {@link DesignatedContact} to {@link XjcDomainContactType}. */
private static XjcDomainContactType convertDesignatedContact(DesignatedContact model) {
XjcDomainContactType bean = new XjcDomainContactType();
ContactResource contact = model.getContactId().getLinked().get();
bean.setType(XjcDomainContactAttrType.fromValue(model.getType().toString().toLowerCase()));
bean.setValue(contact.getContactId());
return bean;
}
private DomainResourceToXjcConverter() {}
}

121
java/google/registry/rde/EscrowTaskRunner.java Normal file
View file

@ -0,0 +1,121 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import com.google.domain.registry.model.registry.Registry;
import com.google.domain.registry.model.registry.RegistryCursor;
import com.google.domain.registry.model.registry.RegistryCursor.CursorType;
import com.google.domain.registry.model.server.Lock;
import com.google.domain.registry.request.HttpException.NoContentException;
import com.google.domain.registry.request.HttpException.ServiceUnavailableException;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.request.RequestParameters;
import com.google.domain.registry.util.Clock;
import com.google.domain.registry.util.FormattingLogger;
import com.googlecode.objectify.VoidWork;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import java.util.concurrent.Callable;
import javax.inject.Inject;
/**
* Runner applying guaranteed reliability to an {@link EscrowTask}.
*
* <p>This class implements the <i>Locking Rolling Cursor</i> pattern, which solves the problem of
* how to reliably execute App Engine tasks which can't be made idempotent.
*
* <p>{@link Lock} is used to ensure only one task executes at a time for a given
* {@code LockedCursorTask} subclass + TLD combination. This is necessary because App Engine tasks
* might double-execute. Normally tasks solve this by being idempotent, but that's not possible for
* RDE, which writes to a GCS filename with a deterministic name. So the datastore is used to to
* guarantee isolation. If we can't acquire the lock, it means the task is already running, so
* {@link NoContentException} is thrown to cancel the task.
*
* <p>The specific date for which the deposit is generated depends on the current position of the
* {@link RegistryCursor}. If the cursor is set to tomorrow, we do nothing and return 204 No
* Content. If the cursor is set to today, then we create a deposit for today and advance the
* cursor. If the cursor is set to yesterday or earlier, then we create a deposit for that date,
* advance the cursor, but we <i>do not</i> make any attempt to catch the cursor up to the current
* time. Therefore <b>you must</b> set the cron interval to something less than the desired
* interval, so the cursor can catch up. For example, if the task is supposed to run daily, you
* should configure cron to execute it every twelve hours, or possibly less.
*/
class EscrowTaskRunner {
/** Callback interface for objects managed by {@link EscrowTaskRunner}. */
public interface EscrowTask {
/**
* Performs task logic while the lock is held.
*
* @param watermark the logical time for a point-in-time view of datastore
*/
abstract void runWithLock(DateTime watermark) throws Exception;
}
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject Clock clock;
@Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@Inject EscrowTaskRunner() {}
/**
* Acquires lock, checks cursor, invokes {@code task}, and advances cursor.
*
* @param task the task to run
* @param registry the {@link Registry} that we are performing escrow for
* @param timeout time when we assume failure, kill the task (and instance) and release the lock
* @param cursorType the cursor to advance on success, indicating the next required runtime
* @param interval how far to advance the cursor (e.g. a day for RDE, a week for BRDA)
*/
void lockRunAndRollForward(
final EscrowTask task,
final Registry registry,
Duration timeout,
final CursorType cursorType,
final Duration interval) {
Callable<Void> lockRunner = new Callable<Void>() {
@Override
public Void call() throws Exception {
logger.info("tld=" + registry.getTld());
DateTime startOfToday = clock.nowUtc().withTimeAtStartOfDay();
final DateTime nextRequiredRun = RegistryCursor.load(registry, cursorType).or(startOfToday);
if (nextRequiredRun.isAfter(startOfToday)) {
throw new NoContentException("Already completed");
}
logger.info("cursor=" + nextRequiredRun);
task.runWithLock(nextRequiredRun);
ofy().transact(new VoidWork() {
@Override
public void vrun() {
RegistryCursor.save(registry, cursorType, nextRequiredRun.plus(interval));
}});
return null;
}};
String lockName = String.format("%s %s", task.getClass().getSimpleName(), registry.getTld());
if (!Lock.executeWithLocks(lockRunner, null, tld, timeout, lockName)) {
// This will happen if either: a) the task is double-executed; b) the task takes a long time
// to run and the retry task got executed while the first one is still running. In both
// situations the safest thing to do is to just return 503 so the task gets retried later.
throw new ServiceUnavailableException("Lock in use: " + lockName);
}
}
}

522
java/google/registry/rde/Ghostryde.java Normal file
View file

@ -0,0 +1,522 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static org.bouncycastle.bcpg.CompressionAlgorithmTags.ZLIB;
import static org.bouncycastle.bcpg.SymmetricKeyAlgorithmTags.AES_128;
import static org.bouncycastle.jce.provider.BouncyCastleProvider.PROVIDER_NAME;
import static org.bouncycastle.openpgp.PGPLiteralData.BINARY;
import static org.joda.time.DateTimeZone.UTC;
import com.google.common.io.ByteStreams;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.util.ImprovedInputStream;
import com.google.domain.registry.util.ImprovedOutputStream;
import org.bouncycastle.openpgp.PGPCompressedData;
import org.bouncycastle.openpgp.PGPCompressedDataGenerator;
import org.bouncycastle.openpgp.PGPEncryptedDataGenerator;
import org.bouncycastle.openpgp.PGPEncryptedDataList;
import org.bouncycastle.openpgp.PGPException;
import org.bouncycastle.openpgp.PGPLiteralData;
import org.bouncycastle.openpgp.PGPLiteralDataGenerator;
import org.bouncycastle.openpgp.PGPObjectFactory;
import org.bouncycastle.openpgp.PGPPrivateKey;
import org.bouncycastle.openpgp.PGPPublicKey;
import org.bouncycastle.openpgp.PGPPublicKeyEncryptedData;
import org.bouncycastle.openpgp.bc.BcPGPObjectFactory;
import org.bouncycastle.openpgp.operator.bc.BcPublicKeyDataDecryptorFactory;
import org.bouncycastle.openpgp.operator.bc.BcPublicKeyKeyEncryptionMethodGenerator;
import org.bouncycastle.openpgp.operator.jcajce.JcePGPDataEncryptorBuilder;
import org.joda.time.DateTime;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.NoSuchAlgorithmException;
import java.security.ProviderException;
import java.security.SecureRandom;
import javax.annotation.CheckReturnValue;
import javax.annotation.Nullable;
import javax.annotation.WillCloseWhenClosed;
import javax.annotation.WillNotClose;
import javax.annotation.concurrent.Immutable;
import javax.annotation.concurrent.NotThreadSafe;
import javax.inject.Inject;
/**
* Utility class for reading and writing data in the ghostryde container format.
*
* <p>Whenever we stage sensitive data to cloud storage (like XML RDE deposit data), we
* <a href="http://youtu.be/YPNJjL9iznY">GHOST RYDE IT</a> first to keep it safe from the prying
* eyes of anyone with access to the <a href="https://cloud.google.com/console">Google Cloud
* Console</a>.
*
* <p>This class has an unusual API that's designed to take advantage of Java 7 try-with-resource
* statements to the greatest extent possible, while also maintaining security contracts at
* compile-time.
*
* <p>Here's how you write a file:
*
* <pre> {@code
* File in = new File("lol.txt");
* File out = new File("lol.txt.ghostryde");
* Ghostryde ghost = new Ghostryde(1024);
* try (OutputStream output = new FileOutputStream(out);
* Ghostryde.Encryptor encryptor = ghost.openEncryptor(output, publicKey);
* Ghostryde.Compressor kompressor = ghost.openCompressor(encryptor);
* OutputStream go = ghost.openOutput(kompressor, in.getName(), DateTime.now());
* InputStream input = new FileInputStream(in)) &lbrace;
* ByteStreams.copy(input, go);
* &rbrace;}</pre>
*
* <p>Here's how you read a file:
*
* <pre> {@code
* File in = new File("lol.txt.ghostryde");
* File out = new File("lol.txt");
* Ghostryde ghost = new Ghostryde(1024);
* try (InputStream fileInput = new FileInputStream(in);
* Ghostryde.Decryptor decryptor = ghost.openDecryptor(fileInput, privateKey);
* Ghostryde.Decompressor decompressor = ghost.openDecompressor(decryptor);
* Ghostryde.Input input = ghost.openInput(decompressor);
* OutputStream fileOutput = new FileOutputStream(out)) &lbrace;
* System.out.println("name = " + input.getName());
* System.out.println("modified = " + input.getModified());
* ByteStreams.copy(input, fileOutput);
* &rbrace;}</pre>
*
* <h2>Simple API</h2>
*
* <p>If you're writing test code or are certain your data can fit in memory, you might find these
* static methods more convenient:
*
* <pre> {@code
* byte[] data = "hello kitty".getBytes(UTF_8);
* byte[] blob = Ghostryde.encode(data, publicKey, "lol.txt", DateTime.now());
* Ghostryde.Result result = Ghostryde.decode(blob, privateKey);
* }</pre>
*
* <h2>GhostRYDE Format</h2>
*
* <p>A {@code .ghostryde} file is the exact same thing as a {@code .gpg} file, except the OpenPGP
* message layers will always be present and in a specific order. You can analyse the layers on the
* command-line using the {@code gpg --list-packets blah.ghostryde} command.
*
* <p>Ghostryde is different from RyDE in the sense that ghostryde is only used for <i>internal</i>
* storage; whereas RyDE is meant to protect data being stored by a third-party.
*/
@Immutable
public final class Ghostryde {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
/**
* Compression algorithm to use when creating ghostryde files.
*
* <p>We're going to use ZLIB since it's better than ZIP.
*
* @see org.bouncycastle.bcpg.CompressionAlgorithmTags
*/
static final int COMPRESSION_ALGORITHM = ZLIB;
/**
* Symmetric encryption cipher to use when creating ghostryde files.
*
* <p>We're going to use AES-128 just like {@link RydePgpEncryptionOutputStream}, although we
* aren't forced to use this algorithm by the ICANN RFCs since this is an internal format.
*
* @see org.bouncycastle.bcpg.SymmetricKeyAlgorithmTags
*/
static final int CIPHER = AES_128;
/**
* Unlike {@link RydePgpEncryptionOutputStream}, we're going to enable the integrity packet
* because it makes GnuPG happy. It's also probably necessary to prevent tampering since we
* don't sign ghostryde files.
*/
static final boolean USE_INTEGRITY_PACKET = true;
/**
* The source of random bits. You are strongly discouraged from changing this value because at
* Google it's configured to use {@code /dev/&lbrace;,u&rbrace;random} in production and somehow
* magically go fast and not drain entropy in the testing environment.
*
* @see SecureRandom#getInstance(String)
*/
static final String RANDOM_SOURCE = "NativePRNG";
/**
* Creates a ghostryde file from an in-memory byte array.
*
* @throws PGPException
* @throws IOException
*/
public static byte[] encode(byte[] data, PGPPublicKey key, String name, DateTime modified)
throws IOException, PGPException {
checkNotNull(data, "data");
checkArgument(key.isEncryptionKey(), "not an encryption key");
Ghostryde ghost = new Ghostryde(1024 * 64);
ByteArrayOutputStream output = new ByteArrayOutputStream();
try (Encryptor encryptor = ghost.openEncryptor(output, key);
Compressor kompressor = ghost.openCompressor(encryptor);
OutputStream go = ghost.openOutput(kompressor, name, modified)) {
go.write(data);
}
return output.toByteArray();
}
/**
* Deciphers a ghostryde file from an in-memory byte array.
*
* @throws PGPException
* @throws IOException
*/
public static DecodeResult decode(byte[] data, PGPPrivateKey key)
throws IOException, PGPException {
checkNotNull(data, "data");
Ghostryde ghost = new Ghostryde(1024 * 64);
ByteArrayInputStream dataStream = new ByteArrayInputStream(data);
ByteArrayOutputStream output = new ByteArrayOutputStream();
String name;
DateTime modified;
try (Decryptor decryptor = ghost.openDecryptor(dataStream, key);
Decompressor decompressor = ghost.openDecompressor(decryptor);
Input input = ghost.openInput(decompressor)) {
name = input.getName();
modified = input.getModified();
ByteStreams.copy(input, output);
}
return new DecodeResult(output.toByteArray(), name, modified);
}
/** Result class for the {@link Ghostryde#decode(byte[], PGPPrivateKey)} method. */
@Immutable
public static final class DecodeResult {
private final byte[] data;
private final String name;
private final DateTime modified;
DecodeResult(byte[] data, String name, DateTime modified) {
this.data = checkNotNull(data, "data");
this.name = checkNotNull(name, "name");
this.modified = checkNotNull(modified, "modified");
}
/** Returns the decoded ghostryde content bytes. */
public byte[] getData() {
return data;
}
/** Returns the name of the original file, taken from the literal data packet. */
public String getName() {
return name;
}
/** Returns the time this file was created or modified, take from the literal data packet. */
public DateTime getModified() {
return modified;
}
}
/**
* PGP literal file {@link InputStream}.
*
* @see Ghostryde#openInput(Decompressor)
*/
@NotThreadSafe
public static final class Input extends ImprovedInputStream {
private final String name;
private final DateTime modified;
Input(@WillCloseWhenClosed InputStream input, String name, DateTime modified) {
super(input);
this.name = checkNotNull(name, "name");
this.modified = checkNotNull(modified, "modified");
}
/** Returns the name of the original file, taken from the literal data packet. */
public String getName() {
return name;
}
/** Returns the time this file was created or modified, take from the literal data packet. */
public DateTime getModified() {
return modified;
}
}
/**
* PGP literal file {@link OutputStream}.
*
* <p>This class isn't needed for ordering safety, but is included regardless for consistency and
* to improve the appearance of log messages.
*
* @see Ghostryde#openOutput(Compressor, String, DateTime)
*/
@NotThreadSafe
public static final class Output extends ImprovedOutputStream {
Output(@WillCloseWhenClosed OutputStream os) {
super(os);
}
}
/**
* Encryption {@link OutputStream}.
*
* <p>This type exists to guarantee {@code open*()} methods are called in the correct order.
*
* @see Ghostryde#openEncryptor(OutputStream, PGPPublicKey)
*/
@NotThreadSafe
public static final class Encryptor extends ImprovedOutputStream {
Encryptor(@WillCloseWhenClosed OutputStream os) {
super(os);
}
}
/**
* Decryption {@link InputStream}.
*
* <p>This type exists to guarantee {@code open*()} methods are called in the correct order.
*
* @see Ghostryde#openDecryptor(InputStream, PGPPrivateKey)
*/
@NotThreadSafe
public static final class Decryptor extends ImprovedInputStream {
private final PGPPublicKeyEncryptedData crypt;
Decryptor(@WillCloseWhenClosed InputStream input, PGPPublicKeyEncryptedData crypt) {
super(input);
this.crypt = checkNotNull(crypt, "crypt");
}
/**
* Verifies that the ciphertext wasn't corrupted or tampered with.
*
* <p>Note: If {@link Ghostryde#USE_INTEGRITY_PACKET} is {@code true}, any ghostryde file
* without an integrity packet will be considered invalid and an exception will be thrown.
*
* @throws IllegalStateException to propagate {@link PGPException}
* @throws IOException
*/
@Override
protected void onClose() throws IOException {
if (USE_INTEGRITY_PACKET) {
try {
if (!crypt.verify()) {
throw new PGPException("ghostryde integrity check failed: possible tampering D:");
}
} catch (PGPException e) {
throw new IllegalStateException(e);
}
}
}
}
/**
* Compression {@link OutputStream}.
*
* <p>This type exists to guarantee {@code open*()} methods are called in the correct order.
*
* @see Ghostryde#openCompressor(Encryptor)
*/
@NotThreadSafe
public static final class Compressor extends ImprovedOutputStream {
Compressor(@WillCloseWhenClosed OutputStream os) {
super(os);
}
}
/**
* Decompression {@link InputStream}.
*
* <p>This type exists to guarantee {@code open*()} methods are called in the correct order.
*
* @see Ghostryde#openDecompressor(Decryptor)
*/
@NotThreadSafe
public static final class Decompressor extends ImprovedInputStream {
Decompressor(@WillCloseWhenClosed InputStream input) {
super(input);
}
}
private final int bufferSize;
/** Constructs a new {@link Ghostryde} object. */
@Inject
public Ghostryde(
@Config("rdeGhostrydeBufferSize") int bufferSize) {
checkArgument(bufferSize > 0, "bufferSize");
this.bufferSize = bufferSize;
}
/**
* Opens a new {@link Encryptor} (Writing Step 1/3)
*
* <p>This is the first step in creating a ghostryde file. After this method, you'll want to
* call {@link #openCompressor(Encryptor)}.
*
* @param os is the upstream {@link OutputStream} to which the result is written.
* @param publicKey is the public encryption key of the recipient.
* @throws IOException
* @throws PGPException
*/
@CheckReturnValue
public Encryptor openEncryptor(@WillNotClose OutputStream os, PGPPublicKey publicKey)
throws IOException, PGPException {
PGPEncryptedDataGenerator encryptor = new PGPEncryptedDataGenerator(
new JcePGPDataEncryptorBuilder(CIPHER)
.setWithIntegrityPacket(USE_INTEGRITY_PACKET)
.setSecureRandom(getRandom())
.setProvider(PROVIDER_NAME));
encryptor.addMethod(new BcPublicKeyKeyEncryptionMethodGenerator(publicKey));
return new Encryptor(encryptor.open(os, new byte[bufferSize]));
}
/** Does stuff. */
private SecureRandom getRandom() {
SecureRandom random;
try {
random = SecureRandom.getInstance(RANDOM_SOURCE);
} catch (NoSuchAlgorithmException e) {
throw new ProviderException(e);
}
return random;
}
/**
* Opens a new {@link Compressor} (Writing Step 2/3)
*
* <p>This is the second step in creating a ghostryde file. After this method, you'll want to
* call {@link #openOutput(Compressor, String, DateTime)}.
*
* @param os is the value returned by {@link #openEncryptor(OutputStream, PGPPublicKey)}.
* @throws IOException
* @throws PGPException
*/
@CheckReturnValue
public Compressor openCompressor(@WillNotClose Encryptor os) throws IOException, PGPException {
PGPCompressedDataGenerator kompressor = new PGPCompressedDataGenerator(COMPRESSION_ALGORITHM);
return new Compressor(kompressor.open(os, new byte[bufferSize]));
}
/**
* Opens an {@link OutputStream} to which the actual data should be written (Writing Step 3/3)
*
* <p>This is the third and final step in creating a ghostryde file. You'll want to write data
* to the returned object.
*
* @param os is the value returned by {@link #openCompressor(Encryptor)}.
* @param name is a filename for your data which gets written in the literal tag.
* @param modified is a timestamp for your data which gets written to the literal tags.
* @throws IOException
*/
@CheckReturnValue
public Output openOutput(@WillNotClose Compressor os, String name, DateTime modified)
throws IOException {
return new Output(new PGPLiteralDataGenerator().open(
os, BINARY, name, modified.toDate(), new byte[bufferSize]));
}
/**
* Opens a new {@link Decryptor} (Reading Step 1/3)
*
* <p>This is the first step in opening a ghostryde file. After this method, you'll want to
* call {@link #openDecompressor(Decryptor)}.
*
* @param input is an {@link InputStream} of the ghostryde file data.
* @param privateKey is the private encryption key of the recipient (which is us!)
* @throws IOException
* @throws PGPException
*/
@CheckReturnValue
public Decryptor openDecryptor(@WillNotClose InputStream input, PGPPrivateKey privateKey)
throws IOException, PGPException {
checkNotNull(privateKey, "privateKey");
PGPObjectFactory fact = new BcPGPObjectFactory(checkNotNull(input, "input"));
PGPEncryptedDataList crypts = pgpCast(fact.nextObject(), PGPEncryptedDataList.class);
checkState(crypts.size() > 0);
if (crypts.size() > 1) {
logger.warningfmt("crypts.size() is %d (should be 1)", crypts.size());
}
PGPPublicKeyEncryptedData crypt = pgpCast(crypts.get(0), PGPPublicKeyEncryptedData.class);
if (crypt.getKeyID() != privateKey.getKeyID()) {
throw new PGPException(String.format(
"Message was encrypted for keyid %x but ours is %x",
crypt.getKeyID(), privateKey.getKeyID()));
}
return new Decryptor(
crypt.getDataStream(new BcPublicKeyDataDecryptorFactory(privateKey)),
crypt);
}
/**
* Opens a new {@link Decompressor} (Reading Step 2/3)
*
* <p>This is the second step in reading a ghostryde file. After this method, you'll want to
* call {@link #openInput(Decompressor)}.
*
* @param input is the value returned by {@link #openDecryptor}.
* @throws IOException
* @throws PGPException
*/
@CheckReturnValue
public Decompressor openDecompressor(@WillNotClose Decryptor input)
throws IOException, PGPException {
PGPObjectFactory fact = new BcPGPObjectFactory(checkNotNull(input, "input"));
PGPCompressedData compressed = pgpCast(fact.nextObject(), PGPCompressedData.class);
return new Decompressor(compressed.getDataStream());
}
/**
* Opens a new {@link Input} for reading the original contents (Reading Step 3/3)
*
* <p>This is the final step in reading a ghostryde file. After calling this method, you should
* call the read methods on the returned {@link InputStream}.
*
* @param input is the value returned by {@link #openDecompressor}.
* @throws IOException
* @throws PGPException
*/
@CheckReturnValue
public Input openInput(@WillNotClose Decompressor input) throws IOException, PGPException {
PGPObjectFactory fact = new BcPGPObjectFactory(checkNotNull(input, "input"));
PGPLiteralData literal = pgpCast(fact.nextObject(), PGPLiteralData.class);
DateTime modified = new DateTime(literal.getModificationTime(), UTC);
return new Input(literal.getDataStream(), literal.getFileName(), modified);
}
/** Safely extracts an object from an OpenPGP message. */
private static <T> T pgpCast(@Nullable Object object, Class<T> expect) throws PGPException {
if (object == null) {
throw new PGPException(String.format(
"Expected %s but out of objects", expect.getSimpleName()));
}
if (!expect.isAssignableFrom(object.getClass())) {
throw new PGPException(String.format(
"Expected %s but got %s", expect.getSimpleName(), object.getClass().getSimpleName()));
}
return expect.cast(object);
}
}

75
java/google/registry/rde/HostResourceToXjcConverter.java Normal file
View file

@ -0,0 +1,75 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import com.google.common.net.InetAddresses;
import com.google.domain.registry.model.eppcommon.StatusValue;
import com.google.domain.registry.model.host.HostResource;
import com.google.domain.registry.xjc.host.XjcHostAddrType;
import com.google.domain.registry.xjc.host.XjcHostIpType;
import com.google.domain.registry.xjc.host.XjcHostStatusType;
import com.google.domain.registry.xjc.host.XjcHostStatusValueType;
import com.google.domain.registry.xjc.rdehost.XjcRdeHost;
import com.google.domain.registry.xjc.rdehost.XjcRdeHostElement;
import java.net.Inet6Address;
import java.net.InetAddress;
/** Utility class that turns {@link HostResource} as {@link XjcRdeHostElement}. */
final class HostResourceToXjcConverter {
/** Converts {@link HostResource} to {@link XjcRdeHostElement}. */
static XjcRdeHostElement convert(HostResource host) {
return new XjcRdeHostElement(convertHost(host));
}
/** Converts {@link HostResource} to {@link XjcRdeHost}. */
static XjcRdeHost convertHost(HostResource model) {
XjcRdeHost bean = new XjcRdeHost();
bean.setName(model.getFullyQualifiedHostName());
bean.setRoid(model.getRepoId());
bean.setClID(model.getCurrentSponsorClientId());
bean.setTrDate(model.getLastTransferTime());
bean.setCrDate(model.getCreationTime());
bean.setUpDate(model.getLastEppUpdateTime());
bean.setCrRr(RdeAdapter.convertRr(model.getCreationClientId(), null));
bean.setUpRr(RdeAdapter.convertRr(model.getLastEppUpdateClientId(), null));
bean.setCrRr(RdeAdapter.convertRr(model.getCreationClientId(), null));
for (StatusValue status : model.getStatusValues()) {
bean.getStatuses().add(convertStatusValue(status));
}
for (InetAddress addr : model.getInetAddresses()) {
bean.getAddrs().add(convertInetAddress(addr));
}
return bean;
}
/** Converts {@link StatusValue} to {@link XjcHostStatusType}. */
private static XjcHostStatusType convertStatusValue(StatusValue model) {
XjcHostStatusType bean = new XjcHostStatusType();
bean.setS(XjcHostStatusValueType.fromValue(model.getXmlName()));
return bean;
}
/** Converts {@link InetAddress} to {@link XjcHostAddrType}. */
private static XjcHostAddrType convertInetAddress(InetAddress model) {
XjcHostAddrType bean = new XjcHostAddrType();
bean.setIp(model instanceof Inet6Address ? XjcHostIpType.V_6 : XjcHostIpType.V_4);
bean.setValue(InetAddresses.toAddrString(model));
return bean;
}
private HostResourceToXjcConverter() {}
}

61
java/google/registry/rde/JSchModule.java Normal file
View file

@ -0,0 +1,61 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.appengine.api.ThreadManager;
import com.google.domain.registry.keyring.api.KeyModule.Key;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import dagger.Module;
import dagger.Provides;
/** Dagger module for {@link JSch} which provides SSH/SFTP connectivity. */
@Module
public final class JSchModule {
@Provides
static JSch provideJSch(
@Key("rdeSshClientPrivateKey") String privateKey,
@Key("rdeSshClientPublicKey") String publicKey) {
applyAppEngineKludge();
JSch jsch = new JSch();
try {
jsch.addIdentity(
"rde@charlestonroadregistry.com",
privateKey.getBytes(UTF_8),
publicKey.getBytes(UTF_8),
null);
} catch (JSchException e) {
throw new RuntimeException(e);
}
// TODO(b/13028224): Implement known hosts checking.
JSch.setConfig("StrictHostKeyChecking", "no");
return jsch;
}
/**
* Overrides the threadFactory used in JSch and disable {@link Thread#setName(String)} in order to
* ensure GAE compatibility. By default it uses the default executor, which fails under GAE. This
* is currently a Google-specific patch that needs to be sent upstream.
*/
private static void applyAppEngineKludge() {
JSch.threadFactory = ThreadManager.currentRequestThreadFactory();
JSch.useThreadNames = false;
}
}

48
java/google/registry/rde/JSchSftpChannel.java Normal file
View file

@ -0,0 +1,48 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.Preconditions.checkNotNull;
import com.jcraft.jsch.ChannelSftp;
import java.io.Closeable;
import java.io.IOException;
/**
* {@link ChannelSftp} wrapper that implements {@link Closeable}.
*
* <p>This class acts as syntactic sugar for JSch so we can open and close SFTP connections in a
* way that's friendlier to Java 7 try-resource statements.
*
* @see JSchSshSession#openSftpChannel()
*/
final class JSchSftpChannel implements Closeable {
private final ChannelSftp channel;
JSchSftpChannel(ChannelSftp channel) {
this.channel = checkNotNull(channel, "channel");
}
/** Returns {@link ChannelSftp} instance wrapped by this object. */
public ChannelSftp get() {
return channel;
}
@Override
public void close() throws IOException {
channel.disconnect();
}
}

128
java/google/registry/rde/JSchSshSession.java Normal file
View file

@ -0,0 +1,128 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import com.google.common.base.Splitter;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.util.FormattingLogger;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException;
import org.joda.time.Duration;
import java.io.Closeable;
import java.io.IOException;
import java.net.URI;
import javax.inject.Inject;
/**
* SFTP connection {@link Session} delegate that implements {@link Closeable}.
*
* <p>This class acts as syntactic sugar for JSch so we can open and close SFTP connections in a
* way that's friendlier to Java 7 try-resource statements. Delegate methods are provided on an
* as-needed basis.
*
* @see JSchSftpChannel
* @see RdeUploadAction
* @see com.jcraft.jsch.Session
*/
final class JSchSshSession implements Closeable {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
/** Factory for {@link JSchSshSession}. */
static final class JSchSshSessionFactory {
private final Duration sshTimeout;
@Inject
JSchSshSessionFactory(@Config("sshTimeout") Duration sshTimeout) {
this.sshTimeout = sshTimeout;
}
/**
* Connect to remote SSH endpoint specified by {@code url}.
*
* @throws JSchException if we fail to open the connection.
*/
JSchSshSession create(JSch jsch, URI uri) throws JSchException {
RdeUploadUrl url = RdeUploadUrl.create(uri);
logger.info("Connecting to SSH endpoint: " + url);
Session session = jsch.getSession(
url.getUser().or("domain-registry"),
url.getHost(),
url.getPort());
if (url.getPass().isPresent()) {
session.setPassword(url.getPass().get());
}
session.setTimeout((int) sshTimeout.getMillis());
session.connect((int) sshTimeout.getMillis());
return new JSchSshSession(session, url, (int) sshTimeout.getMillis());
}
}
private final Session session;
private final RdeUploadUrl url;
private final int timeout;
private JSchSshSession(Session session, RdeUploadUrl url, int timeout) {
this.session = session;
this.url = url;
this.timeout = timeout;
}
/**
* Opens a new SFTP channel over this SSH session.
*
* @throws JSchException
* @throws SftpException
* @see JSchSftpChannel
*/
public JSchSftpChannel openSftpChannel() throws JSchException, SftpException {
ChannelSftp chan = (ChannelSftp) session.openChannel("sftp");
chan.connect(timeout);
if (url.getPath().isPresent()) {
String dir = url.getPath().get();
try {
chan.cd(dir);
} catch (SftpException e) {
logger.warning(e.toString());
mkdirs(chan, dir);
chan.cd(dir);
}
}
return new JSchSftpChannel(chan);
}
private void mkdirs(ChannelSftp chan, String dir) throws SftpException {
StringBuilder pathBuilder = new StringBuilder(dir.length());
for (String part : Splitter.on('/').omitEmptyStrings().split(dir)) {
pathBuilder.append(part);
chan.mkdir(pathBuilder.toString());
pathBuilder.append('/');
}
}
/** @see com.jcraft.jsch.Session#disconnect() */
@Override
public void close() throws IOException {
session.disconnect();
}
}

44
java/google/registry/rde/PendingDeposit.java Normal file
View file

@ -0,0 +1,44 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import com.google.auto.value.AutoValue;
import com.google.domain.registry.model.rde.RdeMode;
import com.google.domain.registry.model.registry.RegistryCursor.CursorType;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import java.io.Serializable;
/** Container representing a single RDE or BRDA XML escrow deposit that needs to be created. */
@AutoValue
public abstract class PendingDeposit implements Serializable {
private static final long serialVersionUID = 3141095605225904433L;
public abstract String tld();
public abstract DateTime watermark();
public abstract RdeMode mode();
public abstract CursorType cursor();
public abstract Duration interval();
static PendingDeposit create(
String tld, DateTime watermark, RdeMode mode, CursorType cursor, Duration interval) {
return new AutoValue_PendingDeposit(tld, watermark, mode, cursor, interval);
}
PendingDeposit() {}
}

133
java/google/registry/rde/PendingDepositChecker.java Normal file
View file

@ -0,0 +1,133 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import static com.google.domain.registry.util.DateTimeUtils.isBeforeOrAt;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.model.rde.RdeMode;
import com.google.domain.registry.model.registry.Registries;
import com.google.domain.registry.model.registry.Registry;
import com.google.domain.registry.model.registry.Registry.TldType;
import com.google.domain.registry.model.registry.RegistryCursor;
import com.google.domain.registry.model.registry.RegistryCursor.CursorType;
import com.google.domain.registry.util.Clock;
import com.googlecode.objectify.Work;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import javax.inject.Inject;
/**
* Utility class that determines which RDE or BRDA deposits need to be created.
*
* <p>This class is called by {@link RdeStagingAction} at the beginning of its execution. Since it
* stages everything in a single run, it needs to know what's awaiting deposit.
*
* <p>We start off by getting the list of TLDs with escrow enabled. We then check {@code cursor}
* to see when it when it was due for a deposit. If that's in the past, then we know that we need
* to generate a deposit. If it's really far in the past, we might have to generate multiple
* deposits for that TLD, based on the configured interval.
*
* <p><i>However</i> we will only generate one interval forward per mapreduce, since the reduce
* phase rolls forward a TLD's cursor, and we can't have that happening in parallel.
*
* <p>If no deposits have been made so far, then {@code startingPoint} is used as the watermark
* of the next deposit. If that's a day in the future, then escrow won't start until that date.
* This first deposit time will be set to datastore in a transaction.
*/
public final class PendingDepositChecker {
@Inject Clock clock;
@Inject @Config("brdaDayOfWeek") int brdaDayOfWeek;
@Inject @Config("brdaInterval") Duration brdaInterval;
@Inject @Config("rdeInterval") Duration rdeInterval;
@Inject PendingDepositChecker() {}
/** Returns multimap of TLDs to all RDE and BRDA deposits that need to happen. */
public ImmutableSetMultimap<String, PendingDeposit>
getTldsAndWatermarksPendingDepositForRdeAndBrda() {
return new ImmutableSetMultimap.Builder<String, PendingDeposit>()
.putAll(
getTldsAndWatermarksPendingDeposit(
RdeMode.FULL,
CursorType.RDE_STAGING,
rdeInterval,
clock.nowUtc().withTimeAtStartOfDay()))
.putAll(
getTldsAndWatermarksPendingDeposit(
RdeMode.THIN,
CursorType.BRDA,
brdaInterval,
advanceToDayOfWeek(clock.nowUtc().withTimeAtStartOfDay(), brdaDayOfWeek)))
.build();
}
private ImmutableSetMultimap<String, PendingDeposit> getTldsAndWatermarksPendingDeposit(
RdeMode mode, CursorType cursor, Duration interval, DateTime startingPoint) {
checkArgument(interval.isLongerThan(Duration.ZERO));
ImmutableSetMultimap.Builder<String, PendingDeposit> builder =
new ImmutableSetMultimap.Builder<>();
DateTime now = clock.nowUtc();
for (String tld : Registries.getTldsOfType(TldType.REAL)) {
Registry registry = Registry.get(tld);
if (!registry.getEscrowEnabled()) {
continue;
}
// Avoid creating a transaction unless absolutely necessary.
Optional<DateTime> cursorValue = RegistryCursor.load(registry, cursor);
if (isBeforeOrAt(cursorValue.or(startingPoint), now)) {
DateTime watermark;
if (cursorValue.isPresent()) {
watermark = cursorValue.get();
} else {
watermark = transactionallyInitializeCursor(registry, cursor, startingPoint);
}
if (isBeforeOrAt(watermark, now)) {
builder.put(tld, PendingDeposit.create(tld, watermark, mode, cursor, interval));
}
}
}
return builder.build();
}
private DateTime transactionallyInitializeCursor(
final Registry registry,
final CursorType cursor,
final DateTime initialValue) {
return ofy().transact(new Work<DateTime>() {
@Override
public DateTime run() {
for (DateTime value : RegistryCursor.load(registry, cursor).asSet()) {
return value;
}
RegistryCursor.save(registry, cursor, initialValue);
return initialValue;
}});
}
private static DateTime advanceToDayOfWeek(DateTime date, int dayOfWeek) {
while (date.getDayOfWeek() != dayOfWeek) {
date = date.plusDays(1);
}
return date;
}
}

40
java/google/registry/rde/RdeAdapter.java Normal file
View file

@ -0,0 +1,40 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.Strings.emptyToNull;
import static com.google.common.base.Strings.isNullOrEmpty;
import com.google.domain.registry.xjc.rde.XjcRdeRrType;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
/** Utility class that converts database objects to RDE XML objects. */
final class RdeAdapter {
/** Create {@link XjcRdeRrType} with optional {@code client} attribute. */
@Nullable
@CheckForNull
static XjcRdeRrType convertRr(@Nullable String value, @Nullable String client) {
if (isNullOrEmpty(value)) {
return null;
}
XjcRdeRrType rrType = new XjcRdeRrType();
rrType.setValue(value);
rrType.setClient(emptyToNull(client));
return rrType;
}
}

97
java/google/registry/rde/RdeCounter.java Normal file
View file

@ -0,0 +1,97 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.Predicates.equalTo;
import static com.google.common.base.Predicates.not;
import static com.google.common.collect.Iterables.filter;
import com.google.domain.registry.model.rde.RdeMode;
import com.google.domain.registry.xjc.rde.XjcRdeDepositTypeType;
import com.google.domain.registry.xjc.rdeheader.XjcRdeHeader;
import com.google.domain.registry.xjc.rdeheader.XjcRdeHeaderCount;
import com.google.domain.registry.xjc.rdeheader.XjcRdeHeaderElement;
import com.google.domain.registry.xjc.rdereport.XjcRdeReport;
import org.joda.time.DateTime;
import java.util.EnumMap;
import java.util.EnumSet;
import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.concurrent.NotThreadSafe;
import javax.inject.Inject;
/** Utility class for generating a single {@link XjcRdeHeader} while marshalling a deposit. */
@NotThreadSafe
public final class RdeCounter {
private static final String URI_ESCROW = "draft-arias-noguchi-registry-data-escrow-06";
private static final String URI_MAPPING = "draft-arias-noguchi-dnrd-objects-mapping-05";
private static final int ICANN_REPORT_SPEC_VERSION = 1;
private final EnumMap<RdeResourceType, AtomicLong> counts = new EnumMap<>(RdeResourceType.class);
@Inject
public RdeCounter() {
for (RdeResourceType resourceType : getResourceTypesExcludingHeader()) {
counts.put(resourceType, new AtomicLong());
}
}
/** Increment the count on a given resource. */
public void increment(RdeResourceType type) {
counts.get(type).incrementAndGet();
}
/** Constructs a header containing the sum of {@link #increment(RdeResourceType)} calls. */
public XjcRdeHeader makeHeader(String tld, RdeMode mode) {
XjcRdeHeader header = new XjcRdeHeader();
header.setTld(tld);
for (RdeResourceType resourceType : getResourceTypesExcludingHeader()) {
if (resourceType.getModes().contains(mode)) {
header.getCounts().add(makeCount(resourceType.getUri(), counts.get(resourceType).get()));
}
}
return header;
}
/** Returns an ICANN notification report as a JAXB object. */
public XjcRdeReport
makeReport(String id, DateTime watermark, XjcRdeHeader header, int revision) {
XjcRdeReport report = new XjcRdeReport();
report.setId(id);
report.setKind(XjcRdeDepositTypeType.FULL);
report.setCrDate(watermark);
report.setWatermark(watermark);
report.setVersion(ICANN_REPORT_SPEC_VERSION);
report.setRydeSpecEscrow(URI_ESCROW);
report.setRydeSpecMapping(URI_MAPPING);
report.setResend(revision);
report.setHeader(new XjcRdeHeaderElement(header));
return report;
}
private Iterable<RdeResourceType> getResourceTypesExcludingHeader() {
return filter(EnumSet.allOf(RdeResourceType.class), not(equalTo(RdeResourceType.HEADER)));
}
private static XjcRdeHeaderCount makeCount(String uri, long count) {
XjcRdeHeaderCount bean = new XjcRdeHeaderCount();
bean.setUri(uri);
bean.setValue(count);
return bean;
}
}

165
java/google/registry/rde/RdeMarshaller.java Normal file
View file

@ -0,0 +1,165 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.Verify.verify;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.domain.registry.model.ImmutableObject;
import com.google.domain.registry.model.contact.ContactResource;
import com.google.domain.registry.model.domain.DomainResource;
import com.google.domain.registry.model.host.HostResource;
import com.google.domain.registry.model.rde.RdeMode;
import com.google.domain.registry.model.registrar.Registrar;
import com.google.domain.registry.tldconfig.idn.IdnTable;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.xjc.XjcXmlTransformer;
import com.google.domain.registry.xjc.rde.XjcRdeContentsType;
import com.google.domain.registry.xjc.rde.XjcRdeDeposit;
import com.google.domain.registry.xjc.rde.XjcRdeDepositTypeType;
import com.google.domain.registry.xjc.rde.XjcRdeMenuType;
import com.google.domain.registry.xjc.rdeidn.XjcRdeIdn;
import com.google.domain.registry.xjc.rdeidn.XjcRdeIdnElement;
import com.google.domain.registry.xjc.rdepolicy.XjcRdePolicy;
import com.google.domain.registry.xjc.rdepolicy.XjcRdePolicyElement;
import com.google.domain.registry.xml.XmlException;
import com.google.domain.registry.xml.XmlFragmentMarshaller;
import com.googlecode.objectify.Key;
import org.joda.time.DateTime;
import java.io.ByteArrayOutputStream;
import java.io.Serializable;
import java.util.Collection;
import javax.annotation.concurrent.NotThreadSafe;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.MarshalException;
/** XML document <i>fragment</i> marshaller for RDE. */
@NotThreadSafe
public final class RdeMarshaller implements Serializable {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
private static final long serialVersionUID = 202890386611768455L;
private transient XmlFragmentMarshaller memoizedMarshaller;
/** Returns top-portion of XML document. */
public String makeHeader(
String depositId, DateTime watermark, Collection<String> uris, int revision) {
// We can't make JAXB marshal half an element. So we're going to use a kludge where we provide
// it with the minimum data necessary to marshal a deposit, and then cut it up by manually.
XjcRdeMenuType menu = new XjcRdeMenuType();
menu.setVersion("1.0");
menu.getObjURIs().addAll(uris);
XjcRdePolicy policy = new XjcRdePolicy();
policy.setScope("this-will-be-trimmed");
policy.setElement("/make/strict/validation/pass");
XjcRdeContentsType contents = new XjcRdeContentsType();
contents.getContents().add(new XjcRdePolicyElement(policy));
XjcRdeDeposit deposit = new XjcRdeDeposit();
deposit.setId(depositId);
deposit.setWatermark(watermark);
deposit.setType(XjcRdeDepositTypeType.FULL);
if (revision > 0) {
deposit.setResend(revision);
}
deposit.setRdeMenu(menu);
deposit.setContents(contents);
ByteArrayOutputStream os = new ByteArrayOutputStream();
try {
XjcXmlTransformer.marshalStrict(deposit, os, UTF_8);
} catch (XmlException e) {
throw new RuntimeException(e);
}
String rdeDocument = os.toString();
String marker = "<rde:contents>\n";
int startOfContents = rdeDocument.indexOf(marker);
verify(startOfContents > 0, "Bad RDE document:\n%s", rdeDocument);
return rdeDocument.substring(0, startOfContents + marker.length());
}
/** Returns bottom-portion of XML document. */
public String makeFooter() {
return "\n</rde:contents>\n</rde:deposit>\n";
}
/** Turns XJC element into XML fragment, with schema validation. */
public String marshalStrictlyOrDie(JAXBElement<?> element) {
try {
return getMarshaller().marshal(element);
} catch (MarshalException e) {
throw new RuntimeException(e);
}
}
/** Turns {@link ContactResource} object into an XML fragment. */
public DepositFragment marshalContact(ContactResource contact) {
return marshalResource(RdeResourceType.CONTACT, contact,
ContactResourceToXjcConverter.convert(contact));
}
/** Turns {@link DomainResource} object into an XML fragment. */
public DepositFragment marshalDomain(DomainResource domain, RdeMode mode) {
return marshalResource(RdeResourceType.DOMAIN, domain,
DomainResourceToXjcConverter.convert(domain, mode));
}
/** Turns {@link HostResource} object into an XML fragment. */
public DepositFragment marshalHost(HostResource host) {
return marshalResource(RdeResourceType.HOST, host,
HostResourceToXjcConverter.convert(host));
}
/** Turns {@link Registrar} object into an XML fragment. */
public DepositFragment marshalRegistrar(Registrar registrar) {
return marshalResource(RdeResourceType.REGISTRAR, registrar,
RegistrarToXjcConverter.convert(registrar));
}
/** Turns {@link IdnTable} object into an XML fragment. */
public String marshalIdn(IdnTable idn) {
XjcRdeIdn bean = new XjcRdeIdn();
bean.setId(idn.getName());
bean.setUrl(idn.getUrl().toString());
bean.setUrlPolicy(idn.getPolicy().toString());
return marshalStrictlyOrDie(new XjcRdeIdnElement(bean));
}
private DepositFragment marshalResource(
RdeResourceType type, ImmutableObject resource, JAXBElement<?> element) {
String xml = "";
String error = "";
try {
xml = getMarshaller().marshal(element);
} catch (MarshalException e) {
error = String.format("RDE XML schema validation failed: %s\n%s%s\n",
Key.create(resource),
e.getLinkedException(),
getMarshaller().marshalLenient(element));
logger.severe(error);
}
return DepositFragment.create(type, xml, error);
}
private XmlFragmentMarshaller getMarshaller() {
return memoizedMarshaller != null
? memoizedMarshaller
: (memoizedMarshaller = XjcXmlTransformer.get().createFragmentMarshaller());
}
}

70
java/google/registry/rde/RdeModule.java Normal file
View file

@ -0,0 +1,70 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import com.google.appengine.api.taskqueue.Queue;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.request.RequestParameters;
import dagger.Module;
import dagger.Provides;
import org.joda.time.DateTime;
import javax.inject.Named;
import javax.servlet.http.HttpServletRequest;
/**
* Dagger module for RDE package.
*
* @see "com.google.domain.registry.module.backend.BackendComponent"
*/
@Module
public final class RdeModule {
static final String PARAM_WATERMARK = "watermark";
@Provides
@Parameter(PARAM_WATERMARK)
static DateTime provideWatermark(HttpServletRequest req) {
return DateTime.parse(RequestParameters.extractRequiredParameter(req, PARAM_WATERMARK));
}
@Provides
@Named("brda")
static Queue provideQueueBrda() {
return getQueue("brda");
}
@Provides
@Named("rde-report")
static Queue provideQueueRdeReport() {
return getQueue("rde-report");
}
@Provides
@Named("rde-staging")
static Queue provideQueueRdeStaging() {
return getQueue("rde-staging");
}
@Provides
@Named("rde-upload")
static Queue provideQueueRdeUpload() {
return getQueue("rde-upload");
}
}

102
java/google/registry/rde/RdeReportAction.java Normal file
View file

@ -0,0 +1,102 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.Verify.verify;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static com.google.domain.registry.model.rde.RdeMode.FULL;
import static com.google.domain.registry.request.Action.Method.POST;
import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.io.ByteStreams;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.gcs.GcsUtils;
import com.google.domain.registry.keyring.api.KeyModule.Key;
import com.google.domain.registry.model.rde.RdeNamingUtils;
import com.google.domain.registry.model.registry.Registry;
import com.google.domain.registry.model.registry.RegistryCursor;
import com.google.domain.registry.model.registry.RegistryCursor.CursorType;
import com.google.domain.registry.rde.EscrowTaskRunner.EscrowTask;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.HttpException.NoContentException;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.request.RequestParameters;
import com.google.domain.registry.request.Response;
import com.google.domain.registry.util.FormattingLogger;
import org.bouncycastle.openpgp.PGPException;
import org.bouncycastle.openpgp.PGPPrivateKey;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import java.io.IOException;
import java.io.InputStream;
import javax.inject.Inject;
/**
* Action that uploads a small XML RDE report to ICANN after {@link RdeUploadAction} has finished.
*/
@Action(path = RdeReportAction.PATH, method = POST)
public final class RdeReportAction implements Runnable, EscrowTask {
static final String PATH = "/_dr/task/rdeReport";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject GcsUtils gcsUtils;
@Inject Ghostryde ghostryde;
@Inject EscrowTaskRunner runner;
@Inject Response response;
@Inject RdeReporter reporter;
@Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@Inject @Config("rdeBucket") String bucket;
@Inject @Config("rdeInterval") Duration interval;
@Inject @Config("rdeReportLockTimeout") Duration timeout;
@Inject @Key("rdeStagingDecryptionKey") PGPPrivateKey stagingDecryptionKey;
@Inject RdeReportAction() {}
@Override
public void run() {
runner.lockRunAndRollForward(this, Registry.get(tld), timeout, CursorType.RDE_REPORT, interval);
}
@Override
public void runWithLock(DateTime watermark) throws Exception {
DateTime stagingCursor =
RegistryCursor.load(Registry.get(tld), CursorType.RDE_UPLOAD).or(START_OF_TIME);
if (!stagingCursor.isAfter(watermark)) {
logger.infofmt("tld=%s reportCursor=%s uploadCursor=%s", tld, watermark, stagingCursor);
throw new NoContentException("Waiting for RdeUploadAction to complete");
}
String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, 0);
GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
verify(gcsUtils.existsAndNotEmpty(reportFilename), "Missing file: %s", reportFilename);
reporter.send(readReportFromGcs(reportFilename));
response.setContentType(PLAIN_TEXT_UTF_8);
response.setPayload(String.format("OK %s %s\n", tld, watermark));
}
/** Reads and decrypts the XML file from cloud storage. */
private byte[] readReportFromGcs(GcsFilename reportFilename) throws IOException, PGPException {
try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename);
Ghostryde.Decryptor decryptor = ghostryde.openDecryptor(gcsInput, stagingDecryptionKey);
Ghostryde.Decompressor decompressor = ghostryde.openDecompressor(decryptor);
Ghostryde.Input xmlInput = ghostryde.openInput(decompressor)) {
return ByteStreams.toByteArray(xmlInput);
}
}
}

124
java/google/registry/rde/RdeReporter.java Normal file
View file

@ -0,0 +1,124 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
import static com.google.common.io.BaseEncoding.base64;
import static com.google.common.net.HttpHeaders.AUTHORIZATION;
import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
import static com.google.domain.registry.util.DomainNameUtils.canonicalizeDomainName;
import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.config.RegistryConfig;
import com.google.domain.registry.keyring.api.KeyModule.Key;
import com.google.domain.registry.request.HttpException.InternalServerErrorException;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.util.UrlFetchException;
import com.google.domain.registry.xjc.XjcXmlTransformer;
import com.google.domain.registry.xjc.iirdea.XjcIirdeaResponseElement;
import com.google.domain.registry.xjc.iirdea.XjcIirdeaResult;
import com.google.domain.registry.xjc.rdeheader.XjcRdeHeader;
import com.google.domain.registry.xjc.rdereport.XjcRdeReportReport;
import com.google.domain.registry.xml.XmlException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import javax.inject.Inject;
/**
* Class that uploads a decrypted XML deposit report to ICANN's webserver.
*
* @see RdeReportAction
*/
public class RdeReporter {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
/** @see "http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4" */
private static final String REPORT_MIME = "text/xml";
@Inject RegistryConfig config;
@Inject URLFetchService urlFetchService;
@Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix;
@Inject @Key("icannReportingPassword") String password;
@Inject RdeReporter() {}
/** Uploads {@code reportBytes} to ICANN. */
public void send(byte[] reportBytes) throws IOException, XmlException {
XjcRdeReportReport report = XjcXmlTransformer.unmarshal(new ByteArrayInputStream(reportBytes));
XjcRdeHeader header = report.getHeader().getValue();
// Send a PUT request to ICANN's HTTPS server.
URL url = makeReportUrl(header.getTld(), report.getId());
String username = header.getTld() + "_ry";
String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8));
HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d));
req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME));
req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
req.setPayload(reportBytes);
logger.infofmt("Sending report:\n%s", new String(reportBytes, UTF_8));
HTTPResponse rsp = urlFetchService.fetch(req);
switch (rsp.getResponseCode()) {
case SC_OK:
case SC_BAD_REQUEST:
break;
default:
throw new UrlFetchException("PUT failed", req, rsp);
}
// Ensure the XML response is valid.
XjcIirdeaResult result = parseResult(rsp);
if (result.getCode().getValue() != 1000) {
logger.warningfmt("PUT rejected: %d %s\n%s",
result.getCode().getValue(),
result.getMsg(),
result.getDescription());
throw new InternalServerErrorException(result.getMsg());
}
}
/**
* Unmarshals IIRDEA XML result object from {@link HTTPResponse} payload.
*
* @see "http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1"
*/
private XjcIirdeaResult parseResult(HTTPResponse rsp) throws XmlException {
byte[] responseBytes = rsp.getContent();
logger.infofmt("Received response:\n%s", new String(responseBytes, UTF_8));
XjcIirdeaResponseElement response =
XjcXmlTransformer.unmarshal(new ByteArrayInputStream(responseBytes));
XjcIirdeaResult result = response.getResult();
return result;
}
private URL makeReportUrl(String tld, String id) {
try {
return new URL(String.format("%s/%s/%s", reportUrlPrefix, canonicalizeDomainName(tld), id));
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
}
}

66
java/google/registry/rde/RdeResourceType.java Normal file
View file

@ -0,0 +1,66 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.domain.registry.model.rde.RdeMode.FULL;
import static com.google.domain.registry.model.rde.RdeMode.THIN;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Ordering;
import com.google.domain.registry.model.rde.RdeMode;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Set;
/** Types of objects that get embedded in an escrow deposit. */
public enum RdeResourceType {
CONTACT("urn:ietf:params:xml:ns:rdeContact-1.0", EnumSet.of(FULL)),
DOMAIN("urn:ietf:params:xml:ns:rdeDomain-1.0", EnumSet.of(FULL, THIN)),
HOST("urn:ietf:params:xml:ns:rdeHost-1.0", EnumSet.of(FULL)),
REGISTRAR("urn:ietf:params:xml:ns:rdeRegistrar-1.0", EnumSet.of(FULL, THIN)),
IDN("urn:ietf:params:xml:ns:rdeIDN-1.0", EnumSet.of(FULL, THIN)),
HEADER("urn:ietf:params:xml:ns:rdeHeader-1.0", EnumSet.of(FULL, THIN));
private final String uri;
private final Set<RdeMode> modes;
private RdeResourceType(String uri, EnumSet<RdeMode> modes) {
this.uri = uri;
this.modes = Collections.unmodifiableSet(modes);
}
/** Returns RDE XML schema URI specifying resource. */
public String getUri() {
return uri;
}
/** Returns set indicating if resource is stored in BRDA thin deposits. */
public Set<RdeMode> getModes() {
return modes;
}
/** Returns set of resource type URIs included in a deposit {@code mode}. */
public static ImmutableSortedSet<String> getUris(RdeMode mode) {
ImmutableSortedSet.Builder<String> builder =
new ImmutableSortedSet.Builder<>(Ordering.natural());
for (RdeResourceType resourceType : RdeResourceType.values()) {
if (resourceType.getModes().contains(mode)) {
builder.add(resourceType.getUri());
}
}
return builder.build();
}
}

208
java/google/registry/rde/RdeStagingAction.java Normal file
View file

@ -0,0 +1,208 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.domain.registry.util.PipelineUtils.createJobPath;
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.Multimaps;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.mapreduce.MapreduceRunner;
import com.google.domain.registry.mapreduce.inputs.EppResourceInputs;
import com.google.domain.registry.mapreduce.inputs.NullInput;
import com.google.domain.registry.model.EppResource;
import com.google.domain.registry.model.contact.ContactResource;
import com.google.domain.registry.model.host.HostResource;
import com.google.domain.registry.model.index.EppResourceIndex;
import com.google.domain.registry.model.rde.RdeMode;
import com.google.domain.registry.model.registrar.Registrar;
import com.google.domain.registry.model.registry.RegistryCursor;
import com.google.domain.registry.model.registry.RegistryCursor.CursorType;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Response;
import com.google.domain.registry.util.Clock;
import com.google.domain.registry.util.FormattingLogger;
import org.joda.time.Duration;
import javax.inject.Inject;
/**
* MapReduce that idempotently stages escrow deposit XML files on GCS for RDE/BRDA for all TLDs.
*
* <h3>MapReduce Operation</h3>
*
* <p>This task starts by asking {@link PendingDepositChecker} which deposits need to be generated.
* If there's nothing to deposit, we return 204 No Content; otherwise, we fire off a MapReduce job
* and redirect to its status GUI.
*
* <p>The mapreduce job scans every {@link EppResource} in datastore. It maps a point-in-time
* representation of each entity to the escrow XML files in which it should appear.
*
* <p>There is one map worker for each {@code EppResourceIndexBucket} entity group shard. There is
* one reduce worker for each deposit being generated.
*
* <p>{@link ContactResource} and {@link HostResource} are emitted on all TLDs, even when the
* domains on a TLD don't reference them. BRDA {@link RdeMode#THIN thin} deposits exclude contacts
* and hosts entirely.
*
* <p>{@link Registrar} entities, both active and inactive, are included in all deposits. They are
* not rewinded point-in-time.
*
* <p>The XML deposit files generated by this job are humongous. A tiny XML report file is generated
* for each deposit, telling us how much of what it contains.
*
* <p>Once a deposit is successfully generated, an {@link RdeUploadAction} is enqueued which will
* upload it via SFTP to the third-party escrow provider.
*
* <p>To generate escrow deposits manually and locally, use the {@code registry_tool} command
* {@code GenerateEscrowDepositCommand}.
*
* <h3>Logging</h3>
*
* <p>To identify the reduce worker request for a deposit in App Engine's log viewer, you can use
* search text like {@code tld=soy}, {@code watermark=2015-01-01}, and {@code mode=FULL}.
*
* <h3>Error Handling</h3>
*
* <p>Valid model objects might not be valid to the RDE XML schema. A single invalid object will
* cause the whole deposit to fail. You need to check the logs, find out which entities are broken,
* and perform datastore surgery.
*
* <p>If a deposit fails, an error is emitted to the logs for each broken entity. It tells you the
* key and shows you its representation in lenient XML.
*
* <p>Failed deposits will be retried indefinitely. This is because RDE and BRDA each have a
* {@link RegistryCursor} for each TLD. Even if the cursor lags for days, it'll catch up gradually
* on its own, once the data becomes valid.
*
* <p>The third-party escrow provider will validate each deposit we send them. They do both schema
* validation and reference checking.
*
* <p>This job does not perform reference checking. Administrators can do this locally with the
* {@code ValidateEscrowDepositCommand} command in {@code registry_tool}.
*
* <h3>Cursors</h3>
*
* <p>Deposits are generated serially for a given (tld, mode) pair. A deposit is never started
* beyond the cursor. Once a deposit is completed, its cursor is rolled forward transactionally.
*
* <p>The mode determines which cursor is used. {@link CursorType#RDE_STAGING} is used for thick
* deposits and {@link CursorType#BRDA} is used for thin deposits.
*
* <p>Use the {@code ListCursorsCommand} and {@code UpdateCursorsCommand} commands to administrate
* with these cursors.
*
* <h3>Security</h3>
*
* <p>The deposit and report are encrypted using {@link Ghostryde}. Administrators can use the
* {@code GhostrydeCommand} command in {@code registry_tool} to view them.
*
* <p>Unencrypted XML fragments are stored temporarily between the map and reduce steps. The
* ghostryde encryption on the full archived deposits makes life a little more difficult for an
* attacker. But security ultimately depends on the bucket.
*
* <h3>Idempotency</h3>
*
* <p>We lock the reduce tasks. This is necessary because: a) App Engine tasks might get double
* executed; and b) Cloud Storage file handles get committed on close <i>even if our code throws an
* exception.</i>
*
* <p>Deposits are generated serially for a given (watermark, mode) pair. A deposit is never started
* beyond the cursor. Once a deposit is completed, its cursor is rolled forward transactionally.
* Duplicate jobs may exist {@code <=cursor}. So a transaction will not bother changing the cursor
* if it's already been rolled forward.
*
* <p>Enqueueing {@code RdeUploadAction} is also part of the cursor transaction. This is necessary
* because the first thing the upload task does is check the staging cursor to verify it's been
* completed, so we can't enqueue before we roll. We also can't enqueue after the roll, because then
* if enqueueing fails, the upload might never be enqueued.
*
* <h3>Determinism</h3>
*
* <p>The filename of an escrow deposit is determistic for a given (TLD, watermark,
* {@linkplain RdeMode mode}) triplet. Its generated contents is deterministic in all the ways that
* we care about. Its view of the database is strongly consistent.
*
* <p>This is because:
* <ol>
* <li>{@code EppResource} queries are strongly consistent thanks to {@link EppResourceIndex}
* <li>{@code EppResource} entities are rewinded to the point-in-time of the watermark
* </ol>
*
* <p>Here's what's not deterministic:
* <ul>
* <li>Ordering of XML fragments. We don't care about this.
* <li>Information about registrars. There's no point-in-time for these objects. So in order to
* guarantee referential correctness of your deposits, you must never delete a registrar entity.
* </ul>
*
* @see "https://tools.ietf.org/html/draft-arias-noguchi-registry-data-escrow-06"
* @see "https://tools.ietf.org/html/draft-arias-noguchi-dnrd-objects-mapping-05"
*/
@Action(path = "/_dr/task/rdeStaging")
public final class RdeStagingAction implements Runnable {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject Clock clock;
@Inject PendingDepositChecker pendingDepositChecker;
@Inject RdeStagingReducer reducer;
@Inject Response response;
@Inject MapreduceRunner mrRunner;
@Inject @Config("transactionCooldown") Duration transactionCooldown;
@Inject RdeStagingAction() {}
@Override
public void run() {
ImmutableSetMultimap<String, PendingDeposit> pendings = ImmutableSetMultimap.copyOf(
Multimaps.filterValues(
pendingDepositChecker.getTldsAndWatermarksPendingDepositForRdeAndBrda(),
new Predicate<PendingDeposit>() {
@Override
public boolean apply(PendingDeposit pending) {
if (clock.nowUtc().isBefore(pending.watermark().plus(transactionCooldown))) {
logger.infofmt("Ignoring within %s cooldown: %s", transactionCooldown, pending);
return false;
} else {
return true;
}
}}));
if (pendings.isEmpty()) {
String message = "Nothing needs to be deposited";
logger.info(message);
response.setStatus(SC_NO_CONTENT);
response.setPayload(message);
return;
}
for (PendingDeposit pending : pendings.values()) {
logger.infofmt("%s", pending);
}
response.sendJavaScriptRedirect(createJobPath(mrRunner
.setJobName("Stage escrow deposits for all TLDs")
.setModuleName("backend")
.setDefaultReduceShards(pendings.size())
.runMapreduce(
new RdeStagingMapper(pendings),
reducer,
ImmutableList.of(
// Add an extra shard that maps over a null resource. See the mapper code for why.
new NullInput<EppResource>(),
EppResourceInputs.createEntityInput(EppResource.class)))));
}
}

193
java/google/registry/rde/RdeStagingMapper.java Normal file
View file

@ -0,0 +1,193 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.Strings.nullToEmpty;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.auto.value.AutoValue;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.Maps;
import com.google.domain.registry.model.EppResource;
import com.google.domain.registry.model.EppResourceUtils;
import com.google.domain.registry.model.contact.ContactResource;
import com.google.domain.registry.model.domain.DomainResource;
import com.google.domain.registry.model.host.HostResource;
import com.google.domain.registry.model.rde.RdeMode;
import com.google.domain.registry.model.registrar.Registrar;
import com.googlecode.objectify.Result;
import org.joda.time.DateTime;
import java.util.HashMap;
import java.util.Map;
/** Mapper for {@link RdeStagingAction}. */
public final class RdeStagingMapper extends Mapper<EppResource, PendingDeposit, DepositFragment> {
private static final long serialVersionUID = -1518185703789372524L;
private final ImmutableSetMultimap<String, PendingDeposit> pendings;
private final RdeMarshaller marshaller = new RdeMarshaller();
RdeStagingMapper(ImmutableSetMultimap<String, PendingDeposit> pendings) {
this.pendings = pendings;
}
@Override
public final void map(final EppResource resource) {
// The mapreduce has one special input that provides a null resource. This is used as a sentinel
// to indicate that we should emit the Registrar objects on this map shard, as these need to be
// added to every deposit. It is important that these be emitted as part of the mapreduce and
// not added in a separate stage, because the reducer only runs if there is at least one value
// emitted from the mapper. Without this, a cursor might never advance because no EppResource
// entity exists at the watermark.
if (resource == null) {
for (Registrar registrar : Registrar.loadAll()) {
DepositFragment fragment = marshaller.marshalRegistrar(registrar);
for (PendingDeposit pending : pendings.values()) {
emit(pending, fragment);
}
}
return;
}
// Skip polymorphic entities that share datastore kind.
if (!(resource instanceof ContactResource
|| resource instanceof DomainResource
|| resource instanceof HostResource)) {
return;
}
// Skip prober data.
if (nullToEmpty(resource.getCreationClientId()).startsWith("prober-")
|| nullToEmpty(resource.getCurrentSponsorClientId()).startsWith("prober-")
|| nullToEmpty(resource.getLastEppUpdateClientId()).startsWith("prober-")) {
return;
}
// Contacts and hosts get emitted on all TLDs, even if domains don't reference them.
boolean shouldEmitOnAllTlds = !(resource instanceof DomainResource);
// Get the set of all TLDs to which this resource should be emitted.
ImmutableSet<String> tlds =
shouldEmitOnAllTlds
? pendings.keySet()
: ImmutableSet.of(((DomainResource) resource).getTld());
// Get the set of all point-in-time watermarks we need, to minimize rewinding.
ImmutableSet<DateTime> dates =
FluentIterable
.from(shouldEmitOnAllTlds
? pendings.values()
: pendings.get(((DomainResource) resource).getTld()))
.transform(new Function<PendingDeposit, DateTime>() {
@Override
public DateTime apply(PendingDeposit pending) {
return pending.watermark();
}})
.toSet();
// Launch asynchronous fetches of point-in-time representations of resource.
ImmutableMap<DateTime, Result<EppResource>> resourceAtTimes =
ImmutableMap.copyOf(Maps.asMap(dates,
new Function<DateTime, Result<EppResource>>() {
@Override
public Result<EppResource> apply(DateTime input) {
return EppResourceUtils.loadAtPointInTime(resource, input);
}}));
// Convert resource to an XML fragment for each watermark/mode pair lazily and cache the result.
Fragmenter fragmenter = new Fragmenter(resourceAtTimes);
// Emit resource as an XML fragment for all TLDs and modes pending deposit.
for (String tld : tlds) {
for (PendingDeposit pending : pendings.get(tld)) {
// Hosts and contacts don't get included in BRDA deposits.
if (pending.mode() == RdeMode.THIN
&& (resource instanceof ContactResource
|| resource instanceof HostResource)) {
continue;
}
for (DepositFragment fragment
: fragmenter.marshal(pending.watermark(), pending.mode()).asSet()) {
emit(pending, fragment);
}
}
}
// Avoid running out of memory.
ofy().clearSessionCache();
}
/** Loading cache that turns a resource into XML for the various points in time and modes. */
private class Fragmenter {
private final Map<WatermarkModePair, Optional<DepositFragment>> cache = new HashMap<>();
private final ImmutableMap<DateTime, Result<EppResource>> resourceAtTimes;
Fragmenter(ImmutableMap<DateTime, Result<EppResource>> resourceAtTimes) {
this.resourceAtTimes = resourceAtTimes;
}
Optional<DepositFragment> marshal(DateTime watermark, RdeMode mode) {
Optional<DepositFragment> result = cache.get(WatermarkModePair.create(watermark, mode));
if (result != null) {
return result;
}
EppResource resource = resourceAtTimes.get(watermark).now();
if (resource == null) {
result = Optional.absent();
cache.put(WatermarkModePair.create(watermark, RdeMode.FULL), result);
cache.put(WatermarkModePair.create(watermark, RdeMode.THIN), result);
return result;
}
if (resource instanceof DomainResource) {
result = Optional.of(marshaller.marshalDomain((DomainResource) resource, mode));
cache.put(WatermarkModePair.create(watermark, mode), result);
return result;
} else if (resource instanceof ContactResource) {
result = Optional.of(marshaller.marshalContact((ContactResource) resource));
cache.put(WatermarkModePair.create(watermark, RdeMode.FULL), result);
cache.put(WatermarkModePair.create(watermark, RdeMode.THIN), result);
return result;
} else if (resource instanceof HostResource) {
result = Optional.of(marshaller.marshalHost((HostResource) resource));
cache.put(WatermarkModePair.create(watermark, RdeMode.FULL), result);
cache.put(WatermarkModePair.create(watermark, RdeMode.THIN), result);
return result;
} else {
throw new AssertionError(resource.toString());
}
}
}
/** Map key for {@link Fragmenter} cache. */
@AutoValue
abstract static class WatermarkModePair {
abstract DateTime watermark();
abstract RdeMode mode();
static WatermarkModePair create(DateTime watermark, RdeMode mode) {
return new AutoValue_RdeStagingMapper_WatermarkModePair(watermark, mode);
}
}
}

226
java/google/registry/rde/RdeStagingReducer.java Normal file
View file

@ -0,0 +1,226 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
import static com.google.common.base.Verify.verify;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.Reducer;
import com.google.appengine.tools.mapreduce.ReducerInput;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.gcs.GcsUtils;
import com.google.domain.registry.keyring.api.KeyModule;
import com.google.domain.registry.keyring.api.PgpHelper;
import com.google.domain.registry.model.rde.RdeMode;
import com.google.domain.registry.model.rde.RdeNamingUtils;
import com.google.domain.registry.model.rde.RdeRevision;
import com.google.domain.registry.model.registry.Registry;
import com.google.domain.registry.model.registry.RegistryCursor;
import com.google.domain.registry.model.server.Lock;
import com.google.domain.registry.request.RequestParameters;
import com.google.domain.registry.tldconfig.idn.IdnTableEnum;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.util.TaskEnqueuer;
import com.google.domain.registry.xjc.rdeheader.XjcRdeHeader;
import com.google.domain.registry.xjc.rdeheader.XjcRdeHeaderElement;
import com.google.domain.registry.xml.XmlException;
import com.googlecode.objectify.VoidWork;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.openpgp.PGPException;
import org.bouncycastle.openpgp.PGPPublicKey;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.security.Security;
import java.util.Iterator;
import java.util.concurrent.Callable;
import javax.inject.Inject;
/** Reducer for {@link RdeStagingAction}. */
public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFragment, Void> {
private static final long serialVersionUID = -3366189042770402345L;
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
private final RdeMarshaller marshaller = new RdeMarshaller();
@Inject TaskEnqueuer taskEnqueuer;
@Inject @Config("gcsBufferSize") int gcsBufferSize;
@Inject @Config("rdeBucket") String bucket;
@Inject @Config("rdeGhostrydeBufferSize") int ghostrydeBufferSize;
@Inject @Config("rdeStagingLockTimeout") Duration lockTimeout;
@Inject @KeyModule.Key("rdeStagingEncryptionKey") byte[] stagingKeyBytes;
@Inject RdeStagingReducer() {}
@Override
public void reduce(final PendingDeposit key, final ReducerInput<DepositFragment> fragments) {
Callable<Void> lockRunner = new Callable<Void>() {
@Override
public Void call() throws Exception {
reduceWithLock(key, fragments);
return null;
}};
String lockName = String.format("RdeStaging %s", key.tld());
if (!Lock.executeWithLocks(lockRunner, null, null, lockTimeout, lockName)) {
logger.warningfmt("Lock in use: %s", lockName);
}
}
private void reduceWithLock(final PendingDeposit key, Iterator<DepositFragment> fragments) {
logger.infofmt("RdeStagingReducer %s", key);
// Normally this is done by BackendServlet but it's not present in MapReduceServlet.
Security.addProvider(new BouncyCastleProvider());
// Construct things that Dagger would inject if this wasn't serialized.
Ghostryde ghostryde = new Ghostryde(ghostrydeBufferSize);
PGPPublicKey stagingKey = PgpHelper.loadPublicKeyBytes(stagingKeyBytes);
GcsUtils cloudStorage =
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
RdeCounter counter = new RdeCounter();
// Determine some basic things about the deposit.
final RdeMode mode = key.mode();
final String tld = key.tld();
final DateTime watermark = key.watermark();
final int revision = RdeRevision.getNextRevision(tld, watermark, mode);
String id = RdeUtil.timestampToId(watermark);
String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
GcsFilename xmlFilename = new GcsFilename(bucket, prefix + ".xml.ghostryde");
GcsFilename xmlLengthFilename = new GcsFilename(bucket, prefix + ".xml.length");
GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
// These variables will be populated as we write the deposit XML and used for other files.
boolean failed = false;
long xmlLength;
XjcRdeHeader header;
// Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
logger.infofmt("Writing %s", xmlFilename);
try (OutputStream gcsOutput = cloudStorage.openOutputStream(xmlFilename);
Ghostryde.Encryptor encryptor = ghostryde.openEncryptor(gcsOutput, stagingKey);
Ghostryde.Compressor kompressor = ghostryde.openCompressor(encryptor);
Ghostryde.Output gOutput = ghostryde.openOutput(kompressor, prefix + ".xml", watermark);
Writer output = new OutputStreamWriter(gOutput, UTF_8)) {
// Output the top portion of the XML document.
output.write(marshaller.makeHeader(id, watermark, RdeResourceType.getUris(mode), revision));
// Output XML fragments emitted to us by RdeStagingMapper while counting them.
while (fragments.hasNext()) {
DepositFragment fragment = fragments.next();
if (!fragment.xml().isEmpty()) {
output.write(fragment.xml());
counter.increment(fragment.type());
}
if (!fragment.error().isEmpty()) {
failed = true;
logger.severe(fragment.error());
}
}
for (IdnTableEnum idn : IdnTableEnum.values()) {
output.write(marshaller.marshalIdn(idn.getTable()));
counter.increment(RdeResourceType.IDN);
}
// Output XML that says how many resources were emitted.
header = counter.makeHeader(tld, mode);
output.write(marshaller.marshalStrictlyOrDie(new XjcRdeHeaderElement(header)));
// Output the bottom of the XML document.
output.write(marshaller.makeFooter());
// And we're done! How many raw XML bytes did we write?
output.flush();
xmlLength = gOutput.getBytesWritten();
} catch (IOException | PGPException e) {
throw new RuntimeException(e);
}
// If an entity was broken, abort after writing as much logs/deposit data as possible.
verify(!failed);
// Write a file to GCS containing the byte length (ASCII) of the raw unencrypted XML.
//
// This is necessary because RdeUploadAction creates a tar file which requires that the length
// be outputted. We don't want to have to decrypt the entire ghostryde file to determine the
// length, so we just save it separately.
logger.infofmt("Writing %s", xmlLengthFilename);
try (OutputStream gcsOutput = cloudStorage.openOutputStream(xmlLengthFilename)) {
gcsOutput.write(Long.toString(xmlLength).getBytes(US_ASCII));
} catch (IOException e) {
throw new RuntimeException(e);
}
// Write a tiny XML file to GCS containing some information about the deposit.
//
// This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
if (mode == RdeMode.FULL) {
logger.infofmt("Writing %s", reportFilename);
String innerName = prefix + "-report.xml";
try (OutputStream gcsOutput = cloudStorage.openOutputStream(reportFilename);
Ghostryde.Encryptor encryptor = ghostryde.openEncryptor(gcsOutput, stagingKey);
Ghostryde.Compressor kompressor = ghostryde.openCompressor(encryptor);
Ghostryde.Output output = ghostryde.openOutput(kompressor, innerName, watermark)) {
counter.makeReport(id, watermark, header, revision).marshal(output, UTF_8);
} catch (IOException | PGPException | XmlException e) {
throw new RuntimeException(e);
}
}
// Now that we're done, kick off RdeUploadAction and roll forward the cursor transactionally.
ofy().transact(new VoidWork() {
@Override
public void vrun() {
Registry registry = Registry.get(tld);
DateTime position = RegistryCursor.load(registry, key.cursor()).get();
DateTime newPosition = key.watermark().plus(key.interval());
if (!position.isBefore(newPosition)) {
logger.warning("Cursor has already been rolled forward.");
return;
}
verify(position.equals(key.watermark()),
"Partial ordering of RDE deposits broken: %s %s", position, key);
RegistryCursor.save(registry, key.cursor(), newPosition);
logger.infofmt("Rolled forward %s on %s cursor to %s", key.cursor(), tld, newPosition);
RdeRevision.saveRevision(tld, watermark, mode, revision);
if (mode == RdeMode.FULL) {
taskEnqueuer.enqueue(getQueue("rde-upload"),
withUrl(RdeUploadAction.PATH)
.param(RequestParameters.PARAM_TLD, tld));
} else {
taskEnqueuer.enqueue(getQueue("brda"),
withUrl(BrdaCopyAction.PATH)
.param(RequestParameters.PARAM_TLD, tld)
.param(RdeModule.PARAM_WATERMARK, watermark.toString()));
}
}});
}
}

221
java/google/registry/rde/RdeUploadAction.java Normal file
View file

@ -0,0 +1,221 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
import static com.google.common.base.Verify.verify;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import static com.google.domain.registry.model.rde.RdeMode.FULL;
import static com.google.domain.registry.request.Action.Method.POST;
import static com.google.domain.registry.util.DateTimeUtils.START_OF_TIME;
import static com.jcraft.jsch.ChannelSftp.OVERWRITE;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Arrays.asList;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.io.ByteStreams;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.gcs.GcsUtils;
import com.google.domain.registry.keyring.api.KeyModule.Key;
import com.google.domain.registry.model.rde.RdeNamingUtils;
import com.google.domain.registry.model.rde.RdeRevision;
import com.google.domain.registry.model.registry.Registry;
import com.google.domain.registry.model.registry.RegistryCursor;
import com.google.domain.registry.model.registry.RegistryCursor.CursorType;
import com.google.domain.registry.rde.EscrowTaskRunner.EscrowTask;
import com.google.domain.registry.rde.JSchSshSession.JSchSshSessionFactory;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.HttpException.ServiceUnavailableException;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.request.RequestParameters;
import com.google.domain.registry.request.Response;
import com.google.domain.registry.util.Clock;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.util.TaskEnqueuer;
import com.google.domain.registry.util.TeeOutputStream;
import com.googlecode.objectify.VoidWork;
import com.jcraft.jsch.JSch;
import org.bouncycastle.openpgp.PGPKeyPair;
import org.bouncycastle.openpgp.PGPPrivateKey;
import org.bouncycastle.openpgp.PGPPublicKey;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import javax.inject.Inject;
import javax.inject.Named;
/**
* Action that securely uploads an RDE XML file from Cloud Storage to a trusted third party (such as
* Iron Mountain) via SFTP.
*
* <p>This action is invoked by {@link RdeStagingAction} once it's created the files we need. The
* date is calculated from {@link CursorType#RDE_UPLOAD}.
*
* <p>Once this action completes, it rolls the cursor forward a day and triggers
* {@link RdeReportAction}.
*/
@Action(path = RdeUploadAction.PATH, method = POST)
public final class RdeUploadAction implements Runnable, EscrowTask {
static final String PATH = "/_dr/task/rdeUpload";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject Clock clock;
@Inject GcsUtils gcsUtils;
@Inject Ghostryde ghostryde;
@Inject EscrowTaskRunner runner;
@Inject JSch jsch;
@Inject JSchSshSessionFactory jschSshSessionFactory;
@Inject Response response;
@Inject RydePgpCompressionOutputStreamFactory pgpCompressionFactory;
@Inject RydePgpEncryptionOutputStreamFactory pgpEncryptionFactory;
@Inject RydePgpFileOutputStreamFactory pgpFileFactory;
@Inject RydePgpSigningOutputStreamFactory pgpSigningFactory;
@Inject RydeTarOutputStreamFactory tarFactory;
@Inject TaskEnqueuer taskEnqueuer;
@Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@Inject @Config("rdeBucket") String bucket;
@Inject @Config("rdeInterval") Duration interval;
@Inject @Config("rdeUploadLockTimeout") Duration timeout;
@Inject @Config("rdeUploadSftpCooldown") Duration sftpCooldown;
@Inject @Config("rdeUploadUrl") URI uploadUrl;
@Inject @Key("rdeReceiverKey") PGPPublicKey receiverKey;
@Inject @Key("rdeSigningKey") PGPKeyPair signingKey;
@Inject @Key("rdeStagingDecryptionKey") PGPPrivateKey stagingDecryptionKey;
@Inject @Named("rde-report") Queue reportQueue;
@Inject RdeUploadAction() {}
@Override
public void run() {
runner.lockRunAndRollForward(this, Registry.get(tld), timeout, CursorType.RDE_UPLOAD, interval);
taskEnqueuer.enqueue(
reportQueue,
withUrl(RdeReportAction.PATH).param(RequestParameters.PARAM_TLD, tld));
}
@Override
public void runWithLock(DateTime watermark) throws Exception {
DateTime stagingCursor =
RegistryCursor.load(Registry.get(tld), CursorType.RDE_STAGING).or(START_OF_TIME);
if (!stagingCursor.isAfter(watermark)) {
logger.infofmt("tld=%s uploadCursor=%s stagingCursor=%s", tld, watermark, stagingCursor);
throw new ServiceUnavailableException("Waiting for RdeStagingAction to complete");
}
DateTime sftpCursor =
RegistryCursor.load(Registry.get(tld), CursorType.RDE_UPLOAD_SFTP).or(START_OF_TIME);
if (sftpCursor.plus(sftpCooldown).isAfter(clock.nowUtc())) {
// Fail the task good and hard so it retries until the cooldown passes.
logger.infofmt("tld=%s cursor=%s sftpCursor=%s", tld, watermark, sftpCursor);
throw new ServiceUnavailableException("SFTP cooldown has not yet passed");
}
int revision = RdeRevision.getNextRevision(tld, watermark, FULL) - 1;
verify(revision >= 0, "RdeRevision was not set on generated deposit");
String name = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
GcsFilename xmlFilename = new GcsFilename(bucket, name + ".xml.ghostryde");
GcsFilename xmlLengthFilename = new GcsFilename(bucket, name + ".xml.length");
GcsFilename reportFilename = new GcsFilename(bucket, name + "-report.xml.ghostryde");
verifyFileExists(xmlFilename);
verifyFileExists(xmlLengthFilename);
verifyFileExists(reportFilename);
upload(xmlFilename, readXmlLength(xmlLengthFilename), watermark, name);
ofy().transact(new VoidWork() {
@Override
public void vrun() {
RegistryCursor.save(
Registry.get(tld), CursorType.RDE_UPLOAD_SFTP, ofy().getTransactionTime());
}
});
response.setContentType(PLAIN_TEXT_UTF_8);
response.setPayload(String.format("OK %s %s\n", tld, watermark));
}
/**
* Performs a blocking upload of a cloud storage XML file to escrow provider, converting
* it to the RyDE format along the way by applying tar+compress+encrypt+sign, and saving the
* created RyDE file on GCS for future reference.
*
* <p>This is done by layering a bunch of {@link java.io.FilterOutputStream FilterOutputStreams}
* on top of each other in reverse order that turn XML bytes into a RyDE file while
* simultaneously uploading it to the SFTP endpoint, and then using {@link ByteStreams#copy} to
* blocking-copy bytes from the cloud storage {@code InputStream} to the RyDE/SFTP pipeline.
*
* <p>In psuedoshell, the whole process looks like the following:
*
* <pre> {@code
* gcs read $xmlFile \ # Get GhostRyDE from cloud storage.
* | decrypt | decompress \ # Convert it to XML.
* | tar | file | compress | encrypt | sign /tmp/sig \ # Convert it to a RyDE file.
* | tee gs://bucket/$rydeFilename.ryde \ # Save a copy of the RyDE file to GCS.
* | sftp put $dstUrl/$rydeFilename.ryde \ # Upload to SFTP server.
* && sftp put $dstUrl/$rydeFilename.sig </tmp/sig \ # Upload detached signature.
* && cat /tmp/sig > gs://bucket/$rydeFilename.sig # Save a copy of signature to GCS.
* }</pre>
*/
private void upload(
GcsFilename xmlFile, long xmlLength, DateTime watermark, String name) throws Exception {
logger.infofmt("Uploading %s to %s", xmlFile, uploadUrl);
try (InputStream gcsInput = gcsUtils.openInputStream(xmlFile);
Ghostryde.Decryptor decryptor = ghostryde.openDecryptor(gcsInput, stagingDecryptionKey);
Ghostryde.Decompressor decompressor = ghostryde.openDecompressor(decryptor);
Ghostryde.Input xmlInput = ghostryde.openInput(decompressor)) {
try (JSchSshSession session = jschSshSessionFactory.create(jsch, uploadUrl);
JSchSftpChannel ftpChan = session.openSftpChannel()) {
byte[] signature;
String rydeFilename = name + ".ryde";
GcsFilename rydeGcsFilename = new GcsFilename(bucket, rydeFilename);
try (OutputStream ftpOutput = ftpChan.get().put(rydeFilename, OVERWRITE);
OutputStream gcsOutput = gcsUtils.openOutputStream(rydeGcsFilename);
TeeOutputStream teeOutput = new TeeOutputStream(asList(ftpOutput, gcsOutput));
RydePgpSigningOutputStream signer = pgpSigningFactory.create(teeOutput, signingKey)) {
try (OutputStream encryptLayer = pgpEncryptionFactory.create(signer, receiverKey);
OutputStream kompressor = pgpCompressionFactory.create(encryptLayer);
OutputStream fileLayer = pgpFileFactory.create(kompressor, watermark, name + ".tar");
OutputStream tarLayer =
tarFactory.create(fileLayer, xmlLength, watermark, name + ".xml")) {
ByteStreams.copy(xmlInput, tarLayer);
}
signature = signer.getSignature();
logger.infofmt("uploaded %,d bytes: %s.ryde", signer.getBytesWritten(), name);
}
String sigFilename = name + ".sig";
gcsUtils.createFromBytes(new GcsFilename(bucket, sigFilename), signature);
ftpChan.get().put(new ByteArrayInputStream(signature), sigFilename);
logger.infofmt("uploaded %,d bytes: %s.sig", signature.length, name);
}
}
}
/** Reads the contents of a file from Cloud Storage that contains nothing but an integer. */
private long readXmlLength(GcsFilename xmlLengthFilename) throws IOException {
try (InputStream input = gcsUtils.openInputStream(xmlLengthFilename)) {
return Long.parseLong(new String(ByteStreams.toByteArray(input), UTF_8).trim());
}
}
private void verifyFileExists(GcsFilename filename) {
verify(gcsUtils.existsAndNotEmpty(filename), "Missing file: %s", filename);
}
}

216
java/google/registry/rde/RdeUploadUrl.java Normal file
View file

@ -0,0 +1,216 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Strings.isNullOrEmpty;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableMap;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URLEncoder;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nullable;
import javax.annotation.concurrent.Immutable;
/**
* Class representing the remote destination for a deposit upload (like an SFTP server).
*
* <p>This class provides validity contracts, default values, and syntactic sugar which make it
* preferable to using a plain {@link URI}.
*
* @see java.net.URI
* @see RdeUploadAction
*/
@Immutable
final class RdeUploadUrl implements Comparable<RdeUploadUrl> {
public static final Protocol SFTP = new Protocol("sftp", 22);
private static final Map<String, Protocol> ALLOWED_PROTOCOLS = ImmutableMap.of("sftp", SFTP);
private final Protocol protocol;
private final URI uri;
/**
* Constructs and validates a new {@link RdeUploadUrl} instance.
*
* @see java.net.URI#create(String)
*/
public static RdeUploadUrl create(URI uri) {
checkArgument(!isNullOrEmpty(uri.getScheme()) && !isNullOrEmpty(uri.getHost()),
"Incomplete url: %s", uri);
Protocol protocol = ALLOWED_PROTOCOLS.get(uri.getScheme());
checkArgument(protocol != null, "Unsupported scheme: %s", uri);
return new RdeUploadUrl(protocol, uri);
}
/** @see #create(URI) */
private RdeUploadUrl(Protocol protocol, URI uri) {
this.protocol = checkNotNull(protocol, "protocol");
this.uri = checkNotNull(uri, "uri");
}
/** Returns username from URL userinfo. */
public Optional<String> getUser() {
String userInfo = uri.getUserInfo();
if (isNullOrEmpty(userInfo)) {
return Optional.absent();
}
int idx = userInfo.indexOf(':');
if (idx != -1) {
return Optional.of(userInfo.substring(0, idx));
} else {
return Optional.of(userInfo);
}
}
/** Returns password from URL userinfo (if specified). */
public Optional<String> getPass() {
String userInfo = uri.getUserInfo();
if (isNullOrEmpty(userInfo)) {
return Optional.absent();
}
int idx = userInfo.indexOf(':');
if (idx != -1) {
return Optional.of(userInfo.substring(idx + 1));
} else {
return Optional.absent();
}
}
/** Returns hostname or IP without port. */
public String getHost() {
return uri.getHost();
}
/** Returns network port or default for protocol if not specified. */
public int getPort() {
return uri.getPort() != -1 ? uri.getPort() : getProtocol().getPort();
}
/** Returns the protocol of this URL. */
public Protocol getProtocol() {
return protocol;
}
/** Returns path element of URL (if present). */
public Optional<String> getPath() {
String path = uri.getPath();
if (isNullOrEmpty(path) || path.equals("/")) {
return Optional.absent();
} else {
return Optional.of(path.substring(1));
}
}
/** Returns URL as ASCII text with password concealed (if any). */
@Override
public String toString() {
String result = getProtocol().getName() + "://";
if (getUser().isPresent()) {
result += urlencode(getUser().get());
if (getPass().isPresent()) {
result += ":****";
}
result += "@";
}
result += getHost();
if (getPort() != getProtocol().getPort()) {
result += String.format(":%d", getPort());
}
result += "/";
result += getPath().or("");
return result;
}
/**
* Simplified wrapper around Java's daft URL encoding API.
*
* @return an ASCII string that's escaped in a conservative manner for safe storage within any
* component of a URL. Non-ASCII characters are converted to UTF-8 bytes before being
* encoded. No choice of charset is provided because the W3C says we should use UTF-8.
* @see URLEncoder#encode(String, String)
*/
private static String urlencode(String str) {
try {
return URLEncoder.encode(str, "UTF-8");
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
/** @see java.net.URI#compareTo(java.net.URI) */
@Override
public int compareTo(RdeUploadUrl rhs) {
return uri.compareTo(checkNotNull(rhs).uri);
}
/** @see java.net.URI#equals(Object) */
@Override
public boolean equals(@Nullable Object object) {
return object == this
|| object instanceof RdeUploadUrl
&& Objects.equals(uri, ((RdeUploadUrl) object).uri);
}
/** @see java.net.URI#hashCode() */
@Override
public int hashCode() {
return Objects.hashCode(uri);
}
/** Used to store default settings for {@link #ALLOWED_PROTOCOLS}. */
@Immutable
public static final class Protocol {
private final String name;
private final int port;
public Protocol(String name, int port) {
checkArgument(0 < port && port < 65536, "bad port: %s", port);
this.name = checkNotNull(name, "name");
this.port = port;
}
/** Returns lowercase name of protocol. */
public String getName() {
return name;
}
/** Returns the standardized port number assigned to this protocol. */
public int getPort() {
return port;
}
/** @see Object#equals(Object) */
@Override
public boolean equals(@Nullable Object object) {
return object == this
|| object instanceof Protocol
&& port == ((Protocol) object).port
&& Objects.equals(name, ((Protocol) object).name);
}
/** @see Object#hashCode() */
@Override
public int hashCode() {
return Objects.hash(name, port);
}
}
}

99
java/google/registry/rde/RdeUtil.java Normal file
View file

@ -0,0 +1,99 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.domain.registry.util.HexDumper.dumpHex;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.io.BaseEncoding;
import com.google.domain.registry.xjc.rde.XjcRdeRrType;
import com.google.domain.registry.xml.XmlException;
import org.joda.time.DateTime;
import org.joda.time.ReadableInstant;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/** Helper methods for RDE. */
public final class RdeUtil {
/** Number of bytes in head of XML deposit that will contain the information we want. */
private static final int PEEK_SIZE = 2048;
/** Regular expression for extracting creation timestamp from a raw XML deposit. */
private static final Pattern WATERMARK_PATTERN = Pattern.compile("[<:]watermark>\\s*([^<\\s]+)");
/** Standard ISO date/time formatter without milliseconds. Used for watermarks. */
private static final DateTimeFormatter DATETIME_FORMATTER =
ISODateTimeFormat.dateTimeNoMillis().withZoneUTC();
/**
* Look at some bytes from {@code xmlInput} to ensure it appears to be a FULL XML deposit and
* then use a regular expression to extract the watermark timestamp which is returned.
*
* @throws IOException
* @throws XmlException
*/
public static DateTime peekWatermark(BufferedInputStream xmlInput)
throws IOException, XmlException {
xmlInput.mark(PEEK_SIZE);
byte[] peek = new byte[PEEK_SIZE];
if (xmlInput.read(peek) != PEEK_SIZE) {
throw new IOException(String.format("Failed to peek %,d bytes on input file", PEEK_SIZE));
}
xmlInput.reset();
String peekStr = new String(peek, UTF_8);
if (!peekStr.contains("urn:ietf:params:xml:ns:rde-1.0")) {
throw new XmlException(String.format(
"Does not appear to be an XML RDE deposit\n%s", dumpHex(peek)));
}
if (!peekStr.contains("type=\"FULL\"")) {
throw new XmlException("Only FULL XML RDE deposits suppported at this time");
}
Matcher watermarkMatcher = WATERMARK_PATTERN.matcher(peekStr);
if (!watermarkMatcher.find()) {
throw new XmlException("Could not find RDE watermark in XML");
}
DateTime watermark = DATETIME_FORMATTER.parseDateTime(watermarkMatcher.group(1));
return watermark;
}
/**
* Generates an ID matching the regex {@code \w&lbrace;1,13&rbrace; } from a millisecond
* timestamp.
*
* <p>This routine works by turning the number of UTC milliseconds from the UNIX epoch into a
* big-endian byte-array which is then converted to a base32 string without padding that's no
* longer than 13 chars because {@code 13 = Ceiling[Log[32, 2^64]]}. How lucky!
*/
public static String timestampToId(ReadableInstant timestamp) {
byte[] bytes = ByteBuffer.allocate(8).putLong(timestamp.getMillis()).array();
return BaseEncoding.base32().omitPadding().encode(bytes);
}
static XjcRdeRrType makeXjcRdeRrType(String clientId) {
XjcRdeRrType bean = new XjcRdeRrType();
bean.setValue(clientId);
return bean;
}
private RdeUtil() {}
}

187
java/google/registry/rde/RegistrarToXjcConverter.java Normal file
View file

@ -0,0 +1,187 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.MoreObjects.firstNonNull;
import com.google.domain.registry.model.registrar.Registrar;
import com.google.domain.registry.model.registrar.RegistrarAddress;
import com.google.domain.registry.xjc.contact.XjcContactE164Type;
import com.google.domain.registry.xjc.rderegistrar.XjcRdeRegistrar;
import com.google.domain.registry.xjc.rderegistrar.XjcRdeRegistrarAddrType;
import com.google.domain.registry.xjc.rderegistrar.XjcRdeRegistrarElement;
import com.google.domain.registry.xjc.rderegistrar.XjcRdeRegistrarPostalInfoEnumType;
import com.google.domain.registry.xjc.rderegistrar.XjcRdeRegistrarPostalInfoType;
import com.google.domain.registry.xjc.rderegistrar.XjcRdeRegistrarStatusType;
import com.google.domain.registry.xjc.rderegistrar.XjcRdeRegistrarWhoisInfoType;
import java.math.BigInteger;
/** Utility class that turns {@link Registrar} as {@link XjcRdeRegistrarElement}. */
final class RegistrarToXjcConverter {
private static final String UNKNOWN_EMAIL = "unknown@crr.com";
private static final String UNKNOWN_CITY = "Unknown";
private static final String UNKNOWN_ZIP = "00000";
private static final String UNKNOWN_CC = "US";
/** Converts {@link Registrar} to {@link XjcRdeRegistrarElement}. */
static XjcRdeRegistrarElement convert(Registrar registrar) {
return new XjcRdeRegistrarElement(convertRegistrar(registrar));
}
/** Converts {@link Registrar} to {@link XjcRdeRegistrar}. */
static XjcRdeRegistrar convertRegistrar(Registrar model) {
XjcRdeRegistrar bean = new XjcRdeRegistrar();
// o An <id> element that contains the Registry-unique identifier of
// the registrar object. This <id> has a superordinate relationship
// to a subordinate <clID>, <crRr> or <upRr> of domain, contact and
// host objects.
bean.setId(model.getClientIdentifier());
// o An <name> element that contains the name of the registrar.
bean.setName(model.getRegistrarName());
// o An OPTIONAL <gurid> element that contains the ID assigned by
// ICANN.
Long ianaId = model.getIanaIdentifier();
if (ianaId != null) {
bean.setGurid(BigInteger.valueOf(ianaId));
}
// o A <status> element that contains the operational status of the
// registrar. Possible values are: ok, readonly and terminated.
switch (model.getState()) {
case ACTIVE:
bean.setStatus(XjcRdeRegistrarStatusType.OK);
break;
case PENDING:
case SUSPENDED:
bean.setStatus(XjcRdeRegistrarStatusType.READONLY);
break;
default:
throw new IllegalStateException(String.format("Bad state: %s", model.getState()));
}
// o One or two <postalInfo> elements that contain postal- address
// information. Two elements are provided so that address
// information can be provided in both internationalized and
// localized forms; a "type" attribute is used to identify the two
// forms. If an internationalized form (type="int") is provided,
// element content MUST be represented in a subset of UTF-8 that can
// be represented in the 7-bit US-ASCII character set. If a
// localized form (type="loc") is provided, element content MAY be
// represented in unrestricted UTF-8.
RegistrarAddress localizedAddress = model.getLocalizedAddress();
if (localizedAddress != null) {
bean.getPostalInfos().add(convertPostalInfo(false, localizedAddress));
}
RegistrarAddress internationalizedAddress = model.getInternationalizedAddress();
if (internationalizedAddress != null) {
bean.getPostalInfos().add(convertPostalInfo(true, internationalizedAddress));
}
// o An OPTIONAL <voice> element that contains the registrar's voice
// telephone number.
// XXX: Make Registrar use PhoneNumber.
if (model.getPhoneNumber() != null) {
XjcContactE164Type phone = new XjcContactE164Type();
phone.setValue(model.getPhoneNumber());
bean.setVoice(phone);
}
// o An OPTIONAL <fax> element that contains the registrar's facsimile
// telephone number.
if (model.getFaxNumber() != null) {
XjcContactE164Type fax = new XjcContactE164Type();
fax.setValue(model.getFaxNumber());
bean.setFax(fax);
}
// o An <email> element that contains the registrar's email address.
bean.setEmail(firstNonNull(model.getEmailAddress(), UNKNOWN_EMAIL));
// o An OPTIONAL <url> element that contains the registrar's URL.
bean.setUrl(model.getUrl());
// o An OPTIONAL <whoisInfo> elements that contains whois information.
// The <whoisInfo> element contains the following child elements:
//
// * An OPTIONAL <name> element that contains the name of the
// registrar WHOIS server listening on TCP port 43 as specified in
// [RFC3912].
//
// * An OPTIONAL <url> element that contains the name of the
// registrar WHOIS server listening on TCP port 80/443.
if (model.getWhoisServer() != null) {
XjcRdeRegistrarWhoisInfoType whoisInfo = new XjcRdeRegistrarWhoisInfoType();
whoisInfo.setName(model.getWhoisServer());
bean.setWhoisInfo(whoisInfo);
}
// o A <crDate> element that contains the date and time of registrar-
// object creation.
bean.setCrDate(model.getCreationTime());
// o An OPTIONAL <upDate> element that contains the date and time of
// the most recent RDE registrar-object modification. This element
// MUST NOT be present if the rdeRegistrar object has never been
// modified.
bean.setUpDate(model.getLastUpdateTime());
return bean;
}
private static XjcRdeRegistrarPostalInfoType convertPostalInfo(
boolean isInt, RegistrarAddress model) {
XjcRdeRegistrarPostalInfoType bean = new XjcRdeRegistrarPostalInfoType();
bean.setType(isInt
? XjcRdeRegistrarPostalInfoEnumType.INT
: XjcRdeRegistrarPostalInfoEnumType.LOC);
bean.setAddr(convertAddress(model));
return bean;
}
private static XjcRdeRegistrarAddrType convertAddress(RegistrarAddress model) {
XjcRdeRegistrarAddrType bean = new XjcRdeRegistrarAddrType();
// * A <addr> element that contains address information associated
// with the registrar. The <addr> element contains the following
// child elements:
//
// + One, two, or three OPTIONAL <street> elements that contain
// the registrar's street address.
bean.getStreets().addAll(model.getStreet());
// + A <city> element that contains the registrar's city.
bean.setCity(firstNonNull(model.getCity(), UNKNOWN_CITY));
// + An OPTIONAL <sp> element that contains the registrar's state
// or province.
bean.setSp(model.getState());
// + An OPTIONAL <pc> element that contains the registrar's
// postal code.
bean.setPc(firstNonNull(model.getZip(), UNKNOWN_ZIP));
// + A <cc> element that contains the registrar's country code.
bean.setCc(firstNonNull(model.getCountryCode(), UNKNOWN_CC));
return bean;
}
private RegistrarToXjcConverter() {}
}

59
java/google/registry/rde/RydePgpCompressionOutputStream.java Normal file
View file

@ -0,0 +1,59 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static org.bouncycastle.bcpg.CompressionAlgorithmTags.ZIP;
import com.google.auto.factory.AutoFactory;
import com.google.auto.factory.Provided;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.util.ImprovedOutputStream;
import org.bouncycastle.openpgp.PGPCompressedDataGenerator;
import org.bouncycastle.openpgp.PGPException;
import java.io.IOException;
import java.io.OutputStream;
import javax.annotation.WillNotClose;
/**
* OpenPGP compression service that wraps an {@link OutputStream}.
*
* <p>This uses the ZIP compression algorithm per the ICANN escrow specification.
*/
@AutoFactory(allowSubclasses = true)
public class RydePgpCompressionOutputStream extends ImprovedOutputStream {
/**
* Creates a new instance that compresses data.
*
* @param os is the upstream {@link OutputStream} which is not closed by this object
* @throws RuntimeException to rethrow {@link PGPException} and {@link IOException}
*/
public RydePgpCompressionOutputStream(
@Provided @Config("rdeRydeBufferSize") Integer bufferSize,
@WillNotClose OutputStream os) {
super(createDelegate(bufferSize, os));
}
private static OutputStream createDelegate(int bufferSize, OutputStream os) {
try {
return new PGPCompressedDataGenerator(ZIP).open(os, new byte[bufferSize]);
} catch (IOException | PGPException e) {
throw new RuntimeException(e);
}
}
}

120
java/google/registry/rde/RydePgpEncryptionOutputStream.java Normal file
View file

@ -0,0 +1,120 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static org.bouncycastle.bcpg.SymmetricKeyAlgorithmTags.AES_128;
import static org.bouncycastle.jce.provider.BouncyCastleProvider.PROVIDER_NAME;
import com.google.auto.factory.AutoFactory;
import com.google.auto.factory.Provided;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.util.ImprovedOutputStream;
import org.bouncycastle.openpgp.PGPEncryptedDataGenerator;
import org.bouncycastle.openpgp.PGPException;
import org.bouncycastle.openpgp.PGPPublicKey;
import org.bouncycastle.openpgp.operator.jcajce.JcePGPDataEncryptorBuilder;
import org.bouncycastle.openpgp.operator.jcajce.JcePublicKeyKeyEncryptionMethodGenerator;
import java.io.IOException;
import java.io.OutputStream;
import java.security.NoSuchAlgorithmException;
import java.security.ProviderException;
import java.security.SecureRandom;
import javax.annotation.WillNotClose;
/**
* OpenPGP encryption service that wraps an {@link OutputStream}.
*
* <p>This uses 128-bit AES (Rijndael) as the symmetric encryption algorithm. This is the only key
* strength ICANN allows. The other valid algorithms are TripleDES and CAST5 per RFC 4880. It's
* probably for the best that we're not using AES-256 since it's been weakened over the years to
* potentially being worse than AES-128.
*
* <p>The key for the symmetric algorithm is generated by a random number generator which SHOULD
* come from {@code /dev/random} (see: {@link sun.security.provider.NativePRNG}) but Java doesn't
* offer any guarantees that {@link SecureRandom} isn't pseudo-random.
*
* <p>The asymmetric algorithm is whatever one is associated with the {@link PGPPublicKey} object
* you provide. That should be either RSA or DSA, per the ICANN escrow spec. The underlying
* {@link PGPEncryptedDataGenerator} class uses PGP Cipher Feedback Mode to chain blocks. No
* integrity packet is used.
*
* @see <a href="http://tools.ietf.org/html/rfc4880">RFC 4880 (OpenPGP Message Format)</a>
* @see <a href="http://en.wikipedia.org/wiki/Advanced_Encryption_Standard">AES (Wikipedia)</a>
*/
@AutoFactory(allowSubclasses = true)
public class RydePgpEncryptionOutputStream extends ImprovedOutputStream {
/**
* The symmetric encryption algorithm to use. Do not change this value without checking the
* RFCs to make sure the encryption algorithm and strength combination is allowed.
*
* @see org.bouncycastle.bcpg.SymmetricKeyAlgorithmTags
*/
private static final int CIPHER = AES_128;
/**
* This option adds an additional checksum to the OpenPGP message. From what I can tell, this is
* meant to fix a bug that made a certain type of message tampering possible. GPG will actually
* complain on the command line when decrypting a message without this feature.
*
* <p>However I'm reasonably certain that this is not required if you have a signature (and
* remember to use it!) and the ICANN requirements document do not mention this. So we're going
* to leave it out.
*/
private static final boolean USE_INTEGRITY_PACKET = false;
/**
* The source of random bits. This should not be changed at Google because it uses dev random
* in production, and the testing environment is configured to make this go fast and not drain
* system entropy.
*
* @see SecureRandom#getInstance(String)
*/
private static final String RANDOM_SOURCE = "NativePRNG";
/**
* Creates a new instance that encrypts data for the owner of {@code receiverKey}.
*
* @param os is the upstream {@link OutputStream} which is not closed by this object
* @throws IllegalArgumentException if {@code publicKey} is invalid
* @throws RuntimeException to rethrow {@link PGPException} and {@link IOException}
*/
public RydePgpEncryptionOutputStream(
@Provided @Config("rdeRydeBufferSize") Integer bufferSize,
@WillNotClose OutputStream os,
PGPPublicKey receiverKey) {
super(createDelegate(bufferSize, os, receiverKey));
}
private static
OutputStream createDelegate(int bufferSize, OutputStream os, PGPPublicKey receiverKey) {
try {
PGPEncryptedDataGenerator encryptor = new PGPEncryptedDataGenerator(
new JcePGPDataEncryptorBuilder(CIPHER)
.setWithIntegrityPacket(USE_INTEGRITY_PACKET)
.setSecureRandom(SecureRandom.getInstance(RANDOM_SOURCE))
.setProvider(PROVIDER_NAME));
encryptor.addMethod(new JcePublicKeyKeyEncryptionMethodGenerator(receiverKey));
return encryptor.open(os, new byte[bufferSize]);
} catch (NoSuchAlgorithmException e) {
throw new ProviderException(e);
} catch (IOException | PGPException e) {
throw new RuntimeException(e);
}
}
}

71
java/google/registry/rde/RydePgpFileOutputStream.java Normal file
View file

@ -0,0 +1,71 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.Preconditions.checkArgument;
import static org.bouncycastle.openpgp.PGPLiteralData.BINARY;
import com.google.auto.factory.AutoFactory;
import com.google.auto.factory.Provided;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.util.ImprovedOutputStream;
import org.bouncycastle.openpgp.PGPLiteralDataGenerator;
import org.joda.time.DateTime;
import java.io.IOException;
import java.io.OutputStream;
import javax.annotation.WillNotClose;
/**
* OpenPGP literal data layer generator that wraps {@link OutputStream}.
*
* <p>OpenPGP messages are like an onion; there can be many layers like compression and encryption.
* It's important to wrap out plaintext in a literal data layer such as this so the code that's
* unwrapping the onion knows when to stop.
*
* <p>According to escrow spec, the PGP message should contain a single tar file.
*/
@AutoFactory(allowSubclasses = true)
public class RydePgpFileOutputStream extends ImprovedOutputStream {
/**
* Creates a new instance for a particular file.
*
* @param os is the upstream {@link OutputStream} which is not closed by this object
* @throws IllegalArgumentException if {@code filename} isn't a {@code .tar} file
* @throws RuntimeException to rethrow {@link IOException}
*/
public RydePgpFileOutputStream(
@Provided @Config("rdeRydeBufferSize") Integer bufferSize,
@WillNotClose OutputStream os,
DateTime modified,
String filename) {
super(createDelegate(bufferSize, os, modified, filename));
}
private static OutputStream
createDelegate(int bufferSize, OutputStream os, DateTime modified, String filename) {
try {
checkArgument(filename.endsWith(".tar"),
"Ryde PGP message should contain a tar file.");
return new PGPLiteralDataGenerator().open(
os, BINARY, filename, modified.toDate(), new byte[bufferSize]);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

112
java/google/registry/rde/RydePgpSigningOutputStream.java Normal file
View file

@ -0,0 +1,112 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static org.bouncycastle.bcpg.HashAlgorithmTags.SHA256;
import static org.bouncycastle.bcpg.PublicKeyAlgorithmTags.RSA_GENERAL;
import static org.bouncycastle.openpgp.PGPSignature.BINARY_DOCUMENT;
import com.google.auto.factory.AutoFactory;
import com.google.domain.registry.util.ImprovedOutputStream;
import org.bouncycastle.openpgp.PGPException;
import org.bouncycastle.openpgp.PGPKeyPair;
import org.bouncycastle.openpgp.PGPPublicKey;
import org.bouncycastle.openpgp.PGPSignatureGenerator;
import org.bouncycastle.openpgp.PGPSignatureSubpacketGenerator;
import org.bouncycastle.openpgp.operator.bc.BcPGPContentSignerBuilder;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Iterator;
import javax.annotation.WillNotClose;
/**
* OpenPGP detached signing service that wraps an {@link OutputStream}.
*
* <p>This is the outermost layer. It signs the resulting file without modifying its bytes,
* instead generating an out-of-band {@code .asc} signature file. This is basically a SHA-256
* checksum of the deposit file that's signed with our RSA private key. This allows the people
* who receive a deposit to check the signature against our public key so they can know the
* data hasn't been forged.
*/
@AutoFactory(allowSubclasses = true)
public class RydePgpSigningOutputStream extends ImprovedOutputStream {
private final PGPSignatureGenerator signer;
/**
* Create a signer that wraps {@code os} and generates a detached signature using
* {@code signingKey}. After closing, you should call {@link #getSignature()} to get the detached
* signature.
*
* @param os is the upstream {@link OutputStream} which is not closed by this object
* @throws RuntimeException to rethrow {@link PGPException}
*/
public RydePgpSigningOutputStream(
@WillNotClose OutputStream os,
PGPKeyPair signingKey) {
super(os, false, -1);
try {
signer = new PGPSignatureGenerator(
new BcPGPContentSignerBuilder(RSA_GENERAL, SHA256));
signer.init(BINARY_DOCUMENT, signingKey.getPrivateKey());
} catch (PGPException e) {
throw new RuntimeException(e);
}
addUserInfoToSignature(signingKey.getPublicKey(), signer);
}
/** Returns the byte contents for the detached {@code .asc} signature file. */
public byte[] getSignature() throws IOException, PGPException {
return signer.generate().getEncoded();
}
/** @see ImprovedOutputStream#write(int) */
@Override
public void write(int b) throws IOException {
super.write(b);
signer.update((byte) b);
}
/** @see ImprovedOutputStream#write(byte[], int, int) */
@Override
public void write(byte[] b, int off, int len) throws IOException {
super.write(b, off, len);
signer.update(b, off, len);
}
/**
* Add user ID to signature file.
*
* <p>This adds information about the identity of the signer to the signature file. It's not
* required, but I'm guessing it could be a lifesaver if somewhere down the road, people lose
* track of the public keys and need to figure out how to verify a couple blobs. This would at
* least tell them which key to download from the MIT keyserver.
*
* <p>But the main reason why I'm using this is because I copied it from the code of another
* googler who was also uncertain about the precise reason why it's needed.
*/
private static void addUserInfoToSignature(PGPPublicKey publicKey, PGPSignatureGenerator signer) {
@SuppressWarnings("unchecked") // safe by specification.
Iterator<String> uidIter = publicKey.getUserIDs();
if (uidIter.hasNext()) {
PGPSignatureSubpacketGenerator spg = new PGPSignatureSubpacketGenerator();
spg.setSignerUserID(false, uidIter.next());
signer.setHashedSubpackets(spg.generate());
}
}
}

70
java/google/registry/rde/RydeTarOutputStream.java Normal file
View file

@ -0,0 +1,70 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.rde;
import static com.google.common.base.Preconditions.checkArgument;
import com.google.auto.factory.AutoFactory;
import com.google.domain.registry.util.ImprovedOutputStream;
import com.google.domain.registry.util.PosixTarHeader;
import org.joda.time.DateTime;
import java.io.IOException;
import java.io.OutputStream;
import javax.annotation.WillNotClose;
/**
* Single-file POSIX tar archive creator that wraps an {@link OutputStream}.
*/
@AutoFactory(allowSubclasses = true)
public class RydeTarOutputStream extends ImprovedOutputStream {
/**
* Creates a new instance that outputs a tar archive.
*
* @param os is the upstream {@link OutputStream} which is not closed by this object
* @param size is the length in bytes of the one file, which you will write to this object
* @param modified is the {@link PosixTarHeader.Builder#setMtime mtime} you want to set
* @param filename is the name of the one file that will be contained in this archive
* @throws RuntimeException to rethrow {@link IOException}
* @throws IllegalArgumentException if {@code size} is negative
*/
public RydeTarOutputStream(
@WillNotClose OutputStream os, long size, DateTime modified, String filename) {
super(os, false, size);
checkArgument(size >= 0);
checkArgument(filename.endsWith(".xml"),
"Ryde expects tar archive to contain a filename with an '.xml' extension.");
try {
os.write(new PosixTarHeader.Builder()
.setName(filename)
.setSize(size)
.setMtime(modified)
.build()
.getBytes());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/** Writes the end of archive marker. */
@Override
public void onClose() throws IOException {
// Round up to a 512-byte boundary and another 1024-bytes to indicate end of archive.
write(new byte[1024 + 512 - (int) (getBytesWritten() % 512L)]);
}
}

23
java/google/registry/rde/package-info.java Normal file
View file

@ -0,0 +1,23 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/**
* Registry Data Escrow
*
* <p>This is a cron job that puts our database in a giant XML file and uploads it to a third party.
* Read the {@link com.google.domain.registry.rde.RdeStagingAction RdeStagingAction} javadoc to
* learn more.
*/
@javax.annotation.ParametersAreNonnullByDefault
package com.google.domain.registry.rde;