zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-15 08:57:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

Delete everything related to RDE import

Browse source 
This code was never finished or fully working anyway.  It would require
substantial reworking for the Registry 3.0 migration because it's closely tied
to the Datastore model and App Engine MapReduce framework, both of which will be
going away.  We can bring back some of these deleted test files as necessary
if/when we rewrite RDE import for the new schema.

On the plus side, in a relational database, RDE import will be much simpler.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=231265578
This commit is contained in:
mcilwain 2019-01-28 12:08:47 -08:00 • committed by Ben McIlwain
parent f0c677b18b
commit 5dedc1e889
90 changed files with 0 additions and 15750 deletions
Download patch file Download diff file Expand all files Collapse all files

12
java/google/registry/config/RegistryConfig.java
View file

@ -725,18 +725,6 @@ public final class RegistryConfig {
return projectId + "-rde";
}
/**
* Returns the Google Cloud Storage bucket for importing escrow files.
*
* @see google.registry.rde.imports.RdeContactImportAction
* @see google.registry.rde.imports.RdeHostImportAction
*/
@Provides
@Config("rdeImportBucket")
public static String provideRdeImportBucket(@Config("projectId") String projectId) {
return projectId + "-rde-import";
}
/**
* Amount of time between RDE deposits.
*

1
java/google/registry/module/backend/BUILD
View file

@ -31,7 +31,6 @@ java_library(
"//java/google/registry/module",
"//java/google/registry/monitoring/whitebox",
"//java/google/registry/rde",
"//java/google/registry/rde/imports",
"//java/google/registry/reporting",
"//java/google/registry/reporting/billing",
"//java/google/registry/reporting/icann",

10
java/google/registry/module/backend/BackendRequestComponent.java
View file

@ -59,11 +59,6 @@ import google.registry.rde.RdeReportAction;
import google.registry.rde.RdeReporter;
import google.registry.rde.RdeStagingAction;
import google.registry.rde.RdeUploadAction;
import google.registry.rde.imports.RdeContactImportAction;
import google.registry.rde.imports.RdeDomainImportAction;
import google.registry.rde.imports.RdeHostImportAction;
import google.registry.rde.imports.RdeHostLinkAction;
import google.registry.rde.imports.RdeImportsModule;
import google.registry.reporting.ReportingModule;
import google.registry.reporting.billing.BillingModule;
import google.registry.reporting.billing.CopyDetailReportsAction;
@ -104,7 +99,6 @@ import google.registry.tmch.TmchSmdrlAction;
IcannReportingModule.class,
MapreduceModule.class,
RdeModule.class,
RdeImportsModule.class,
ReportingModule.class,
RequestModule.class,
SheetModule.class,
@ -139,10 +133,6 @@ interface BackendRequestComponent {
PublishDnsUpdatesAction publishDnsUpdatesAction();
PublishSpec11ReportAction publishSpec11ReportAction();
ReadDnsQueueAction readDnsQueueAction();
RdeContactImportAction rdeContactImportAction();
RdeDomainImportAction rdeDomainImportAction();
RdeHostImportAction rdeHostImportAction();
RdeHostLinkAction rdeHostLinkAction();
RdeReportAction rdeReportAction();
RdeStagingAction rdeStagingAction();
RdeUploadAction rdeUploadAction();

39
java/google/registry/rde/imports/BUILD
View file

@ -1,39 +0,0 @@
package(
default_visibility = ["//visibility:public"],
)
licenses(["notice"]) # Apache 2.0
java_library(
name = "imports",
srcs = glob(["*.java"]),
deps = [
"//java/google/registry/config",
"//java/google/registry/dns",
"//java/google/registry/flows",
"//java/google/registry/gcs",
"//java/google/registry/mapreduce",
"//java/google/registry/model",
"//java/google/registry/pricing",
"//java/google/registry/request",
"//java/google/registry/request/auth",
"//java/google/registry/util",
"//java/google/registry/xjc",
"//java/google/registry/xml",
"//third_party/jaxb",
"//third_party/objectify:objectify-v4_1",
"@com_google_appengine_api_1_0_sdk",
"@com_google_appengine_tools_appengine_gcs_client",
"@com_google_appengine_tools_appengine_mapreduce",
"@com_google_auto_factory",
"@com_google_auto_value",
"@com_google_code_findbugs_jsr305",
"@com_google_dagger",
"@com_google_flogger",
"@com_google_flogger_system_backend",
"@com_google_guava",
"@javax_servlet_api",
"@joda_time",
"@org_joda_money",
],
)

161
java/google/registry/rde/imports/RdeContactImportAction.java
View file

@ -1,161 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
import static google.registry.model.ofy.ObjectifyService.ofy;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.collect.ImmutableList;
import com.google.common.flogger.FluentLogger;
import google.registry.config.RegistryConfig.Config;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.gcs.GcsUtils;
import google.registry.mapreduce.MapreduceRunner;
import google.registry.model.contact.ContactResource;
import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.SystemClock;
import google.registry.xjc.JaxbFragment;
import google.registry.xjc.rdecontact.XjcRdeContact;
import google.registry.xjc.rdecontact.XjcRdeContactElement;
import java.util.Optional;
import javax.inject.Inject;
/**
* A mapreduce that imports contacts from an escrow file.
*
* <p>Specify the escrow file to import with the "path" parameter.
*/
@Action(
service = Action.Service.BACKEND,
path = "/_dr/task/importRdeContacts",
auth = Auth.AUTH_INTERNAL_ONLY)
public class RdeContactImportAction implements Runnable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final GcsService GCS_SERVICE =
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
protected final MapreduceRunner mrRunner;
protected final Response response;
protected final String importBucketName;
protected final String importFileName;
protected final Optional<Integer> mapShards;
@Inject
public RdeContactImportAction(
MapreduceRunner mrRunner,
Response response,
@Config("rdeImportBucket") String importBucketName,
@Parameter("path") String importFileName,
@Parameter(PARAM_MAP_SHARDS) Optional<Integer> mapShards) {
this.mrRunner = mrRunner;
this.response = response;
this.importBucketName = importBucketName;
this.importFileName = importFileName;
this.mapShards = mapShards;
}
@Override
public void run() {
mrRunner
.setJobName("Import contacts from escrow file")
.setModuleName("backend")
.runMapOnly(createMapper(), ImmutableList.of(createInput()))
.sendLinkToMapreduceConsole(response);
}
/**
* Creates a new {@link RdeContactInput}
*/
private RdeContactInput createInput() {
return new RdeContactInput(mapShards, importBucketName, importFileName);
}
/**
* Creates a new {@link RdeContactImportMapper}
*/
private RdeContactImportMapper createMapper() {
return new RdeContactImportMapper(importBucketName);
}
/** Mapper to import contacts from an escrow file. */
public static class RdeContactImportMapper
extends Mapper<JaxbFragment<XjcRdeContactElement>, Void, Void> {
private static final long serialVersionUID = -7645091075256589374L;
private final String importBucketName;
private transient RdeImportUtils importUtils;
public RdeContactImportMapper(String importBucketName) {
this.importBucketName = importBucketName;
}
private RdeImportUtils getImportUtils() {
if (importUtils == null) {
importUtils = createRdeImportUtils();
}
return importUtils;
}
/**
* Creates a new instance of RdeImportUtils.
*/
private RdeImportUtils createRdeImportUtils() {
return new RdeImportUtils(
ofy(),
new SystemClock(),
importBucketName,
new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize()));
}
@Override
public void map(JaxbFragment<XjcRdeContactElement> fragment) {
final XjcRdeContact xjcContact = fragment.getInstance().getValue();
try {
logger.atInfo().log("Converting xml for contact %s", xjcContact.getId());
// Record number of attempted map operations
getContext().incrementCounter("contact imports attempted");
logger.atInfo().log("Saving contact %s", xjcContact.getId());
ofy()
.transact(
() -> {
ContactResource contact =
XjcToContactResourceConverter.convertContact(xjcContact);
getImportUtils().importEppResource(contact);
});
// Record number of contacts imported
getContext().incrementCounter("contacts saved");
logger.atInfo().log("Contact %s was imported successfully", xjcContact.getId());
} catch (ResourceExistsException e) {
// Record the number of contacts already in the registry
getContext().incrementCounter("existing contacts skipped");
logger.atInfo().log("Contact %s already exists", xjcContact.getId());
} catch (Exception e) {
// Record the number of contacts with unexpected errors
getContext().incrementCounter("contact import errors");
logger.atSevere().withCause(e).log(
"Error importing contact %s; xml=%s", xjcContact.getId(), xjcContact);
}
}
}
}

120
java/google/registry/rde/imports/RdeContactInput.java
View file

@ -1,120 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static com.google.common.math.IntMath.divide;
import static java.math.RoundingMode.CEILING;
import static java.math.RoundingMode.FLOOR;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.Input;
import com.google.appengine.tools.mapreduce.InputReader;
import com.google.common.collect.ImmutableList;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.gcs.GcsUtils;
import google.registry.model.contact.ContactResource;
import google.registry.rde.imports.RdeParser.RdeHeader;
import google.registry.xjc.JaxbFragment;
import google.registry.xjc.rdecontact.XjcRdeContactElement;
import java.io.InputStream;
import java.util.List;
import java.util.Optional;
/**
* A MapReduce {@link Input} that imports {@link ContactResource} objects from an escrow file.
*
* <p>If a mapShards parameter has been specified, up to that many readers will be created
* so that each map shard has one reader. If a mapShards parameter has not been specified, a
* default number of readers will be created.
*/
public class RdeContactInput extends Input<JaxbFragment<XjcRdeContactElement>> {
private static final long serialVersionUID = -366966393494008712L;
private static final GcsService GCS_SERVICE =
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
/**
* Default number of readers if map shards are not specified.
*/
private static final int DEFAULT_READERS = 50;
/**
* Minimum number of records per reader.
*/
private static final int MINIMUM_RECORDS_PER_READER = 100;
/**
* Optional argument to explicitly specify the number of readers.
*/
private final int numReaders;
private final String importBucketName;
private final String importFileName;
public RdeContactInput(Optional<Integer> mapShards, String importBucketName,
String importFileName) {
this.numReaders = mapShards.orElse(DEFAULT_READERS);
this.importBucketName = importBucketName;
this.importFileName = importFileName;
}
@Override
public List<? extends InputReader<JaxbFragment<XjcRdeContactElement>>> createReaders() {
int numReaders = this.numReaders;
RdeHeader header = newParser().getHeader();
int numberOfContacts = header.getContactCount().intValue();
if (numberOfContacts / numReaders < MINIMUM_RECORDS_PER_READER) {
numReaders = divide(numberOfContacts, MINIMUM_RECORDS_PER_READER, FLOOR);
// use at least one reader
numReaders = Math.max(numReaders, 1);
}
ImmutableList.Builder<RdeContactReader> builder = new ImmutableList.Builder<>();
int contactsPerReader =
Math.max(MINIMUM_RECORDS_PER_READER, divide(numberOfContacts, numReaders, CEILING));
int offset = 0;
for (int i = 0; i < numReaders; i++) {
builder = builder.add(newReader(offset, contactsPerReader));
offset += contactsPerReader;
}
return builder.build();
}
private RdeContactReader newReader(int offset, int maxResults) {
return new RdeContactReader(importBucketName, importFileName, offset, maxResults);
}
private RdeParser newParser() {
GcsUtils utils = new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize());
GcsFilename filename = new GcsFilename(importBucketName, importFileName);
try (InputStream xmlInput = utils.openInputStream(filename)) {
return new RdeParser(xmlInput);
} catch (Exception e) {
throw new InitializationException(
String.format("Error opening rde file %s/%s", importBucketName, importFileName), e);
}
}
/**
* Thrown when the input cannot initialize properly.
*/
private static class InitializationException extends RuntimeException {
public InitializationException(String message, Throwable cause) {
super(message, cause);
}
}
}

112
java/google/registry/rde/imports/RdeContactReader.java
View file

@ -1,112 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.InputReader;
import com.google.common.flogger.FluentLogger;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.gcs.GcsUtils;
import google.registry.xjc.JaxbFragment;
import google.registry.xjc.rdecontact.XjcRdeContactElement;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.util.NoSuchElementException;
import javax.annotation.concurrent.NotThreadSafe;
/** Mapreduce {@link InputReader} for reading contacts from escrow files */
@NotThreadSafe
public class RdeContactReader extends InputReader<JaxbFragment<XjcRdeContactElement>>
implements Serializable {
private static final long serialVersionUID = -3688793834175577691L;
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final GcsService GCS_SERVICE =
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
final String importBucketName;
final String importFileName;
final int offset;
final int maxResults;
private int count = 0;
transient RdeParser parser;
/**
* Creates a new instance of {@link RdeParser}
*/
private RdeParser newParser() {
GcsUtils utils = new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize());
GcsFilename filename = new GcsFilename(importBucketName, importFileName);
InputStream xmlInput = utils.openInputStream(filename);
try {
RdeParser parser = new RdeParser(xmlInput);
// skip the file offset and count
// if count is greater than 0, the reader has been rehydrated after doing some work.
// skip any already processed records.
parser.skipContacts(offset + count);
return parser;
} catch (Exception e) {
throw new RuntimeException(
String.format("Error opening rde file %s/%s", importBucketName, importFileName), e);
}
}
public RdeContactReader(
String importBucketName,
String importFileName,
int offset,
int maxResults) {
this.importBucketName = importBucketName;
this.importFileName = importFileName;
this.offset = offset;
this.maxResults = maxResults;
}
@Override
public JaxbFragment<XjcRdeContactElement> next() {
if (count < maxResults) {
if (parser == null) {
parser = newParser();
if (parser.isAtContact()) {
return readContact();
}
}
if (parser.nextContact()) {
return readContact();
}
}
throw new NoSuchElementException();
}
private JaxbFragment<XjcRdeContactElement> readContact() {
count++;
return JaxbFragment.create(new XjcRdeContactElement(parser.getContact()));
}
@Override
public void endSlice() throws IOException {
super.endSlice();
if (parser != null) {
parser.close();
}
}
}

292
java/google/registry/rde/imports/RdeDomainImportAction.java
View file

@ -1,292 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static google.registry.flows.domain.DomainTransferUtils.createLosingTransferPollMessage;
import static google.registry.flows.domain.DomainTransferUtils.createPendingTransferData;
import static google.registry.flows.domain.DomainTransferUtils.createTransferServerApproveEntities;
import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
import static google.registry.model.domain.DomainBase.extendRegistrationWithCap;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.pricing.PricingEngineProxy.getDomainRenewCost;
import static google.registry.rde.imports.RdeImportUtils.createAutoRenewBillingEventForDomainImport;
import static google.registry.rde.imports.RdeImportUtils.createAutoRenewPollMessageForDomainImport;
import static google.registry.rde.imports.RdeImportUtils.createHistoryEntryForDomainImport;
import static google.registry.rde.imports.RdeImportsModule.PATH;
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.flogger.FluentLogger;
import google.registry.config.RegistryConfig.Config;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.dns.DnsQueue;
import google.registry.gcs.GcsUtils;
import google.registry.mapreduce.MapreduceRunner;
import google.registry.model.billing.BillingEvent;
import google.registry.model.domain.DomainBase;
import google.registry.model.domain.Period;
import google.registry.model.domain.Period.Unit;
import google.registry.model.domain.rgp.GracePeriodStatus;
import google.registry.model.poll.PollMessage;
import google.registry.model.reporting.HistoryEntry;
import google.registry.model.transfer.TransferData;
import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
import google.registry.model.transfer.TransferStatus;
import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.StringGenerator;
import google.registry.util.SystemClock;
import google.registry.xjc.JaxbFragment;
import google.registry.xjc.rdedomain.XjcRdeDomain;
import google.registry.xjc.rdedomain.XjcRdeDomainElement;
import java.util.Optional;
import javax.inject.Inject;
import javax.inject.Named;
import org.joda.money.Money;
import org.joda.time.DateTime;
/**
* A mapreduce that imports domains from an escrow file.
*
* <p>Specify the escrow file to import with the "path" parameter.
*/
@Action(
service = Action.Service.BACKEND,
path = "/_dr/task/importRdeDomains",
auth = Auth.AUTH_INTERNAL_ONLY)
public class RdeDomainImportAction implements Runnable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final GcsService GCS_SERVICE =
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
protected final MapreduceRunner mrRunner;
protected final Response response;
protected final String importBucketName;
protected final String importFileName;
protected final Optional<Integer> mapShards;
protected final StringGenerator stringGenerator;
@Inject
public RdeDomainImportAction(
MapreduceRunner mrRunner,
Response response,
@Config("rdeImportBucket") String importBucketName,
@Parameter(PATH) String importFileName,
@Parameter(PARAM_MAP_SHARDS) Optional<Integer> mapShards,
@Named("base64StringGenerator") StringGenerator stringGenerator) {
this.mrRunner = mrRunner;
this.response = response;
this.importBucketName = importBucketName;
this.importFileName = importFileName;
this.mapShards = mapShards;
this.stringGenerator = stringGenerator;
}
@Override
public void run() {
logger.atInfo().log(
"Launching domains import mapreduce: bucket=%s, filename=%s",
this.importBucketName, this.importFileName);
mrRunner
.setJobName("Import domains from escrow file")
.setModuleName("backend")
.runMapOnly(createMapper(), ImmutableList.of(createInput()))
.sendLinkToMapreduceConsole(response);
}
/**
* Creates a new {@link RdeDomainInput}
*/
private RdeDomainInput createInput() {
return new RdeDomainInput(mapShards, importBucketName, importFileName);
}
/**
* Creates a new {@link RdeDomainImportMapper}
*/
private RdeDomainImportMapper createMapper() {
return new RdeDomainImportMapper(importBucketName, stringGenerator);
}
/** Mapper to import domains from an escrow file. */
public static class RdeDomainImportMapper
extends Mapper<JaxbFragment<XjcRdeDomainElement>, Void, Void> {
private static final long serialVersionUID = -7645091075256589374L;
private final String importBucketName;
private final StringGenerator stringGenerator;
private transient RdeImportUtils importUtils;
private transient DnsQueue dnsQueue;
public RdeDomainImportMapper(String importBucketName, StringGenerator stringGenerator) {
this.importBucketName = importBucketName;
this.stringGenerator = stringGenerator;
}
private RdeImportUtils getImportUtils() {
if (importUtils == null) {
importUtils = createRdeImportUtils();
}
return importUtils;
}
private DnsQueue getDnsQueue() {
if (dnsQueue == null) {
dnsQueue = DnsQueue.create();
}
return dnsQueue;
}
/**
* Creates a new instance of RdeImportUtils.
*/
private RdeImportUtils createRdeImportUtils() {
return new RdeImportUtils(
ofy(),
new SystemClock(),
importBucketName,
new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize()));
}
@Override
public void map(JaxbFragment<XjcRdeDomainElement> fragment) {
final XjcRdeDomain xjcDomain = fragment.getInstance().getValue();
try {
// Record number of attempted map operations
getContext().incrementCounter("domain imports attempted");
logger.atInfo().log("Saving domain %s", xjcDomain.getName());
ofy().transact(() -> saveDomain(xjcDomain));
// Record the number of domains imported
getContext().incrementCounter("domains saved");
logger.atInfo().log("Domain %s was imported successfully", xjcDomain.getName());
} catch (ResourceExistsException e) {
// Record the number of domains already in the registry
getContext().incrementCounter("existing domains skipped");
logger.atInfo().log("Domain %s already exists", xjcDomain.getName());
} catch (Exception e) {
getContext().incrementCounter("domain import errors");
logger.atSevere().withCause(e).log(
"Error processing domain %s; xml=%s", xjcDomain.getName(), xjcDomain);
}
}
private void saveDomain(XjcRdeDomain xjcDomain) {
HistoryEntry historyEntry = createHistoryEntryForDomainImport(xjcDomain);
BillingEvent.Recurring autorenewBillingEvent =
createAutoRenewBillingEventForDomainImport(xjcDomain, historyEntry);
PollMessage.Autorenew autorenewPollMessage =
createAutoRenewPollMessageForDomainImport(xjcDomain, historyEntry);
DomainBase domain =
XjcToDomainBaseConverter.convertDomain(
xjcDomain, autorenewBillingEvent, autorenewPollMessage, stringGenerator);
getDnsQueue().addDomainRefreshTask(domain.getFullyQualifiedDomainName());
// Keep a list of "extra objects" that need to be saved along with the domain
// and add to it if necessary.
ImmutableSet<Object> extraEntitiesToSave =
getImportUtils().createIndexesForEppResource(domain);
// Create speculative server approval entities for pending transfers
if (domain.getTransferData().getTransferStatus() == TransferStatus.PENDING) {
TransferData transferData = domain.getTransferData();
checkArgumentNotNull(
transferData,
"Domain %s is in pending transfer but has no transfer data",
domain.getFullyQualifiedDomainName());
Money transferCost =
getDomainRenewCost(
domain.getFullyQualifiedDomainName(),
transferData.getPendingTransferExpirationTime(),
1);
DateTime automaticTransferTime = transferData.getPendingTransferExpirationTime();
// If the transfer will occur within the autorenew grace period, it should
// subsume the autorenew, so we don't add the normal extra year. See the
// original logic in DomainTransferRequestFlow (which is very similar) for
// more information. That said, note that here we stop 1 millisecond before
// the actual transfer time to avoid hitting the transfer-handling part of
// cloneProjectedAtTime(), since unlike in the DomainTransferRequestFlow case,
// this domain already has a pending transfer.
DomainBase domainAtTransferTime =
domain.cloneProjectedAtTime(automaticTransferTime.minusMillis(1));
boolean inAutorenewGraceAtTransfer =
!domainAtTransferTime.getGracePeriodsOfType(GracePeriodStatus.AUTO_RENEW).isEmpty();
int extraYears = inAutorenewGraceAtTransfer ? 0 : 1;
// Construct the capped new expiration time.
DateTime serverApproveNewExpirationTime =
extendRegistrationWithCap(
automaticTransferTime,
domainAtTransferTime.getRegistrationExpirationTime(),
extraYears);
// Create speculative entities in anticipation of an automatic server
// approval.
ImmutableSet<TransferServerApproveEntity> serverApproveEntities =
createTransferServerApproveEntities(
automaticTransferTime,
serverApproveNewExpirationTime,
historyEntry,
domain,
historyEntry.getTrid(),
transferData.getGainingClientId(),
Optional.of(transferCost),
transferData.getTransferRequestTime());
transferData =
createPendingTransferData(
transferData.asBuilder(), serverApproveEntities, Period.create(1, Unit.YEARS));
// Create a poll message to notify the losing registrar that a transfer was
// requested.
PollMessage requestPollMessage =
createLosingTransferPollMessage(
domain.getRepoId(), transferData, serverApproveNewExpirationTime, historyEntry)
.asBuilder()
.setEventTime(transferData.getTransferRequestTime())
.build();
domain = domain.asBuilder().setTransferData(transferData).build();
autorenewBillingEvent =
autorenewBillingEvent
.asBuilder()
.setRecurrenceEndTime(transferData.getPendingTransferExpirationTime())
.build();
autorenewPollMessage =
autorenewPollMessage
.asBuilder()
.setAutorenewEndTime(transferData.getPendingTransferExpirationTime())
.build();
extraEntitiesToSave =
new ImmutableSet.Builder<>()
.add(requestPollMessage)
.addAll(extraEntitiesToSave)
.addAll(serverApproveEntities)
.build();
} // End pending transfer check
ofy()
.save()
.entities(
new ImmutableSet.Builder<>()
.add(domain, historyEntry, autorenewBillingEvent, autorenewPollMessage)
.addAll(extraEntitiesToSave)
.build())
.now();
}
}
}

120
java/google/registry/rde/imports/RdeDomainInput.java
View file

@ -1,120 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static com.google.common.math.IntMath.divide;
import static java.math.RoundingMode.CEILING;
import static java.math.RoundingMode.FLOOR;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.Input;
import com.google.appengine.tools.mapreduce.InputReader;
import com.google.common.collect.ImmutableList;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.gcs.GcsUtils;
import google.registry.model.domain.DomainBase;
import google.registry.rde.imports.RdeParser.RdeHeader;
import google.registry.xjc.JaxbFragment;
import google.registry.xjc.rdedomain.XjcRdeDomainElement;
import java.io.InputStream;
import java.util.List;
import java.util.Optional;
/**
* A MapReduce {@link Input} that imports {@link DomainBase} objects from an escrow file.
*
* <p>If a mapShards parameter has been specified, up to that many readers will be created
* so that each map shard has one reader. If a mapShards parameter has not been specified, a
* default number of readers will be created.
*/
public class RdeDomainInput extends Input<JaxbFragment<XjcRdeDomainElement>> {
private static final long serialVersionUID = -366966393494008712L;
private static final GcsService GCS_SERVICE =
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
/**
* Default number of readers if map shards are not specified.
*/
private static final int DEFAULT_READERS = 50;
/**
* Minimum number of records per reader.
*/
private static final int MINIMUM_RECORDS_PER_READER = 100;
/**
* Optional argument to explicitly specify the number of readers.
*/
private final int numReaders;
private final String importBucketName;
private final String importFileName;
public RdeDomainInput(
Optional<Integer> mapShards, String importBucketName, String importFileName) {
this.numReaders = mapShards.orElse(DEFAULT_READERS);
this.importBucketName = importBucketName;
this.importFileName = importFileName;
}
@Override
public List<? extends InputReader<JaxbFragment<XjcRdeDomainElement>>> createReaders() {
int numReaders = this.numReaders;
RdeHeader header = newParser().getHeader();
int numberOfDomains = header.getDomainCount().intValue();
if (numberOfDomains / numReaders < MINIMUM_RECORDS_PER_READER) {
numReaders = divide(numberOfDomains, MINIMUM_RECORDS_PER_READER, FLOOR);
// use at least one reader
numReaders = Math.max(numReaders, 1);
}
ImmutableList.Builder<RdeDomainReader> builder = new ImmutableList.Builder<>();
int domainsPerReader =
Math.max(MINIMUM_RECORDS_PER_READER, divide(numberOfDomains, numReaders, CEILING));
int offset = 0;
for (int i = 0; i < numReaders; i++) {
builder = builder.add(newReader(offset, domainsPerReader));
offset += domainsPerReader;
}
return builder.build();
}
private RdeDomainReader newReader(int offset, int maxResults) {
return new RdeDomainReader(importBucketName, importFileName, offset, maxResults);
}
private RdeParser newParser() {
GcsUtils utils = new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize());
GcsFilename filename = new GcsFilename(importBucketName, importFileName);
try (InputStream xmlInput = utils.openInputStream(filename)) {
return new RdeParser(xmlInput);
} catch (Exception e) {
throw new InitializationException(
String.format("Error opening rde file %s/%s", importBucketName, importFileName), e);
}
}
/**
* Thrown when the input cannot initialize properly.
*/
private static class InitializationException extends RuntimeException {
public InitializationException(String message, Throwable cause) {
super(message, cause);
}
}
}

109
java/google/registry/rde/imports/RdeDomainReader.java
View file

@ -1,109 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.InputReader;
import com.google.common.flogger.FluentLogger;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.gcs.GcsUtils;
import google.registry.xjc.JaxbFragment;
import google.registry.xjc.rdedomain.XjcRdeDomainElement;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.util.NoSuchElementException;
/** Mapreduce {@link InputReader} for reading domains from escrow files */
public class RdeDomainReader extends InputReader<JaxbFragment<XjcRdeDomainElement>>
implements Serializable {
private static final long serialVersionUID = -2175777052970160122L;
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final GcsService GCS_SERVICE =
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
final String importBucketName;
final String importFileName;
final int offset;
final int maxResults;
private int count = 0;
transient RdeParser parser;
/**
* Creates a new instance of {@link RdeParser}
*/
private RdeParser newParser() {
GcsUtils utils = new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize());
GcsFilename filename = new GcsFilename(importBucketName, importFileName);
InputStream xmlInput = utils.openInputStream(filename);
try {
RdeParser parser = new RdeParser(xmlInput);
// skip the file offset and count
// if count is greater than 0, the reader has been rehydrated after doing some work.
// skip any already processed records.
parser.skipDomains(offset + count);
return parser;
} catch (Exception e) {
throw new RuntimeException(
String.format("Error opening rde file %s/%s", importBucketName, importFileName), e);
}
}
public RdeDomainReader(
String importBucketName,
String importFileName,
int offset,
int maxResults) {
this.importBucketName = importBucketName;
this.importFileName = importFileName;
this.offset = offset;
this.maxResults = maxResults;
}
@Override
public JaxbFragment<XjcRdeDomainElement> next() {
if (count < maxResults) {
if (parser == null) {
parser = newParser();
if (parser.isAtDomain()) {
return readDomain();
}
}
if (parser.nextDomain()) {
return readDomain();
}
}
throw new NoSuchElementException();
}
private JaxbFragment<XjcRdeDomainElement> readDomain() {
count++;
return JaxbFragment.create(new XjcRdeDomainElement(parser.getDomain()));
}
@Override
public void endSlice() throws IOException {
super.endSlice();
if (parser != null) {
parser.close();
}
}
}

147
java/google/registry/rde/imports/RdeHostImportAction.java
View file

@ -1,147 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
import static google.registry.model.ofy.ObjectifyService.ofy;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.collect.ImmutableList;
import com.google.common.flogger.FluentLogger;
import google.registry.config.RegistryConfig.Config;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.gcs.GcsUtils;
import google.registry.mapreduce.MapreduceRunner;
import google.registry.model.host.HostResource;
import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.SystemClock;
import google.registry.xjc.JaxbFragment;
import google.registry.xjc.rdehost.XjcRdeHost;
import google.registry.xjc.rdehost.XjcRdeHostElement;
import java.util.Optional;
import javax.inject.Inject;
/**
* A mapreduce that imports hosts from an escrow file.
*
* <p>Specify the escrow file to import with the "path" parameter.
*/
@Action(
service = Action.Service.BACKEND,
path = "/_dr/task/importRdeHosts",
auth = Auth.AUTH_INTERNAL_ONLY)
public class RdeHostImportAction implements Runnable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final GcsService GCS_SERVICE =
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
private final MapreduceRunner mrRunner;
private final Response response;
private final String importBucketName;
private final String importFileName;
private final Optional<Integer> mapShards;
@Inject
public RdeHostImportAction(
MapreduceRunner mrRunner,
Response response,
@Config("rdeImportBucket") String importBucketName,
@Parameter("path") String importFileName,
@Parameter(PARAM_MAP_SHARDS) Optional<Integer> mapShards) {
this.mrRunner = mrRunner;
this.response = response;
this.importBucketName = importBucketName;
this.importFileName = importFileName;
this.mapShards = mapShards;
}
@Override
public void run() {
mrRunner
.setJobName("Import hosts from escrow file")
.setModuleName("backend")
.runMapOnly(
new RdeHostImportMapper(importBucketName),
ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName)))
.sendLinkToMapreduceConsole(response);
}
/** Mapper to import hosts from an escrow file. */
public static class RdeHostImportMapper
extends Mapper<JaxbFragment<XjcRdeHostElement>, Void, Void> {
private static final long serialVersionUID = -2898753709127134419L;
private final String importBucketName;
private transient RdeImportUtils importUtils;
public RdeHostImportMapper(String importBucketName) {
this.importBucketName = importBucketName;
}
private RdeImportUtils getImportUtils() {
if (importUtils == null) {
importUtils = createRdeImportUtils();
}
return importUtils;
}
/**
* Creates a new instance of RdeImportUtils.
*/
private RdeImportUtils createRdeImportUtils() {
return new RdeImportUtils(
ofy(),
new SystemClock(),
importBucketName,
new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize()));
}
@Override
public void map(JaxbFragment<XjcRdeHostElement> fragment) {
final XjcRdeHost xjcHost = fragment.getInstance().getValue();
try {
// Record number of attempted map operations
getContext().incrementCounter("host imports attempted");
logger.atInfo().log("Saving host %s", xjcHost.getName());
ofy()
.transact(
() -> {
HostResource host = XjcToHostResourceConverter.convert(xjcHost);
getImportUtils().importEppResource(host);
});
// Record number of hosts imported
getContext().incrementCounter("hosts saved");
logger.atInfo().log("Host %s was imported successfully", xjcHost.getName());
} catch (ResourceExistsException e) {
// Record the number of hosts already in the registry
getContext().incrementCounter("existing hosts skipped");
logger.atInfo().log("Host %s already exists", xjcHost.getName());
} catch (Exception e) {
// Record the number of hosts with unexpected errors
getContext().incrementCounter("host import errors");
logger.atSevere().withCause(e).log(
"Error processing host %s; xml=%s", xjcHost.getName(), xjcHost);
}
}
}
}

120
java/google/registry/rde/imports/RdeHostInput.java
View file

@ -1,120 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static com.google.common.base.Preconditions.checkArgument;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.Input;
import com.google.appengine.tools.mapreduce.InputReader;
import com.google.common.collect.ImmutableList;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.gcs.GcsUtils;
import google.registry.model.host.HostResource;
import google.registry.rde.imports.RdeParser.RdeHeader;
import google.registry.xjc.JaxbFragment;
import google.registry.xjc.rdehost.XjcRdeHostElement;
import java.io.InputStream;
import java.util.List;
import java.util.Optional;
/**
* A MapReduce {@link Input} that imports {@link HostResource} objects from an escrow file.
*
* <p>If a mapShards parameter has been specified, up to that many readers will be created
* so that each map shard has one reader. If a mapShards parameter has not been specified, a
* default number of readers will be created.
*/
public class RdeHostInput extends Input<JaxbFragment<XjcRdeHostElement>> {
private static final long serialVersionUID = 9218225041307602452L;
private static final GcsService GCS_SERVICE =
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
/**
* Default number of readers if map shards are not specified.
*/
private static final int DEFAULT_READERS = 50;
/**
* Minimum number of records per reader.
*/
private static final int MINIMUM_RECORDS_PER_READER = 100;
/**
* Optional argument to explicitly specify the number of readers.
*/
private final int numReaders;
private final String importBucketName;
private final String importFileName;
public RdeHostInput(Optional<Integer> mapShards, String importBucketName,
String importFileName) {
this.numReaders = mapShards.orElse(DEFAULT_READERS);
checkArgument(numReaders > 0, "Number of shards must be greater than zero");
this.importBucketName = importBucketName;
this.importFileName = importFileName;
}
@Override
public List<? extends InputReader<JaxbFragment<XjcRdeHostElement>>> createReaders() {
int numReaders = this.numReaders;
RdeHeader header = createParser().getHeader();
int numberOfHosts = header.getHostCount().intValue();
if (numberOfHosts / numReaders < MINIMUM_RECORDS_PER_READER) {
numReaders = numberOfHosts / MINIMUM_RECORDS_PER_READER;
// use at least one reader
numReaders = Math.max(numReaders, 1);
}
ImmutableList.Builder<RdeHostReader> builder = new ImmutableList.Builder<>();
int hostsPerReader =
Math.max(MINIMUM_RECORDS_PER_READER, (int) Math.ceil((double) numberOfHosts / numReaders));
int offset = 0;
for (int i = 0; i < numReaders; i++) {
builder = builder.add(createReader(offset, hostsPerReader));
offset += hostsPerReader;
}
return builder.build();
}
private RdeHostReader createReader(int offset, int maxResults) {
return new RdeHostReader(importBucketName, importFileName, offset, maxResults);
}
private RdeParser createParser() {
GcsUtils utils = new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize());
GcsFilename filename = new GcsFilename(importBucketName, importFileName);
InputStream xmlInput = utils.openInputStream(filename);
try {
return new RdeParser(xmlInput);
} catch (Exception e) {
throw new InitializationException(
String.format("Error opening rde file %s/%s", importBucketName, importFileName), e);
}
}
/**
* Thrown when the input cannot initialize properly.
*/
private static class InitializationException extends RuntimeException {
public InitializationException(String message, Throwable cause) {
super(message, cause);
}
}
}

215
java/google/registry/rde/imports/RdeHostLinkAction.java
View file

@ -1,215 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static com.google.common.base.Preconditions.checkState;
import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
import static google.registry.model.EppResourceUtils.loadByForeignKey;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.registry.Registries.findTldForName;
import static java.util.stream.Collectors.joining;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.collect.ImmutableList;
import com.google.common.flogger.FluentLogger;
import com.google.common.net.InternetDomainName;
import com.googlecode.objectify.Key;
import google.registry.config.RegistryConfig.Config;
import google.registry.flows.host.HostFlowUtils;
import google.registry.mapreduce.MapreduceRunner;
import google.registry.model.EppResource;
import google.registry.model.domain.DomainBase;
import google.registry.model.eppcommon.StatusValue;
import google.registry.model.host.HostResource;
import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.xjc.JaxbFragment;
import google.registry.xjc.rdehost.XjcRdeHost;
import google.registry.xjc.rdehost.XjcRdeHostElement;
import java.util.Optional;
import javax.inject.Inject;
import org.joda.time.DateTime;
/**
* A mapreduce that links hosts from an escrow file to their superordinate domains.
*
* <p>This mapreduce is run as the last step of the process of importing escrow files. For each host
* in the escrow file, the corresponding {@link HostResource} record in Datastore is linked to its
* superordinate {@link DomainBase} only if it is an in-zone host. This is necessary because all
* hosts must exist before domains can be imported, due to references in host objects, and domains
* must exist before hosts can be linked to their superordinate domains.
*
* <p>Specify the escrow file to import with the "path" parameter.
*/
@Action(
service = Action.Service.BACKEND,
path = "/_dr/task/linkRdeHosts",
auth = Auth.AUTH_INTERNAL_ONLY)
public class RdeHostLinkAction implements Runnable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final MapreduceRunner mrRunner;
private final Response response;
private final String importBucketName;
private final String importFileName;
private final Optional<Integer> mapShards;
@Inject
public RdeHostLinkAction(
MapreduceRunner mrRunner,
Response response,
@Config("rdeImportBucket") String importBucketName,
@Parameter("path") String importFileName,
@Parameter(PARAM_MAP_SHARDS) Optional<Integer> mapShards) {
this.mrRunner = mrRunner;
this.response = response;
this.importBucketName = importBucketName;
this.importFileName = importFileName;
this.mapShards = mapShards;
}
@Override
public void run() {
mrRunner
.setJobName("Link hosts from escrow file")
.setModuleName("backend")
.runMapOnly(
new RdeHostPostImportMapper(),
ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName)))
.sendLinkToMapreduceConsole(response);
}
/** Mapper to link hosts from an escrow file to their superordinate domains. */
public static class RdeHostPostImportMapper
extends Mapper<JaxbFragment<XjcRdeHostElement>, Void, Void> {
private static final long serialVersionUID = -2898753709127134419L;
@Override
public void map(JaxbFragment<XjcRdeHostElement> fragment) {
// Record number of attempted map operations
getContext().incrementCounter("post-import hosts read");
final XjcRdeHost xjcHost = fragment.getInstance().getValue();
logger.atInfo().log("Attempting to link superordinate domain for host %s", xjcHost.getName());
try {
final InternetDomainName hostName = InternetDomainName.from(xjcHost.getName());
HostLinkResult hostLinkResult = ofy().transact(() -> {
Optional<DomainBase> superordinateDomain =
lookupSuperordinateDomain(hostName, ofy().getTransactionTime());
// if suporordinateDomain is absent, this is an out of zone host and can't be linked.
// absent is only returned for out of zone hosts, and an exception is thrown for in
// zone hosts with no superordinate domain.
if (!superordinateDomain.isPresent()) {
return HostLinkResult.HOST_OUT_OF_ZONE;
}
if (superordinateDomain.get().getStatusValues().contains(StatusValue.PENDING_DELETE)) {
return HostLinkResult.SUPERORDINATE_DOMAIN_IN_PENDING_DELETE;
}
Key<DomainBase> superordinateDomainKey = Key.create(superordinateDomain.get());
// link host to superordinate domain and set time of last superordinate change to
// the time of the import
HostResource host =
ofy().load().now(Key.create(HostResource.class, xjcHost.getRoid()));
if (host == null) {
return HostLinkResult.HOST_NOT_FOUND;
}
// link domain to subordinate host
ofy().save().<EppResource>entities(
host.asBuilder().setSuperordinateDomain(superordinateDomainKey)
.setLastSuperordinateChange(ofy().getTransactionTime())
.build(),
superordinateDomain.get().asBuilder()
.addSubordinateHost(host.getFullyQualifiedHostName()).build());
return HostLinkResult.HOST_LINKED;
});
// increment counter and log appropriately based on result of transaction
switch (hostLinkResult) {
case HOST_LINKED:
getContext().incrementCounter("post-import hosts linked");
logger.atInfo().log(
"Successfully linked host %s to superordinate domain", xjcHost.getName());
// Record number of hosts successfully linked
break;
case HOST_NOT_FOUND:
getContext().incrementCounter("hosts not found");
logger.atSevere().log(
"Host with name %s and repoid %s not found", xjcHost.getName(), xjcHost.getRoid());
break;
case SUPERORDINATE_DOMAIN_IN_PENDING_DELETE:
getContext()
.incrementCounter(
"post-import hosts with superordinate domains in pending delete");
logger.atInfo().log(
"Host %s has a superordinate domain in pending delete", xjcHost.getName());
break;
case HOST_OUT_OF_ZONE:
getContext().incrementCounter("post-import hosts out of zone");
logger.atInfo().log("Host %s is out of zone", xjcHost.getName());
break;
}
} catch (RuntimeException e) {
// Record the number of hosts with unexpected errors
getContext().incrementCounter("post-import host errors");
logger.atSevere().withCause(e).log(
"Error linking host %s; xml=%s", xjcHost.getName(), xjcHost);
}
}
/**
* Return the {@link DomainBase} this host is subordinate to, or absent for out of zone
* hosts.
*
* <p>We use this instead of {@link HostFlowUtils#lookupSuperordinateDomain} because we don't
* want to use the EPP exception classes for the case when the superordinate domain doesn't
* exist or isn't active.
*
* @throws IllegalStateException for hosts without superordinate domains
*/
private static Optional<DomainBase> lookupSuperordinateDomain(
InternetDomainName hostName, DateTime now) {
Optional<InternetDomainName> tld = findTldForName(hostName);
// out of zone hosts cannot be linked
if (!tld.isPresent()) {
return Optional.empty();
}
// This is a subordinate host
String domainName =
hostName
.parts()
.stream()
.skip(hostName.parts().size() - (tld.get().parts().size() + 1))
.collect(joining("."));
Optional<DomainBase> superordinateDomain =
loadByForeignKey(DomainBase.class, domainName, now);
// Hosts can't be linked if domains import hasn't been run
checkState(
superordinateDomain.isPresent(),
"Superordinate domain does not exist or is deleted: %s",
domainName);
return superordinateDomain;
}
}
private enum HostLinkResult {
HOST_NOT_FOUND,
HOST_OUT_OF_ZONE,
SUPERORDINATE_DOMAIN_IN_PENDING_DELETE,
HOST_LINKED;
}
}

112
java/google/registry/rde/imports/RdeHostReader.java
View file

@ -1,112 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.InputReader;
import com.google.common.flogger.FluentLogger;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.gcs.GcsUtils;
import google.registry.xjc.JaxbFragment;
import google.registry.xjc.rdehost.XjcRdeHostElement;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.util.NoSuchElementException;
import javax.annotation.concurrent.NotThreadSafe;
/** Mapreduce {@link InputReader} for reading hosts from escrow files */
@NotThreadSafe
public class RdeHostReader extends InputReader<JaxbFragment<XjcRdeHostElement>>
implements Serializable {
private static final long serialVersionUID = 3037264959150412846L;
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final GcsService GCS_SERVICE =
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
final String importBucketName;
final String importFileName;
final int offset;
final int maxResults;
private int count = 0;
transient RdeParser parser;
/**
* Creates a new instance of {@link RdeParser}
*/
private RdeParser newParser() {
GcsUtils utils = new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize());
GcsFilename filename = new GcsFilename(importBucketName, importFileName);
InputStream xmlInput = utils.openInputStream(filename);
try {
RdeParser parser = new RdeParser(xmlInput);
// skip the file offset and count
// if count is greater than 0, the reader has been rehydrated after doing some work.
// skip any already processed records.
parser.skipHosts(offset + count);
return parser;
} catch (Exception e) {
throw new RuntimeException(
String.format("Error opening RDE file %s/%s", importBucketName, importFileName), e);
}
}
public RdeHostReader(
String importBucketName,
String importFileName,
int offset,
int maxResults) {
this.importBucketName = importBucketName;
this.importFileName = importFileName;
this.offset = offset;
this.maxResults = maxResults;
}
@Override
public JaxbFragment<XjcRdeHostElement> next() {
if (count < maxResults) {
if (parser == null) {
parser = newParser();
if (parser.isAtHost()) {
return readHost();
}
}
if (parser.nextHost()) {
return readHost();
}
}
throw new NoSuchElementException();
}
private JaxbFragment<XjcRdeHostElement> readHost() {
count++;
return JaxbFragment.create(new XjcRdeHostElement(parser.getHost()));
}
@Override
public void endSlice() throws IOException {
super.endSlice();
if (parser != null) {
parser.close();
}
}
}

239
java/google/registry/rde/imports/RdeImportUtils.java
View file

@ -1,239 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.registry.Registry.TldState.PREDELEGATION;
import static google.registry.util.DateTimeUtils.END_OF_TIME;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static google.registry.util.PreconditionsUtils.checkArgumentPresent;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.collect.ImmutableSet;
import com.google.common.flogger.FluentLogger;
import com.google.common.io.BaseEncoding;
import com.googlecode.objectify.Key;
import google.registry.config.RegistryConfig.Config;
import google.registry.flows.ServerTridProvider;
import google.registry.flows.ServerTridProviderImpl;
import google.registry.gcs.GcsUtils;
import google.registry.model.EppResource;
import google.registry.model.EppResource.ForeignKeyedEppResource;
import google.registry.model.billing.BillingEvent;
import google.registry.model.billing.BillingEvent.Flag;
import google.registry.model.billing.BillingEvent.Reason;
import google.registry.model.domain.DomainBase;
import google.registry.model.eppcommon.Trid;
import google.registry.model.index.EppResourceIndex;
import google.registry.model.index.ForeignKeyIndex;
import google.registry.model.ofy.Ofy;
import google.registry.model.poll.PollMessage;
import google.registry.model.registrar.Registrar;
import google.registry.model.registry.Registry;
import google.registry.model.registry.Registry.RegistryNotFoundException;
import google.registry.model.registry.Registry.TldState;
import google.registry.model.reporting.HistoryEntry;
import google.registry.util.Clock;
import google.registry.xjc.XjcXmlTransformer;
import google.registry.xjc.rdedomain.XjcRdeDomain;
import google.registry.xjc.rdedomain.XjcRdeDomainElement;
import google.registry.xjc.rderegistrar.XjcRdeRegistrar;
import google.registry.xml.XmlException;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.UUID;
import javax.inject.Inject;
import javax.xml.bind.JAXBException;
import javax.xml.stream.XMLStreamException;
/**
* Utility functions for escrow file import.
*/
public class RdeImportUtils {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final Ofy ofy;
private final Clock clock;
private final String escrowBucketName;
private final GcsUtils gcsUtils;
private static final ServerTridProvider serverTridProvider = new ServerTridProviderImpl();
@Inject
public RdeImportUtils(
Ofy ofy, Clock clock, @Config("rdeImportBucket") String escrowBucketName, GcsUtils gcsUtils) {
this.ofy = ofy;
this.clock = clock;
this.gcsUtils = gcsUtils;
this.escrowBucketName = escrowBucketName;
}
public <T extends EppResource & ForeignKeyedEppResource> ImmutableSet<Object>
createIndexesForEppResource(T resource) {
@SuppressWarnings("unchecked")
ForeignKeyIndex<T> existingForeignKeyIndex =
ForeignKeyIndex.load(
(Class<T>) resource.getClass(), resource.getForeignKey(), START_OF_TIME);
// ForeignKeyIndex should never have existed, since existing resource was not found.
checkState(
existingForeignKeyIndex == null,
"New %s resource has existing foreign key index; foreignKey=%s, repoId=%s",
resource.getClass().getCanonicalName(),
resource.getForeignKey(),
resource.getRepoId());
return ImmutableSet.of(ForeignKeyIndex.create(resource, resource.getDeletionTime()),
EppResourceIndex.create(Key.create(resource)));
}
/**
* Imports a resource from an escrow file.
*
* <p>The resource will only be imported if it has not been previously imported.
*
* <p>If the resource is imported, {@link ForeignKeyIndex} and {@link EppResourceIndex} are also
* created.
*/
public <T extends EppResource & ForeignKeyedEppResource> void importEppResource(
final T resource) {
Object existing = ofy.load().key(Key.create(resource)).now();
if (existing != null) {
// This will roll back the transaction and prevent duplicate history entries from being saved.
throw new ResourceExistsException();
}
ofy.save().entities(new ImmutableSet.Builder<>()
.add(resource)
.addAll(createIndexesForEppResource(resource))
.build());
logger.atInfo().log(
"Imported %s resource - ROID=%s, id=%s",
resource.getClass().getSimpleName(), resource.getRepoId(), resource.getForeignKey());
}
/**
* Validates an escrow file for import.
*
* <p>Before an escrow file is imported into the registry, the following conditions must be met:
*
* <ul>
* <li>The TLD must already exist in the registry
* <li>The TLD must be in the PREDELEGATION state
* <li>Each registrar must already exist in the registry
* <li>Each IDN table referenced must already exist in the registry
* </ul>
*
* <p>If any of the above conditions is not true, an {@link IllegalStateException} will be thrown.
*
* @param escrowFilePath Path to the escrow file to validate
* @throws IOException If the escrow file cannot be read
* @throws IllegalArgumentException if the escrow file cannot be imported
*/
public void validateEscrowFileForImport(String escrowFilePath) throws IOException {
// TODO (wolfgang@donuts.co): Add validation method for IDN tables
try (InputStream input =
gcsUtils.openInputStream(new GcsFilename(escrowBucketName, escrowFilePath))) {
try (RdeParser parser = new RdeParser(input)) {
// validate that tld exists and is in PREDELEGATION state
String tld = parser.getHeader().getTld();
try {
Registry registry = Registry.get(tld);
TldState currentState = registry.getTldState(clock.nowUtc());
checkArgument(
currentState == PREDELEGATION,
"TLD '%s' is in state %s and cannot be imported",
tld,
currentState);
} catch (RegistryNotFoundException e) {
throw new IllegalArgumentException(
String.format("TLD '%s' not found in the registry", tld));
}
// validate that all registrars exist
while (parser.nextRegistrar()) {
XjcRdeRegistrar registrar = parser.getRegistrar();
checkArgumentPresent(
Registrar.loadByClientIdCached(registrar.getId()),
"Registrar '%s' not found in the registry",
registrar.getId());
}
} catch (XMLStreamException | JAXBException e) {
throw new IllegalArgumentException(
String.format("Invalid XML file: '%s'", escrowFilePath), e);
}
}
}
/** Generates a random {@link Trid} for rde import. */
public static Trid generateTridForImport() {
// Client trids must be a token between 3 and 64 characters long
// Base64 encoded UUID string meets this requirement
return Trid.create(
"Import_" + BaseEncoding.base64().encode(UUID.randomUUID().toString().getBytes(US_ASCII)),
serverTridProvider.createServerTrid());
}
public static BillingEvent.Recurring createAutoRenewBillingEventForDomainImport(
XjcRdeDomain domain, HistoryEntry historyEntry) {
return new BillingEvent.Recurring.Builder()
.setReason(Reason.RENEW)
.setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
.setTargetId(domain.getRoid())
.setClientId(domain.getClID())
.setEventTime(domain.getExDate())
.setRecurrenceEndTime(END_OF_TIME)
.setParent(historyEntry)
.build();
}
public static PollMessage.Autorenew createAutoRenewPollMessageForDomainImport(
XjcRdeDomain domain, HistoryEntry historyEntry) {
return new PollMessage.Autorenew.Builder()
.setTargetId(domain.getRoid())
.setClientId(domain.getClID())
.setEventTime(domain.getExDate())
.setMsg("Domain was auto-renewed.")
.setParent(historyEntry)
.build();
}
public static HistoryEntry createHistoryEntryForDomainImport(XjcRdeDomain domain) {
XjcRdeDomainElement element = new XjcRdeDomainElement(domain);
return new HistoryEntry.Builder()
.setType(HistoryEntry.Type.RDE_IMPORT)
.setClientId(domain.getClID())
.setTrid(generateTridForImport())
.setModificationTime(ofy().getTransactionTime())
.setXmlBytes(getObjectXml(element))
.setBySuperuser(true)
.setReason("RDE Import")
.setRequestedByRegistrar(false)
.setParent(Key.create(null, DomainBase.class, domain.getRoid()))
.build();
}
public static byte[] getObjectXml(Object jaxbElement) {
try {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
XjcXmlTransformer.marshalLenient(jaxbElement, bout, UTF_8);
return bout.toByteArray();
} catch (XmlException e) {
throw new RuntimeException(e);
}
}
}

38
java/google/registry/rde/imports/RdeImportsModule.java
View file

@ -1,38 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import dagger.Module;
import dagger.Provides;
import google.registry.request.Parameter;
import google.registry.request.RequestParameters;
import javax.servlet.http.HttpServletRequest;
/**
* Dagger module for RDE imports package.
*
* @see "google.registry.module.backend.BackendRequestComponent"
*/
@Module
public final class RdeImportsModule {
static final String PATH = "path";
@Provides
@Parameter(PATH)
static String providePath(HttpServletRequest req) {
return RequestParameters.extractRequiredParameter(req, PATH);
}
}

591
java/google/registry/rde/imports/RdeParser.java
View file

@ -1,591 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import google.registry.xjc.rdecontact.XjcRdeContact;
import google.registry.xjc.rdecontact.XjcRdeContactElement;
import google.registry.xjc.rdedomain.XjcRdeDomain;
import google.registry.xjc.rdedomain.XjcRdeDomainElement;
import google.registry.xjc.rdeeppparams.XjcRdeEppParams;
import google.registry.xjc.rdeeppparams.XjcRdeEppParamsElement;
import google.registry.xjc.rdeheader.XjcRdeHeader;
import google.registry.xjc.rdeheader.XjcRdeHeaderCount;
import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
import google.registry.xjc.rdehost.XjcRdeHost;
import google.registry.xjc.rdehost.XjcRdeHostElement;
import google.registry.xjc.rdeidn.XjcRdeIdn;
import google.registry.xjc.rdeidn.XjcRdeIdnElement;
import google.registry.xjc.rdenndn.XjcRdeNndn;
import google.registry.xjc.rdenndn.XjcRdeNndnElement;
import google.registry.xjc.rderegistrar.XjcRdeRegistrar;
import google.registry.xjc.rderegistrar.XjcRdeRegistrarElement;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Optional;
import javax.annotation.concurrent.NotThreadSafe;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
/**
* RDE escrow deposit file parser
*
* <p>{@link RdeParser} parses escrow deposit files as a sequence of elements. The parser will first
* parse and cache the RDE header before any other elements, so all calls to {@link #getHeader} will
* return the header even if the parser has advanced beyond it.
*
* <p>{@link RdeParser} currently supports parsing the following rde elements as jaxb objects:
* <ul>
* <li>Contact</li>
* <li>Host</li>
* <li>Domain</li>
* <li>Registrar</li>
* <li>Tld</li>
* <li>IDN table reference</li>
* <li>EPP Params</li>
* <li>NNDN</li>
* </ul>
*
* <p>Any calls to {@link #nextDomain}, {@link #nextHost}, etc. will advance the parser to the next
* element in the file, if any additional elements of that type exist. Since the order of these
* elements is not known at the time the file is read, client code should only try to parse one type
* of element at a time. Parsing of additional element types should be performed by creating a new
* parser.
*/
@NotThreadSafe
public class RdeParser implements Closeable {
private static final String RDE_DOMAIN_URI = "urn:ietf:params:xml:ns:rdeDomain-1.0";
private static final String RDE_HOST_URI = "urn:ietf:params:xml:ns:rdeHost-1.0";
private static final String RDE_CONTACT_URI = "urn:ietf:params:xml:ns:rdeContact-1.0";
private static final String RDE_REGISTRAR_URI = "urn:ietf:params:xml:ns:rdeRegistrar-1.0";
private static final String RDE_IDN_URI = "urn:ietf:params:xml:ns:rdeIDN-1.0";
private static final String RDE_NNDN_URI = "urn:ietf:params:xml:ns:rdeNNDN-1.0";
private static final String RDE_EPP_PARAMS_URI = "urn:ietf:params:xml:ns:rdeEppParams-1.0";
private static final String RDE_HEADER_URI = "urn:ietf:params:xml:ns:rdeHeader-1.0";
/** List of packages to initialize JAXBContext. **/
private static final String JAXB_CONTEXT_PACKAGES = Joiner.on(":")
.join(Arrays.asList(
"google.registry.xjc.contact",
"google.registry.xjc.domain",
"google.registry.xjc.host",
"google.registry.xjc.mark",
"google.registry.xjc.rde",
"google.registry.xjc.rdecontact",
"google.registry.xjc.rdedomain",
"google.registry.xjc.rdeeppparams",
"google.registry.xjc.rdeheader",
"google.registry.xjc.rdeidn",
"google.registry.xjc.rdenndn",
"google.registry.xjc.rderegistrar",
"google.registry.xjc.smd"));
/**
* Convenient immutable java representation of an RDE header
*/
public static class RdeHeader {
private final ImmutableMap<String, Long> counts;
private final String tld;
public String getTld() {
return tld;
}
public Long getDomainCount() {
return Optional.ofNullable(counts.get(RDE_DOMAIN_URI)).orElse(0L);
}
public Long getHostCount() {
return Optional.ofNullable(counts.get(RDE_HOST_URI)).orElse(0L);
}
public Long getContactCount() {
return Optional.ofNullable(counts.get(RDE_CONTACT_URI)).orElse(0L);
}
public Long getRegistrarCount() {
return Optional.ofNullable(counts.get(RDE_REGISTRAR_URI)).orElse(0L);
}
public Long getIdnCount() {
return Optional.ofNullable(counts.get(RDE_IDN_URI)).orElse(0L);
}
public Long getNndnCount() {
return Optional.ofNullable(counts.get(RDE_NNDN_URI)).orElse(0L);
}
public Long getEppParamsCount() {
return Optional.ofNullable(counts.get(RDE_EPP_PARAMS_URI)).orElse(0L);
}
private RdeHeader(XjcRdeHeader header) {
this.tld = header.getTld();
ImmutableMap.Builder<String, Long> builder = new ImmutableMap.Builder<>();
for (XjcRdeHeaderCount count : header.getCounts()) {
builder = builder.put(count.getUri(), count.getValue());
}
counts = builder.build();
}
}
private final InputStream xmlInput;
private final XMLStreamReader reader;
private final Unmarshaller unmarshaller;
private RdeHeader header;
/**
* Creates a new instance of {@link RdeParser}
*
* @param xmlInput Contents of the escrow deposit file
* @throws JAXBException
*/
public RdeParser(InputStream xmlInput) throws XMLStreamException, JAXBException {
this.xmlInput = xmlInput;
this.unmarshaller = JAXBContext.newInstance(JAXB_CONTEXT_PACKAGES).createUnmarshaller();
this.reader = XMLInputFactory.newInstance().createXMLStreamReader(xmlInput);
this.header = new RdeHeader(readHeader());
}
/**
* Attempts to read the RDE header as a jaxb object.
*
* @throws IllegalStateException if no RDE header is found in the file
*/
private XjcRdeHeader readHeader() {
if (!nextElement(RDE_HEADER_URI, "header")) {
throw new IllegalStateException("No RDE Header found");
}
XjcRdeHeaderElement element = (XjcRdeHeaderElement) unmarshalElement(RDE_HEADER_URI, "header");
return element.getValue();
}
/**
* Unmarshals the current element into a Jaxb element.
*
* @param uri Element URI
* @param name Element Name
* @return Jaxb Element
* @throws IllegalStateException if the parser is not at the specified element
*/
private Object unmarshalElement(String uri, String name) {
checkArgumentNotNull(name, "name cannot be null");
checkArgumentNotNull(uri, "uri cannot be null");
try {
if (isAtElement(uri, name)) {
return unmarshaller.unmarshal(reader);
} else {
throw new IllegalStateException(String.format("Not at element %s:%s", uri, name));
}
} catch (IllegalStateException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* Checks if the parser is at an instance of the specified element.
*
* @param uri Element URI
* @param name Element Name
* @return true if the parser is at an instance of the element, false otherwise
*/
private boolean isAtElement(String uri, String name) {
return reader.getEventType() == XMLStreamReader.START_ELEMENT
&& uri.equals(reader.getNamespaceURI()) && name.equals(reader.getName().getLocalPart());
}
/**
* Attempts to advance to the next instance of the specified element.
*
* <p>The parser may skip over other types of elements while advancing to the next instance of the
* specified element.
*
* @param uri Element URI
* @param name Element Name
* @return true if the parser advanced to the element, false otherwise.
*/
private boolean nextElement(String uri, String name) {
try {
while (reader.hasNext()) {
reader.next();
if (isAtElement(uri, name)) {
return true;
}
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return false;
}
public RdeHeader getHeader() {
return header;
}
/**
* Attempts to skip over a number of specified elements.
*
* <p>If the parser is not currently at one of the specified elements, it will advance to the next
* instance before skipping any.
*
* <p>In the process of skipping over a specific type of element, other elements may be skipped as
* well. Elements of types other than that specified by the uri and name will not be counted.
*
* @param numberOfElements Number of elements to skip
* @param uri Element URI
* @param name Element Name
* @return Number of elements that were skipped
*/
private int skipElements(int numberOfElements, String uri, String name) {
checkArgument(
numberOfElements >= 0, "number of elements must be greater than or equal to zero");
// don't do any skipping if numberOfElements is 0
if (numberOfElements == 0) {
return 0;
}
// unless the parser is at one of the specified elements,
// the first call to nextElement() will advance to the first
// element to be skipped
int skipped = 0;
if (isAtElement(uri, name)) {
skipped = 1;
}
while (nextElement(uri, name) && skipped < numberOfElements) {
skipped++;
}
return skipped;
}
/**
* Advances parser to the next contact element.
*
* <p>The parser may skip over other types of elements while advancing to the next contact
* element.
*
* @return true if the parser advanced to a contact element, false otherwise
*/
public boolean nextContact() {
return nextElement(RDE_CONTACT_URI, "contact");
}
/**
* Checks if the parser is at a contact element.
*
* @return true if the parser is at a contact element, false otherwise
*/
public boolean isAtContact() {
return isAtElement(RDE_CONTACT_URI, "contact");
}
/**
* Attempts to skip over a number of contacts.
*
* <p>If the parser is not currently at a contact element, it will advance to the next instance
* before skipping any.
*
* <p>In the process of skipping over a contact element, other elements may be skipped as well.
* Elements of types other than contact elements will not be counted.
*
* @return Number of contact elements that were skipped
*/
public int skipContacts(int numberOfContacts) {
return skipElements(numberOfContacts, RDE_CONTACT_URI, "contact");
}
/**
* Gets the current contact.
*
* <p>The parser must be at a contact element before this method is called, or an
* {@link IllegalStateException} will be thrown. Check the return value of {@link #isAtContact} or
* {@link #nextContact} to determine if the parser is at a contact element.
*
* @return Jaxb Contact
* @throws IllegalStateException if the parser is not at a contact element
*/
public XjcRdeContact getContact() {
XjcRdeContactElement element =
(XjcRdeContactElement) unmarshalElement(RDE_CONTACT_URI, "contact");
return element.getValue();
}
/**
* Advances parser to the next domain element.
*
* <p>The parser may skip over other types of elements while advancing to the next domain element.
*
* @return true if the parser advanced to a domain element, false otherwise
*/
public boolean nextDomain() {
return nextElement(RDE_DOMAIN_URI, "domain");
}
/**
* Checks if the parser is at a domain element.
*
* @return true if the parser is at a domain element, false otherwise
*/
public boolean isAtDomain() {
return isAtElement(RDE_DOMAIN_URI, "domain");
}
/**
* Attempts to skip over a number of domains.
*
* <p>If the parser is not currently at a domain element, it will advance to the next instance
* before skipping any.
*
* <p>In the process of skipping over a domain element, other elements may be skipped as well.
* Elements of types other than domain elements will not be counted.
*
* @return Number of domain elements that were skipped
*/
public int skipDomains(int numberOfDomains) {
return skipElements(numberOfDomains, RDE_DOMAIN_URI, "domain");
}
/**
* Gets the current domain.
*
* <p>The parser must be at a domain element before this method is called, or an
* {@link IllegalStateException} will be thrown. Check the return value of {@link #isAtDomain} or
* {@link #nextDomain} to determine if the parser is at a domain element.
*
* @return Jaxb Domain
* @throws IllegalStateException if the parser is not at a domain element
*/
public XjcRdeDomain getDomain() {
XjcRdeDomainElement element = (XjcRdeDomainElement) unmarshalElement(RDE_DOMAIN_URI, "domain");
return element.getValue();
}
/**
* Advances parser to the next host element.
*
* <p>The parser may skip over other types of elements while advancing to the next host element.
*
* @return true if the parser advanced to a host element, false otherwise
*/
public boolean nextHost() {
return nextElement(RDE_HOST_URI, "host");
}
/**
* Checks if the parser is at a host element.
*
* @return true if the parser is at a host element, false otherwise
*/
public boolean isAtHost() {
return isAtElement(RDE_HOST_URI, "host");
}
/**
* Attempts to skip over a number of hosts.
*
* <p>If the parser is not currently at a host element, it will advance to the next instance
* before skipping any.
*
* <p>In the process of skipping over a host element, other elements may be skipped as well.
* Elements of types other than host elements will not be counted.
*
* @return Number of host elements that were skipped
*/
public int skipHosts(int numberOfHosts) {
return skipElements(numberOfHosts, RDE_HOST_URI, "host");
}
/**
* Gets the current host.
*
* <p>The parser must be at a host element before this method is called, or an
* {@link IllegalStateException} will be thrown. Check the return value of {@link #isAtHost} or
* {@link #nextHost} to determine if the parser is at a host element.
*
* @return Jaxb Host
* @throws IllegalStateException if the parser is not at a host element
*/
public XjcRdeHost getHost() {
XjcRdeHostElement element = (XjcRdeHostElement) unmarshalElement(RDE_HOST_URI, "host");
return element.getValue();
}
/**
* Advances parser to the next registrar element.
*
* <p>The parser may skip over other types of elements while advancing to the next registrar
* element.
*
* @return true if the parser advanced to a registrar element, false otherwise
*/
public boolean nextRegistrar() {
return nextElement(RDE_REGISTRAR_URI, "registrar");
}
/**
* Checks if the parser is at a registrar element.
*
* @return true if the parser is at a registrar element, false otherwise
*/
public boolean isAtRegistrar() {
return isAtElement(RDE_REGISTRAR_URI, "registrar");
}
/**
* Gets the current registrar.
*
* <p>The parser must be at a registrar element before this method is called, or an
* {@link IllegalStateException} will be thrown. Check the return value of {@link #isAtRegistrar}
* or {@link #nextRegistrar} to determine if the parser is at a registrar element.
*
* @return Jaxb Registrar
* @throws IllegalStateException if the parser is not at a registrar element
*/
public XjcRdeRegistrar getRegistrar() {
XjcRdeRegistrarElement element =
(XjcRdeRegistrarElement) unmarshalElement(RDE_REGISTRAR_URI, "registrar");
return element.getValue();
}
/**
* Advances parser to the next IDN element.
*
* <p>The parser may skip over other types of elements while advancing to the next IDN element.
*
* @return true if the parser advanced to a IDN element, false otherwise
*/
public boolean nextIdn() {
return nextElement(RDE_IDN_URI, "idnTableRef");
}
/**
* Checks if the parser is at a IDN element.
*
* @return true if the parser is at a IDN element, false otherwise
*/
public boolean isAtIdn() {
return isAtElement(RDE_IDN_URI, "idnTableRef");
}
/**
* Gets the current IDN.
*
* <p>The parser must be at a IDN element before this method is called, or an
* {@link IllegalStateException} will be thrown. Check the return value of {@link #isAtIdn} or
* {@link #nextIdn} to determine if the parser is at a IDN element.
*
* @return Jaxb IDN
* @throws IllegalStateException if the parser is not at a IDN element
*/
public XjcRdeIdn getIdn() {
XjcRdeIdnElement element = (XjcRdeIdnElement) unmarshalElement(RDE_IDN_URI, "idnTableRef");
return element.getValue();
}
/**
* Advances parser to the next NNDN element.
*
* <p>The parser may skip over other types of elements while advancing to the next NNDN element.
*
* @return true if the parser advanced to a NNDN element, false otherwise
*/
public boolean nextNndn() {
return nextElement(RDE_NNDN_URI, "NNDN");
}
/**
* Checks if the parser is at a NNDN element.
*
* @return true if the parser is at a NNDN element, false otherwise
*/
public boolean isAtNndn() {
return isAtElement(RDE_NNDN_URI, "NNDN");
}
/**
* Gets the current NNDN.
*
* <p>The parser must be at a NNDN element before this method is called, or an
* {@link IllegalStateException} will be thrown. Check the return value of {@link #isAtNndn} or
* {@link #nextNndn} to determine if the parser is at a NNDN element.
*
* @return Jaxb NNDN
* @throws IllegalStateException if the parser is not at a NNDN element
*/
public XjcRdeNndn getNndn() {
XjcRdeNndnElement element = (XjcRdeNndnElement) unmarshalElement(RDE_NNDN_URI, "NNDN");
return element.getValue();
}
/**
* Advances parser to the next eppParams element.
*
* <p>The parser may skip over other types of elements while advancing to the next eppParams
* element.
*
* @return true if the parser advanced to a eppParams element, false otherwise
*/
public boolean nextEppParams() {
return nextElement(RDE_EPP_PARAMS_URI, "eppParams");
}
/**
* Checks if the parser is at a eppParams element.
*
* @return true if the parser is at a eppParams element, false otherwise
*/
public boolean isAtEppParams() {
return isAtElement(RDE_EPP_PARAMS_URI, "eppParams");
}
/**
* Gets the current eppParams.
*
* <p>The parser must be at a eppParams element before this method is called, or an
* {@link IllegalStateException} will be thrown. Check the return value of {@link #isAtEppParams}
* or {@link #nextEppParams} to determine if the parser is at a eppParams element.
*
* @return Jaxb EppParams
* @throws IllegalStateException if the parser is not at a eppParams element
*/
public XjcRdeEppParams getEppParams() {
XjcRdeEppParamsElement element =
(XjcRdeEppParamsElement) unmarshalElement(RDE_EPP_PARAMS_URI, "eppParams");
return element.getValue();
}
/**
* Closes the underlying InputStream
*
* @throws IOException if the underlying stream throws {@link IOException} on close.
*/
@Override
public void close() throws IOException {
xmlInput.close();
}
}

23
java/google/registry/rde/imports/ResourceExistsException.java
View file

@ -1,23 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
/**
* Indicates that a resource already exists and was not imported.
*/
public class ResourceExistsException extends RuntimeException {
private static final long serialVersionUID = -9180381693364904061L;
}

200
java/google/registry/rde/imports/XjcToContactResourceConverter.java
View file

@ -1,200 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static com.google.common.base.Predicates.equalTo;
import static com.google.common.base.Predicates.not;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.rde.imports.RdeImportUtils.generateTridForImport;
import com.google.common.collect.ImmutableList;
import com.googlecode.objectify.Key;
import google.registry.model.contact.ContactAddress;
import google.registry.model.contact.ContactPhoneNumber;
import google.registry.model.contact.ContactResource;
import google.registry.model.contact.Disclose;
import google.registry.model.contact.Disclose.PostalInfoChoice;
import google.registry.model.contact.PostalInfo;
import google.registry.model.eppcommon.StatusValue;
import google.registry.model.reporting.HistoryEntry;
import google.registry.model.transfer.TransferData;
import google.registry.model.transfer.TransferStatus;
import google.registry.util.XmlToEnumMapper;
import google.registry.xjc.contact.XjcContactAddrType;
import google.registry.xjc.contact.XjcContactDiscloseType;
import google.registry.xjc.contact.XjcContactE164Type;
import google.registry.xjc.contact.XjcContactIntLocType;
import google.registry.xjc.contact.XjcContactPostalInfoEnumType;
import google.registry.xjc.contact.XjcContactPostalInfoType;
import google.registry.xjc.contact.XjcContactStatusType;
import google.registry.xjc.rdecontact.XjcRdeContact;
import google.registry.xjc.rdecontact.XjcRdeContactElement;
import google.registry.xjc.rdecontact.XjcRdeContactTransferDataType;
import javax.annotation.Nullable;
/** Utility class that converts an {@link XjcRdeContact} into a {@link ContactResource}. */
final class XjcToContactResourceConverter extends XjcToEppResourceConverter {
private static final XmlToEnumMapper<PostalInfo.Type> POSTAL_INFO_TYPE_MAPPER =
XmlToEnumMapper.create(PostalInfo.Type.values());
private static final XmlToEnumMapper<TransferStatus> TRANSFER_STATUS_MAPPER =
XmlToEnumMapper.create(TransferStatus.values());
/** Converts {@link XjcRdeContact} to {@link ContactResource}. */
static ContactResource convertContact(XjcRdeContact contact) {
ofy().save().entity(
new HistoryEntry.Builder()
.setType(HistoryEntry.Type.RDE_IMPORT)
.setClientId(contact.getClID())
.setTrid(generateTridForImport())
.setModificationTime(ofy().getTransactionTime())
.setXmlBytes(getObjectXml(new XjcRdeContactElement(contact)))
.setBySuperuser(true)
.setReason("RDE Import")
.setRequestedByRegistrar(false)
.setParent(Key.create(null, ContactResource.class, contact.getRoid()))
.build());
return new ContactResource.Builder()
.setRepoId(contact.getRoid())
.setStatusValues(
contact
.getStatuses()
.stream()
.map(XjcToContactResourceConverter::convertStatusValue)
.filter(not(equalTo(StatusValue.LINKED)))
.collect(toImmutableSet()))
.setLocalizedPostalInfo(
getPostalInfoOfType(contact.getPostalInfos(), XjcContactPostalInfoEnumType.LOC))
.setInternationalizedPostalInfo(
getPostalInfoOfType(contact.getPostalInfos(), XjcContactPostalInfoEnumType.INT))
.setContactId(contact.getId())
.setPersistedCurrentSponsorClientId(contact.getClID())
.setCreationClientId(contact.getCrRr() == null ? null : contact.getCrRr().getValue())
.setLastEppUpdateClientId(contact.getUpRr() == null ? null : contact.getUpRr().getValue())
.setCreationTime(contact.getCrDate())
.setLastEppUpdateTime(contact.getUpDate())
.setLastTransferTime(contact.getTrDate())
.setVoiceNumber(convertPhoneNumber(contact.getVoice()))
.setFaxNumber(convertPhoneNumber(contact.getFax()))
.setEmailAddress(contact.getEmail())
.setDisclose(convertDisclose(contact.getDisclose()))
.setTransferData(convertTransferData(contact.getTrnData()))
.build();
}
/**
* Extracts a {@link PostalInfo} from an {@link Iterable} of {@link XjcContactPostalInfoEnumType}.
*/
@Nullable
private static PostalInfo getPostalInfoOfType(
Iterable<XjcContactPostalInfoType> postalInfos, XjcContactPostalInfoEnumType type) {
for (XjcContactPostalInfoType postalInfo : postalInfos) {
if (postalInfo.getType() == type) {
return convertPostalInfo(postalInfo);
}
}
return null;
}
/** Converts {@link XjcRdeContactTransferDataType} to {@link TransferData}. */
private static TransferData convertTransferData(
@Nullable XjcRdeContactTransferDataType transferData) {
if (transferData == null) {
return TransferData.EMPTY;
}
return new TransferData.Builder()
.setTransferStatus(TRANSFER_STATUS_MAPPER.xmlToEnum(transferData.getTrStatus().value()))
.setGainingClientId(transferData.getReRr().getValue())
.setLosingClientId(transferData.getAcRr().getValue())
.setTransferRequestTime(transferData.getReDate())
.setPendingTransferExpirationTime(transferData.getAcDate())
.build();
}
/** Converts {@link XjcContactAddrType} to {@link ContactAddress}. */
private static ContactAddress convertAddress(XjcContactAddrType address) {
return new ContactAddress.Builder()
.setStreet(ImmutableList.copyOf(address.getStreets()))
.setCity(address.getCity())
.setState(address.getSp())
.setZip(address.getPc())
.setCountryCode(address.getCc())
.build();
}
/** Converts {@link XjcContactDiscloseType} to {@link Disclose}. */
@Nullable
private static Disclose convertDisclose(@Nullable XjcContactDiscloseType disclose) {
if (disclose == null) {
return null;
}
return new Disclose.Builder()
.setFlag(disclose.isFlag())
.setNames(
disclose
.getNames()
.stream()
.map(XjcToContactResourceConverter::convertPostalInfoChoice)
.collect(toImmutableList()))
.setOrgs(
disclose
.getOrgs()
.stream()
.map(XjcToContactResourceConverter::convertPostalInfoChoice)
.collect(toImmutableList()))
.setAddrs(
disclose
.getAddrs()
.stream()
.map(XjcToContactResourceConverter::convertPostalInfoChoice)
.collect(toImmutableList()))
.build();
}
/** Converts {@link XjcContactE164Type} to {@link ContactPhoneNumber}. */
@Nullable
private static ContactPhoneNumber convertPhoneNumber(@Nullable XjcContactE164Type phoneNumber) {
if (phoneNumber == null) {
return null;
}
return new ContactPhoneNumber.Builder()
.setPhoneNumber(phoneNumber.getValue())
.setExtension(phoneNumber.getX())
.build();
}
/** Converts {@link PostalInfoChoice} to {@link XjcContactIntLocType}. */
private static PostalInfoChoice convertPostalInfoChoice(XjcContactIntLocType choice) {
return PostalInfoChoice.create(POSTAL_INFO_TYPE_MAPPER.xmlToEnum(choice.getType().value()));
}
/** Converts {@link XjcContactPostalInfoType} to {@link PostalInfo}. */
private static PostalInfo convertPostalInfo(XjcContactPostalInfoType postalInfo) {
return new PostalInfo.Builder()
.setName(postalInfo.getName())
.setOrg(postalInfo.getOrg())
.setAddress(convertAddress(postalInfo.getAddr()))
.setType(POSTAL_INFO_TYPE_MAPPER.xmlToEnum(postalInfo.getType().value()))
.build();
}
/** Converts {@link XjcContactStatusType} to {@link StatusValue}. */
private static StatusValue convertStatusValue(XjcContactStatusType statusType) {
return StatusValue.fromXmlName(statusType.getS().value());
}
private XjcToContactResourceConverter() {}
}

255
java/google/registry/rde/imports/XjcToDomainBaseConverter.java
View file

@ -1,255 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
import static org.joda.time.DateTimeZone.UTC;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableSet;
import com.google.common.net.InternetDomainName;
import com.googlecode.objectify.Key;
import google.registry.model.billing.BillingEvent;
import google.registry.model.contact.ContactResource;
import google.registry.model.domain.DesignatedContact;
import google.registry.model.domain.DomainAuthInfo;
import google.registry.model.domain.DomainBase;
import google.registry.model.domain.GracePeriod;
import google.registry.model.domain.rgp.GracePeriodStatus;
import google.registry.model.domain.secdns.DelegationSignerData;
import google.registry.model.eppcommon.AuthInfo.PasswordAuth;
import google.registry.model.eppcommon.StatusValue;
import google.registry.model.host.HostResource;
import google.registry.model.index.ForeignKeyIndex;
import google.registry.model.poll.PollMessage;
import google.registry.model.registry.Registries;
import google.registry.model.registry.Registry;
import google.registry.model.transfer.TransferData;
import google.registry.model.transfer.TransferStatus;
import google.registry.util.StringGenerator;
import google.registry.util.XmlToEnumMapper;
import google.registry.xjc.domain.XjcDomainContactType;
import google.registry.xjc.domain.XjcDomainNsType;
import google.registry.xjc.domain.XjcDomainStatusType;
import google.registry.xjc.rdedomain.XjcRdeDomain;
import google.registry.xjc.rdedomain.XjcRdeDomainElement;
import google.registry.xjc.rdedomain.XjcRdeDomainTransferDataType;
import google.registry.xjc.rgp.XjcRgpStatusType;
import google.registry.xjc.secdns.XjcSecdnsDsDataType;
import java.util.function.Function;
import org.joda.time.DateTime;
/** Utility class that converts an {@link XjcRdeDomainElement} into a {@link DomainBase}. */
final class XjcToDomainBaseConverter extends XjcToEppResourceConverter {
private static final XmlToEnumMapper<TransferStatus> TRANSFER_STATUS_MAPPER =
XmlToEnumMapper.create(TransferStatus.values());
private static final Function<String, Key<HostResource>> HOST_OBJ_CONVERTER =
fullyQualifiedHostName -> {
// host names are always lower case
fullyQualifiedHostName = canonicalizeDomainName(fullyQualifiedHostName);
Key<HostResource> key =
ForeignKeyIndex.loadAndGetKey(
HostResource.class, fullyQualifiedHostName, DateTime.now(UTC));
checkState(
key != null,
String.format("HostResource not found with name '%s'", fullyQualifiedHostName));
return key;
};
/** Converts {@link XjcRgpStatusType} to {@link GracePeriod} */
private static class GracePeriodConverter implements Function<XjcRgpStatusType, GracePeriod> {
private final XjcRdeDomain domain;
private final Key<BillingEvent.Recurring> autoRenewBillingEvent;
private final Registry tld;
GracePeriodConverter(XjcRdeDomain domain, Key<BillingEvent.Recurring> autoRenewBillingEvent) {
this.domain = domain;
this.autoRenewBillingEvent = autoRenewBillingEvent;
this.tld =
Registry.get(
Registries.findTldForNameOrThrow(InternetDomainName.from(domain.getName()))
.toString());
}
@Override
public GracePeriod apply(XjcRgpStatusType gracePeriodStatus) {
switch (gracePeriodStatus.getS()) {
case ADD_PERIOD:
return GracePeriod.createWithoutBillingEvent(
GracePeriodStatus.ADD,
domain.getCrDate().plus(this.tld.getAddGracePeriodLength()),
domain.getCrRr().getValue());
case AUTO_RENEW_PERIOD:
return GracePeriod.createForRecurring(
GracePeriodStatus.AUTO_RENEW,
domain.getUpDate().plus(this.tld.getAutoRenewGracePeriodLength()),
domain.getClID(),
autoRenewBillingEvent);
case PENDING_DELETE:
return GracePeriod.createWithoutBillingEvent(
GracePeriodStatus.PENDING_DELETE,
domain.getUpDate().plus(this.tld.getPendingDeleteLength()),
domain.getClID());
case REDEMPTION_PERIOD:
return GracePeriod.createWithoutBillingEvent(
GracePeriodStatus.REDEMPTION,
domain.getUpDate().plus(this.tld.getRedemptionGracePeriodLength()),
domain.getClID());
case RENEW_PERIOD:
return GracePeriod.createWithoutBillingEvent(
GracePeriodStatus.RENEW,
domain.getUpDate().plus(this.tld.getRenewGracePeriodLength()),
domain.getClID());
case TRANSFER_PERIOD:
return GracePeriod.createWithoutBillingEvent(
GracePeriodStatus.TRANSFER,
domain.getUpDate().plus(this.tld.getTransferGracePeriodLength()),
domain.getTrnData().getReRr().getValue());
default:
throw new IllegalArgumentException(
"Unsupported grace period status: " + gracePeriodStatus.getS());
}
}
}
/** Converts {@link XjcRdeDomain} to {@link DomainBase}. */
static DomainBase convertDomain(
XjcRdeDomain domain,
BillingEvent.Recurring autoRenewBillingEvent,
PollMessage.Autorenew autoRenewPollMessage,
StringGenerator stringGenerator) {
GracePeriodConverter gracePeriodConverter =
new GracePeriodConverter(domain, Key.create(autoRenewBillingEvent));
DomainBase.Builder builder =
new DomainBase.Builder()
.setFullyQualifiedDomainName(canonicalizeDomainName(domain.getName()))
.setRepoId(domain.getRoid())
.setIdnTableName(domain.getIdnTableId())
.setPersistedCurrentSponsorClientId(domain.getClID())
.setCreationClientId(domain.getCrRr().getValue())
.setCreationTime(domain.getCrDate())
.setAutorenewPollMessage(Key.create(autoRenewPollMessage))
.setAutorenewBillingEvent(Key.create(autoRenewBillingEvent))
.setRegistrationExpirationTime(domain.getExDate())
.setLastEppUpdateTime(domain.getUpDate())
.setLastEppUpdateClientId(domain.getUpRr() == null ? null : domain.getUpRr().getValue())
.setLastTransferTime(domain.getTrDate())
.setStatusValues(
domain
.getStatuses()
.stream()
.map(XjcToDomainBaseConverter::convertStatusType)
.collect(toImmutableSet()))
.setNameservers(convertNameservers(domain.getNs()))
.setGracePeriods(
domain
.getRgpStatuses()
.stream()
.map(gracePeriodConverter)
.collect(toImmutableSet()))
.setContacts(
domain
.getContacts()
.stream()
.map(XjcToDomainBaseConverter::convertContactType)
.collect(toImmutableSet()))
.setDsData(
domain.getSecDNS() == null
? ImmutableSet.of()
: domain
.getSecDNS()
.getDsDatas()
.stream()
.map(XjcToDomainBaseConverter::convertSecdnsDsDataType)
.collect(toImmutableSet()))
.setTransferData(convertDomainTransferData(domain.getTrnData()))
// authInfo pw must be a token between 6 and 16 characters in length
// generate a token of 16 characters as the default authInfo pw
.setAuthInfo(
DomainAuthInfo.create(
PasswordAuth.create(stringGenerator.createString(16), domain.getRoid())));
checkArgumentNotNull(
domain.getRegistrant(), "Registrant is missing for domain '%s'", domain.getName());
builder = builder.setRegistrant(convertRegistrant(domain.getRegistrant()));
return builder.build();
}
/** Returns {@link Key} for registrant from foreign key */
private static Key<ContactResource> convertRegistrant(String contactId) {
Key<ContactResource> key =
ForeignKeyIndex.loadAndGetKey(ContactResource.class, contactId, DateTime.now(UTC));
checkState(key != null, "Registrant not found: '%s'", contactId);
return key;
}
/** Converts {@link XjcDomainNsType} to <code>ImmutableSet<Key<HostResource>></code>. */
private static ImmutableSet<Key<HostResource>> convertNameservers(XjcDomainNsType ns) {
// If no hosts are specified, return an empty set
if (ns == null || (ns.getHostAttrs() == null && ns.getHostObjs() == null)) {
return ImmutableSet.of();
}
// Domain linked hosts must be specified by host object, not host attributes.
checkArgument(
ns.getHostAttrs() == null || ns.getHostAttrs().isEmpty(),
"Host attributes are not yet supported");
return ns.getHostObjs().stream().map(HOST_OBJ_CONVERTER).collect(toImmutableSet());
}
/** Converts {@link XjcRdeDomainTransferDataType} to {@link TransferData}. */
private static TransferData convertDomainTransferData(XjcRdeDomainTransferDataType data) {
if (data == null) {
return TransferData.EMPTY;
}
return new TransferData.Builder()
.setTransferStatus(TRANSFER_STATUS_MAPPER.xmlToEnum(data.getTrStatus().value()))
.setGainingClientId(data.getReRr().getValue())
.setLosingClientId(data.getAcRr().getValue())
.setTransferRequestTime(data.getReDate())
.setPendingTransferExpirationTime(data.getAcDate())
.setTransferredRegistrationExpirationTime(data.getExDate())
.build();
}
/** Converts {@link XjcDomainStatusType} to {@link StatusValue}. */
private static StatusValue convertStatusType(XjcDomainStatusType type) {
return StatusValue.fromXmlName(type.getS().value());
}
/** Converts {@link XjcSecdnsDsDataType} to {@link DelegationSignerData}. */
private static DelegationSignerData convertSecdnsDsDataType(XjcSecdnsDsDataType secdns) {
return DelegationSignerData.create(
secdns.getKeyTag(), secdns.getAlg(), secdns.getDigestType(), secdns.getDigest());
}
/** Converts {@link XjcDomainContactType} to {@link DesignatedContact}. */
private static DesignatedContact convertContactType(XjcDomainContactType contact) {
String contactId = contact.getValue();
Key<ContactResource> key =
ForeignKeyIndex.loadAndGetKey(ContactResource.class, contactId, DateTime.now(UTC));
checkState(key != null, "Contact not found: '%s'", contactId);
DesignatedContact.Type type =
DesignatedContact.Type.valueOf(Ascii.toUpperCase(contact.getType().toString()));
return DesignatedContact.create(type, key);
}
private XjcToDomainBaseConverter() {}
}

38
java/google/registry/rde/imports/XjcToEppResourceConverter.java
View file

@ -1,38 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static java.nio.charset.StandardCharsets.UTF_8;
import google.registry.model.EppResource;
import google.registry.xjc.XjcXmlTransformer;
import google.registry.xml.XmlException;
import java.io.ByteArrayOutputStream;
/**
* Base class for Jaxb object to {@link EppResource} converters
*/
public abstract class XjcToEppResourceConverter {
protected static byte[] getObjectXml(Object jaxbElement) {
try {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
XjcXmlTransformer.marshalLenient(jaxbElement, bout, UTF_8);
return bout.toByteArray();
} catch (XmlException e) {
throw new RuntimeException(e);
}
}
}

89
java/google/registry/rde/imports/XjcToHostResourceConverter.java
View file

@ -1,89 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static com.google.common.base.Predicates.in;
import static com.google.common.base.Predicates.not;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.rde.imports.RdeImportUtils.generateTridForImport;
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import com.google.common.collect.ImmutableSet;
import com.google.common.net.InetAddresses;
import com.googlecode.objectify.Key;
import google.registry.model.eppcommon.StatusValue;
import google.registry.model.host.HostResource;
import google.registry.model.reporting.HistoryEntry;
import google.registry.xjc.host.XjcHostAddrType;
import google.registry.xjc.host.XjcHostStatusType;
import google.registry.xjc.rdehost.XjcRdeHost;
import google.registry.xjc.rdehost.XjcRdeHostElement;
import java.net.InetAddress;
/** Utility class that converts an {@link XjcRdeHost} into a {@link HostResource}. */
public class XjcToHostResourceConverter extends XjcToEppResourceConverter {
static HostResource convert(XjcRdeHost host) {
// TODO(b/35384052): Handle subordinate hosts correctly by setting superordinateDomaina and
// lastSuperordinateChange fields.
// First create and save history entry
ofy().save().entity(
new HistoryEntry.Builder()
.setType(HistoryEntry.Type.RDE_IMPORT)
.setClientId(host.getClID())
.setTrid(generateTridForImport())
.setModificationTime(ofy().getTransactionTime())
.setXmlBytes(getObjectXml(new XjcRdeHostElement(host)))
.setBySuperuser(true)
.setReason("RDE Import")
.setRequestedByRegistrar(false)
.setParent(Key.create(null, HostResource.class, host.getRoid()))
.build());
return new HostResource.Builder()
.setFullyQualifiedHostName(canonicalizeDomainName(host.getName()))
.setRepoId(host.getRoid())
.setPersistedCurrentSponsorClientId(host.getClID())
.setLastTransferTime(host.getTrDate())
.setCreationTime(host.getCrDate())
.setLastEppUpdateTime(host.getUpDate())
.setCreationClientId(host.getCrRr().getValue())
.setLastEppUpdateClientId(host.getUpRr() == null ? null : host.getUpRr().getValue())
.setStatusValues(
host.getStatuses()
.stream()
.map(XjcToHostResourceConverter::convertStatusType)
.filter(not(in(ImmutableSet.of(StatusValue.LINKED, StatusValue.PENDING_TRANSFER))))
.collect(toImmutableSet()))
.setInetAddresses(
host.getAddrs()
.stream()
.map(XjcToHostResourceConverter::convertAddrType)
.collect(toImmutableSet()))
.build();
}
/** Converts {@link XjcHostStatusType} to {@link StatusValue}. */
private static StatusValue convertStatusType(XjcHostStatusType status) {
return StatusValue.fromXmlName(status.getS().value());
}
/** Converts {@link XjcHostAddrType} to {@link InetAddress}. */
private static InetAddress convertAddrType(XjcHostAddrType addr) {
return InetAddresses.forString(addr.getValue());
}
private XjcToHostResourceConverter() {}
}