Add host linker mapreduce for RDE imports

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=147025088
This commit is contained in:
Wolfgang Meyers 2017-02-09 05:49:24 -08:00 committed by Ben McIlwain
parent 6cdb3d81d3
commit da1f83a7b4
15 changed files with 353 additions and 29 deletions

View file

@ -281,6 +281,12 @@
<url-pattern>/_dr/task/importRdeDomains</url-pattern>
</servlet-mapping>
<!-- Mapreduce to link hosts from escrow file to superordinate domains -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>
<url-pattern>/_dr/task/linkRdeHosts</url-pattern>
</servlet-mapping>
<!-- Security config -->
<security-constraint>
<web-resource-collection>

View file

@ -61,6 +61,7 @@ import google.registry.rde.RdeUploadAction;
import google.registry.rde.imports.RdeContactImportAction;
import google.registry.rde.imports.RdeDomainImportAction;
import google.registry.rde.imports.RdeHostImportAction;
import google.registry.rde.imports.RdeHostLinkAction;
import google.registry.rde.imports.RdeImportsModule;
import google.registry.request.RequestComponentBuilder;
import google.registry.request.RequestModule;
@ -117,6 +118,7 @@ interface BackendRequestComponent {
RdeContactImportAction rdeContactImportAction();
RdeDomainImportAction rdeDomainImportAction();
RdeHostImportAction rdeHostImportAction();
RdeHostLinkAction rdeHostLinkAction();
RdeReportAction rdeReportAction();
RdeStagingAction rdeStagingAction();
RdeUploadAction rdeUploadAction();

View file

@ -148,19 +148,13 @@ public class RdeContactImportAction implements Runnable {
logger.infofmt("Contact %s was imported successfully", xjcContact.getId());
} catch (ResourceExistsException e) {
// Record the number of contacts already in the registry
getContext().incrementCounter("contacts skipped");
getContext().incrementCounter("existing contacts skipped");
logger.infofmt("Contact %s already exists", xjcContact.getId());
} catch (Exception e) {
// Record the number of contacts with unexpected errors
getContext().incrementCounter("contact import errors");
throw new ContactImportException(xjcContact.getId(), xjcContact.toString(), e);
logger.severefmt(e, "Error importing contact %s; xml=%s", xjcContact.getId(), xjcContact);
}
}
}
private static class ContactImportException extends RuntimeException {
ContactImportException(String contactId, String xml, Throwable cause) {
super(String.format("Error importing contact %s; xml=%s", contactId, xml), cause);
}
}
}

View file

@ -163,18 +163,12 @@ public class RdeDomainImportAction implements Runnable {
logger.infofmt("Domain %s was imported successfully", xjcDomain.getName());
} catch (ResourceExistsException e) {
// Record the number of domains already in the registry
getContext().incrementCounter("domains skipped");
getContext().incrementCounter("existing domains skipped");
logger.infofmt("Domain %s already exists", xjcDomain.getName());
} catch (Exception e) {
getContext().incrementCounter("domain import errors");
throw new DomainImportException(xjcDomain.getName(), xjcDomain.toString(), e);
logger.severefmt(e, "Error processing domain %s; xml=%s", xjcDomain.getName(), xjcDomain);
}
}
}
private static class DomainImportException extends RuntimeException {
DomainImportException(String domainName, String xml, Throwable cause) {
super(String.format("Error processing domain %s; xml=%s", domainName, xml), cause);
}
}
}

View file

@ -117,7 +117,6 @@ public class RdeHostImportAction implements Runnable {
public void map(JaxbFragment<XjcRdeHostElement> fragment) {
final XjcRdeHost xjcHost = fragment.getInstance().getValue();
try {
logger.infofmt("Converting xml for host %s", xjcHost.getName());
// Record number of attempted map operations
getContext().incrementCounter("host imports attempted");
logger.infofmt("Saving host %s", xjcHost.getName());
@ -134,19 +133,13 @@ public class RdeHostImportAction implements Runnable {
logger.infofmt("Host %s was imported successfully", xjcHost.getName());
} catch (ResourceExistsException e) {
// Record the number of hosts already in the registry
getContext().incrementCounter("hosts skipped");
getContext().incrementCounter("existing hosts skipped");
logger.infofmt("Host %s already exists", xjcHost.getName());
} catch (Exception e) {
// Record the number of hosts with unexpected errors
getContext().incrementCounter("host import errors");
throw new HostImportException(xjcHost.getName(), xjcHost.toString(), e);
logger.severefmt(e, "Error processing host %s; xml=%s", xjcHost.getName(), xjcHost);
}
}
}
private static class HostImportException extends RuntimeException {
HostImportException(String hostName, String xml, Throwable cause) {
super(String.format("Error processing host %s; xml=%s", hostName, xml), cause);
}
}
}

View file

@ -0,0 +1,168 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static com.google.common.base.Preconditions.checkState;
import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
import static google.registry.model.EppResourceUtils.loadByForeignKey;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.registry.Registries.findTldForName;
import static google.registry.util.PipelineUtils.createJobPath;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.net.InternetDomainName;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.VoidWork;
import google.registry.config.RegistryConfig.Config;
import google.registry.mapreduce.MapreduceRunner;
import google.registry.model.domain.DomainResource;
import google.registry.model.host.HostResource;
import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.util.FormattingLogger;
import google.registry.xjc.JaxbFragment;
import google.registry.xjc.rdehost.XjcRdeHost;
import google.registry.xjc.rdehost.XjcRdeHostElement;
import javax.inject.Inject;
import org.joda.time.DateTime;
/**
* A mapreduce that links hosts from an escrow file to their superordinate domains.
*
* <p>This mapreduce is run as the last step of the process of importing escrow files. For each host
* in the escrow file, the corresponding {@link HostResource} record in the datastore is linked to
* its superordinate {@link DomainResource} only if it is an in-zone host. This is necessary because
* all hosts must exist before domains can be imported, due to references in host objects, and
* domains must exist before hosts can be linked to their superordinate domains.
*
* <p>Specify the escrow file to import with the "path" parameter.
*/
@Action(path = "/_dr/task/linkRdeHosts")
public class RdeHostLinkAction implements Runnable {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
private final MapreduceRunner mrRunner;
private final Response response;
private final String importBucketName;
private final String importFileName;
private final Optional<Integer> mapShards;
@Inject
public RdeHostLinkAction(
MapreduceRunner mrRunner,
Response response,
@Config("rdeImportBucket") String importBucketName,
@Parameter("path") String importFileName,
@Parameter(PARAM_MAP_SHARDS) Optional<Integer> mapShards) {
this.mrRunner = mrRunner;
this.response = response;
this.importBucketName = importBucketName;
this.importFileName = importFileName;
this.mapShards = mapShards;
}
@Override
public void run() {
response.sendJavaScriptRedirect(createJobPath(mrRunner
.setJobName("Link hosts from escrow file")
.setModuleName("backend")
.runMapOnly(
new RdeHostPostImportMapper(),
ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName)))));
}
/** Mapper to link hosts from an escrow file to their superordinate domains. */
public static class RdeHostPostImportMapper
extends Mapper<JaxbFragment<XjcRdeHostElement>, Void, Void> {
private static final long serialVersionUID = -2898753709127134419L;
@Override
public void map(JaxbFragment<XjcRdeHostElement> fragment) {
// Record number of attempted map operations
getContext().incrementCounter("post-import hosts read");
final XjcRdeHost xjcHost = fragment.getInstance().getValue();
logger.infofmt("Attempting to link superordinate domain for host %s", xjcHost.getName());
try {
InternetDomainName hostName = InternetDomainName.from(xjcHost.getName());
Optional<DomainResource> superordinateDomain =
lookupSuperordinateDomain(hostName, DateTime.now());
// if suporordinateDomain is null, this is an out of zone host and can't be linked
if (!superordinateDomain.isPresent()) {
getContext().incrementCounter("post-import hosts out of zone");
logger.infofmt("Host %s is out of zone", xjcHost.getName());
return;
}
// at this point, the host is definitely in zone and should be linked
getContext().incrementCounter("post-import hosts in zone");
final Key<DomainResource> superordinateDomainKey = Key.create(superordinateDomain.get());
ofy().transact(new VoidWork() {
@Override
public void vrun() {
HostResource host =
ofy().load().now(Key.create(HostResource.class, xjcHost.getRoid()));
ofy().save()
.entity(host.asBuilder().setSuperordinateDomain(superordinateDomainKey).build());
}
});
logger.infofmt(
"Successfully linked host %s to superordinate domain %s",
xjcHost.getName(),
superordinateDomain.get().getFullyQualifiedDomainName());
// Record number of hosts successfully linked
getContext().incrementCounter("post-import hosts linked");
} catch (Exception e) {
// Record the number of hosts with unexpected errors
getContext().incrementCounter("post-import host errors");
throw new HostLinkException(xjcHost.getName(), xjcHost.toString(), e);
}
}
/**
* Return the {@link DomainResource} this host is subordinate to, or absent for out of zone
* hosts.
*
* @throws IllegalStateException for hosts without superordinate domains
*/
private static Optional<DomainResource> lookupSuperordinateDomain(
InternetDomainName hostName, DateTime now) {
Optional<InternetDomainName> tld = findTldForName(hostName);
// out of zone hosts cannot be linked
if (!tld.isPresent()) {
return Optional.absent();
}
// This is a subordinate host
String domainName = Joiner.on('.').join(Iterables.skip(
hostName.parts(), hostName.parts().size() - (tld.get().parts().size() + 1)));
DomainResource superordinateDomain = loadByForeignKey(DomainResource.class, domainName, now);
// Hosts can't be linked if domains import hasn't been run
checkState(
superordinateDomain != null, "Superordinate domain does not exist: %s", domainName);
return Optional.of(superordinateDomain);
}
}
private static class HostLinkException extends RuntimeException {
HostLinkException(String hostname, String xml, Throwable cause) {
super(String.format("Error linking host %s; xml=%s", hostname, xml), cause);
}
}
}

View file

@ -20,6 +20,7 @@ import static com.google.common.collect.Iterables.transform;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.rde.imports.RdeImportUtils.generateTridForImport;
import static google.registry.util.DateTimeUtils.END_OF_TIME;
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
import com.google.common.base.Ascii;
@ -97,6 +98,8 @@ final class XjcToDomainResourceConverter extends XjcToEppResourceConverter {
new Function<String, Key<HostResource>>() {
@Override
public Key<HostResource> apply(String fullyQualifiedHostName) {
// host names are always lower case
fullyQualifiedHostName = canonicalizeDomainName(fullyQualifiedHostName);
Key<HostResource> key =
ForeignKeyIndex.loadAndGetKey(
HostResource.class, fullyQualifiedHostName, DateTime.now());
@ -193,7 +196,7 @@ final class XjcToDomainResourceConverter extends XjcToEppResourceConverter {
new GracePeriodConverter(domain, Key.create(autoRenewBillingEvent));
DomainResource.Builder builder =
new DomainResource.Builder()
.setFullyQualifiedDomainName(domain.getName())
.setFullyQualifiedDomainName(canonicalizeDomainName(domain.getName()))
.setRepoId(domain.getRoid())
.setIdnTableName(domain.getIdnTableId())
.setCurrentSponsorClientId(domain.getClID())

View file

@ -18,6 +18,7 @@ import static com.google.common.base.Predicates.equalTo;
import static com.google.common.base.Predicates.not;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.rde.imports.RdeImportUtils.generateTridForImport;
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import com.google.common.base.Function;
import com.google.common.collect.FluentIterable;
@ -69,7 +70,7 @@ public class XjcToHostResourceConverter extends XjcToEppResourceConverter {
.setParent(Key.create(null, HostResource.class, host.getRoid()))
.build());
return new HostResource.Builder()
.setFullyQualifiedHostName(host.getName())
.setFullyQualifiedHostName(canonicalizeDomainName(host.getName()))
.setRepoId(host.getRoid())
.setCurrentSponsorClientId(host.getClID())
.setLastTransferTime(host.getTrDate())

View file

@ -16,6 +16,7 @@ package google.registry.rde.imports;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.testing.DatastoreHelper.getHistoryEntries;
import static google.registry.testing.DatastoreHelper.newHostResource;
import static google.registry.testing.DatastoreHelper.persistResource;
@ -66,6 +67,7 @@ public class RdeHostImportActionTest extends MapreduceTestCase<RdeHostImportActi
@Before
public void before() throws Exception {
createTld("test");
response = new FakeResponse();
mrRunner = new MapreduceRunner(Optional.<Integer>absent(), Optional.<Integer>absent());
action = new RdeHostImportAction(

View file

@ -0,0 +1,108 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde.imports;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.testing.DatastoreHelper.newHostResource;
import static google.registry.testing.DatastoreHelper.persistActiveDomain;
import static google.registry.testing.DatastoreHelper.persistResource;
import static google.registry.util.DateTimeUtils.END_OF_TIME;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.common.base.Optional;
import com.google.common.io.ByteSource;
import com.google.common.io.ByteStreams;
import com.googlecode.objectify.Key;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.gcs.GcsUtils;
import google.registry.mapreduce.MapreduceRunner;
import google.registry.model.domain.DomainResource;
import google.registry.model.host.HostResource;
import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
import google.registry.request.Response;
import google.registry.testing.FakeResponse;
import google.registry.testing.mapreduce.MapreduceTestCase;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link RdeHostLinkAction}. */
@RunWith(JUnit4.class)
public class RdeHostLinkActionTest extends MapreduceTestCase<RdeHostLinkAction> {
private static final ByteSource DEPOSIT_1_HOST = RdeImportsTestData.get("deposit_1_host.xml");
private static final String IMPORT_BUCKET_NAME = "import-bucket";
private static final String IMPORT_FILE_NAME = "escrow-file.xml";
private static final GcsService GCS_SERVICE =
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
private MapreduceRunner mrRunner;
private Response response;
private final Optional<Integer> mapShards = Optional.absent();
@Before
public void before() throws Exception {
createTld("test");
response = new FakeResponse();
mrRunner = new MapreduceRunner(Optional.<Integer>absent(), Optional.<Integer>absent());
action =
new RdeHostLinkAction(mrRunner, response, IMPORT_BUCKET_NAME, IMPORT_FILE_NAME, mapShards);
}
@Test
public void test_mapreduceSuccessfullyLinksHost() throws Exception {
// Create host and domain first
persistResource(
newHostResource("ns1.example1.test")
.asBuilder()
.setRepoId("Hns1_example1_test-TEST")
.build());
DomainResource superordinateDomain = persistActiveDomain("example1.test");
ForeignKeyDomainIndex.create(superordinateDomain, END_OF_TIME);
Key<DomainResource> superOrdinateDomainKey = Key.create(superordinateDomain);
pushToGcs(DEPOSIT_1_HOST);
runMapreduce();
List<HostResource> hosts = ofy().load().type(HostResource.class).list();
assertThat(hosts).hasSize(1);
assertThat(hosts.get(0).getSuperordinateDomain()).isEqualTo(superOrdinateDomainKey);
}
private void runMapreduce() throws Exception {
action.run();
executeTasksUntilEmpty("mapreduce");
}
private static void pushToGcs(ByteSource source) throws IOException {
try (OutputStream outStream =
new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize())
.openOutputStream(new GcsFilename(IMPORT_BUCKET_NAME, IMPORT_FILE_NAME));
InputStream inStream = source.openStream()) {
ByteStreams.copy(inStream, outStream);
}
}
}

View file

@ -132,6 +132,16 @@ public class XjcToDomainResourceConverterTest {
assertThat(domain.getAuthInfo().getPw().getValue()).isEqualTo("0123456789abcdef");
}
/** Verifies that uppercase domain names are converted to lowercase */
@Test
public void testConvertDomainResourceUpperCase() throws Exception {
persistActiveContact("jd1234");
persistActiveContact("sh8013");
final XjcRdeDomain xjcDomain = loadDomainFromRdeXml("domain_fragment_ucase.xml");
DomainResource domain = convertDomainInTransaction(xjcDomain);
assertThat(domain.getFullyQualifiedDomainName()).isEqualTo("example1.example");
}
@Test
public void testConvertDomainResourceAddPeriod() throws Exception {
persistActiveContact("jd1234");

View file

@ -51,6 +51,8 @@ import org.junit.runners.JUnit4;
public class XjcToHostResourceConverterTest extends ShardableTestCase {
private static final ByteSource HOST_XML = RdeImportsTestData.get("host_fragment.xml");
private static final ByteSource HOST_XML_UCASE =
RdeImportsTestData.get("host_fragment_ucase.xml");
// List of packages to initialize JAXBContext
private static final String JAXB_CONTEXT_PACKAGES = Joiner.on(":")
@ -103,6 +105,14 @@ public class XjcToHostResourceConverterTest extends ShardableTestCase {
assertThat(host.getLastTransferTime()).isEqualTo(DateTime.parse("2008-10-03T09:34:00.0Z"));
}
/** Verifies that uppercase host names are converted to lowercase */
@Test
public void testConvertHostResourceUpperCase() throws Exception {
XjcRdeHost xjcHost = loadHostFromRdeXml(HOST_XML_UCASE);
HostResource host = convertHostInTransaction(xjcHost);
assertThat(host.getFullyQualifiedHostName()).isEqualTo("ns1.example1.test");
}
@Test
public void testConvertHostResourceHistoryEntry() throws Exception {
XjcRdeHost xjcHost = loadHostFromRdeXml();
@ -134,7 +144,11 @@ public class XjcToHostResourceConverterTest extends ShardableTestCase {
}
private XjcRdeHost loadHostFromRdeXml() throws Exception {
try (InputStream ins = HOST_XML.openStream()) {
return loadHostFromRdeXml(HOST_XML);
}
private XjcRdeHost loadHostFromRdeXml(ByteSource source) throws Exception {
try (InputStream ins = source.openStream()) {
return ((XjcRdeHostElement) unmarshaller.unmarshal(ins)).getValue();
}
}

View file

@ -9,7 +9,7 @@
<rdeDom:contact type="tech">sh8013</rdeDom:contact>
<rdeDom:ns>
<domain:hostObj>ns1.example.net</domain:hostObj>
<domain:hostObj>ns2.example.net</domain:hostObj>
<domain:hostObj>NS2.EXAMPLE.NET</domain:hostObj>
</rdeDom:ns>
<rdeDom:clID>RegistrarX</rdeDom:clID>
<rdeDom:crRr client="jdoe">RegistrarX</rdeDom:crRr>

View file

@ -0,0 +1,14 @@
<rdeDom:domain
xmlns:domain="urn:ietf:params:xml:ns:domain-1.0"
xmlns:rdeDom="urn:ietf:params:xml:ns:rdeDomain-1.0">
<rdeDom:name>EXAMPLE1.EXAMPLE</rdeDom:name>
<rdeDom:roid>Dexample1-TEST</rdeDom:roid>
<rdeDom:status s="ok"/>
<rdeDom:registrant>jd1234</rdeDom:registrant>
<rdeDom:contact type="admin">sh8013</rdeDom:contact>
<rdeDom:contact type="tech">sh8013</rdeDom:contact>
<rdeDom:clID>RegistrarX</rdeDom:clID>
<rdeDom:crRr client="jdoe">RegistrarX</rdeDom:crRr>
<rdeDom:crDate>1999-04-03T22:00:00.0Z</rdeDom:crDate>
<rdeDom:exDate>2015-04-03T22:00:00.0Z</rdeDom:exDate>
</rdeDom:domain>

View file

@ -0,0 +1,15 @@
<rdeHost:host xmlns:rdeHost="urn:ietf:params:xml:ns:rdeHost-1.0">
<rdeHost:name>NS1.EXAMPLE1.TEST</rdeHost:name>
<rdeHost:roid>Hns1_example1_test-TEST</rdeHost:roid>
<rdeHost:status s="ok"/>
<rdeHost:status s="linked"/>
<rdeHost:addr ip="v4">192.0.2.2</rdeHost:addr>
<rdeHost:addr ip="v4">192.0.2.29</rdeHost:addr>
<rdeHost:addr ip="v6">1080:0:0:0:8:800:200C:417A</rdeHost:addr>
<rdeHost:clID>RegistrarX</rdeHost:clID>
<rdeHost:crRr>RegistrarX</rdeHost:crRr>
<rdeHost:crDate>1999-05-08T12:10:00.0Z</rdeHost:crDate>
<rdeHost:upRr>RegistrarX</rdeHost:upRr>
<rdeHost:upDate>2009-10-03T09:34:00.0Z</rdeHost:upDate>
<rdeHost:trDate>2008-10-03T09:34:00.0Z</rdeHost:trDate>
</rdeHost:host>