mirror of
https://github.com/google/nomulus.git
synced 2025-05-02 04:57:51 +02:00
2547313ef9
Attending to this old bug will improve our ability to perform zone comparisons between Datastore and the DNS provider. Right now, zone comparison finds some bogus differences, because the TTL we send to the DNS subsystem doesn't match the TTL we use when generating our local dump files. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=164635557
330 lines
13 KiB
Java
330 lines
13 KiB
Java
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package google.registry.tools.server;
|
|
|
|
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
|
import static com.google.common.base.Predicates.notNull;
|
|
import static com.google.common.collect.Iterators.filter;
|
|
import static com.google.common.io.BaseEncoding.base16;
|
|
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
|
import static google.registry.model.EppResourceUtils.loadAtPointInTime;
|
|
import static google.registry.model.ofy.ObjectifyService.ofy;
|
|
import static google.registry.request.Action.Method.POST;
|
|
import static google.registry.util.PipelineUtils.createJobPath;
|
|
import static java.nio.charset.StandardCharsets.UTF_8;
|
|
import static org.joda.time.DateTimeZone.UTC;
|
|
|
|
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
|
import com.google.appengine.tools.cloudstorage.RetryParams;
|
|
import com.google.appengine.tools.mapreduce.Mapper;
|
|
import com.google.appengine.tools.mapreduce.Reducer;
|
|
import com.google.appengine.tools.mapreduce.ReducerInput;
|
|
import com.google.common.base.Function;
|
|
import com.google.common.collect.FluentIterable;
|
|
import com.google.common.collect.ImmutableList;
|
|
import com.google.common.collect.ImmutableMap;
|
|
import com.google.common.collect.ImmutableSet;
|
|
import google.registry.config.RegistryConfig.Config;
|
|
import google.registry.gcs.GcsUtils;
|
|
import google.registry.mapreduce.MapreduceRunner;
|
|
import google.registry.mapreduce.inputs.NullInput;
|
|
import google.registry.model.EppResource;
|
|
import google.registry.model.domain.DomainResource;
|
|
import google.registry.model.domain.secdns.DelegationSignerData;
|
|
import google.registry.model.host.HostResource;
|
|
import google.registry.request.Action;
|
|
import google.registry.request.HttpException.BadRequestException;
|
|
import google.registry.request.JsonActionRunner;
|
|
import google.registry.request.auth.Auth;
|
|
import google.registry.util.Clock;
|
|
import java.io.IOException;
|
|
import java.io.OutputStream;
|
|
import java.io.OutputStreamWriter;
|
|
import java.io.PrintWriter;
|
|
import java.io.Writer;
|
|
import java.net.Inet4Address;
|
|
import java.net.InetAddress;
|
|
import java.util.Iterator;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import javax.inject.Inject;
|
|
import org.joda.time.DateTime;
|
|
import org.joda.time.Duration;
|
|
|
|
/**
|
|
* MapReduce that requests generation of BIND zone files for a set of TLDs at a given time.
|
|
*
|
|
* <p>Zone files for each requested TLD are written to GCS. TLDs without entries produce zone files
|
|
* with only a header. The export time must be at least two minutes in the past and no more than
|
|
* 29 days in the past, and must be at midnight UTC.
|
|
*/
|
|
@Action(
|
|
path = GenerateZoneFilesAction.PATH,
|
|
method = POST,
|
|
auth = Auth.AUTH_INTERNAL_OR_ADMIN
|
|
)
|
|
public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonAction {
|
|
|
|
public static final String PATH = "/_dr/task/generateZoneFiles";
|
|
|
|
/** Format for the zone file name. */
|
|
private static final String FILENAME_FORMAT = "%s-%s.zone";
|
|
|
|
/** Format for the GCS path to a file. */
|
|
private static final String GCS_PATH_FORMAT = "gs://%s/%s";
|
|
|
|
/** Format for the zone file header. */
|
|
private static final String HEADER_FORMAT = "$ORIGIN\t%s.\n\n";
|
|
|
|
/** Format for NS records. */
|
|
private static final String NS_FORMAT = "%s\t%d\tIN\tNS\t%s.\n";
|
|
|
|
/** Format for DS records. */
|
|
private static final String DS_FORMAT = "%s\t%d\tIN\tDS\t%d %d %d %s\n";
|
|
|
|
/** Format for A and AAAA records. */
|
|
private static final String A_FORMAT = "%s\t%d\tIN\t%s\t%s\n";
|
|
|
|
@Inject MapreduceRunner mrRunner;
|
|
@Inject JsonActionRunner jsonActionRunner;
|
|
@Inject @Config("zoneFilesBucket") String bucket;
|
|
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
|
@Inject @Config("commitLogDatastoreRetention") Duration datastoreRetention;
|
|
@Inject @Config("dnsDefaultATtl") Duration dnsDefaultATtl;
|
|
@Inject @Config("dnsDefaultNsTtl") Duration dnsDefaultNsTtl;
|
|
@Inject @Config("dnsDefaultDsTtl") Duration dnsDefaultDsTtl;
|
|
@Inject Clock clock;
|
|
@Inject GenerateZoneFilesAction() {}
|
|
|
|
@Override
|
|
public void run() {
|
|
jsonActionRunner.run(this);
|
|
}
|
|
|
|
@Override
|
|
public Map<String, Object> handleJsonRequest(Map<String, ?> json) {
|
|
@SuppressWarnings("unchecked")
|
|
ImmutableSet<String> tlds = ImmutableSet.copyOf((List<String>) json.get("tlds"));
|
|
final DateTime exportTime = DateTime.parse(json.get("exportTime").toString());
|
|
// We disallow exporting within the past 2 minutes because there might be outstanding writes.
|
|
// We can only reliably call loadAtPointInTime at times that are UTC midnight and >
|
|
// datastoreRetention ago in the past.
|
|
DateTime now = clock.nowUtc();
|
|
if (exportTime.isAfter(now.minusMinutes(2))) {
|
|
throw new BadRequestException("Invalid export time: must be > 2 minutes ago");
|
|
}
|
|
if (exportTime.isBefore(now.minus(datastoreRetention))) {
|
|
throw new BadRequestException(String.format(
|
|
"Invalid export time: must be < %d days ago",
|
|
datastoreRetention.getStandardDays()));
|
|
}
|
|
if (!exportTime.equals(exportTime.toDateTime(UTC).withTimeAtStartOfDay())) {
|
|
throw new BadRequestException("Invalid export time: must be midnight UTC");
|
|
}
|
|
String jobId = mrRunner
|
|
.setJobName("Generate bind file stanzas")
|
|
.setModuleName("tools")
|
|
.setDefaultReduceShards(tlds.size())
|
|
.runMapreduce(
|
|
new GenerateBindFileMapper(
|
|
tlds, exportTime, dnsDefaultATtl, dnsDefaultNsTtl, dnsDefaultDsTtl),
|
|
new GenerateBindFileReducer(bucket, exportTime, gcsBufferSize),
|
|
ImmutableList.of(
|
|
new NullInput<EppResource>(),
|
|
createEntityInput(DomainResource.class, HostResource.class)));
|
|
ImmutableList<String> filenames = FluentIterable.from(tlds)
|
|
.transform(
|
|
new Function<String, String>() {
|
|
@Override
|
|
public String apply(String tld) {
|
|
return String.format(
|
|
GCS_PATH_FORMAT,
|
|
bucket,
|
|
String.format(FILENAME_FORMAT, tld, exportTime));
|
|
}})
|
|
.toList();
|
|
return ImmutableMap.<String, Object>of(
|
|
"jobPath", createJobPath(jobId),
|
|
"filenames", filenames);
|
|
}
|
|
|
|
/** Mapper to find domains and hosts that were active at a given time. */
|
|
static class GenerateBindFileMapper extends Mapper<EppResource, String, String> {
|
|
|
|
private static final long serialVersionUID = 4647941823789859913L;
|
|
|
|
private final ImmutableSet<String> tlds;
|
|
private final DateTime exportTime;
|
|
private final Duration dnsDefaultATtl;
|
|
private final Duration dnsDefaultNsTtl;
|
|
private final Duration dnsDefaultDsTtl;
|
|
|
|
GenerateBindFileMapper(
|
|
ImmutableSet<String> tlds,
|
|
DateTime exportTime,
|
|
Duration dnsDefaultATtl,
|
|
Duration dnsDefaultNsTtl,
|
|
Duration dnsDefaultDsTtl) {
|
|
this.tlds = tlds;
|
|
this.exportTime = exportTime;
|
|
this.dnsDefaultATtl = dnsDefaultATtl;
|
|
this.dnsDefaultNsTtl = dnsDefaultNsTtl;
|
|
this.dnsDefaultDsTtl = dnsDefaultDsTtl;
|
|
}
|
|
|
|
@Override
|
|
public void map(EppResource resource) {
|
|
if (resource == null) { // Force the reducer to always generate a bind header for each tld.
|
|
for (String tld : tlds) {
|
|
emit(tld, null);
|
|
}
|
|
} else if (resource instanceof DomainResource) {
|
|
mapDomain((DomainResource) resource);
|
|
} else {
|
|
mapHost((HostResource) resource);
|
|
}
|
|
}
|
|
|
|
private void mapDomain(DomainResource domain) {
|
|
// Domains never change their tld, so we can check if it's from the wrong tld right away.
|
|
if (tlds.contains(domain.getTld())) {
|
|
domain = loadAtPointInTime(domain, exportTime).now();
|
|
// A null means the domain was deleted (or not created) at this time.
|
|
if (domain != null && domain.shouldPublishToDns()) {
|
|
String stanza = domainStanza(domain, exportTime, dnsDefaultNsTtl, dnsDefaultDsTtl);
|
|
if (!stanza.isEmpty()) {
|
|
emit(domain.getTld(), stanza);
|
|
getContext().incrementCounter(domain.getTld() + " domains");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private void mapHost(HostResource host) {
|
|
host = loadAtPointInTime(host, exportTime).now();
|
|
if (host != null) { // A null means the host was deleted (or not created) at this time.
|
|
// Find a matching tld. Hosts might change their tld, so check after the point-in-time load.
|
|
String fullyQualifiedHostName = host.getFullyQualifiedHostName();
|
|
for (String tld : tlds) {
|
|
if (fullyQualifiedHostName.endsWith("." + tld)) {
|
|
String stanza = hostStanza(host, dnsDefaultATtl);
|
|
if (!stanza.isEmpty()) {
|
|
emit(tld, stanza);
|
|
getContext().incrementCounter(tld + " hosts");
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Reducer to write zone files to GCS. */
|
|
static class GenerateBindFileReducer extends Reducer<String, String, Void> {
|
|
|
|
private static final long serialVersionUID = -8489050680083119352L;
|
|
|
|
private final String bucket;
|
|
private final DateTime exportTime;
|
|
private final int gcsBufferSize;
|
|
|
|
GenerateBindFileReducer(String bucket, DateTime exportTime, int gcsBufferSize) {
|
|
this.bucket = bucket;
|
|
this.exportTime = exportTime;
|
|
this.gcsBufferSize = gcsBufferSize;
|
|
}
|
|
|
|
@Override
|
|
public void reduce(String tld, ReducerInput<String> stanzas) {
|
|
String stanzaCounter = tld + " stanzas";
|
|
GcsFilename filename =
|
|
new GcsFilename(bucket, String.format(FILENAME_FORMAT, tld, exportTime));
|
|
GcsUtils cloudStorage =
|
|
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
|
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
|
|
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8);
|
|
PrintWriter writer = new PrintWriter(osWriter)) {
|
|
writer.printf(HEADER_FORMAT, tld);
|
|
for (Iterator<String> stanzaIter = filter(stanzas, notNull()); stanzaIter.hasNext(); ) {
|
|
writer.println(stanzaIter.next());
|
|
getContext().incrementCounter(stanzaCounter);
|
|
}
|
|
writer.flush();
|
|
} catch (IOException e) {
|
|
throw new RuntimeException(e);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Generates DNS records for a domain (NS and DS).
|
|
*
|
|
* These look like this:
|
|
* {@code
|
|
* foo.tld 180 IN NS ns.example.com.
|
|
* foo.tld 86400 IN DS 1 2 3 000102
|
|
* }
|
|
*/
|
|
private static String domainStanza(
|
|
DomainResource domain,
|
|
DateTime exportTime,
|
|
Duration dnsDefaultNsTtl,
|
|
Duration dnsDefaultDsTtl) {
|
|
StringBuilder result = new StringBuilder();
|
|
for (HostResource nameserver : ofy().load().keys(domain.getNameservers()).values()) {
|
|
result.append(String.format(
|
|
NS_FORMAT,
|
|
domain.getFullyQualifiedDomainName(),
|
|
dnsDefaultNsTtl.getStandardSeconds(),
|
|
// Load the nameservers at the export time in case they've been renamed or deleted.
|
|
loadAtPointInTime(nameserver, exportTime).now().getFullyQualifiedHostName()));
|
|
}
|
|
for (DelegationSignerData dsData : domain.getDsData()) {
|
|
result.append(String.format(
|
|
DS_FORMAT,
|
|
domain.getFullyQualifiedDomainName(),
|
|
dnsDefaultDsTtl.getStandardSeconds(),
|
|
dsData.getKeyTag(),
|
|
dsData.getAlgorithm(),
|
|
dsData.getDigestType(),
|
|
base16().encode((dsData.getDigest()))));
|
|
}
|
|
return result.toString();
|
|
}
|
|
|
|
/**
|
|
* Generates DNS records for a domain (A and AAAA).
|
|
*
|
|
* <p>These look like this:
|
|
* {@code
|
|
* ns.foo.tld 3600 IN A 127.0.0.1
|
|
* ns.foo.tld 3600 IN AAAA 0:0:0:0:0:0:0:1
|
|
* }
|
|
*/
|
|
private static String hostStanza(HostResource host, Duration dnsDefaultATtl) {
|
|
StringBuilder result = new StringBuilder();
|
|
for (InetAddress addr : host.getInetAddresses()) {
|
|
// must be either IPv4 or IPv6
|
|
String rrSetClass = (addr instanceof Inet4Address) ? "A" : "AAAA";
|
|
result.append(String.format(
|
|
A_FORMAT,
|
|
host.getFullyQualifiedHostName(),
|
|
dnsDefaultATtl.getStandardSeconds(),
|
|
rrSetClass,
|
|
addr.getHostAddress()));
|
|
}
|
|
return result.toString();
|
|
}
|
|
}
|