diff --git a/apiserving/discoverydata/cloud/dns/BUILD b/apiserving/discoverydata/cloud/dns/BUILD new file mode 100644 index 000000000..a70e5acd5 --- /dev/null +++ b/apiserving/discoverydata/cloud/dns/BUILD @@ -0,0 +1,9 @@ +package(default_visibility = ["//visibility:public"]) + +java_library( + name = "cloud_dns_v2beta1_versioned", + exports = ["@google_api_services_dns//jar"], + runtime_deps = [ + "@google_api_client//jar", + ], +) diff --git a/java/google/registry/dns/writer/clouddns/BUILD b/java/google/registry/dns/writer/clouddns/BUILD new file mode 100644 index 000000000..41d487764 --- /dev/null +++ b/java/google/registry/dns/writer/clouddns/BUILD @@ -0,0 +1,27 @@ +package( + default_visibility = ["//java/google/registry:registry_project"], +) + +licenses(["notice"]) # Apache 2.0 + + +java_library( + name = "clouddns", + srcs = glob(["*.java"]), + deps = [ + "//apiserving/discoverydata/cloud/dns:cloud_dns_v2beta1_versioned", + "//java/com/google/api/client/googleapis/json", + "//java/com/google/api/client/http", + "//java/com/google/api/client/json", + "//java/com/google/common/annotations", + "//java/com/google/common/base", + "//java/com/google/common/collect", + "//java/com/google/common/net", + "//java/com/google/common/util/concurrent", + "//third_party/java/dagger", + "//third_party/java/joda_time", + "//java/google/registry/config", + "//java/google/registry/model", + "//java/google/registry/util", + ], +) diff --git a/java/google/registry/dns/writer/clouddns/CloudDnsModule.java b/java/google/registry/dns/writer/clouddns/CloudDnsModule.java new file mode 100644 index 000000000..62cb2b408 --- /dev/null +++ b/java/google/registry/dns/writer/clouddns/CloudDnsModule.java @@ -0,0 +1,42 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.dns.writer.clouddns; + +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.services.dns.Dns; +import com.google.api.services.dns.DnsScopes; +import com.google.common.base.Function; +import dagger.Module; +import dagger.Provides; +import google.registry.config.ConfigModule.Config; +import java.util.Set; + +/** Dagger module for Google Cloud DNS service connection objects. */ +@Module +public final class CloudDnsModule { + + @Provides + static Dns provideDns( + HttpTransport transport, + JsonFactory jsonFactory, + Function, ? extends HttpRequestInitializer> credential, + @Config("projectId") String projectId) { + return new Dns.Builder(transport, jsonFactory, credential.apply(DnsScopes.all())) + .setApplicationName(projectId) + .build(); + } +} diff --git a/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java b/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java new file mode 100644 index 000000000..6d3ebb184 --- /dev/null +++ b/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java @@ -0,0 +1,385 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.dns.writer.clouddns; + +import static com.google.common.base.Preconditions.checkArgument; +import static google.registry.model.EppResourceUtils.loadByUniqueId; + +import com.google.api.client.googleapis.json.GoogleJsonError.ErrorInfo; +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.services.dns.Dns; +import com.google.api.services.dns.model.Change; +import com.google.api.services.dns.model.ResourceRecordSet; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSet.Builder; +import com.google.common.net.InternetDomainName; +import com.google.common.util.concurrent.RateLimiter; +import google.registry.config.ConfigModule.Config; +import google.registry.model.dns.DnsWriter; +import google.registry.model.dns.DnsWriterZone; +import google.registry.model.domain.DomainResource; +import google.registry.model.domain.secdns.DelegationSignerData; +import google.registry.model.host.HostResource; +import google.registry.model.registry.Registries; +import google.registry.util.Clock; +import google.registry.util.FormattingLogger; +import google.registry.util.Retrier; +import java.io.IOException; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import javax.inject.Inject; +import org.joda.time.Duration; + +/** + * {@link DnsWriter} implementation that talks to Google Cloud DNS. + * + * @see "https://cloud.google.com/dns/docs/" + */ +class CloudDnsWriter implements DnsWriter { + + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); + + // This is the default max QPS for Cloud DNS. It can be increased by contacting the team + // via the Quotas page on the Cloud Console. + // TODO(shikhman): inject the RateLimiter + private static final int CLOUD_DNS_MAX_QPS = 20; + private static final RateLimiter rateLimiter = RateLimiter.create(CLOUD_DNS_MAX_QPS); + private static final ImmutableSet RETRYABLE_EXCEPTION_REASONS = + ImmutableSet.of("preconditionFailed", "notFound", "alreadyExists"); + + private final Clock clock; + // TODO(shikhman): This uses @Config("transientFailureRetries") which may not be tuned for this + // application. + private final Retrier retrier; + private final Duration defaultTtl; + private final String projectId; + private final String zoneName; + private final Dns dnsConnection; + private final ImmutableMap.Builder> + desiredRecordsBuilder = new ImmutableMap.Builder<>(); + + @Inject + CloudDnsWriter( + Dns dnsConnection, + @Config("projectId") String projectId, + @DnsWriterZone String zoneName, + @Config("dnsDefaultTtl") Duration defaultTtl, + Clock clock, + Retrier retrier) { + this.dnsConnection = dnsConnection; + this.projectId = projectId; + this.zoneName = zoneName; + this.defaultTtl = defaultTtl; + this.clock = clock; + this.retrier = retrier; + } + + /** Publish the domain and all subordinate hosts. */ + @Override + public void publishDomain(String domainName) { + // Canonicalize name + String absoluteDomainName = getAbsoluteHostName(domainName); + + // Load the target domain. Note that it can be null if this domain was just deleted. + Optional domainResource = + Optional.fromNullable(loadByUniqueId(DomainResource.class, domainName, clock.nowUtc())); + + // Return early if no DNS records should be published. + // desiredRecordsBuilder is populated with an empty set to indicate that all existing records + // should be deleted. + if (!domainResource.isPresent() || !domainResource.get().shouldPublishToDns()) { + desiredRecordsBuilder.put(absoluteDomainName, ImmutableSet.of()); + return; + } + + ImmutableSet.Builder domainRecords = new ImmutableSet.Builder<>(); + + // Construct DS records (if any). + Set dsData = domainResource.get().getDsData(); + if (!dsData.isEmpty()) { + HashSet dsRrData = new HashSet<>(); + for (DelegationSignerData ds : dsData) { + dsRrData.add(ds.toRrData()); + } + + if (!dsRrData.isEmpty()) { + domainRecords.add( + new ResourceRecordSet() + .setName(absoluteDomainName) + .setTtl((int) defaultTtl.getStandardSeconds()) + .setType("DS") + .setKind("dns#resourceRecordSet") + .setRrdatas(ImmutableList.copyOf(dsRrData))); + } + } + + + // Construct NS records (if any). + Set nameserverData = domainResource.get().loadNameserverFullyQualifiedHostNames(); + if (!nameserverData.isEmpty()) { + HashSet nsRrData = new HashSet<>(); + for (String hostName : nameserverData) { + nsRrData.add(getAbsoluteHostName(hostName)); + + // Construct glue records for subordinate NS hostnames (if any) + if (hostName.endsWith(domainName)) { + publishSubordinateHost(hostName); + } + } + + if (!nsRrData.isEmpty()) { + domainRecords.add( + new ResourceRecordSet() + .setName(absoluteDomainName) + .setTtl((int) defaultTtl.getStandardSeconds()) + .setType("NS") + .setKind("dns#resourceRecordSet") + .setRrdatas(ImmutableList.copyOf(nsRrData))); + } + } + + desiredRecordsBuilder.put(absoluteDomainName, domainRecords.build()); + logger.finefmt( + "Will write %s records for domain %s", domainRecords.build().size(), absoluteDomainName); + } + + private void publishSubordinateHost(String hostName) { + logger.infofmt("Publishing glue records for %s", hostName); + // Canonicalize name + String absoluteHostName = getAbsoluteHostName(hostName); + + // Load the target host. Note that it can be null if this host was just deleted. + // desiredRecords is populated with an empty set to indicate that all existing records + // should be deleted. + Optional host = + Optional.fromNullable(loadByUniqueId(HostResource.class, hostName, clock.nowUtc())); + + // Return early if the host is deleted. + if (!host.isPresent()) { + desiredRecordsBuilder.put(absoluteHostName, ImmutableSet.of()); + return; + } + + ImmutableSet.Builder domainRecords = new ImmutableSet.Builder<>(); + + // Construct A and AAAA records (if any). + HashSet aRrData = new HashSet<>(); + HashSet aaaaRrData = new HashSet<>(); + for (InetAddress ip : host.get().getInetAddresses()) { + if (ip instanceof Inet4Address) { + aRrData.add(ip.toString()); + } else { + checkArgument(ip instanceof Inet6Address); + aaaaRrData.add(ip.toString()); + } + } + + if (!aRrData.isEmpty()) { + domainRecords.add( + new ResourceRecordSet() + .setName(absoluteHostName) + .setTtl((int) defaultTtl.getStandardSeconds()) + .setType("A") + .setKind("dns#resourceRecordSet") + .setRrdatas(ImmutableList.copyOf(aRrData))); + } + + if (!aaaaRrData.isEmpty()) { + domainRecords.add( + new ResourceRecordSet() + .setName(absoluteHostName) + .setTtl((int) defaultTtl.getStandardSeconds()) + .setType("AAAA") + .setKind("dns#resourceRecordSet") + .setRrdatas(ImmutableList.copyOf(aaaaRrData))); + } + + desiredRecordsBuilder.put(absoluteHostName, domainRecords.build()); + } + + /** + * Publish A/AAAA records to Cloud DNS. + * + *

Cloud DNS has no API for glue -- A/AAAA records are automatically matched to their + * corresponding NS records to serve glue. + */ + @Override + public void publishHost(String hostName) { + // Get the superordinate domain name of the host. + InternetDomainName host = InternetDomainName.from(hostName); + Optional tld = Registries.findTldForName(host); + + // Host not managed by our registry, no need to update DNS. + if (!tld.isPresent()) { + logger.severefmt("publishHost called for invalid host %s", hostName); + return; + } + + // Extract the superordinate domain name. The TLD and host may have several dots so this + // must calculate a sublist. + ImmutableList hostParts = host.parts(); + ImmutableList tldParts = tld.get().parts(); + ImmutableList domainParts = + hostParts.subList(hostParts.size() - tldParts.size() - 1, hostParts.size()); + String domain = Joiner.on(".").join(domainParts); + + // Refresh the superordinate domain, since we shouldn't be publishing glue records if we are not + // authoritative for the superordinate domain. + publishDomain(domain); + } + + /** + * Sync changes in a zone requested by publishDomain and publishHost to Cloud DNS. + * + *

The zone for the TLD must exist first in Cloud DNS and must be DNSSEC enabled. + * + *

The relevant resource records (including those of all subordinate hosts) will be retrieved + * and the operation will be retried until the state of the retrieved zone data matches the + * representation built via this writer. + */ + @Override + public void close() { + close(desiredRecordsBuilder.build()); + } + + @VisibleForTesting + void close(ImmutableMap> desiredRecords) { + retrier.callWithRetry(getMutateZoneCallback(desiredRecords), ZoneStateException.class); + logger.info("Wrote to Cloud DNS"); + } + + /** + * Get a callback to mutate the zone with the provided {@code desiredRecords}. + */ + @VisibleForTesting + Callable getMutateZoneCallback( + final ImmutableMap> desiredRecords) { + return new Callable() { + @Override + public Void call() throws IOException, ZoneStateException { + // Fetch all existing records for names that this writer is trying to modify + Builder existingRecords = new Builder<>(); + for (String domainName : desiredRecords.keySet()) { + List existingRecordsForDomain = + getResourceRecordsForDomain(domainName); + existingRecords.addAll(existingRecordsForDomain); + + // Fetch glue records for in-bailiwick nameservers + for (ResourceRecordSet record : existingRecordsForDomain) { + if (!record.getType().equals("NS")) { + continue; + } + for (String hostName : record.getRrdatas()) { + if (hostName.endsWith(domainName) && !hostName.equals(domainName)) { + existingRecords.addAll(getResourceRecordsForDomain(hostName)); + } + } + } + } + + // Flatten the desired records into one set. + Builder flattenedDesiredRecords = new Builder<>(); + for (ImmutableSet records : desiredRecords.values()) { + flattenedDesiredRecords.addAll(records); + } + + // Delete all existing records and add back the desired records + updateResourceRecords(flattenedDesiredRecords.build(), existingRecords.build()); + return null; + } + }; + } + + /** + * Fetch the {@link ResourceRecordSet}s for the given domain name under this zone. + * + *

The provided domain should be in absolute form. + * + * @throws IOException if the operation could not be completed successfully + */ + private List getResourceRecordsForDomain(String domainName) + throws IOException { + logger.finefmt("Fetching records for %s", domainName); + Dns.ResourceRecordSets.List listRecordsRequest = + dnsConnection.resourceRecordSets().list(projectId, zoneName).setName(domainName); + + rateLimiter.acquire(); + return listRecordsRequest.execute().getRrsets(); + } + + /** + * Update {@link ResourceRecordSet}s under this zone. + * + *

This call should be used in conjunction with getResourceRecordsForDomain in a get-and-set + * retry loop. + * + * @throws IOException if the operation could not be completed successfully due to an + * uncorrectable error. + * @throws ZoneStateException if the operation could not be completely successfully because the + * records to delete do not exist, already exist or have been modified with different + * attributes since being queried. + * @see "https://cloud.google.com/dns/troubleshooting" for a list of errors produced by the Google + * Cloud DNS API. + */ + private void updateResourceRecords( + ImmutableSet additions, ImmutableSet deletions) + throws IOException, ZoneStateException { + Change change = new Change().setAdditions(additions.asList()).setDeletions(deletions.asList()); + + rateLimiter.acquire(); + try { + dnsConnection.changes().create(projectId, zoneName, change).execute(); + } catch (GoogleJsonResponseException e) { + List errors = e.getDetails().getErrors(); + // We did something really wrong here, just give up and re-throw + if (errors.size() > 1) { + throw e; + } + String errorReason = errors.get(0).getReason(); + + if (RETRYABLE_EXCEPTION_REASONS.contains(errorReason)) { + throw new ZoneStateException(errorReason); + } else { + throw e; + } + } + } + + /** + * Returns the presentation format ending in a dot used for an absolute hostname. + * + * @param hostName the fully qualified hostname + */ + private static String getAbsoluteHostName(String hostName) { + return hostName.endsWith(".") ? hostName : hostName + "."; + } + + /** Zone state on Cloud DNS does not match the expected state. */ + static class ZoneStateException extends RuntimeException { + public ZoneStateException(String reason) { + super("Zone state on Cloud DNS does not match the expected state: " + reason); + } + } +} diff --git a/java/google/registry/model/domain/secdns/DelegationSignerData.java b/java/google/registry/model/domain/secdns/DelegationSignerData.java index 1f2080f0c..0d9a6e3a4 100644 --- a/java/google/registry/model/domain/secdns/DelegationSignerData.java +++ b/java/google/registry/model/domain/secdns/DelegationSignerData.java @@ -17,6 +17,7 @@ package google.registry.model.domain.secdns; import com.google.common.annotations.VisibleForTesting; import com.googlecode.objectify.annotation.Embed; import google.registry.model.ImmutableObject; +import javax.xml.bind.DatatypeConverter; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlType; import javax.xml.bind.annotation.adapters.HexBinaryAdapter; @@ -90,4 +91,15 @@ public class DelegationSignerData public int compareTo(DelegationSignerData other) { return Integer.compare(getKeyTag(), other.getKeyTag()); } + + /** + * Returns the presentation format of this DS record. + * + * @see RFC 4034 Section 5.3 + */ + public String toRrData() { + return String.format( + "%d %d %d %s", + this.keyTag, this.algorithm, this.digestType, DatatypeConverter.printHexBinary(digest)); + } } diff --git a/java/google/registry/repositories.bzl b/java/google/registry/repositories.bzl index 4b2bfe9d1..f7bb17ad0 100644 --- a/java/google/registry/repositories.bzl +++ b/java/google/registry/repositories.bzl @@ -202,6 +202,12 @@ def domain_registry_repositories(): sha1 = "4f1ee62be6b1b7258560ee7808094292798ef718", ) + native.maven_jar( + name = "google_api_services_dns", + artifact = "com.google.apis:google-api-services-dns:v2beta1-rev2-1.21.0", + sha1 = "8ea36fec19051f41afdf2cb9ca6a08af929530a6", + ) + native.maven_jar( name = "google_api_services_drive", artifact = "com.google.apis:google-api-services-drive:v2-rev160-1.19.1",