diff --git a/core/gradle.lockfile b/core/gradle.lockfile index 1bc06ef65..6945e75c9 100644 --- a/core/gradle.lockfile +++ b/core/gradle.lockfile @@ -349,7 +349,7 @@ org.apache.mina:mina-core:2.1.6=testCompileClasspath,testRuntimeClasspath org.apache.sshd:sshd-core:2.0.0=testCompileClasspath,testRuntimeClasspath org.apache.sshd:sshd-scp:2.0.0=testCompileClasspath,testRuntimeClasspath org.apache.sshd:sshd-sftp:2.0.0=testCompileClasspath,testRuntimeClasspath -org.apache.tomcat:tomcat-annotations-api:11.0.0-M15=testCompileClasspath,testRuntimeClasspath +org.apache.tomcat:tomcat-annotations-api:11.0.0-M16=testCompileClasspath,testRuntimeClasspath org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath org.bouncycastle:bcpg-jdk15on:1.67=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.bouncycastle:bcpkix-jdk15on:1.67=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/core/src/main/java/google/registry/bsa/UploadBsaUnavailableDomainsAction.java b/core/src/main/java/google/registry/bsa/UploadBsaUnavailableDomainsAction.java new file mode 100644 index 000000000..530e3c494 --- /dev/null +++ b/core/src/main/java/google/registry/bsa/UploadBsaUnavailableDomainsAction.java @@ -0,0 +1,220 @@ +// Copyright 2024 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.bsa; + +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static google.registry.model.tld.Tld.isEnrolledWithBsa; +import static google.registry.model.tld.Tlds.getTldEntitiesOfType; +import static google.registry.model.tld.label.ReservedList.loadReservedLists; +import static google.registry.persistence.transaction.TransactionManagerFactory.replicaTm; +import static google.registry.request.Action.Method.POST; +import static java.nio.charset.StandardCharsets.US_ASCII; + +import com.google.api.client.http.HttpStatusCodes; +import com.google.cloud.storage.BlobId; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Ordering; +import com.google.common.flogger.FluentLogger; +import com.google.common.hash.Hashing; +import com.google.common.io.ByteSource; +import google.registry.bsa.api.BsaCredential; +import google.registry.config.RegistryConfig.Config; +import google.registry.gcs.GcsUtils; +import google.registry.model.tld.Tld; +import google.registry.model.tld.Tld.TldType; +import google.registry.model.tld.label.ReservedList; +import google.registry.request.Action; +import google.registry.request.Action.Service; +import google.registry.request.auth.Auth; +import google.registry.util.Clock; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.zip.GZIPOutputStream; +import javax.inject.Inject; +import okhttp3.MediaType; +import okhttp3.MultipartBody; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.joda.time.DateTime; + +/** + * Daily action that uploads unavailable domain names on applicable TLDs to BSA. + * + *

The upload is a single zipped text file containing combined details for all BSA-enrolled TLDs. + * The text is a newline-delimited list of punycoded fully qualified domain names, and contains all + * domains on each TLD that are registered and/or reserved. + * + *

The file is also uploaded to GCS to preserve it as a record for ourselves. + */ +@Action( + service = Service.BSA, + path = "/_dr/task/uploadBsaUnavailableNames", + method = POST, + auth = Auth.AUTH_API_ADMIN) +public class UploadBsaUnavailableDomainsAction implements Runnable { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + Clock clock; + + BsaCredential bsaCredential; + + GcsUtils gcsUtils; + + String gcsBucket; + + String apiUrl; + + google.registry.request.Response response; + + @Inject + public UploadBsaUnavailableDomainsAction( + Clock clock, + BsaCredential bsaCredential, + GcsUtils gcsUtils, + @Config("bsaUnavailableDomainsGcsBucket") String gcsBucket, + @Config("bsaUploadUnavailableDomainsUrl") String apiUrl, + google.registry.request.Response response) { + this.clock = clock; + this.bsaCredential = bsaCredential; + this.gcsUtils = gcsUtils; + this.gcsBucket = gcsBucket; + this.apiUrl = apiUrl; + this.response = response; + } + + @Override + public void run() { + DateTime runTime = clock.nowUtc(); + String unavailableDomains = + Joiner.on("\n").join(replicaTm().transact(() -> getUnavailableDomains(runTime))); + uploadToGcs(unavailableDomains, runTime); + uploadToBsa(unavailableDomains, runTime); + } + + /** Uploads the unavailable domains list to GCS in the unavailable domains bucket. */ + void uploadToGcs(String unavailableDomains, DateTime runTime) { + logger.atInfo().log("Uploading unavailable names file to GCS in bucket %s", gcsBucket); + BlobId blobId = BlobId.of(gcsBucket, createFilename(runTime)); + try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId); + Writer osWriter = new OutputStreamWriter(gcsOutput, US_ASCII)) { + osWriter.write(unavailableDomains); + } catch (Exception e) { + logger.atSevere().withCause(e).log( + "Error writing BSA unavailable domains to GCS; skipping to BSA upload ..."); + } + } + + void uploadToBsa(String unavailableDomains, DateTime runTime) { + try { + byte[] gzippedContents = gzipUnavailableDomains(unavailableDomains); + String sha512Hash = ByteSource.wrap(gzippedContents).hash(Hashing.sha512()).toString(); + String filename = createFilename(runTime); + OkHttpClient client = new OkHttpClient().newBuilder().build(); + + RequestBody body = + new MultipartBody.Builder() + .setType(MultipartBody.FORM) + .addFormDataPart( + "zone", + null, + RequestBody.create( + String.format("{\"checkSum\": \"%s\"}", sha512Hash).getBytes(US_ASCII), + MediaType.parse("application/json"))) + .addFormDataPart( + "file", + String.format("%s.gz", filename), + RequestBody.create(gzippedContents, MediaType.parse("application/octet-stream"))) + .build(); + + Request request = + new Request.Builder() + .url(apiUrl) + .method("POST", body) + .addHeader("Authorization", "Bearer " + bsaCredential.getAuthToken()) + .build(); + + logger.atInfo().log( + "Uploading unavailable domains list %s to %s with hash %s", filename, apiUrl, sha512Hash); + try (Response uploadResponse = client.newCall(request).execute()) { + logger.atInfo().log( + "Received response with code %s from server: %s", + uploadResponse.code(), + uploadResponse.body() == null ? "(none)" : uploadResponse.body().string()); + } + } catch (IOException e) { + logger.atSevere().withCause(e).log("Error while attempting to upload to BSA, aborting."); + response.setStatus(HttpStatusCodes.STATUS_CODE_SERVER_ERROR); + response.setPayload("Error while attempting to upload to BSA: " + e.getMessage()); + } + } + + private byte[] gzipUnavailableDomains(String unavailableDomains) throws IOException { + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { + gzipOutputStream.write(unavailableDomains.getBytes(US_ASCII)); + } + return byteArrayOutputStream.toByteArray(); + } + } + + private static String createFilename(DateTime runTime) { + return String.format("unavailable_domains_%s.txt", runTime.toString()); + } + + private ImmutableSortedSet getUnavailableDomains(DateTime runTime) { + ImmutableSet bsaEnabledTlds = + getTldEntitiesOfType(TldType.REAL).stream() + .filter(tld -> isEnrolledWithBsa(tld, runTime)) + .collect(toImmutableSet()); + + ImmutableSortedSet.Builder unavailableDomains = + new ImmutableSortedSet.Builder<>(Ordering.natural()); + for (Tld tld : bsaEnabledTlds) { + for (ReservedList reservedList : loadReservedLists(tld.getReservedListNames())) { + if (reservedList.getShouldPublish()) { + unavailableDomains.addAll( + reservedList.getReservedListEntries().keySet().stream() + .map(label -> toDomain(label, tld)) + .collect(toImmutableSet())); + } + } + } + + unavailableDomains.addAll( + replicaTm() + .query( + "SELECT domainName FROM Domain " + + "WHERE tld IN :tlds " + + "AND deletionTime > :now ", + String.class) + .setParameter( + "tlds", bsaEnabledTlds.stream().map(Tld::getTldStr).collect(toImmutableSet())) + .setParameter("now", runTime) + .getResultList()); + return unavailableDomains.build(); + } + + private static String toDomain(String domainLabel, Tld tld) { + return String.format("%s.%s", domainLabel, tld.getTldStr()); + } +} diff --git a/core/src/main/java/google/registry/config/RegistryConfig.java b/core/src/main/java/google/registry/config/RegistryConfig.java index 4fb9e1171..30d0391a5 100644 --- a/core/src/main/java/google/registry/config/RegistryConfig.java +++ b/core/src/main/java/google/registry/config/RegistryConfig.java @@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; import dagger.Module; import dagger.Provides; +import google.registry.bsa.UploadBsaUnavailableDomainsAction; import google.registry.dns.ReadDnsRefreshRequestsAction; import google.registry.model.common.DnsRefreshRequest; import google.registry.persistence.transaction.JpaTransactionManager; @@ -270,6 +271,18 @@ public final class RegistryConfig { return projectId + "-domain-lists"; } + /** + * The GCS bucket for exporting lists of unavailable names for the BSA. + * + * @see UploadBsaUnavailableDomainsAction + */ + @Provides + @Config("bsaUnavailableDomainsGcsBucket") + public static String provideBsaUnavailableNamesGcsBucket( + @Config("projectId") String projectId) { + return projectId + "-bsa-unavailable-domains"; + } + /** * Returns the Google Cloud Storage bucket for staging BRDA escrow deposits. * @@ -1495,6 +1508,12 @@ public final class RegistryConfig { return String.format("%s?%s", config.bsa.unblockableDomainsUrl, "action=remove"); } + @Provides + @Config("bsaUploadUnavailableDomainsUrl") + public static String provideBsaUploadUnavailableDomainsUrl(RegistryConfigSettings config) { + return config.bsa.uploadUnavailableDomainsUrl; + } + private static String formatComments(String text) { return Splitter.on('\n').omitEmptyStrings().trimResults().splitToList(text).stream() .map(s -> "# " + s) diff --git a/core/src/main/java/google/registry/config/RegistryConfigSettings.java b/core/src/main/java/google/registry/config/RegistryConfigSettings.java index 065253e3b..93e801a5f 100644 --- a/core/src/main/java/google/registry/config/RegistryConfigSettings.java +++ b/core/src/main/java/google/registry/config/RegistryConfigSettings.java @@ -279,5 +279,6 @@ public class RegistryConfigSettings { public Map dataUrls; public String orderStatusUrl; public String unblockableDomainsUrl; + public String uploadUnavailableDomainsUrl; } } diff --git a/core/src/main/java/google/registry/config/files/default-config.yaml b/core/src/main/java/google/registry/config/files/default-config.yaml index db356b4fa..927a3ca47 100644 --- a/core/src/main/java/google/registry/config/files/default-config.yaml +++ b/core/src/main/java/google/registry/config/files/default-config.yaml @@ -643,3 +643,5 @@ bsa: orderStatusUrl: "https://" # Http endpoint for reporting changes in the set of unblockable domains. unblockableDomainsUrl: "https://" + # API endpoint for uploading the list of unavailable domain names. + uploadUnavailableDomainsUrl: "https://" diff --git a/core/src/main/java/google/registry/model/tld/label/ReservedList.java b/core/src/main/java/google/registry/model/tld/label/ReservedList.java index a643f9238..d28825beb 100644 --- a/core/src/main/java/google/registry/model/tld/label/ReservedList.java +++ b/core/src/main/java/google/registry/model/tld/label/ReservedList.java @@ -63,15 +63,13 @@ public final class ReservedList extends BaseDomainLabelList { /** - * Mapping from domain name to its reserved list info. + * Mapping from domain label to its reserved list info. * *

This field requires special treatment since we want to lazy load it. We have to remove it * from the immutability contract so we can modify it after construction and we have to handle the * database processing on our own so we can detach it after load. */ - @Insignificant - @Transient - Map reservedListMap; + @Insignificant @Transient Map reservedListMap; @Column(nullable = false) boolean shouldPublish = true; @@ -269,7 +267,7 @@ public final class ReservedList } /** Loads and returns the reserved lists with the given names, skipping those that don't exist. */ - private static ImmutableSet loadReservedLists( + public static ImmutableSet loadReservedLists( ImmutableSet reservedListNames) { return cache.getAll(reservedListNames).values().stream() .filter(Optional::isPresent) diff --git a/core/src/main/java/google/registry/module/bsa/BsaRequestComponent.java b/core/src/main/java/google/registry/module/bsa/BsaRequestComponent.java index 5d7412aea..d9811f183 100644 --- a/core/src/main/java/google/registry/module/bsa/BsaRequestComponent.java +++ b/core/src/main/java/google/registry/module/bsa/BsaRequestComponent.java @@ -18,21 +18,22 @@ import dagger.Module; import dagger.Subcomponent; import google.registry.bsa.BsaDownloadAction; import google.registry.bsa.BsaRefreshAction; +import google.registry.bsa.UploadBsaUnavailableDomainsAction; +import google.registry.request.Modules.UrlConnectionServiceModule; import google.registry.request.RequestComponentBuilder; import google.registry.request.RequestModule; import google.registry.request.RequestScope; @RequestScope -@Subcomponent( - modules = { - RequestModule.class, - }) +@Subcomponent(modules = {RequestModule.class, UrlConnectionServiceModule.class}) interface BsaRequestComponent { BsaDownloadAction bsaDownloadAction(); BsaRefreshAction bsaRefreshAction(); + UploadBsaUnavailableDomainsAction uploadBsaUnavailableDomains(); + @Subcomponent.Builder abstract class Builder implements RequestComponentBuilder { diff --git a/core/src/test/resources/google/registry/module/bsa/bsa_routing.txt b/core/src/test/resources/google/registry/module/bsa/bsa_routing.txt index 2dae71deb..fb58c1173 100644 --- a/core/src/test/resources/google/registry/module/bsa/bsa_routing.txt +++ b/core/src/test/resources/google/registry/module/bsa/bsa_routing.txt @@ -1,3 +1,4 @@ -PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY -/_dr/task/bsaDownload BsaDownloadAction GET,POST n API APP ADMIN -/_dr/task/bsaRefresh BsaRefreshAction GET,POST n API APP ADMIN +PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY +/_dr/task/bsaDownload BsaDownloadAction GET,POST n API APP ADMIN +/_dr/task/bsaRefresh BsaRefreshAction GET,POST n API APP ADMIN +/_dr/task/uploadBsaUnavailableNames UploadBsaUnavailableDomainsAction POST n API APP ADMIN