zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-07-22 18:55:58 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add a daily batch action to upload unavailable domains to BSA (#2265)

Browse source
This commit is contained in:
Ben McIlwain 2024-01-09 14:52:07 -05:00 • committed by GitHub
parent f8ac7afc33
commit e79c63142a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 255 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

2
core/gradle.lockfile
View file

@ -349,7 +349,7 @@ org.apache.mina:mina-core:2.1.6=testCompileClasspath,testRuntimeClasspath
org.apache.sshd:sshd-core:2.0.0=testCompileClasspath,testRuntimeClasspath
org.apache.sshd:sshd-scp:2.0.0=testCompileClasspath,testRuntimeClasspath
org.apache.sshd:sshd-sftp:2.0.0=testCompileClasspath,testRuntimeClasspath
org.apache.tomcat:tomcat-annotations-api:11.0.0-M15=testCompileClasspath,testRuntimeClasspath
org.apache.tomcat:tomcat-annotations-api:11.0.0-M16=testCompileClasspath,testRuntimeClasspath
org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath
org.bouncycastle:bcpg-jdk15on:1.67=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.bouncycastle:bcpkix-jdk15on:1.67=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath

220
core/src/main/java/google/registry/bsa/UploadBsaUnavailableDomainsAction.java Normal file
View file

@ -0,0 +1,220 @@
// Copyright 2024 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.bsa;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static google.registry.model.tld.Tld.isEnrolledWithBsa;
import static google.registry.model.tld.Tlds.getTldEntitiesOfType;
import static google.registry.model.tld.label.ReservedList.loadReservedLists;
import static google.registry.persistence.transaction.TransactionManagerFactory.replicaTm;
import static google.registry.request.Action.Method.POST;
import static java.nio.charset.StandardCharsets.US_ASCII;
import com.google.api.client.http.HttpStatusCodes;
import com.google.cloud.storage.BlobId;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Ordering;
import com.google.common.flogger.FluentLogger;
import com.google.common.hash.Hashing;
import com.google.common.io.ByteSource;
import google.registry.bsa.api.BsaCredential;
import google.registry.config.RegistryConfig.Config;
import google.registry.gcs.GcsUtils;
import google.registry.model.tld.Tld;
import google.registry.model.tld.Tld.TldType;
import google.registry.model.tld.label.ReservedList;
import google.registry.request.Action;
import google.registry.request.Action.Service;
import google.registry.request.auth.Auth;
import google.registry.util.Clock;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.zip.GZIPOutputStream;
import javax.inject.Inject;
import okhttp3.MediaType;
import okhttp3.MultipartBody;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import org.joda.time.DateTime;
/**
* Daily action that uploads unavailable domain names on applicable TLDs to BSA.
*
* <p>The upload is a single zipped text file containing combined details for all BSA-enrolled TLDs.
* The text is a newline-delimited list of punycoded fully qualified domain names, and contains all
* domains on each TLD that are registered and/or reserved.
*
* <p>The file is also uploaded to GCS to preserve it as a record for ourselves.
*/
@Action(
service = Service.BSA,
path = "/_dr/task/uploadBsaUnavailableNames",
method = POST,
auth = Auth.AUTH_API_ADMIN)
public class UploadBsaUnavailableDomainsAction implements Runnable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
Clock clock;
BsaCredential bsaCredential;
GcsUtils gcsUtils;
String gcsBucket;
String apiUrl;
google.registry.request.Response response;
@Inject
public UploadBsaUnavailableDomainsAction(
Clock clock,
BsaCredential bsaCredential,
GcsUtils gcsUtils,
@Config("bsaUnavailableDomainsGcsBucket") String gcsBucket,
@Config("bsaUploadUnavailableDomainsUrl") String apiUrl,
google.registry.request.Response response) {
this.clock = clock;
this.bsaCredential = bsaCredential;
this.gcsUtils = gcsUtils;
this.gcsBucket = gcsBucket;
this.apiUrl = apiUrl;
this.response = response;
}
@Override
public void run() {
DateTime runTime = clock.nowUtc();
String unavailableDomains =
Joiner.on("\n").join(replicaTm().transact(() -> getUnavailableDomains(runTime)));
uploadToGcs(unavailableDomains, runTime);
uploadToBsa(unavailableDomains, runTime);
}
/** Uploads the unavailable domains list to GCS in the unavailable domains bucket. */
void uploadToGcs(String unavailableDomains, DateTime runTime) {
logger.atInfo().log("Uploading unavailable names file to GCS in bucket %s", gcsBucket);
BlobId blobId = BlobId.of(gcsBucket, createFilename(runTime));
try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId);
Writer osWriter = new OutputStreamWriter(gcsOutput, US_ASCII)) {
osWriter.write(unavailableDomains);
} catch (Exception e) {
logger.atSevere().withCause(e).log(
"Error writing BSA unavailable domains to GCS; skipping to BSA upload ...");
}
}
void uploadToBsa(String unavailableDomains, DateTime runTime) {
try {
byte[] gzippedContents = gzipUnavailableDomains(unavailableDomains);
String sha512Hash = ByteSource.wrap(gzippedContents).hash(Hashing.sha512()).toString();
String filename = createFilename(runTime);
OkHttpClient client = new OkHttpClient().newBuilder().build();
RequestBody body =
new MultipartBody.Builder()
.setType(MultipartBody.FORM)
.addFormDataPart(
"zone",
null,
RequestBody.create(
String.format("{\"checkSum\": \"%s\"}", sha512Hash).getBytes(US_ASCII),
MediaType.parse("application/json")))
.addFormDataPart(
"file",
String.format("%s.gz", filename),
RequestBody.create(gzippedContents, MediaType.parse("application/octet-stream")))
.build();
Request request =
new Request.Builder()
.url(apiUrl)
.method("POST", body)
.addHeader("Authorization", "Bearer " + bsaCredential.getAuthToken())
.build();
logger.atInfo().log(
"Uploading unavailable domains list %s to %s with hash %s", filename, apiUrl, sha512Hash);
try (Response uploadResponse = client.newCall(request).execute()) {
logger.atInfo().log(
"Received response with code %s from server: %s",
uploadResponse.code(),
uploadResponse.body() == null ? "(none)" : uploadResponse.body().string());
}
} catch (IOException e) {
logger.atSevere().withCause(e).log("Error while attempting to upload to BSA, aborting.");
response.setStatus(HttpStatusCodes.STATUS_CODE_SERVER_ERROR);
response.setPayload("Error while attempting to upload to BSA: " + e.getMessage());
}
}
private byte[] gzipUnavailableDomains(String unavailableDomains) throws IOException {
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) {
gzipOutputStream.write(unavailableDomains.getBytes(US_ASCII));
}
return byteArrayOutputStream.toByteArray();
}
}
private static String createFilename(DateTime runTime) {
return String.format("unavailable_domains_%s.txt", runTime.toString());
}
private ImmutableSortedSet<String> getUnavailableDomains(DateTime runTime) {
ImmutableSet<Tld> bsaEnabledTlds =
getTldEntitiesOfType(TldType.REAL).stream()
.filter(tld -> isEnrolledWithBsa(tld, runTime))
.collect(toImmutableSet());
ImmutableSortedSet.Builder<String> unavailableDomains =
new ImmutableSortedSet.Builder<>(Ordering.natural());
for (Tld tld : bsaEnabledTlds) {
for (ReservedList reservedList : loadReservedLists(tld.getReservedListNames())) {
if (reservedList.getShouldPublish()) {
unavailableDomains.addAll(
reservedList.getReservedListEntries().keySet().stream()
.map(label -> toDomain(label, tld))
.collect(toImmutableSet()));
}
}
}
unavailableDomains.addAll(
replicaTm()
.query(
"SELECT domainName FROM Domain "
+ "WHERE tld IN :tlds "
+ "AND deletionTime > :now ",
String.class)
.setParameter(
"tlds", bsaEnabledTlds.stream().map(Tld::getTldStr).collect(toImmutableSet()))
.setParameter("now", runTime)
.getResultList());
return unavailableDomains.build();
}
private static String toDomain(String domainLabel, Tld tld) {
return String.format("%s.%s", domainLabel, tld.getTldStr());
}
}

19
core/src/main/java/google/registry/config/RegistryConfig.java
View file

@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import dagger.Module;
import dagger.Provides;
import google.registry.bsa.UploadBsaUnavailableDomainsAction;
import google.registry.dns.ReadDnsRefreshRequestsAction;
import google.registry.model.common.DnsRefreshRequest;
import google.registry.persistence.transaction.JpaTransactionManager;
@ -270,6 +271,18 @@ public final class RegistryConfig {
return projectId + "-domain-lists";
}
/**
* The GCS bucket for exporting lists of unavailable names for the BSA.
*
* @see UploadBsaUnavailableDomainsAction
*/
@Provides
@Config("bsaUnavailableDomainsGcsBucket")
public static String provideBsaUnavailableNamesGcsBucket(
@Config("projectId") String projectId) {
return projectId + "-bsa-unavailable-domains";
}
/**
* Returns the Google Cloud Storage bucket for staging BRDA escrow deposits.
*
@ -1495,6 +1508,12 @@ public final class RegistryConfig {
return String.format("%s?%s", config.bsa.unblockableDomainsUrl, "action=remove");
}
@Provides
@Config("bsaUploadUnavailableDomainsUrl")
public static String provideBsaUploadUnavailableDomainsUrl(RegistryConfigSettings config) {
return config.bsa.uploadUnavailableDomainsUrl;
}
private static String formatComments(String text) {
return Splitter.on('\n').omitEmptyStrings().trimResults().splitToList(text).stream()
.map(s -> "# " + s)

1
core/src/main/java/google/registry/config/RegistryConfigSettings.java
View file

@ -279,5 +279,6 @@ public class RegistryConfigSettings {
public Map<String, String> dataUrls;
public String orderStatusUrl;
public String unblockableDomainsUrl;
public String uploadUnavailableDomainsUrl;
}
}

2
core/src/main/java/google/registry/config/files/default-config.yaml
View file

@ -643,3 +643,5 @@ bsa:
orderStatusUrl: "https://"
# Http endpoint for reporting changes in the set of unblockable domains.
unblockableDomainsUrl: "https://"
# API endpoint for uploading the list of unavailable domain names.
uploadUnavailableDomainsUrl: "https://"

8
core/src/main/java/google/registry/model/tld/label/ReservedList.java
View file

@ -63,15 +63,13 @@ public final class ReservedList
extends BaseDomainLabelList<ReservationType, ReservedList.ReservedListEntry> {
/**
* Mapping from domain name to its reserved list info.
* Mapping from domain label to its reserved list info.
*
* <p>This field requires special treatment since we want to lazy load it. We have to remove it
* from the immutability contract so we can modify it after construction and we have to handle the
* database processing on our own so we can detach it after load.
*/
@Insignificant
@Transient
Map<String, ReservedListEntry> reservedListMap;
@Insignificant @Transient Map<String, ReservedListEntry> reservedListMap;
@Column(nullable = false)
boolean shouldPublish = true;
@ -269,7 +267,7 @@ public final class ReservedList
}
/** Loads and returns the reserved lists with the given names, skipping those that don't exist. */
private static ImmutableSet<ReservedList> loadReservedLists(
public static ImmutableSet<ReservedList> loadReservedLists(
ImmutableSet<String> reservedListNames) {
return cache.getAll(reservedListNames).values().stream()
.filter(Optional::isPresent)

9
core/src/main/java/google/registry/module/bsa/BsaRequestComponent.java
View file

@ -18,21 +18,22 @@ import dagger.Module;
import dagger.Subcomponent;
import google.registry.bsa.BsaDownloadAction;
import google.registry.bsa.BsaRefreshAction;
import google.registry.bsa.UploadBsaUnavailableDomainsAction;
import google.registry.request.Modules.UrlConnectionServiceModule;
import google.registry.request.RequestComponentBuilder;
import google.registry.request.RequestModule;
import google.registry.request.RequestScope;
@RequestScope
@Subcomponent(
modules = {
RequestModule.class,
})
@Subcomponent(modules = {RequestModule.class, UrlConnectionServiceModule.class})
interface BsaRequestComponent {
BsaDownloadAction bsaDownloadAction();
BsaRefreshAction bsaRefreshAction();
UploadBsaUnavailableDomainsAction uploadBsaUnavailableDomains();
@Subcomponent.Builder
abstract class Builder implements RequestComponentBuilder<BsaRequestComponent> {

7
core/src/test/resources/google/registry/module/bsa/bsa_routing.txt
View file

@ -1,3 +1,4 @@
PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY
/_dr/task/bsaDownload BsaDownloadAction GET,POST n API APP ADMIN
/_dr/task/bsaRefresh BsaRefreshAction GET,POST n API APP ADMIN
PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY
/_dr/task/bsaDownload BsaDownloadAction GET,POST n API APP ADMIN
/_dr/task/bsaRefresh BsaRefreshAction GET,POST n API APP ADMIN
/_dr/task/uploadBsaUnavailableNames UploadBsaUnavailableDomainsAction POST n API APP ADMIN