mirror of
https://github.com/google/nomulus.git
synced 2025-07-26 04:28:34 +02:00
Fix BsaRefreshAction bugs (#2294)
* Fix BsaRefreshAction bugs Added functional tests for BsaRefreshAction, which checks for changes in domain registration and reservation, and apply them to the Unblockable domain list. Fixed a few bugs exposed by the tests. Also refactored a few other tests.
This commit is contained in:
parent
c414e38a98
commit
f61579b350
14 changed files with 509 additions and 74 deletions
|
@ -54,7 +54,7 @@ public class BsaRefreshAction implements Runnable {
|
|||
private final GcsClient gcsClient;
|
||||
private final BsaReportSender bsaReportSender;
|
||||
private final int transactionBatchSize;
|
||||
private final Duration domainTxnMaxDuration;
|
||||
private final Duration domainCreateTxnCommitTimeLag;
|
||||
private final BsaLock bsaLock;
|
||||
private final Clock clock;
|
||||
private final Response response;
|
||||
|
@ -65,7 +65,7 @@ public class BsaRefreshAction implements Runnable {
|
|||
GcsClient gcsClient,
|
||||
BsaReportSender bsaReportSender,
|
||||
@Config("bsaTxnBatchSize") int transactionBatchSize,
|
||||
@Config("domainCreateTxnCommitTimeLag") Duration domainTxnMaxDuration,
|
||||
@Config("domainCreateTxnCommitTimeLag") Duration domainCreateTxnCommitTimeLag,
|
||||
BsaLock bsaLock,
|
||||
Clock clock,
|
||||
Response response) {
|
||||
|
@ -73,7 +73,7 @@ public class BsaRefreshAction implements Runnable {
|
|||
this.gcsClient = gcsClient;
|
||||
this.bsaReportSender = bsaReportSender;
|
||||
this.transactionBatchSize = transactionBatchSize;
|
||||
this.domainTxnMaxDuration = domainTxnMaxDuration;
|
||||
this.domainCreateTxnCommitTimeLag = domainCreateTxnCommitTimeLag;
|
||||
this.bsaLock = bsaLock;
|
||||
this.clock = clock;
|
||||
this.response = response;
|
||||
|
@ -109,17 +109,21 @@ public class BsaRefreshAction implements Runnable {
|
|||
RefreshSchedule schedule = maybeSchedule.get();
|
||||
DomainsRefresher refresher =
|
||||
new DomainsRefresher(
|
||||
schedule.prevRefreshTime(), clock.nowUtc(), domainTxnMaxDuration, transactionBatchSize);
|
||||
schedule.prevRefreshTime(),
|
||||
clock.nowUtc(),
|
||||
domainCreateTxnCommitTimeLag,
|
||||
transactionBatchSize);
|
||||
switch (schedule.stage()) {
|
||||
case CHECK_FOR_CHANGES:
|
||||
ImmutableList<UnblockableDomainChange> blockabilityChanges =
|
||||
refresher.checkForBlockabilityChanges();
|
||||
// Always write even if no change. Easier for manual inspection of GCS bucket.
|
||||
gcsClient.writeRefreshChanges(schedule.jobName(), blockabilityChanges.stream());
|
||||
if (blockabilityChanges.isEmpty()) {
|
||||
logger.atInfo().log("No change to Unblockable domains found.");
|
||||
schedule.updateJobStage(RefreshStage.DONE);
|
||||
return null;
|
||||
}
|
||||
gcsClient.writeRefreshChanges(schedule.jobName(), blockabilityChanges.stream());
|
||||
schedule.updateJobStage(RefreshStage.APPLY_CHANGES);
|
||||
// Fall through
|
||||
case APPLY_CHANGES:
|
||||
|
@ -132,10 +136,12 @@ public class BsaRefreshAction implements Runnable {
|
|||
case UPLOAD_REMOVALS:
|
||||
try (Stream<UnblockableDomainChange> changes =
|
||||
gcsClient.readRefreshChanges(schedule.jobName())) {
|
||||
// Unblockables with changes in REASON are removed then added back. That is why they are
|
||||
// included in this stage.
|
||||
Optional<String> report =
|
||||
JsonSerializations.toUnblockableDomainsRemovalReport(
|
||||
changes
|
||||
.filter(UnblockableDomainChange::isDelete)
|
||||
.filter(UnblockableDomainChange::isChangeOrDelete)
|
||||
.map(UnblockableDomainChange::domainName));
|
||||
if (report.isPresent()) {
|
||||
gcsClient.logRemovedUnblockableDomainsReport(
|
||||
|
@ -153,12 +159,12 @@ public class BsaRefreshAction implements Runnable {
|
|||
Optional<String> report =
|
||||
JsonSerializations.toUnblockableDomainsReport(
|
||||
changes
|
||||
.filter(UnblockableDomainChange::isAddOrChange)
|
||||
.filter(UnblockableDomainChange::isNewOrChange)
|
||||
.map(UnblockableDomainChange::newValue));
|
||||
if (report.isPresent()) {
|
||||
gcsClient.logRemovedUnblockableDomainsReport(
|
||||
schedule.jobName(), LINE_SPLITTER.splitToStream(report.get()));
|
||||
bsaReportSender.removeUnblockableDomainsUpdates(report.get());
|
||||
bsaReportSender.addUnblockableDomainsUpdates(report.get());
|
||||
} else {
|
||||
logger.atInfo().log("No new Unblockable domains to add.");
|
||||
}
|
||||
|
|
|
@ -30,7 +30,7 @@ public class BsaStringUtils {
|
|||
public static final Splitter LINE_SPLITTER = Splitter.on('\n');
|
||||
|
||||
public static String getLabelInDomain(String domainName) {
|
||||
List<String> parts = DOMAIN_SPLITTER.limit(1).splitToList(domainName);
|
||||
List<String> parts = DOMAIN_SPLITTER.splitToList(domainName);
|
||||
checkArgument(!parts.isEmpty(), "Not a valid domain: [%s]", domainName);
|
||||
return parts.get(0);
|
||||
}
|
||||
|
|
|
@ -21,6 +21,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
|||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
|
||||
import com.google.errorprone.annotations.CanIgnoreReturnValue;
|
||||
import google.registry.persistence.transaction.TransactionManager.ThrowingRunnable;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
/**
|
||||
|
@ -39,6 +40,11 @@ public final class BsaTransactions {
|
|||
return tm().transact(work, TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
|
||||
public static void bsaTransact(ThrowingRunnable work) {
|
||||
verify(!isInTransaction(), "May only be used for top-level transactions.");
|
||||
tm().transact(work, TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
|
||||
@CanIgnoreReturnValue
|
||||
public static <T> T bsaQuery(Callable<T> work) {
|
||||
verify(!isInTransaction(), "May only be used for top-level transactions.");
|
||||
|
|
|
@ -161,7 +161,7 @@ public class GcsClient {
|
|||
}
|
||||
|
||||
Stream<UnblockableDomainChange> readRefreshChanges(String jobName) {
|
||||
BlobId blobId = getBlobId(jobName, UNBLOCKABLE_DOMAINS_FILE);
|
||||
BlobId blobId = getBlobId(jobName, REFRESHED_UNBLOCKABLE_DOMAINS_FILE);
|
||||
return readStream(blobId).map(UnblockableDomainChange::deserialize);
|
||||
}
|
||||
|
||||
|
|
|
@ -28,9 +28,9 @@ import java.util.List;
|
|||
// TODO(1/15/2024): rename to UnblockableDomain.
|
||||
@AutoValue
|
||||
public abstract class UnblockableDomain {
|
||||
abstract String domainName();
|
||||
public abstract String domainName();
|
||||
|
||||
abstract Reason reason();
|
||||
public abstract Reason reason();
|
||||
|
||||
/** Reasons why a valid domain name cannot be blocked. */
|
||||
public enum Reason {
|
||||
|
|
|
@ -52,12 +52,16 @@ public abstract class UnblockableDomainChange {
|
|||
return UnblockableDomain.of(unblockable().domainName(), newReason().get());
|
||||
}
|
||||
|
||||
public boolean isAddOrChange() {
|
||||
public boolean isNewOrChange() {
|
||||
return newReason().isPresent();
|
||||
}
|
||||
|
||||
public boolean isChangeOrDelete() {
|
||||
return !isNew();
|
||||
}
|
||||
|
||||
public boolean isDelete() {
|
||||
return !this.isAddOrChange();
|
||||
return !this.isNewOrChange();
|
||||
}
|
||||
|
||||
public boolean isNew() {
|
||||
|
|
|
@ -105,6 +105,10 @@ class BsaUnblockableDomain {
|
|||
return new BsaUnblockableDomain(parts.get(0), parts.get(1), reason);
|
||||
}
|
||||
|
||||
static BsaUnblockableDomain of(UnblockableDomain unblockable) {
|
||||
return of(unblockable.domainName(), Reason.valueOf(unblockable.reason().name()));
|
||||
}
|
||||
|
||||
static VKey<BsaUnblockableDomain> vKey(String domainName) {
|
||||
ImmutableList<String> parts = ImmutableList.copyOf(DOMAIN_SPLITTER.splitToList(domainName));
|
||||
verify(parts.size() == 2, "Invalid domain name: %s", domainName);
|
||||
|
|
|
@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
|
|||
import static com.google.common.collect.ImmutableMap.toImmutableMap;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.bsa.BsaTransactions.bsaQuery;
|
||||
import static google.registry.bsa.BsaTransactions.bsaTransact;
|
||||
import static google.registry.bsa.ReservedDomainsUtils.getAllReservedNames;
|
||||
import static google.registry.bsa.ReservedDomainsUtils.isReservedDomain;
|
||||
import static google.registry.bsa.persistence.Queries.batchReadUnblockables;
|
||||
|
@ -44,6 +45,7 @@ import google.registry.model.domain.Domain;
|
|||
import google.registry.model.tld.Tld;
|
||||
import google.registry.model.tld.Tld.TldType;
|
||||
import google.registry.util.BatchedStreams;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
@ -107,6 +109,8 @@ public final class DomainsRefresher {
|
|||
|
||||
ImmutableSet<String> upgradedDomains =
|
||||
upgrades.stream().map(UnblockableDomainChange::domainName).collect(toImmutableSet());
|
||||
// Upgrades are collected in its own txn after downgrades so need reconciliation. Conflicts are
|
||||
// unlikely but possible: domains may be recreated or reserved names added during the interval.
|
||||
ImmutableList<UnblockableDomainChange> trueDowngrades =
|
||||
downgrades.stream()
|
||||
.filter(c -> !upgradedDomains.contains(c.domainName()))
|
||||
|
@ -198,22 +202,50 @@ public final class DomainsRefresher {
|
|||
}
|
||||
|
||||
public ImmutableList<UnblockableDomainChange> getNewUnblockables() {
|
||||
return bsaQuery(
|
||||
() -> {
|
||||
// TODO(weiminyu): both methods below use `queryBsaLabelByLabels`. Should combine.
|
||||
ImmutableSet<String> newCreated = getNewlyCreatedUnblockables(prevRefreshStartTime, now);
|
||||
ImmutableSet<String> newReserved =
|
||||
getNewlyReservedUnblockables(now, transactionBatchSize);
|
||||
SetView<String> reservedNotCreated = Sets.difference(newReserved, newCreated);
|
||||
return Streams.concat(
|
||||
newCreated.stream()
|
||||
.map(name -> UnblockableDomain.of(name, Reason.REGISTERED))
|
||||
.map(UnblockableDomainChange::ofNew),
|
||||
reservedNotCreated.stream()
|
||||
.map(name -> UnblockableDomain.of(name, Reason.RESERVED))
|
||||
.map(UnblockableDomainChange::ofNew))
|
||||
.collect(toImmutableList());
|
||||
});
|
||||
// TODO(weiminyu): both methods below use `queryBsaLabelByLabels`. Should combine in a single
|
||||
// query.
|
||||
HashSet<String> newCreated =
|
||||
Sets.newHashSet(bsaQuery(() -> getNewlyCreatedUnblockables(prevRefreshStartTime, now)));
|
||||
// We cannot identify new reserved unblockables so must look at all of them. There are not many
|
||||
// of these.
|
||||
HashSet<String> allReserved =
|
||||
Sets.newHashSet(bsaQuery(() -> getAllReservedUnblockables(now, transactionBatchSize)));
|
||||
HashSet<String> reservedNotCreated = Sets.newHashSet(Sets.difference(allReserved, newCreated));
|
||||
|
||||
ImmutableList.Builder<UnblockableDomainChange> changes = new ImmutableList.Builder<>();
|
||||
ImmutableList<BsaUnblockableDomain> batch;
|
||||
Optional<BsaUnblockableDomain> lastRead = Optional.empty();
|
||||
do {
|
||||
batch = batchReadUnblockables(lastRead, transactionBatchSize);
|
||||
if (batch.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
lastRead = Optional.of(batch.get(batch.size() - 1));
|
||||
for (BsaUnblockableDomain unblockable : batch) {
|
||||
String domainName = unblockable.domainName();
|
||||
if (unblockable.reason.equals(BsaUnblockableDomain.Reason.REGISTERED)) {
|
||||
newCreated.remove(domainName);
|
||||
reservedNotCreated.remove(domainName);
|
||||
} else {
|
||||
reservedNotCreated.remove(domainName);
|
||||
if (newCreated.remove(domainName)) {
|
||||
changes.add(
|
||||
UnblockableDomainChange.ofChanged(
|
||||
unblockable.toUnblockableDomain(), Reason.REGISTERED));
|
||||
}
|
||||
}
|
||||
}
|
||||
} while (batch.size() == transactionBatchSize);
|
||||
|
||||
Streams.concat(
|
||||
newCreated.stream()
|
||||
.map(name -> UnblockableDomain.of(name, Reason.REGISTERED))
|
||||
.map(UnblockableDomainChange::ofNew),
|
||||
reservedNotCreated.stream()
|
||||
.map(name -> UnblockableDomain.of(name, Reason.RESERVED))
|
||||
.map(UnblockableDomainChange::ofNew))
|
||||
.forEach(changes::add);
|
||||
return changes.build();
|
||||
}
|
||||
|
||||
static ImmutableSet<String> getNewlyCreatedUnblockables(
|
||||
|
@ -225,18 +257,18 @@ public final class DomainsRefresher {
|
|||
.collect(toImmutableSet());
|
||||
ImmutableSet<String> liveDomains =
|
||||
queryNewlyCreatedDomains(bsaEnabledTlds, prevRefreshStartTime, now);
|
||||
return getUnblockedDomainNames(liveDomains);
|
||||
return getBlockedDomainNames(liveDomains);
|
||||
}
|
||||
|
||||
static ImmutableSet<String> getNewlyReservedUnblockables(DateTime now, int batchSize) {
|
||||
static ImmutableSet<String> getAllReservedUnblockables(DateTime now, int batchSize) {
|
||||
Stream<String> allReserved = getAllReservedNames(now);
|
||||
return BatchedStreams.toBatches(allReserved, batchSize)
|
||||
.map(DomainsRefresher::getUnblockedDomainNames)
|
||||
.map(DomainsRefresher::getBlockedDomainNames)
|
||||
.flatMap(ImmutableSet::stream)
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
|
||||
static ImmutableSet<String> getUnblockedDomainNames(ImmutableCollection<String> domainNames) {
|
||||
static ImmutableSet<String> getBlockedDomainNames(ImmutableCollection<String> domainNames) {
|
||||
Map<String, List<String>> labelToNames =
|
||||
domainNames.stream().collect(groupingBy(BsaStringUtils::getLabelInDomain));
|
||||
ImmutableSet<String> bsaLabels =
|
||||
|
@ -244,7 +276,7 @@ public final class DomainsRefresher {
|
|||
.map(BsaLabel::getLabel)
|
||||
.collect(toImmutableSet());
|
||||
return labelToNames.entrySet().stream()
|
||||
.filter(entry -> !bsaLabels.contains(entry.getKey()))
|
||||
.filter(entry -> bsaLabels.contains(entry.getKey()))
|
||||
.map(Entry::getValue)
|
||||
.flatMap(List::stream)
|
||||
.collect(toImmutableSet());
|
||||
|
@ -257,20 +289,21 @@ public final class DomainsRefresher {
|
|||
.collect(
|
||||
groupingBy(
|
||||
change -> change.isDelete() ? "remove" : "change", toImmutableSet())));
|
||||
tm().transact(
|
||||
() -> {
|
||||
if (changesByType.containsKey("remove")) {
|
||||
tm().delete(
|
||||
changesByType.get("remove").stream()
|
||||
.map(c -> BsaUnblockableDomain.vKey(c.domainName()))
|
||||
.collect(toImmutableSet()));
|
||||
}
|
||||
if (changesByType.containsKey("change")) {
|
||||
tm().putAll(
|
||||
changesByType.get("change").stream()
|
||||
.map(UnblockableDomainChange::newValue)
|
||||
.collect(toImmutableSet()));
|
||||
}
|
||||
});
|
||||
bsaTransact(
|
||||
() -> {
|
||||
if (changesByType.containsKey("remove")) {
|
||||
tm().delete(
|
||||
changesByType.get("remove").stream()
|
||||
.map(c -> BsaUnblockableDomain.vKey(c.domainName()))
|
||||
.collect(toImmutableSet()));
|
||||
}
|
||||
if (changesByType.containsKey("change")) {
|
||||
tm().putAll(
|
||||
changesByType.get("change").stream()
|
||||
.map(UnblockableDomainChange::newValue)
|
||||
.map(BsaUnblockableDomain::of)
|
||||
.collect(toImmutableSet()));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
@ -118,15 +118,8 @@ class BsaDownloadFunctionalTest {
|
|||
gcsClient.readBlockList(downloadJob, BlockListType.BLOCK_PLUS)) {
|
||||
assertThat(blockListFile).containsExactly(BSA_CSV_HEADER, "abc,2", "def,3");
|
||||
}
|
||||
ImmutableList<String> persistedLabels =
|
||||
ImmutableList.copyOf(
|
||||
tm().transact(
|
||||
() ->
|
||||
tm().getEntityManager()
|
||||
.createNativeQuery("SELECT label from \"BsaLabel\"")
|
||||
.getResultList()));
|
||||
// TODO(weiminyu): check intermediate files
|
||||
assertThat(persistedLabels).containsExactly("abc", "def");
|
||||
assertThat(getPersistedLabels()).containsExactly("abc", "def");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
@ -0,0 +1,319 @@
|
|||
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.bsa;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth8.assertThat;
|
||||
import static google.registry.bsa.ReservedDomainsTestingUtils.addReservedDomainToList;
|
||||
import static google.registry.bsa.ReservedDomainsTestingUtils.addReservedListsToTld;
|
||||
import static google.registry.bsa.ReservedDomainsTestingUtils.createReservedList;
|
||||
import static google.registry.bsa.ReservedDomainsTestingUtils.removeReservedDomainFromList;
|
||||
import static google.registry.bsa.persistence.BsaTestingUtils.createDownloadScheduler;
|
||||
import static google.registry.bsa.persistence.BsaTestingUtils.persistBsaLabel;
|
||||
import static google.registry.bsa.persistence.BsaTestingUtils.queryUnblockableDomains;
|
||||
import static google.registry.model.tld.Tlds.getTldEntitiesOfType;
|
||||
import static google.registry.model.tld.label.ReservationType.RESERVED_FOR_SPECIFIC_USE;
|
||||
import static google.registry.testing.DatabaseHelper.createTlds;
|
||||
import static google.registry.testing.DatabaseHelper.deleteTestDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.bsa.api.BsaReportSender;
|
||||
import google.registry.bsa.api.UnblockableDomain;
|
||||
import google.registry.bsa.api.UnblockableDomain.Reason;
|
||||
import google.registry.bsa.api.UnblockableDomainChange;
|
||||
import google.registry.bsa.persistence.BsaTestingUtils;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.domain.Domain;
|
||||
import google.registry.model.tld.Tld.TldType;
|
||||
import google.registry.persistence.transaction.JpaTestExtensions;
|
||||
import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeLockHandler;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import java.util.Optional;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
import org.mockito.InOrder;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
/**
|
||||
* Functional tests for refreshing the unblockable domains with recent registration and reservation
|
||||
* changes.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class BsaRefreshFunctionalTest {
|
||||
|
||||
static final DateTime TEST_START_TIME = DateTime.parse("2024-01-01T00:00:00Z");
|
||||
|
||||
static final String RESERVED_LIST_NAME = "reserved";
|
||||
|
||||
private final FakeClock fakeClock = new FakeClock(TEST_START_TIME);
|
||||
|
||||
@RegisterExtension
|
||||
JpaIntegrationWithCoverageExtension jpa =
|
||||
new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension();
|
||||
|
||||
@Mock BsaReportSender bsaReportSender;
|
||||
|
||||
private GcsClient gcsClient;
|
||||
private Response response;
|
||||
private BsaRefreshAction action;
|
||||
|
||||
@BeforeEach
|
||||
void setup() throws Exception {
|
||||
gcsClient =
|
||||
new GcsClient(new GcsUtils(LocalStorageHelper.getOptions()), "my-bucket", "SHA-256");
|
||||
response = new FakeResponse();
|
||||
action =
|
||||
new BsaRefreshAction(
|
||||
BsaTestingUtils.createRefreshScheduler(),
|
||||
gcsClient,
|
||||
bsaReportSender,
|
||||
/* transactionBatchSize= */ 5,
|
||||
/* domainCreateTxnCommitTimeLag= */ Duration.millis(1),
|
||||
new BsaLock(
|
||||
new FakeLockHandler(/* lockSucceeds= */ true), Duration.standardSeconds(30)),
|
||||
fakeClock,
|
||||
response);
|
||||
|
||||
initDb();
|
||||
}
|
||||
|
||||
private String getRefreshJobName(DateTime jobStartTime) {
|
||||
return jobStartTime.toString() + "-refresh";
|
||||
}
|
||||
|
||||
private void initDb() {
|
||||
createTlds("app", "dev");
|
||||
getTldEntitiesOfType(TldType.REAL)
|
||||
.forEach(
|
||||
tld ->
|
||||
persistResource(
|
||||
tld.asBuilder().setBsaEnrollStartTime(Optional.of(START_OF_TIME)).build()));
|
||||
|
||||
createReservedList(RESERVED_LIST_NAME, "dummy", RESERVED_FOR_SPECIFIC_USE);
|
||||
addReservedListsToTld("app", ImmutableList.of(RESERVED_LIST_NAME));
|
||||
|
||||
persistBsaLabel("blocked1");
|
||||
persistBsaLabel("blocked2");
|
||||
// Creates a download record so that refresher will not quit immediately.
|
||||
createDownloadScheduler(fakeClock).schedule().get().updateJobStage(DownloadStage.DONE);
|
||||
fakeClock.advanceOneMilli();
|
||||
}
|
||||
|
||||
@Test
|
||||
void newReservedDomain_addedAsUnblockable() throws Exception {
|
||||
addReservedDomainToList(
|
||||
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
|
||||
String jobName = getRefreshJobName(fakeClock.nowUtc());
|
||||
action.run();
|
||||
UnblockableDomain newUnblockable = UnblockableDomain.of("blocked1.app", Reason.RESERVED);
|
||||
assertThat(queryUnblockableDomains()).containsExactly(newUnblockable);
|
||||
assertThat(gcsClient.readRefreshChanges(jobName))
|
||||
.containsExactly(UnblockableDomainChange.ofNew(newUnblockable));
|
||||
verify(bsaReportSender, never()).removeUnblockableDomainsUpdates(anyString());
|
||||
verify(bsaReportSender, times(1))
|
||||
.addUnblockableDomainsUpdates("{\n \"reserved\": [\n \"blocked1.app\"\n ]\n}");
|
||||
}
|
||||
|
||||
@Test
|
||||
void newRegisteredDomain_addedAsUnblockable() throws Exception {
|
||||
persistActiveDomain("blocked1.dev", fakeClock.nowUtc());
|
||||
persistActiveDomain("dummy.dev", fakeClock.nowUtc());
|
||||
String jobName = getRefreshJobName(fakeClock.nowUtc());
|
||||
action.run();
|
||||
UnblockableDomain newUnblockable = UnblockableDomain.of("blocked1.dev", Reason.REGISTERED);
|
||||
assertThat(queryUnblockableDomains()).containsExactly(newUnblockable);
|
||||
assertThat(gcsClient.readRefreshChanges(jobName))
|
||||
.containsExactly(UnblockableDomainChange.ofNew(newUnblockable));
|
||||
|
||||
verify(bsaReportSender, never()).removeUnblockableDomainsUpdates(anyString());
|
||||
verify(bsaReportSender, times(1))
|
||||
.addUnblockableDomainsUpdates("{\n \"registered\": [\n \"blocked1.dev\"\n ]\n}");
|
||||
}
|
||||
|
||||
@Test
|
||||
void registeredUnblockable_unregistered() {
|
||||
Domain domain = persistActiveDomain("blocked1.dev", fakeClock.nowUtc());
|
||||
action.run();
|
||||
assertThat(queryUnblockableDomains())
|
||||
.containsExactly(UnblockableDomain.of("blocked1.dev", Reason.REGISTERED));
|
||||
fakeClock.advanceOneMilli();
|
||||
deleteTestDomain(domain, fakeClock.nowUtc());
|
||||
fakeClock.advanceOneMilli();
|
||||
|
||||
String jobName = getRefreshJobName(fakeClock.nowUtc());
|
||||
Mockito.reset(bsaReportSender);
|
||||
action.run();
|
||||
assertThat(queryUnblockableDomains()).isEmpty();
|
||||
assertThat(gcsClient.readRefreshChanges(jobName))
|
||||
.containsExactly(
|
||||
UnblockableDomainChange.ofDeleted(
|
||||
UnblockableDomain.of("blocked1.dev", Reason.REGISTERED)));
|
||||
|
||||
verify(bsaReportSender, never()).addUnblockableDomainsUpdates(anyString());
|
||||
verify(bsaReportSender, times(1)).removeUnblockableDomainsUpdates("[\n \"blocked1.dev\"\n]");
|
||||
}
|
||||
|
||||
@Test
|
||||
void reservedUnblockable_noLongerReserved() {
|
||||
addReservedDomainToList(
|
||||
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
|
||||
action.run();
|
||||
assertThat(queryUnblockableDomains())
|
||||
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.RESERVED));
|
||||
fakeClock.advanceOneMilli();
|
||||
removeReservedDomainFromList(RESERVED_LIST_NAME, ImmutableSet.of("blocked1"));
|
||||
|
||||
String jobName = getRefreshJobName(fakeClock.nowUtc());
|
||||
Mockito.reset(bsaReportSender);
|
||||
action.run();
|
||||
assertThat(queryUnblockableDomains()).isEmpty();
|
||||
assertThat(gcsClient.readRefreshChanges(jobName))
|
||||
.containsExactly(
|
||||
UnblockableDomainChange.ofDeleted(
|
||||
UnblockableDomain.of("blocked1.app", Reason.RESERVED)));
|
||||
|
||||
verify(bsaReportSender, never()).addUnblockableDomainsUpdates(anyString());
|
||||
verify(bsaReportSender, times(1)).removeUnblockableDomainsUpdates("[\n \"blocked1.app\"\n]");
|
||||
}
|
||||
|
||||
@Test
|
||||
void registeredAndReservedUnblockable_noLongerRegistered_stillUnblockable() throws Exception {
|
||||
addReservedDomainToList(
|
||||
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
|
||||
Domain domain = persistActiveDomain("blocked1.app", fakeClock.nowUtc());
|
||||
action.run();
|
||||
assertThat(queryUnblockableDomains())
|
||||
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
|
||||
fakeClock.advanceOneMilli();
|
||||
deleteTestDomain(domain, fakeClock.nowUtc());
|
||||
fakeClock.advanceOneMilli();
|
||||
|
||||
String jobName = getRefreshJobName(fakeClock.nowUtc());
|
||||
Mockito.reset(bsaReportSender);
|
||||
action.run();
|
||||
assertThat(queryUnblockableDomains())
|
||||
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.RESERVED));
|
||||
assertThat(gcsClient.readRefreshChanges(jobName))
|
||||
.containsExactly(
|
||||
UnblockableDomainChange.ofChanged(
|
||||
UnblockableDomain.of("blocked1.app", Reason.REGISTERED), Reason.RESERVED));
|
||||
InOrder inOrder = Mockito.inOrder(bsaReportSender);
|
||||
inOrder.verify(bsaReportSender).removeUnblockableDomainsUpdates("[\n \"blocked1.app\"\n]");
|
||||
inOrder
|
||||
.verify(bsaReportSender)
|
||||
.addUnblockableDomainsUpdates("{\n \"reserved\": [\n \"blocked1.app\"\n ]\n}");
|
||||
}
|
||||
|
||||
@Test
|
||||
void reservedUblockable_becomesRegistered_changeToRegisterd() throws Exception {
|
||||
addReservedDomainToList(
|
||||
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
|
||||
action.run();
|
||||
assertThat(queryUnblockableDomains())
|
||||
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.RESERVED));
|
||||
fakeClock.advanceOneMilli();
|
||||
persistActiveDomain("blocked1.app", fakeClock.nowUtc());
|
||||
fakeClock.advanceOneMilli();
|
||||
|
||||
Mockito.reset(bsaReportSender);
|
||||
String jobName = getRefreshJobName(fakeClock.nowUtc());
|
||||
action.run();
|
||||
UnblockableDomain changed = UnblockableDomain.of("blocked1.app", Reason.REGISTERED);
|
||||
assertThat(queryUnblockableDomains()).containsExactly(changed);
|
||||
assertThat(gcsClient.readRefreshChanges(jobName))
|
||||
.containsExactly(
|
||||
UnblockableDomainChange.ofChanged(
|
||||
UnblockableDomain.of("blocked1.app", Reason.RESERVED), Reason.REGISTERED));
|
||||
InOrder inOrder = Mockito.inOrder(bsaReportSender);
|
||||
inOrder.verify(bsaReportSender).removeUnblockableDomainsUpdates("[\n \"blocked1.app\"\n]");
|
||||
inOrder
|
||||
.verify(bsaReportSender)
|
||||
.addUnblockableDomainsUpdates("{\n \"registered\": [\n \"blocked1.app\"\n ]\n}");
|
||||
}
|
||||
|
||||
@Test
|
||||
void newRegisteredAndReservedDomain_addedAsRegisteredUnblockable() throws Exception {
|
||||
addReservedDomainToList(
|
||||
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
|
||||
persistActiveDomain("blocked1.app", fakeClock.nowUtc());
|
||||
String jobName = getRefreshJobName(fakeClock.nowUtc());
|
||||
action.run();
|
||||
UnblockableDomain newUnblockable = UnblockableDomain.of("blocked1.app", Reason.REGISTERED);
|
||||
assertThat(queryUnblockableDomains()).containsExactly(newUnblockable);
|
||||
assertThat(gcsClient.readRefreshChanges(jobName))
|
||||
.containsExactly(UnblockableDomainChange.ofNew(newUnblockable));
|
||||
}
|
||||
|
||||
@Test
|
||||
void registeredAndReservedUnblockable_noLongerReserved_noChange() throws Exception {
|
||||
addReservedDomainToList(
|
||||
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
|
||||
persistActiveDomain("blocked1.app", fakeClock.nowUtc());
|
||||
action.run();
|
||||
assertThat(queryUnblockableDomains())
|
||||
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
|
||||
fakeClock.advanceOneMilli();
|
||||
removeReservedDomainFromList(RESERVED_LIST_NAME, ImmutableSet.of("blocked1"));
|
||||
fakeClock.advanceOneMilli();
|
||||
|
||||
Mockito.reset(bsaReportSender);
|
||||
String jobName = getRefreshJobName(fakeClock.nowUtc());
|
||||
action.run();
|
||||
assertThat(queryUnblockableDomains())
|
||||
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
|
||||
assertThat(gcsClient.readRefreshChanges(jobName)).isEmpty();
|
||||
verifyNoInteractions(bsaReportSender);
|
||||
}
|
||||
|
||||
@Test
|
||||
void registeredUblockable_becomesReserved_noChange() throws Exception {
|
||||
persistActiveDomain("blocked1.app", fakeClock.nowUtc());
|
||||
action.run();
|
||||
assertThat(queryUnblockableDomains())
|
||||
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
|
||||
fakeClock.advanceOneMilli();
|
||||
addReservedDomainToList(
|
||||
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
|
||||
fakeClock.advanceOneMilli();
|
||||
|
||||
Mockito.reset(bsaReportSender);
|
||||
String jobName = getRefreshJobName(fakeClock.nowUtc());
|
||||
action.run();
|
||||
assertThat(queryUnblockableDomains())
|
||||
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
|
||||
assertThat(gcsClient.readRefreshChanges(jobName)).isEmpty();
|
||||
verifyNoInteractions(bsaReportSender);
|
||||
}
|
||||
}
|
|
@ -62,4 +62,43 @@ public final class ReservedDomainsTestingUtils {
|
|||
.build();
|
||||
persistResource(tld.asBuilder().setReservedListsByName(reservedLists).build());
|
||||
}
|
||||
|
||||
public static void addReservedDomainToList(
|
||||
String listName, ImmutableMap<String, ReservationType> reservedLabels) {
|
||||
ImmutableMap<String, ReservedListEntry> existingEntries =
|
||||
ReservedList.get(listName).get().getReservedListEntries();
|
||||
ImmutableMap<String, ReservedListEntry> newEntries =
|
||||
ImmutableMap.copyOf(
|
||||
Maps.transformEntries(
|
||||
reservedLabels, (key, value) -> ReservedListEntry.create(key, value, "")));
|
||||
|
||||
ReservedListDao.save(
|
||||
new ReservedList.Builder()
|
||||
.setName(listName)
|
||||
.setCreationTimestamp(START_OF_TIME)
|
||||
.setShouldPublish(true)
|
||||
.setReservedListMap(
|
||||
new ImmutableMap.Builder<String, ReservedListEntry>()
|
||||
.putAll(existingEntries)
|
||||
.putAll(newEntries)
|
||||
.buildKeepingLast())
|
||||
.build());
|
||||
}
|
||||
|
||||
public static void removeReservedDomainFromList(
|
||||
String listName, ImmutableSet<String> removedLabels) {
|
||||
ImmutableMap<String, ReservedListEntry> existingEntries =
|
||||
ReservedList.get(listName).get().getReservedListEntries();
|
||||
ImmutableMap<String, ReservedListEntry> newEntries =
|
||||
ImmutableMap.copyOf(
|
||||
Maps.filterEntries(existingEntries, entry -> !removedLabels.contains(entry.getKey())));
|
||||
|
||||
ReservedListDao.save(
|
||||
new ReservedList.Builder()
|
||||
.setName(listName)
|
||||
.setCreationTimestamp(START_OF_TIME)
|
||||
.setShouldPublish(true)
|
||||
.setReservedListMap(newEntries)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,8 +14,11 @@
|
|||
|
||||
package google.registry.bsa.persistence;
|
||||
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.bsa.api.UnblockableDomain;
|
||||
import google.registry.util.Clock;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
|
@ -35,7 +38,21 @@ public final class BsaTestingUtils {
|
|||
tm().transact(() -> tm().put(new BsaLabel(domainLabel, BSA_LABEL_CREATION_TIME)));
|
||||
}
|
||||
|
||||
public static void persistUnblockableDomain(UnblockableDomain unblockableDomain) {
|
||||
tm().transact(() -> tm().put(BsaUnblockableDomain.of(unblockableDomain)));
|
||||
}
|
||||
|
||||
public static DownloadScheduler createDownloadScheduler(Clock clock) {
|
||||
return new DownloadScheduler(DEFAULT_DOWNLOAD_INTERVAL, DEFAULT_NOP_INTERVAL, clock);
|
||||
}
|
||||
|
||||
public static RefreshScheduler createRefreshScheduler() {
|
||||
return new RefreshScheduler();
|
||||
}
|
||||
|
||||
public static ImmutableList<UnblockableDomain> queryUnblockableDomains() {
|
||||
return tm().transact(() -> tm().loadAllOf(BsaUnblockableDomain.class)).stream()
|
||||
.map(BsaUnblockableDomain::toUnblockableDomain)
|
||||
.collect(toImmutableList());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -16,6 +16,8 @@ package google.registry.bsa.persistence;
|
|||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth8.assertThat;
|
||||
import static google.registry.bsa.ReservedDomainsTestingUtils.addReservedListsToTld;
|
||||
import static google.registry.bsa.ReservedDomainsTestingUtils.createReservedList;
|
||||
import static google.registry.bsa.persistence.LabelDiffUpdates.applyLabelDiff;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
|
@ -26,7 +28,6 @@ import static org.mockito.ArgumentMatchers.any;
|
|||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Sets;
|
||||
import google.registry.bsa.IdnChecker;
|
||||
|
@ -36,9 +37,6 @@ import google.registry.bsa.api.UnblockableDomain;
|
|||
import google.registry.bsa.persistence.BsaUnblockableDomain.Reason;
|
||||
import google.registry.model.tld.Tld;
|
||||
import google.registry.model.tld.label.ReservationType;
|
||||
import google.registry.model.tld.label.ReservedList;
|
||||
import google.registry.model.tld.label.ReservedList.ReservedListEntry;
|
||||
import google.registry.model.tld.label.ReservedListDao;
|
||||
import google.registry.persistence.transaction.JpaTestExtensions;
|
||||
import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension;
|
||||
import google.registry.testing.FakeClock;
|
||||
|
@ -139,18 +137,8 @@ class LabelDiffUpdatesTest {
|
|||
@Test
|
||||
void applyLabelDiffs_newLabel() {
|
||||
persistActiveDomain("label.app");
|
||||
ReservedListDao.save(
|
||||
new ReservedList.Builder()
|
||||
.setReservedListMap(
|
||||
ImmutableMap.of(
|
||||
"label",
|
||||
ReservedListEntry.create(
|
||||
"label", ReservationType.RESERVED_FOR_SPECIFIC_USE, null)))
|
||||
.setName("page_reserved")
|
||||
.setCreationTimestamp(fakeClock.nowUtc())
|
||||
.build());
|
||||
ReservedList reservedList = ReservedList.get("page_reserved").get();
|
||||
tm().transact(() -> tm().put(page.asBuilder().setReservedLists(reservedList).build()));
|
||||
createReservedList("page_reserved", "label", ReservationType.RESERVED_FOR_SPECIFIC_USE);
|
||||
addReservedListsToTld("page", ImmutableList.of("page_reserved"));
|
||||
|
||||
when(idnChecker.getForbiddingTlds(any()))
|
||||
.thenReturn(Sets.difference(ImmutableSet.of(dev), ImmutableSet.of()).immutableCopy());
|
||||
|
|
|
@ -208,6 +208,32 @@ class QueriesTest {
|
|||
.containsExactly("d1.tld", "d2.tld");
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryNewlyCreatedDomains_onlyDomainsAfterMinCreationTimeReturned() {
|
||||
DateTime testStartTime = fakeClock.nowUtc();
|
||||
createTlds("tld");
|
||||
persistNewRegistrar("TheRegistrar");
|
||||
// time 0:
|
||||
persistResource(
|
||||
newDomain("d1.tld").asBuilder().setCreationTimeForTest(fakeClock.nowUtc()).build());
|
||||
// time 0, deletion time 1
|
||||
persistDomainAsDeleted(
|
||||
newDomain("will-delete.tld").asBuilder().setCreationTimeForTest(fakeClock.nowUtc()).build(),
|
||||
fakeClock.nowUtc().plusMillis(1));
|
||||
fakeClock.advanceOneMilli();
|
||||
// time 1
|
||||
persistResource(
|
||||
newDomain("d2.tld").asBuilder().setCreationTimeForTest(fakeClock.nowUtc()).build());
|
||||
fakeClock.advanceOneMilli();
|
||||
// Now is time 2, ask for domains created since time 1
|
||||
assertThat(
|
||||
bsaQuery(
|
||||
() ->
|
||||
queryNewlyCreatedDomains(
|
||||
ImmutableList.of("tld"), testStartTime.plusMillis(1), fakeClock.nowUtc())))
|
||||
.containsExactly("d2.tld");
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryNewlyCreatedDomains_onlyDomainsInRequestedTldsReturned() {
|
||||
DateTime testStartTime = fakeClock.nowUtc();
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue