mirror of
https://github.com/google/nomulus.git
synced 2025-07-03 09:43:30 +02:00
Remove obsolete scrap commands (#1502)
This commit is contained in:
Ben McIlwain
GitHub
parent
e3cca90846
commit
dc00d4fca7
10 changed files with 0 additions and 1148 deletions
|
|
@ -15,14 +15,8 @@
|
|||
package google.registry.tools;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
|
||||
import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand;
|
||||
import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
|
||||
import google.registry.tools.javascrap.DeleteContactByRoidCommand;
|
||||
import google.registry.tools.javascrap.HardDeleteHostCommand;
|
||||
import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
|
||||
import google.registry.tools.javascrap.RemoveIpAddressCommand;
|
||||
import google.registry.tools.javascrap.ResaveAllTldsCommand;
|
||||
|
||||
/** Container class to create and run remote commands against a Datastore instance. */
|
||||
public final class RegistryTool {
|
||||
|
|
@ -36,8 +30,6 @@ public final class RegistryTool {
|
|||
public static final ImmutableMap<String, Class<? extends Command>> COMMAND_MAP =
|
||||
new ImmutableMap.Builder<String, Class<? extends Command>>()
|
||||
.put("ack_poll_messages", AckPollMessagesCommand.class)
|
||||
.put("backfill_registry_locks", BackfillRegistryLocksCommand.class)
|
||||
.put("backfill_spec11_threat_matches", BackfillSpec11ThreatMatchesCommand.class)
|
||||
.put("canonicalize_labels", CanonicalizeLabelsCommand.class)
|
||||
.put("check_domain", CheckDomainCommand.class)
|
||||
.put("check_domain_claims", CheckDomainClaimsCommand.class)
|
||||
|
|
@ -57,7 +49,6 @@ public final class RegistryTool {
|
|||
.put("curl", CurlCommand.class)
|
||||
.put("dedupe_one_time_billing_event_ids", DedupeOneTimeBillingEventIdsCommand.class)
|
||||
.put("delete_allocation_tokens", DeleteAllocationTokensCommand.class)
|
||||
.put("delete_contact_by_roid", DeleteContactByRoidCommand.class)
|
||||
.put("delete_domain", DeleteDomainCommand.class)
|
||||
.put("delete_host", DeleteHostCommand.class)
|
||||
.put("delete_premium_list", DeletePremiumListCommand.class)
|
||||
|
|
@ -107,12 +98,9 @@ public final class RegistryTool {
|
|||
.put("login", LoginCommand.class)
|
||||
.put("logout", LogoutCommand.class)
|
||||
.put("pending_escrow", PendingEscrowCommand.class)
|
||||
.put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class)
|
||||
.put("registrar_contact", RegistrarContactCommand.class)
|
||||
.put("remove_ip_address", RemoveIpAddressCommand.class)
|
||||
.put("remove_registry_one_key", RemoveRegistryOneKeyCommand.class)
|
||||
.put("renew_domain", RenewDomainCommand.class)
|
||||
.put("resave_all_tlds", ResaveAllTldsCommand.class)
|
||||
.put("resave_entities", ResaveEntitiesCommand.class)
|
||||
.put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class)
|
||||
.put("resave_epp_resource", ResaveEppResourceCommand.class)
|
||||
|
|
|
|||
|
|
@ -42,9 +42,7 @@ import google.registry.request.Modules.URLFetchServiceModule;
|
|||
import google.registry.request.Modules.UrlFetchTransportModule;
|
||||
import google.registry.request.Modules.UserServiceModule;
|
||||
import google.registry.tools.AuthModule.LocalCredentialModule;
|
||||
import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
|
||||
import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
|
||||
import google.registry.tools.javascrap.DeleteContactByRoidCommand;
|
||||
import google.registry.tools.javascrap.HardDeleteHostCommand;
|
||||
import google.registry.util.UtilsModule;
|
||||
import google.registry.whois.NonCachingWhoisModule;
|
||||
|
|
@ -90,8 +88,6 @@ import javax.inject.Singleton;
|
|||
interface RegistryToolComponent {
|
||||
void inject(AckPollMessagesCommand command);
|
||||
|
||||
void inject(BackfillRegistryLocksCommand command);
|
||||
|
||||
void inject(CheckDomainClaimsCommand command);
|
||||
|
||||
void inject(CheckDomainCommand command);
|
||||
|
|
@ -112,8 +108,6 @@ interface RegistryToolComponent {
|
|||
|
||||
void inject(CreateTldCommand command);
|
||||
|
||||
void inject(DeleteContactByRoidCommand command);
|
||||
|
||||
void inject(EncryptEscrowDepositCommand command);
|
||||
|
||||
void inject(EnqueuePollMessageCommand command);
|
||||
|
|
|
|||
|
|
@ -1,157 +0,0 @@
|
|||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.tools.javascrap;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
|
||||
|
||||
import com.beust.jcommander.Parameter;
|
||||
import com.beust.jcommander.Parameters;
|
||||
import com.google.common.collect.ImmutableCollection;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.RegistryLock;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.reporting.HistoryEntryDao;
|
||||
import google.registry.model.tld.RegistryLockDao;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.tools.CommandWithRemoteApi;
|
||||
import google.registry.tools.ConfirmingCommand;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.StringGenerator;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
* Scrap tool to backfill {@link RegistryLock}s for domains previously locked.
|
||||
*
|
||||
* <p>This will save new objects for all existing domains that are locked but don't have any
|
||||
* corresponding lock objects already in the database.
|
||||
*/
|
||||
@Parameters(
|
||||
separators = " =",
|
||||
commandDescription =
|
||||
"Backfills RegistryLock objects for specified domain resource IDs that are locked but don't"
|
||||
+ " already have a corresponding RegistryLock object.")
|
||||
public class BackfillRegistryLocksCommand extends ConfirmingCommand
|
||||
implements CommandWithRemoteApi {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final int VERIFICATION_CODE_LENGTH = 32;
|
||||
|
||||
@Parameter(
|
||||
names = {"--domain_roids"},
|
||||
description = "Comma-separated list of domain roids to check")
|
||||
protected List<String> roids;
|
||||
|
||||
// Inject here so that we can create the command automatically for tests
|
||||
@Inject Clock clock;
|
||||
|
||||
@Inject
|
||||
@Config("registryAdminClientId")
|
||||
String registryAdminClientId;
|
||||
|
||||
@Inject
|
||||
@Named("base58StringGenerator")
|
||||
StringGenerator stringGenerator;
|
||||
|
||||
private ImmutableList<DomainBase> lockedDomains;
|
||||
|
||||
@Override
|
||||
protected String prompt() {
|
||||
checkArgument(
|
||||
roids != null && !roids.isEmpty(), "Must provide non-empty domain_roids argument");
|
||||
lockedDomains =
|
||||
jpaTm().transact(() -> getLockedDomainsWithoutLocks(jpaTm().getTransactionTime()));
|
||||
ImmutableList<String> lockedDomainNames =
|
||||
lockedDomains.stream().map(DomainBase::getDomainName).collect(toImmutableList());
|
||||
return String.format(
|
||||
"Locked domains for which there does not exist a RegistryLock object: %s",
|
||||
lockedDomainNames);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String execute() {
|
||||
ImmutableSet.Builder<String> failedDomainsBuilder = new ImmutableSet.Builder<>();
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
for (DomainBase domainBase : lockedDomains) {
|
||||
try {
|
||||
RegistryLockDao.save(
|
||||
new RegistryLock.Builder()
|
||||
.isSuperuser(true)
|
||||
.setRegistrarId(registryAdminClientId)
|
||||
.setRepoId(domainBase.getRepoId())
|
||||
.setDomainName(domainBase.getDomainName())
|
||||
.setLockCompletionTime(
|
||||
getLockCompletionTimestamp(domainBase, jpaTm().getTransactionTime()))
|
||||
.setVerificationCode(
|
||||
stringGenerator.createString(VERIFICATION_CODE_LENGTH))
|
||||
.build());
|
||||
} catch (Throwable t) {
|
||||
logger.atSevere().withCause(t).log(
|
||||
"Error when creating lock object for domain '%s'.",
|
||||
domainBase.getDomainName());
|
||||
failedDomainsBuilder.add(domainBase.getDomainName());
|
||||
}
|
||||
}
|
||||
});
|
||||
ImmutableSet<String> failedDomains = failedDomainsBuilder.build();
|
||||
if (failedDomains.isEmpty()) {
|
||||
return String.format(
|
||||
"Successfully created lock objects for %d domains.", lockedDomains.size());
|
||||
} else {
|
||||
return String.format(
|
||||
"Successfully created lock objects for %d domains. We failed to create locks "
|
||||
+ "for the following domains: %s",
|
||||
lockedDomains.size() - failedDomains.size(), failedDomains);
|
||||
}
|
||||
}
|
||||
|
||||
private DateTime getLockCompletionTimestamp(DomainBase domainBase, DateTime now) {
|
||||
// Best-effort, if a domain was URS-locked we should use that time
|
||||
// If we can't find that, return now.
|
||||
return HistoryEntryDao.loadHistoryObjectsForResource(domainBase.createVKey()).stream()
|
||||
// sort by modification time descending so we get the most recent one if it was locked twice
|
||||
.sorted(Comparator.comparing(HistoryEntry::getModificationTime).reversed())
|
||||
.filter(entry -> "Uniform Rapid Suspension".equals(entry.getReason()))
|
||||
.findFirst()
|
||||
.map(HistoryEntry::getModificationTime)
|
||||
.orElse(now);
|
||||
}
|
||||
|
||||
private ImmutableList<DomainBase> getLockedDomainsWithoutLocks(DateTime now) {
|
||||
ImmutableList<VKey<DomainBase>> domainKeys =
|
||||
roids.stream().map(roid -> VKey.create(DomainBase.class, roid)).collect(toImmutableList());
|
||||
ImmutableCollection<DomainBase> domains =
|
||||
transactIfJpaTm(() -> tm().loadByKeys(domainKeys)).values();
|
||||
return domains.stream()
|
||||
.filter(d -> d.getDeletionTime().isAfter(now))
|
||||
.filter(d -> d.getStatusValues().containsAll(REGISTRY_LOCK_STATUSES))
|
||||
.filter(d -> !RegistryLockDao.getMostRecentByRepoId(d.getRepoId()).isPresent())
|
||||
.collect(toImmutableList());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,223 +0,0 @@
|
|||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.tools.javascrap;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
|
||||
import com.beust.jcommander.Parameter;
|
||||
import com.beust.jcommander.Parameters;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableListMultimap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.beam.spec11.ThreatMatch;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
|
||||
import google.registry.model.reporting.Spec11ThreatMatchDao;
|
||||
import google.registry.persistence.transaction.QueryComposer;
|
||||
import google.registry.reporting.spec11.RegistrarThreatMatches;
|
||||
import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
|
||||
import google.registry.tools.CommandWithRemoteApi;
|
||||
import google.registry.tools.ConfirmingCommand;
|
||||
import google.registry.util.Clock;
|
||||
import java.io.IOException;
|
||||
import java.util.Comparator;
|
||||
import java.util.function.Function;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.LocalDate;
|
||||
|
||||
/**
|
||||
* Scrap tool to backfill {@link Spec11ThreatMatch} objects from prior days.
|
||||
*
|
||||
* <p>This will load the previously-existing Spec11 files from GCS (looking back to 2019-01-01 (a
|
||||
* rough estimate of when we started using this format) and convert those RegistrarThreatMatches
|
||||
* objects into the new Spec11ThreatMatch format. It will then insert these entries into SQL.
|
||||
*
|
||||
* <p>Note that the script will attempt to find the corresponding {@link DomainBase} object for each
|
||||
* domain name on the day of the scan. It will fail if it cannot find a corresponding domain object,
|
||||
* or if the domain objects were not active at the time of the scan.
|
||||
*/
|
||||
@Parameters(
|
||||
commandDescription =
|
||||
"Backfills Spec11 threat match entries from the old and deprecated GCS JSON files to the "
|
||||
+ "Cloud SQL database.")
|
||||
public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
|
||||
implements CommandWithRemoteApi {
|
||||
|
||||
private static final LocalDate START_DATE = new LocalDate(2019, 1, 1);
|
||||
|
||||
@Parameter(
|
||||
names = {"-o", "--overwrite_existing_dates"},
|
||||
description =
|
||||
"Whether the command will overwrite data that already exists for dates that exist in the "
|
||||
+ "GCS bucket. Defaults to false.")
|
||||
private boolean overrideExistingDates;
|
||||
|
||||
@Inject Spec11RegistrarThreatMatchesParser threatMatchesParser;
|
||||
// Inject the clock for testing purposes
|
||||
@Inject Clock clock;
|
||||
|
||||
@Override
|
||||
protected String prompt() {
|
||||
return String.format("Backfill Spec11 results from %d files?", getDatesToBackfill().size());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String execute() {
|
||||
ImmutableList<LocalDate> dates = getDatesToBackfill();
|
||||
ImmutableListMultimap.Builder<LocalDate, RegistrarThreatMatches> threatMatchesBuilder =
|
||||
new ImmutableListMultimap.Builder<>();
|
||||
for (LocalDate date : dates) {
|
||||
try {
|
||||
// It's OK if the file doesn't exist for a particular date; the result will be empty.
|
||||
threatMatchesBuilder.putAll(date, threatMatchesParser.getRegistrarThreatMatches(date));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(
|
||||
String.format("Error parsing through file with date %s.", date), e);
|
||||
}
|
||||
}
|
||||
ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatches =
|
||||
threatMatchesBuilder.build();
|
||||
// Look up all possible DomainBases for these domain names, any of which can be in the past
|
||||
ImmutableListMultimap<String, DomainBase> domainsByDomainName =
|
||||
getDomainsByDomainName(threatMatches);
|
||||
|
||||
// For each date, convert all threat matches with the proper domain repo ID
|
||||
int totalNumThreats = 0;
|
||||
for (LocalDate date : threatMatches.keySet()) {
|
||||
ImmutableList.Builder<Spec11ThreatMatch> spec11ThreatsBuilder = new ImmutableList.Builder<>();
|
||||
for (RegistrarThreatMatches rtm : threatMatches.get(date)) {
|
||||
rtm.threatMatches().stream()
|
||||
.map(
|
||||
threatMatch ->
|
||||
threatMatchToCloudSqlObject(
|
||||
threatMatch, date, rtm.clientId(), domainsByDomainName))
|
||||
.forEach(spec11ThreatsBuilder::add);
|
||||
}
|
||||
ImmutableList<Spec11ThreatMatch> spec11Threats = spec11ThreatsBuilder.build();
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
Spec11ThreatMatchDao.deleteEntriesByDate(jpaTm(), date);
|
||||
jpaTm().putAll(spec11Threats);
|
||||
});
|
||||
totalNumThreats += spec11Threats.size();
|
||||
}
|
||||
return String.format(
|
||||
"Successfully parsed through %d files with %d threats.", dates.size(), totalNumThreats);
|
||||
}
|
||||
|
||||
/** Returns a per-domain list of possible DomainBase objects, starting with the most recent. */
|
||||
private ImmutableListMultimap<String, DomainBase> getDomainsByDomainName(
|
||||
ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatchesByDate) {
|
||||
return threatMatchesByDate.values().stream()
|
||||
.map(RegistrarThreatMatches::threatMatches)
|
||||
.flatMap(ImmutableList::stream)
|
||||
.map(ThreatMatch::fullyQualifiedDomainName)
|
||||
.distinct()
|
||||
.collect(
|
||||
flatteningToImmutableListMultimap(
|
||||
Function.identity(),
|
||||
(domainName) -> {
|
||||
ImmutableList<DomainBase> domains = loadDomainsForFqdn(domainName);
|
||||
checkState(
|
||||
!domains.isEmpty(),
|
||||
"Domain name %s had no associated DomainBase objects.",
|
||||
domainName);
|
||||
return domains.stream()
|
||||
.sorted(Comparator.comparing(DomainBase::getCreationTime).reversed());
|
||||
}));
|
||||
}
|
||||
|
||||
/** Loads in all {@link DomainBase} objects for a given FQDN. */
|
||||
private ImmutableList<DomainBase> loadDomainsForFqdn(String fullyQualifiedDomainName) {
|
||||
return transactIfJpaTm(
|
||||
() ->
|
||||
tm().createQueryComposer(DomainBase.class)
|
||||
.where(
|
||||
"fullyQualifiedDomainName",
|
||||
QueryComposer.Comparator.EQ,
|
||||
fullyQualifiedDomainName)
|
||||
.list());
|
||||
}
|
||||
|
||||
/** Converts the previous {@link ThreatMatch} object to {@link Spec11ThreatMatch}. */
|
||||
private Spec11ThreatMatch threatMatchToCloudSqlObject(
|
||||
ThreatMatch threatMatch,
|
||||
LocalDate date,
|
||||
String registrarId,
|
||||
ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
|
||||
DomainBase domain =
|
||||
findDomainAsOfDateOrThrow(
|
||||
threatMatch.fullyQualifiedDomainName(), date, domainsByDomainName);
|
||||
return new Spec11ThreatMatch.Builder()
|
||||
.setThreatTypes(ImmutableSet.of(ThreatType.valueOf(threatMatch.threatType())))
|
||||
.setCheckDate(date)
|
||||
.setRegistrarId(registrarId)
|
||||
.setDomainName(threatMatch.fullyQualifiedDomainName())
|
||||
.setDomainRepoId(domain.getRepoId())
|
||||
.build();
|
||||
}
|
||||
|
||||
/** Returns the DomainBase object as of the particular date, which is likely in the past. */
|
||||
private DomainBase findDomainAsOfDateOrThrow(
|
||||
String domainName,
|
||||
LocalDate date,
|
||||
ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
|
||||
ImmutableList<DomainBase> domains = domainsByDomainName.get(domainName);
|
||||
for (DomainBase domain : domains) {
|
||||
// We only know the date (not datetime) of the threat scan, so we approximate
|
||||
LocalDate creationDate = domain.getCreationTime().toLocalDate();
|
||||
LocalDate deletionDate = domain.getDeletionTime().toLocalDate();
|
||||
if (!date.isBefore(creationDate) && !date.isAfter(deletionDate)) {
|
||||
return domain;
|
||||
}
|
||||
}
|
||||
throw new IllegalStateException(
|
||||
String.format("Could not find a DomainBase valid for %s on day %s.", domainName, date));
|
||||
}
|
||||
|
||||
/** Returns the list of dates between {@link #START_DATE} and now (UTC), inclusive. */
|
||||
private ImmutableList<LocalDate> getDatesToBackfill() {
|
||||
ImmutableSet<LocalDate> datesToSkip =
|
||||
overrideExistingDates ? ImmutableSet.of() : getExistingDates();
|
||||
ImmutableList.Builder<LocalDate> result = new ImmutableList.Builder<>();
|
||||
LocalDate endDate = clock.nowUtc().toLocalDate();
|
||||
for (LocalDate currentDate = START_DATE;
|
||||
!currentDate.isAfter(endDate);
|
||||
currentDate = currentDate.plusDays(1)) {
|
||||
if (!datesToSkip.contains(currentDate)) {
|
||||
result.add(currentDate);
|
||||
}
|
||||
}
|
||||
return result.build();
|
||||
}
|
||||
|
||||
private ImmutableSet<LocalDate> getExistingDates() {
|
||||
return jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
jpaTm()
|
||||
.query(
|
||||
"SELECT DISTINCT stm.checkDate FROM Spec11ThreatMatch stm", LocalDate.class)
|
||||
.getResultStream()
|
||||
.collect(toImmutableSet()));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.tools.javascrap;
|
||||
|
||||
import static com.google.common.base.Verify.verify;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
|
||||
import com.beust.jcommander.Parameter;
|
||||
import com.beust.jcommander.Parameters;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
import google.registry.model.eppcommon.StatusValue;
|
||||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex;
|
||||
import google.registry.tools.CommandWithRemoteApi;
|
||||
import google.registry.tools.ConfirmingCommand;
|
||||
import google.registry.util.SystemClock;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Deletes a {@link google.registry.model.contact.ContactResource} by its ROID.
|
||||
*
|
||||
* <p>This is a short-term tool for race condition clean up while the bug is being fixed.
|
||||
*/
|
||||
@Parameters(separators = " =", commandDescription = "Delete a contact by its ROID.")
|
||||
public class DeleteContactByRoidCommand extends ConfirmingCommand implements CommandWithRemoteApi {
|
||||
|
||||
@Parameter(names = "--roid", description = "The roid of the contact to be deleted.")
|
||||
String roid;
|
||||
|
||||
@Parameter(
|
||||
names = "--contact_id",
|
||||
description = "The user provided contactId, for verification purpose.")
|
||||
String contactId;
|
||||
|
||||
ImmutableList<Key<?>> toDelete;
|
||||
|
||||
@Override
|
||||
protected void init() {
|
||||
System.out.printf("Deleting %s, which refers to %s.\n", roid, contactId);
|
||||
tm().transact(
|
||||
() -> {
|
||||
Key<ContactResource> targetKey = Key.create(ContactResource.class, roid);
|
||||
ContactResource targetContact = auditedOfy().load().key(targetKey).now();
|
||||
verify(
|
||||
Objects.equals(targetContact.getContactId(), contactId),
|
||||
"contactId does not match.");
|
||||
verify(
|
||||
Objects.equals(targetContact.getStatusValues(), ImmutableSet.of(StatusValue.OK)));
|
||||
System.out.println("Target contact has the expected contactId");
|
||||
String canonicalResource =
|
||||
ForeignKeyIndex.load(ContactResource.class, contactId, new SystemClock().nowUtc())
|
||||
.getResourceKey()
|
||||
.getOfyKey()
|
||||
.getName();
|
||||
verify(!Objects.equals(canonicalResource, roid), "Contact still in ForeignKeyIndex.");
|
||||
System.out.printf(
|
||||
"It is safe to delete %s, since the contactId is mapped to a different entry in"
|
||||
+ " the Foreign key index (%s).\n\n",
|
||||
roid, canonicalResource);
|
||||
|
||||
List<Object> ancestors =
|
||||
auditedOfy().load().ancestor(Key.create(ContactResource.class, roid)).list();
|
||||
|
||||
System.out.println("Ancestor query returns: ");
|
||||
for (Object entity : ancestors) {
|
||||
System.out.println(Key.create(entity));
|
||||
}
|
||||
|
||||
ImmutableSet<String> deletetableKinds =
|
||||
ImmutableSet.of("HistoryEntry", "ContactResource");
|
||||
toDelete =
|
||||
ancestors.stream()
|
||||
.map(Key::create)
|
||||
.filter(key -> deletetableKinds.contains(key.getKind()))
|
||||
.collect(ImmutableList.toImmutableList());
|
||||
|
||||
EppResourceIndex eppResourceIndex =
|
||||
auditedOfy().load().entity(EppResourceIndex.create(targetKey)).now();
|
||||
verify(eppResourceIndex.getKey().equals(targetKey), "Wrong EppResource Index loaded");
|
||||
System.out.printf("\n\nEppResourceIndex found (%s).\n", Key.create(eppResourceIndex));
|
||||
|
||||
toDelete =
|
||||
new ImmutableList.Builder<Key<?>>()
|
||||
.addAll(toDelete)
|
||||
.add(Key.create(eppResourceIndex))
|
||||
.build();
|
||||
|
||||
System.out.printf("\n\nAbout to delete %s entities:\n", toDelete.size());
|
||||
toDelete.forEach(System.out::println);
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String execute() {
|
||||
tm().transact(() -> auditedOfy().delete().keys(toDelete).now());
|
||||
return "Done";
|
||||
}
|
||||
}
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.tools.javascrap;
|
||||
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
|
||||
import com.beust.jcommander.Parameters;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.RegistrarAddress;
|
||||
import google.registry.tools.MutatingCommand;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Scrap tool to update Registrars with null registrarName or localizedAddress fields.
|
||||
*
|
||||
* <p>This sets a null registrarName to the key name, and null localizedAddress fields to fake data.
|
||||
*/
|
||||
@Parameters(
|
||||
separators = " =",
|
||||
commandDescription = "Populate previously null required registrar fields."
|
||||
)
|
||||
public class PopulateNullRegistrarFieldsCommand extends MutatingCommand {
|
||||
|
||||
@Override
|
||||
protected void init() {
|
||||
for (Registrar registrar : Registrar.loadAll()) {
|
||||
Registrar.Builder changeBuilder = registrar.asBuilder();
|
||||
changeBuilder.setRegistrarName(
|
||||
firstNonNull(registrar.getRegistrarName(), registrar.getRegistrarId()));
|
||||
|
||||
RegistrarAddress address = registrar.getLocalizedAddress();
|
||||
if (address == null) {
|
||||
changeBuilder.setLocalizedAddress(
|
||||
new RegistrarAddress.Builder()
|
||||
.setCity("Fakington")
|
||||
.setCountryCode("US")
|
||||
.setState("FL")
|
||||
.setZip("12345")
|
||||
.setStreet(ImmutableList.of("123 Fake Street"))
|
||||
.build());
|
||||
} else {
|
||||
changeBuilder.setLocalizedAddress(
|
||||
new RegistrarAddress.Builder()
|
||||
.setCity(firstNonNull(address.getCity(), "Fakington"))
|
||||
.setCountryCode(firstNonNull(address.getCountryCode(), "US"))
|
||||
.setState(firstNonNull(address.getState(), "FL"))
|
||||
.setZip(firstNonNull(address.getZip(), "12345"))
|
||||
.setStreet(firstNonNull(address.getStreet(), ImmutableList.of("123 Fake Street")))
|
||||
.build());
|
||||
}
|
||||
Registrar changedRegistrar = changeBuilder.build();
|
||||
if (!Objects.equals(registrar, changedRegistrar)) {
|
||||
stageEntityChange(registrar, changedRegistrar);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.tools.javascrap;
|
||||
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.beust.jcommander.Parameter;
|
||||
import com.beust.jcommander.Parameters;
|
||||
import com.google.template.soy.data.SoyMapData;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.tools.MutatingEppToolCommand;
|
||||
import google.registry.tools.params.PathParameter;
|
||||
import google.registry.tools.soy.RemoveIpAddressSoyInfo;
|
||||
import java.net.Inet6Address;
|
||||
import java.net.InetAddress;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Command to remove external IP Addresses from HostResources identified by text file listing
|
||||
* resource ids, one per line.
|
||||
*
|
||||
* <p>Written for b/23757755 so we can clean up records with IP addresses that should always be
|
||||
* resolved by hostname.
|
||||
*
|
||||
* <p>The JSON file should contain a list of objects each of which has a "roid" attribute.
|
||||
*/
|
||||
@Parameters(separators = " =", commandDescription = "Remove all IP Addresses.")
|
||||
public class RemoveIpAddressCommand extends MutatingEppToolCommand {
|
||||
public static String registrarId = "CharlestonRoad";
|
||||
|
||||
@Parameter(names = "--roids_file",
|
||||
description = "Text file containing a list of HostResource roids to remove",
|
||||
required = true,
|
||||
validateWith = PathParameter.InputFile.class)
|
||||
private Path roidsFilePath;
|
||||
|
||||
@Override
|
||||
protected void initMutatingEppToolCommand() throws Exception {
|
||||
List<String> roids = Files.readAllLines(roidsFilePath, UTF_8);
|
||||
|
||||
for (String roid : roids) {
|
||||
// Look up the HostResource from its roid.
|
||||
Optional<HostResource> host =
|
||||
transactIfJpaTm(() -> tm().loadByKeyIfPresent(VKey.create(HostResource.class, roid)));
|
||||
if (!host.isPresent()) {
|
||||
System.err.printf("Record for %s not found.\n", roid);
|
||||
continue;
|
||||
}
|
||||
|
||||
ArrayList<SoyMapData> ipAddresses = new ArrayList<>();
|
||||
for (InetAddress address : host.get().getInetAddresses()) {
|
||||
SoyMapData dataMap = new SoyMapData(
|
||||
"address", address.getHostAddress(),
|
||||
"version", address instanceof Inet6Address ? "v6" : "v4");
|
||||
ipAddresses.add(dataMap);
|
||||
}
|
||||
|
||||
// Build and execute the EPP command.
|
||||
setSoyTemplate(
|
||||
RemoveIpAddressSoyInfo.getInstance(), RemoveIpAddressSoyInfo.REMOVE_IP_ADDRESS);
|
||||
addSoyRecord(
|
||||
registrarId,
|
||||
new SoyMapData(
|
||||
"name", host.get().getHostName(),
|
||||
"ipAddresses", ipAddresses,
|
||||
"requestedByRegistrar", registrarId));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.tools.javascrap;
|
||||
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
|
||||
import com.beust.jcommander.Parameters;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.tools.CommandWithRemoteApi;
|
||||
|
||||
/** Scrap command to resave all Registry entities. */
|
||||
@Parameters(commandDescription = "Resave all TLDs")
|
||||
public class ResaveAllTldsCommand implements CommandWithRemoteApi {
|
||||
@Override
|
||||
public void run() throws Exception {
|
||||
tm().transact(() -> tm().putAll(tm().loadAllOf(Registry.class)));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,173 +0,0 @@
|
|||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.tools.javascrap;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth8.assertThat;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistNewRegistrar;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.SqlHelper.getMostRecentRegistryLockByRepoId;
|
||||
import static google.registry.testing.SqlHelper.getMostRecentVerifiedRegistryLockByRepoId;
|
||||
import static google.registry.testing.SqlHelper.getRegistryLocksByRegistrarId;
|
||||
import static google.registry.testing.SqlHelper.saveRegistryLock;
|
||||
import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.truth.Truth8;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.RegistryLock;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.testing.DeterministicStringGenerator;
|
||||
import google.registry.testing.DualDatabaseTest;
|
||||
import google.registry.testing.TestOfyAndSql;
|
||||
import google.registry.tools.CommandTestCase;
|
||||
import google.registry.util.StringGenerator.Alphabets;
|
||||
import java.util.Optional;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
|
||||
/** Unit tests for {@link BackfillRegistryLocksCommand}. */
|
||||
@DualDatabaseTest
|
||||
class BackfillRegistryLocksCommandTest extends CommandTestCase<BackfillRegistryLocksCommand> {
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
persistNewRegistrar("adminreg", "Admin Registrar", Registrar.Type.REAL, 693L);
|
||||
createTld("tld");
|
||||
command.registryAdminClientId = "adminreg";
|
||||
command.clock = fakeClock;
|
||||
command.stringGenerator = new DeterministicStringGenerator(Alphabets.BASE_58);
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSimpleBackfill() throws Exception {
|
||||
DomainBase domain = persistLockedDomain("example.tld");
|
||||
Truth8.assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId())).isEmpty();
|
||||
|
||||
runCommandForced("--domain_roids", domain.getRepoId());
|
||||
|
||||
Optional<RegistryLock> lockOptional = getMostRecentRegistryLockByRepoId(domain.getRepoId());
|
||||
Truth8.assertThat(lockOptional).isPresent();
|
||||
Truth8.assertThat(lockOptional.get().getLockCompletionTime()).isPresent();
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testBackfill_onlyLockedDomains() throws Exception {
|
||||
DomainBase neverLockedDomain = persistActiveDomain("neverlocked.tld");
|
||||
DomainBase previouslyLockedDomain = persistLockedDomain("unlocked.tld");
|
||||
persistResource(previouslyLockedDomain.asBuilder().setStatusValues(ImmutableSet.of()).build());
|
||||
DomainBase lockedDomain = persistLockedDomain("locked.tld");
|
||||
|
||||
runCommandForced(
|
||||
"--domain_roids",
|
||||
String.format(
|
||||
"%s,%s,%s",
|
||||
neverLockedDomain.getRepoId(),
|
||||
previouslyLockedDomain.getRepoId(),
|
||||
lockedDomain.getRepoId()));
|
||||
|
||||
ImmutableList<RegistryLock> locks = getRegistryLocksByRegistrarId("adminreg");
|
||||
assertThat(locks).hasSize(1);
|
||||
assertThat(Iterables.getOnlyElement(locks).getDomainName()).isEqualTo("locked.tld");
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testBackfill_skipsDeletedDomains() throws Exception {
|
||||
DomainBase domain = persistDeletedDomain("example.tld", fakeClock.nowUtc());
|
||||
persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build());
|
||||
fakeClock.advanceBy(Duration.standardSeconds(1));
|
||||
runCommandForced("--domain_roids", domain.getRepoId());
|
||||
Truth8.assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId())).isEmpty();
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testBackfill_skipsDomains_ifLockAlreadyExists() throws Exception {
|
||||
DomainBase domain = persistLockedDomain("example.tld");
|
||||
|
||||
RegistryLock previousLock =
|
||||
saveRegistryLock(
|
||||
new RegistryLock.Builder()
|
||||
.isSuperuser(true)
|
||||
.setRegistrarId("adminreg")
|
||||
.setRepoId(domain.getRepoId())
|
||||
.setDomainName(domain.getDomainName())
|
||||
.setLockCompletionTime(fakeClock.nowUtc())
|
||||
.setVerificationCode(command.stringGenerator.createString(32))
|
||||
.build());
|
||||
|
||||
fakeClock.advanceBy(Duration.standardDays(1));
|
||||
runCommandForced("--domain_roids", domain.getRepoId());
|
||||
|
||||
assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId()).get().getLockCompletionTime())
|
||||
.isEqualTo(previousLock.getLockCompletionTime());
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testBackfill_usesUrsTime_ifExists() throws Exception {
|
||||
DateTime ursTime = fakeClock.nowUtc();
|
||||
DomainBase ursDomain = persistLockedDomain("urs.tld");
|
||||
persistResource(
|
||||
new DomainHistory.Builder()
|
||||
.setBySuperuser(true)
|
||||
.setRegistrarId("adminreg")
|
||||
.setModificationTime(ursTime)
|
||||
.setDomain(ursDomain)
|
||||
.setReason("Uniform Rapid Suspension")
|
||||
.setType(HistoryEntry.Type.DOMAIN_UPDATE)
|
||||
.setRequestedByRegistrar(false)
|
||||
.build());
|
||||
DomainBase nonUrsDomain = persistLockedDomain("nonurs.tld");
|
||||
persistResource(
|
||||
new DomainHistory.Builder()
|
||||
.setBySuperuser(true)
|
||||
.setRegistrarId("adminreg")
|
||||
.setDomain(nonUrsDomain)
|
||||
.setType(HistoryEntry.Type.DOMAIN_UPDATE)
|
||||
.setRequestedByRegistrar(false)
|
||||
.setModificationTime(ursTime)
|
||||
.build());
|
||||
|
||||
fakeClock.advanceBy(Duration.standardDays(10));
|
||||
runCommandForced(
|
||||
"--domain_roids", String.format("%s,%s", ursDomain.getRepoId(), nonUrsDomain.getRepoId()));
|
||||
|
||||
RegistryLock ursLock = getMostRecentVerifiedRegistryLockByRepoId(ursDomain.getRepoId()).get();
|
||||
assertThat(ursLock.getLockCompletionTime()).hasValue(ursTime);
|
||||
RegistryLock nonUrsLock =
|
||||
getMostRecentVerifiedRegistryLockByRepoId(nonUrsDomain.getRepoId()).get();
|
||||
assertThat(nonUrsLock.getLockCompletionTime()).hasValue(fakeClock.nowUtc());
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testFailure_mustProvideDomainRoids() {
|
||||
assertThat(assertThrows(IllegalArgumentException.class, this::runCommandForced))
|
||||
.hasMessageThat()
|
||||
.isEqualTo("Must provide non-empty domain_roids argument");
|
||||
}
|
||||
|
||||
private static DomainBase persistLockedDomain(String domainName) {
|
||||
DomainBase domain = persistActiveDomain(domainName);
|
||||
return persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,274 +0,0 @@
|
|||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.tools.javascrap;
|
||||
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
|
||||
import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.deleteResource;
|
||||
import static google.registry.testing.DatabaseHelper.insertInDb;
|
||||
import static google.registry.testing.DatabaseHelper.newDomainBase;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Iterables;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
|
||||
import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
|
||||
import google.registry.testing.DualDatabaseTest;
|
||||
import google.registry.testing.TestOfyAndSql;
|
||||
import google.registry.tools.CommandTestCase;
|
||||
import java.io.IOException;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.LocalDate;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
|
||||
/** Tests for {@link BackfillSpec11ThreatMatchesCommand}. */
|
||||
@DualDatabaseTest
|
||||
public class BackfillSpec11ThreatMatchesCommandTest
|
||||
extends CommandTestCase<BackfillSpec11ThreatMatchesCommand> {
|
||||
|
||||
private static final LocalDate CURRENT_DATE = DateTime.parse("2020-11-22").toLocalDate();
|
||||
private final Spec11RegistrarThreatMatchesParser threatMatchesParser =
|
||||
mock(Spec11RegistrarThreatMatchesParser.class);
|
||||
|
||||
private DomainBase domainA;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
createTld("com");
|
||||
domainA = persistActiveDomain("a.com");
|
||||
persistActiveDomain("b.com");
|
||||
persistActiveDomain("c.com");
|
||||
fakeClock.setTo(CURRENT_DATE.toDateTimeAtStartOfDay());
|
||||
command.threatMatchesParser = threatMatchesParser;
|
||||
command.clock = fakeClock;
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(any(LocalDate.class)))
|
||||
.thenReturn(ImmutableSet.of());
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_singleFile() throws Exception {
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
|
||||
.thenReturn(sampleThreatMatches());
|
||||
runCommandForced();
|
||||
assertInStdout("Backfill Spec11 results from 692 files?");
|
||||
assertInStdout("Successfully parsed through 692 files with 3 threats.");
|
||||
verifyExactlyThreeEntriesInDbFromLastDay();
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_sameDomain_multipleDays() throws Exception {
|
||||
// If the same domains show up on multiple days, there should be multiple entries for them
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
|
||||
.thenReturn(sampleThreatMatches());
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(LocalDate.parse("2019-01-01")))
|
||||
.thenReturn(sampleThreatMatches());
|
||||
runCommandForced();
|
||||
assertInStdout("Backfill Spec11 results from 692 files?");
|
||||
assertInStdout("Successfully parsed through 692 files with 6 threats.");
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
ImmutableList<Spec11ThreatMatch> threatMatches =
|
||||
jpaTm().loadAllOf(Spec11ThreatMatch.class);
|
||||
assertThat(threatMatches).hasSize(6);
|
||||
assertThat(
|
||||
threatMatches.stream()
|
||||
.map(Spec11ThreatMatch::getDomainName)
|
||||
.collect(toImmutableSet()))
|
||||
.containsExactly("a.com", "b.com", "c.com");
|
||||
assertThat(
|
||||
threatMatches.stream()
|
||||
.map(Spec11ThreatMatch::getCheckDate)
|
||||
.collect(toImmutableSet()))
|
||||
.containsExactly(CURRENT_DATE, LocalDate.parse("2019-01-01"));
|
||||
});
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_empty() throws Exception {
|
||||
runCommandForced();
|
||||
assertInStdout("Backfill Spec11 results from 692 files?");
|
||||
assertInStdout("Successfully parsed through 692 files with 0 threats.");
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_sameDayTwice() throws Exception {
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
|
||||
.thenReturn(sampleThreatMatches());
|
||||
runCommandForced();
|
||||
runCommandForced();
|
||||
verifyExactlyThreeEntriesInDbFromLastDay();
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_threeDomainsForDomainName() throws Exception {
|
||||
// We should use the repo ID from the proper DomainBase object at the scan's point in time.
|
||||
// First, domain was created at START_OF_TIME and deleted one year ago
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build());
|
||||
|
||||
// Next, domain was created six months ago and deleted two months ago
|
||||
DomainBase secondSave =
|
||||
persistResource(
|
||||
newDomainBase("a.com")
|
||||
.asBuilder()
|
||||
.setCreationTimeForTest(now.minusMonths(6))
|
||||
.setDeletionTime(now.minusMonths(2))
|
||||
.build());
|
||||
|
||||
// Lastly, domain was created one month ago and is still valid
|
||||
DomainBase thirdSave =
|
||||
persistResource(
|
||||
newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build());
|
||||
|
||||
// If the scan result was from three months ago, we should use the second save
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3)))
|
||||
.thenReturn(sampleThreatMatches());
|
||||
runCommandForced();
|
||||
String threatMatchRepoId =
|
||||
jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
jpaTm().loadAllOf(Spec11ThreatMatch.class).stream()
|
||||
.filter((match) -> match.getDomainName().equals("a.com"))
|
||||
.findFirst()
|
||||
.get()
|
||||
.getDomainRepoId());
|
||||
assertThat(threatMatchRepoId).isNotEqualTo(domainA.getRepoId());
|
||||
assertThat(threatMatchRepoId).isEqualTo(secondSave.getRepoId());
|
||||
assertThat(threatMatchRepoId).isNotEqualTo(thirdSave.getRepoId());
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_skipsExistingDatesWithoutOverwrite() throws Exception {
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
|
||||
.thenReturn(sampleThreatMatches());
|
||||
Spec11ThreatMatch previous =
|
||||
new Spec11ThreatMatch.Builder()
|
||||
.setCheckDate(CURRENT_DATE)
|
||||
.setDomainName("previous.tld")
|
||||
.setDomainRepoId("1-DOMAIN")
|
||||
.setRegistrarId("TheRegistrar")
|
||||
.setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
|
||||
.build();
|
||||
insertInDb(previous);
|
||||
|
||||
runCommandForced();
|
||||
ImmutableList<Spec11ThreatMatch> threatMatches =
|
||||
jpaTm().transact(() -> jpaTm().loadAllOf(Spec11ThreatMatch.class));
|
||||
assertAboutImmutableObjects()
|
||||
.that(Iterables.getOnlyElement(threatMatches))
|
||||
.isEqualExceptFields(previous, "id");
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_overwritesExistingDatesWhenSpecified() throws Exception {
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
|
||||
.thenReturn(sampleThreatMatches());
|
||||
Spec11ThreatMatch previous =
|
||||
new Spec11ThreatMatch.Builder()
|
||||
.setCheckDate(CURRENT_DATE)
|
||||
.setDomainName("previous.tld")
|
||||
.setDomainRepoId("1-DOMAIN")
|
||||
.setRegistrarId("TheRegistrar")
|
||||
.setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
|
||||
.build();
|
||||
insertInDb(previous);
|
||||
|
||||
runCommandForced("--overwrite_existing_dates");
|
||||
verifyExactlyThreeEntriesInDbFromLastDay();
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testFailure_oneFileFails() throws Exception {
|
||||
// If there are any exceptions, we should fail loud and fast
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
|
||||
.thenReturn(sampleThreatMatches());
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE.minusDays(1)))
|
||||
.thenThrow(new IOException("hi"));
|
||||
RuntimeException runtimeException =
|
||||
assertThrows(RuntimeException.class, this::runCommandForced);
|
||||
assertThat(runtimeException.getCause().getClass()).isEqualTo(IOException.class);
|
||||
assertThat(runtimeException).hasCauseThat().hasMessageThat().isEqualTo("hi");
|
||||
assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(Spec11ThreatMatch.class))).isEmpty();
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testFailure_noDomainForDomainName() throws Exception {
|
||||
deleteResource(domainA);
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
|
||||
.thenReturn(sampleThreatMatches());
|
||||
assertThat(assertThrows(IllegalStateException.class, this::runCommandForced))
|
||||
.hasMessageThat()
|
||||
.isEqualTo("Domain name a.com had no associated DomainBase objects.");
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testFailure_noDomainAtTimeOfScan() throws Exception {
|
||||
// If the domain existed at some point(s) in time but not the time of the scan, fail.
|
||||
// First, domain was created at START_OF_TIME and deleted one year ago
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build());
|
||||
|
||||
// Second, domain was created one month ago and is still valid
|
||||
persistResource(
|
||||
newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build());
|
||||
|
||||
// If we have a result for this domain from 3 months ago when it didn't exist, fail.
|
||||
when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3)))
|
||||
.thenReturn(sampleThreatMatches());
|
||||
assertThat(assertThrows(IllegalStateException.class, this::runCommandForced))
|
||||
.hasMessageThat()
|
||||
.isEqualTo("Could not find a DomainBase valid for a.com on day 2020-08-22.");
|
||||
}
|
||||
|
||||
private void verifyExactlyThreeEntriesInDbFromLastDay() {
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
ImmutableList<Spec11ThreatMatch> threatMatches =
|
||||
jpaTm().loadAllOf(Spec11ThreatMatch.class);
|
||||
assertThat(threatMatches)
|
||||
.comparingElementsUsing(immutableObjectCorrespondence("id", "domainRepoId"))
|
||||
.containsExactly(
|
||||
expectedThreatMatch("TheRegistrar", "a.com"),
|
||||
expectedThreatMatch("NewRegistrar", "b.com"),
|
||||
expectedThreatMatch("NewRegistrar", "c.com"));
|
||||
});
|
||||
}
|
||||
|
||||
private Spec11ThreatMatch expectedThreatMatch(String registrarId, String domainName) {
|
||||
return new Spec11ThreatMatch.Builder()
|
||||
.setDomainRepoId("ignored")
|
||||
.setDomainName(domainName)
|
||||
.setRegistrarId(registrarId)
|
||||
.setCheckDate(CURRENT_DATE)
|
||||
.setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue