diff --git a/core/src/main/java/google/registry/tools/RegistryTool.java b/core/src/main/java/google/registry/tools/RegistryTool.java index 55802ba0d..976d9ab63 100644 --- a/core/src/main/java/google/registry/tools/RegistryTool.java +++ b/core/src/main/java/google/registry/tools/RegistryTool.java @@ -15,14 +15,8 @@ package google.registry.tools; import com.google.common.collect.ImmutableMap; -import google.registry.tools.javascrap.BackfillRegistryLocksCommand; -import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand; import google.registry.tools.javascrap.CompareEscrowDepositsCommand; -import google.registry.tools.javascrap.DeleteContactByRoidCommand; import google.registry.tools.javascrap.HardDeleteHostCommand; -import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand; -import google.registry.tools.javascrap.RemoveIpAddressCommand; -import google.registry.tools.javascrap.ResaveAllTldsCommand; /** Container class to create and run remote commands against a Datastore instance. */ public final class RegistryTool { @@ -36,8 +30,6 @@ public final class RegistryTool { public static final ImmutableMap> COMMAND_MAP = new ImmutableMap.Builder>() .put("ack_poll_messages", AckPollMessagesCommand.class) - .put("backfill_registry_locks", BackfillRegistryLocksCommand.class) - .put("backfill_spec11_threat_matches", BackfillSpec11ThreatMatchesCommand.class) .put("canonicalize_labels", CanonicalizeLabelsCommand.class) .put("check_domain", CheckDomainCommand.class) .put("check_domain_claims", CheckDomainClaimsCommand.class) @@ -57,7 +49,6 @@ public final class RegistryTool { .put("curl", CurlCommand.class) .put("dedupe_one_time_billing_event_ids", DedupeOneTimeBillingEventIdsCommand.class) .put("delete_allocation_tokens", DeleteAllocationTokensCommand.class) - .put("delete_contact_by_roid", DeleteContactByRoidCommand.class) .put("delete_domain", DeleteDomainCommand.class) .put("delete_host", DeleteHostCommand.class) .put("delete_premium_list", DeletePremiumListCommand.class) @@ -107,12 +98,9 @@ public final class RegistryTool { .put("login", LoginCommand.class) .put("logout", LogoutCommand.class) .put("pending_escrow", PendingEscrowCommand.class) - .put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class) .put("registrar_contact", RegistrarContactCommand.class) - .put("remove_ip_address", RemoveIpAddressCommand.class) .put("remove_registry_one_key", RemoveRegistryOneKeyCommand.class) .put("renew_domain", RenewDomainCommand.class) - .put("resave_all_tlds", ResaveAllTldsCommand.class) .put("resave_entities", ResaveEntitiesCommand.class) .put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class) .put("resave_epp_resource", ResaveEppResourceCommand.class) diff --git a/core/src/main/java/google/registry/tools/RegistryToolComponent.java b/core/src/main/java/google/registry/tools/RegistryToolComponent.java index c075ca453..a2f8f57ff 100644 --- a/core/src/main/java/google/registry/tools/RegistryToolComponent.java +++ b/core/src/main/java/google/registry/tools/RegistryToolComponent.java @@ -42,9 +42,7 @@ import google.registry.request.Modules.URLFetchServiceModule; import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UserServiceModule; import google.registry.tools.AuthModule.LocalCredentialModule; -import google.registry.tools.javascrap.BackfillRegistryLocksCommand; import google.registry.tools.javascrap.CompareEscrowDepositsCommand; -import google.registry.tools.javascrap.DeleteContactByRoidCommand; import google.registry.tools.javascrap.HardDeleteHostCommand; import google.registry.util.UtilsModule; import google.registry.whois.NonCachingWhoisModule; @@ -90,8 +88,6 @@ import javax.inject.Singleton; interface RegistryToolComponent { void inject(AckPollMessagesCommand command); - void inject(BackfillRegistryLocksCommand command); - void inject(CheckDomainClaimsCommand command); void inject(CheckDomainCommand command); @@ -112,8 +108,6 @@ interface RegistryToolComponent { void inject(CreateTldCommand command); - void inject(DeleteContactByRoidCommand command); - void inject(EncryptEscrowDepositCommand command); void inject(EnqueuePollMessageCommand command); diff --git a/core/src/main/java/google/registry/tools/javascrap/BackfillRegistryLocksCommand.java b/core/src/main/java/google/registry/tools/javascrap/BackfillRegistryLocksCommand.java deleted file mode 100644 index 887ff8bb9..000000000 --- a/core/src/main/java/google/registry/tools/javascrap/BackfillRegistryLocksCommand.java +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools.javascrap; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm; -import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.collect.ImmutableCollection; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.flogger.FluentLogger; -import google.registry.config.RegistryConfig.Config; -import google.registry.model.domain.DomainBase; -import google.registry.model.domain.RegistryLock; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.reporting.HistoryEntryDao; -import google.registry.model.tld.RegistryLockDao; -import google.registry.persistence.VKey; -import google.registry.tools.CommandWithRemoteApi; -import google.registry.tools.ConfirmingCommand; -import google.registry.util.Clock; -import google.registry.util.StringGenerator; -import java.util.Comparator; -import java.util.List; -import javax.inject.Inject; -import javax.inject.Named; -import org.joda.time.DateTime; - -/** - * Scrap tool to backfill {@link RegistryLock}s for domains previously locked. - * - *

This will save new objects for all existing domains that are locked but don't have any - * corresponding lock objects already in the database. - */ -@Parameters( - separators = " =", - commandDescription = - "Backfills RegistryLock objects for specified domain resource IDs that are locked but don't" - + " already have a corresponding RegistryLock object.") -public class BackfillRegistryLocksCommand extends ConfirmingCommand - implements CommandWithRemoteApi { - - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final int VERIFICATION_CODE_LENGTH = 32; - - @Parameter( - names = {"--domain_roids"}, - description = "Comma-separated list of domain roids to check") - protected List roids; - - // Inject here so that we can create the command automatically for tests - @Inject Clock clock; - - @Inject - @Config("registryAdminClientId") - String registryAdminClientId; - - @Inject - @Named("base58StringGenerator") - StringGenerator stringGenerator; - - private ImmutableList lockedDomains; - - @Override - protected String prompt() { - checkArgument( - roids != null && !roids.isEmpty(), "Must provide non-empty domain_roids argument"); - lockedDomains = - jpaTm().transact(() -> getLockedDomainsWithoutLocks(jpaTm().getTransactionTime())); - ImmutableList lockedDomainNames = - lockedDomains.stream().map(DomainBase::getDomainName).collect(toImmutableList()); - return String.format( - "Locked domains for which there does not exist a RegistryLock object: %s", - lockedDomainNames); - } - - @Override - protected String execute() { - ImmutableSet.Builder failedDomainsBuilder = new ImmutableSet.Builder<>(); - jpaTm() - .transact( - () -> { - for (DomainBase domainBase : lockedDomains) { - try { - RegistryLockDao.save( - new RegistryLock.Builder() - .isSuperuser(true) - .setRegistrarId(registryAdminClientId) - .setRepoId(domainBase.getRepoId()) - .setDomainName(domainBase.getDomainName()) - .setLockCompletionTime( - getLockCompletionTimestamp(domainBase, jpaTm().getTransactionTime())) - .setVerificationCode( - stringGenerator.createString(VERIFICATION_CODE_LENGTH)) - .build()); - } catch (Throwable t) { - logger.atSevere().withCause(t).log( - "Error when creating lock object for domain '%s'.", - domainBase.getDomainName()); - failedDomainsBuilder.add(domainBase.getDomainName()); - } - } - }); - ImmutableSet failedDomains = failedDomainsBuilder.build(); - if (failedDomains.isEmpty()) { - return String.format( - "Successfully created lock objects for %d domains.", lockedDomains.size()); - } else { - return String.format( - "Successfully created lock objects for %d domains. We failed to create locks " - + "for the following domains: %s", - lockedDomains.size() - failedDomains.size(), failedDomains); - } - } - - private DateTime getLockCompletionTimestamp(DomainBase domainBase, DateTime now) { - // Best-effort, if a domain was URS-locked we should use that time - // If we can't find that, return now. - return HistoryEntryDao.loadHistoryObjectsForResource(domainBase.createVKey()).stream() - // sort by modification time descending so we get the most recent one if it was locked twice - .sorted(Comparator.comparing(HistoryEntry::getModificationTime).reversed()) - .filter(entry -> "Uniform Rapid Suspension".equals(entry.getReason())) - .findFirst() - .map(HistoryEntry::getModificationTime) - .orElse(now); - } - - private ImmutableList getLockedDomainsWithoutLocks(DateTime now) { - ImmutableList> domainKeys = - roids.stream().map(roid -> VKey.create(DomainBase.class, roid)).collect(toImmutableList()); - ImmutableCollection domains = - transactIfJpaTm(() -> tm().loadByKeys(domainKeys)).values(); - return domains.stream() - .filter(d -> d.getDeletionTime().isAfter(now)) - .filter(d -> d.getStatusValues().containsAll(REGISTRY_LOCK_STATUSES)) - .filter(d -> !RegistryLockDao.getMostRecentByRepoId(d.getRepoId()).isPresent()) - .collect(toImmutableList()); - } -} diff --git a/core/src/main/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommand.java b/core/src/main/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommand.java deleted file mode 100644 index 8176e660b..000000000 --- a/core/src/main/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommand.java +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools.javascrap; - -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap; -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; -import com.google.common.collect.ImmutableSet; -import google.registry.beam.spec11.ThreatMatch; -import google.registry.model.domain.DomainBase; -import google.registry.model.reporting.Spec11ThreatMatch; -import google.registry.model.reporting.Spec11ThreatMatch.ThreatType; -import google.registry.model.reporting.Spec11ThreatMatchDao; -import google.registry.persistence.transaction.QueryComposer; -import google.registry.reporting.spec11.RegistrarThreatMatches; -import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser; -import google.registry.tools.CommandWithRemoteApi; -import google.registry.tools.ConfirmingCommand; -import google.registry.util.Clock; -import java.io.IOException; -import java.util.Comparator; -import java.util.function.Function; -import javax.inject.Inject; -import org.joda.time.LocalDate; - -/** - * Scrap tool to backfill {@link Spec11ThreatMatch} objects from prior days. - * - *

This will load the previously-existing Spec11 files from GCS (looking back to 2019-01-01 (a - * rough estimate of when we started using this format) and convert those RegistrarThreatMatches - * objects into the new Spec11ThreatMatch format. It will then insert these entries into SQL. - * - *

Note that the script will attempt to find the corresponding {@link DomainBase} object for each - * domain name on the day of the scan. It will fail if it cannot find a corresponding domain object, - * or if the domain objects were not active at the time of the scan. - */ -@Parameters( - commandDescription = - "Backfills Spec11 threat match entries from the old and deprecated GCS JSON files to the " - + "Cloud SQL database.") -public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand - implements CommandWithRemoteApi { - - private static final LocalDate START_DATE = new LocalDate(2019, 1, 1); - - @Parameter( - names = {"-o", "--overwrite_existing_dates"}, - description = - "Whether the command will overwrite data that already exists for dates that exist in the " - + "GCS bucket. Defaults to false.") - private boolean overrideExistingDates; - - @Inject Spec11RegistrarThreatMatchesParser threatMatchesParser; - // Inject the clock for testing purposes - @Inject Clock clock; - - @Override - protected String prompt() { - return String.format("Backfill Spec11 results from %d files?", getDatesToBackfill().size()); - } - - @Override - protected String execute() { - ImmutableList dates = getDatesToBackfill(); - ImmutableListMultimap.Builder threatMatchesBuilder = - new ImmutableListMultimap.Builder<>(); - for (LocalDate date : dates) { - try { - // It's OK if the file doesn't exist for a particular date; the result will be empty. - threatMatchesBuilder.putAll(date, threatMatchesParser.getRegistrarThreatMatches(date)); - } catch (IOException e) { - throw new RuntimeException( - String.format("Error parsing through file with date %s.", date), e); - } - } - ImmutableListMultimap threatMatches = - threatMatchesBuilder.build(); - // Look up all possible DomainBases for these domain names, any of which can be in the past - ImmutableListMultimap domainsByDomainName = - getDomainsByDomainName(threatMatches); - - // For each date, convert all threat matches with the proper domain repo ID - int totalNumThreats = 0; - for (LocalDate date : threatMatches.keySet()) { - ImmutableList.Builder spec11ThreatsBuilder = new ImmutableList.Builder<>(); - for (RegistrarThreatMatches rtm : threatMatches.get(date)) { - rtm.threatMatches().stream() - .map( - threatMatch -> - threatMatchToCloudSqlObject( - threatMatch, date, rtm.clientId(), domainsByDomainName)) - .forEach(spec11ThreatsBuilder::add); - } - ImmutableList spec11Threats = spec11ThreatsBuilder.build(); - jpaTm() - .transact( - () -> { - Spec11ThreatMatchDao.deleteEntriesByDate(jpaTm(), date); - jpaTm().putAll(spec11Threats); - }); - totalNumThreats += spec11Threats.size(); - } - return String.format( - "Successfully parsed through %d files with %d threats.", dates.size(), totalNumThreats); - } - - /** Returns a per-domain list of possible DomainBase objects, starting with the most recent. */ - private ImmutableListMultimap getDomainsByDomainName( - ImmutableListMultimap threatMatchesByDate) { - return threatMatchesByDate.values().stream() - .map(RegistrarThreatMatches::threatMatches) - .flatMap(ImmutableList::stream) - .map(ThreatMatch::fullyQualifiedDomainName) - .distinct() - .collect( - flatteningToImmutableListMultimap( - Function.identity(), - (domainName) -> { - ImmutableList domains = loadDomainsForFqdn(domainName); - checkState( - !domains.isEmpty(), - "Domain name %s had no associated DomainBase objects.", - domainName); - return domains.stream() - .sorted(Comparator.comparing(DomainBase::getCreationTime).reversed()); - })); - } - - /** Loads in all {@link DomainBase} objects for a given FQDN. */ - private ImmutableList loadDomainsForFqdn(String fullyQualifiedDomainName) { - return transactIfJpaTm( - () -> - tm().createQueryComposer(DomainBase.class) - .where( - "fullyQualifiedDomainName", - QueryComposer.Comparator.EQ, - fullyQualifiedDomainName) - .list()); - } - - /** Converts the previous {@link ThreatMatch} object to {@link Spec11ThreatMatch}. */ - private Spec11ThreatMatch threatMatchToCloudSqlObject( - ThreatMatch threatMatch, - LocalDate date, - String registrarId, - ImmutableListMultimap domainsByDomainName) { - DomainBase domain = - findDomainAsOfDateOrThrow( - threatMatch.fullyQualifiedDomainName(), date, domainsByDomainName); - return new Spec11ThreatMatch.Builder() - .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(threatMatch.threatType()))) - .setCheckDate(date) - .setRegistrarId(registrarId) - .setDomainName(threatMatch.fullyQualifiedDomainName()) - .setDomainRepoId(domain.getRepoId()) - .build(); - } - - /** Returns the DomainBase object as of the particular date, which is likely in the past. */ - private DomainBase findDomainAsOfDateOrThrow( - String domainName, - LocalDate date, - ImmutableListMultimap domainsByDomainName) { - ImmutableList domains = domainsByDomainName.get(domainName); - for (DomainBase domain : domains) { - // We only know the date (not datetime) of the threat scan, so we approximate - LocalDate creationDate = domain.getCreationTime().toLocalDate(); - LocalDate deletionDate = domain.getDeletionTime().toLocalDate(); - if (!date.isBefore(creationDate) && !date.isAfter(deletionDate)) { - return domain; - } - } - throw new IllegalStateException( - String.format("Could not find a DomainBase valid for %s on day %s.", domainName, date)); - } - - /** Returns the list of dates between {@link #START_DATE} and now (UTC), inclusive. */ - private ImmutableList getDatesToBackfill() { - ImmutableSet datesToSkip = - overrideExistingDates ? ImmutableSet.of() : getExistingDates(); - ImmutableList.Builder result = new ImmutableList.Builder<>(); - LocalDate endDate = clock.nowUtc().toLocalDate(); - for (LocalDate currentDate = START_DATE; - !currentDate.isAfter(endDate); - currentDate = currentDate.plusDays(1)) { - if (!datesToSkip.contains(currentDate)) { - result.add(currentDate); - } - } - return result.build(); - } - - private ImmutableSet getExistingDates() { - return jpaTm() - .transact( - () -> - jpaTm() - .query( - "SELECT DISTINCT stm.checkDate FROM Spec11ThreatMatch stm", LocalDate.class) - .getResultStream() - .collect(toImmutableSet())); - } -} diff --git a/core/src/main/java/google/registry/tools/javascrap/DeleteContactByRoidCommand.java b/core/src/main/java/google/registry/tools/javascrap/DeleteContactByRoidCommand.java deleted file mode 100644 index ce6dc8cd6..000000000 --- a/core/src/main/java/google/registry/tools/javascrap/DeleteContactByRoidCommand.java +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools.javascrap; - -import static com.google.common.base.Verify.verify; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.model.contact.ContactResource; -import google.registry.model.eppcommon.StatusValue; -import google.registry.model.index.EppResourceIndex; -import google.registry.model.index.ForeignKeyIndex; -import google.registry.tools.CommandWithRemoteApi; -import google.registry.tools.ConfirmingCommand; -import google.registry.util.SystemClock; -import java.util.List; -import java.util.Objects; - -/** - * Deletes a {@link google.registry.model.contact.ContactResource} by its ROID. - * - *

This is a short-term tool for race condition clean up while the bug is being fixed. - */ -@Parameters(separators = " =", commandDescription = "Delete a contact by its ROID.") -public class DeleteContactByRoidCommand extends ConfirmingCommand implements CommandWithRemoteApi { - - @Parameter(names = "--roid", description = "The roid of the contact to be deleted.") - String roid; - - @Parameter( - names = "--contact_id", - description = "The user provided contactId, for verification purpose.") - String contactId; - - ImmutableList> toDelete; - - @Override - protected void init() { - System.out.printf("Deleting %s, which refers to %s.\n", roid, contactId); - tm().transact( - () -> { - Key targetKey = Key.create(ContactResource.class, roid); - ContactResource targetContact = auditedOfy().load().key(targetKey).now(); - verify( - Objects.equals(targetContact.getContactId(), contactId), - "contactId does not match."); - verify( - Objects.equals(targetContact.getStatusValues(), ImmutableSet.of(StatusValue.OK))); - System.out.println("Target contact has the expected contactId"); - String canonicalResource = - ForeignKeyIndex.load(ContactResource.class, contactId, new SystemClock().nowUtc()) - .getResourceKey() - .getOfyKey() - .getName(); - verify(!Objects.equals(canonicalResource, roid), "Contact still in ForeignKeyIndex."); - System.out.printf( - "It is safe to delete %s, since the contactId is mapped to a different entry in" - + " the Foreign key index (%s).\n\n", - roid, canonicalResource); - - List ancestors = - auditedOfy().load().ancestor(Key.create(ContactResource.class, roid)).list(); - - System.out.println("Ancestor query returns: "); - for (Object entity : ancestors) { - System.out.println(Key.create(entity)); - } - - ImmutableSet deletetableKinds = - ImmutableSet.of("HistoryEntry", "ContactResource"); - toDelete = - ancestors.stream() - .map(Key::create) - .filter(key -> deletetableKinds.contains(key.getKind())) - .collect(ImmutableList.toImmutableList()); - - EppResourceIndex eppResourceIndex = - auditedOfy().load().entity(EppResourceIndex.create(targetKey)).now(); - verify(eppResourceIndex.getKey().equals(targetKey), "Wrong EppResource Index loaded"); - System.out.printf("\n\nEppResourceIndex found (%s).\n", Key.create(eppResourceIndex)); - - toDelete = - new ImmutableList.Builder>() - .addAll(toDelete) - .add(Key.create(eppResourceIndex)) - .build(); - - System.out.printf("\n\nAbout to delete %s entities:\n", toDelete.size()); - toDelete.forEach(System.out::println); - }); - } - - @Override - protected String execute() { - tm().transact(() -> auditedOfy().delete().keys(toDelete).now()); - return "Done"; - } -} diff --git a/core/src/main/java/google/registry/tools/javascrap/PopulateNullRegistrarFieldsCommand.java b/core/src/main/java/google/registry/tools/javascrap/PopulateNullRegistrarFieldsCommand.java deleted file mode 100644 index 09ac09c9e..000000000 --- a/core/src/main/java/google/registry/tools/javascrap/PopulateNullRegistrarFieldsCommand.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools.javascrap; - -import static com.google.common.base.MoreObjects.firstNonNull; - -import com.beust.jcommander.Parameters; -import com.google.common.collect.ImmutableList; -import google.registry.model.registrar.Registrar; -import google.registry.model.registrar.RegistrarAddress; -import google.registry.tools.MutatingCommand; -import java.util.Objects; - -/** - * Scrap tool to update Registrars with null registrarName or localizedAddress fields. - * - *

This sets a null registrarName to the key name, and null localizedAddress fields to fake data. - */ -@Parameters( - separators = " =", - commandDescription = "Populate previously null required registrar fields." -) -public class PopulateNullRegistrarFieldsCommand extends MutatingCommand { - - @Override - protected void init() { - for (Registrar registrar : Registrar.loadAll()) { - Registrar.Builder changeBuilder = registrar.asBuilder(); - changeBuilder.setRegistrarName( - firstNonNull(registrar.getRegistrarName(), registrar.getRegistrarId())); - - RegistrarAddress address = registrar.getLocalizedAddress(); - if (address == null) { - changeBuilder.setLocalizedAddress( - new RegistrarAddress.Builder() - .setCity("Fakington") - .setCountryCode("US") - .setState("FL") - .setZip("12345") - .setStreet(ImmutableList.of("123 Fake Street")) - .build()); - } else { - changeBuilder.setLocalizedAddress( - new RegistrarAddress.Builder() - .setCity(firstNonNull(address.getCity(), "Fakington")) - .setCountryCode(firstNonNull(address.getCountryCode(), "US")) - .setState(firstNonNull(address.getState(), "FL")) - .setZip(firstNonNull(address.getZip(), "12345")) - .setStreet(firstNonNull(address.getStreet(), ImmutableList.of("123 Fake Street"))) - .build()); - } - Registrar changedRegistrar = changeBuilder.build(); - if (!Objects.equals(registrar, changedRegistrar)) { - stageEntityChange(registrar, changedRegistrar); - } - } - } -} diff --git a/core/src/main/java/google/registry/tools/javascrap/RemoveIpAddressCommand.java b/core/src/main/java/google/registry/tools/javascrap/RemoveIpAddressCommand.java deleted file mode 100644 index 96c0ef6a0..000000000 --- a/core/src/main/java/google/registry/tools/javascrap/RemoveIpAddressCommand.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools.javascrap; - -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.template.soy.data.SoyMapData; -import google.registry.model.host.HostResource; -import google.registry.persistence.VKey; -import google.registry.tools.MutatingEppToolCommand; -import google.registry.tools.params.PathParameter; -import google.registry.tools.soy.RemoveIpAddressSoyInfo; -import java.net.Inet6Address; -import java.net.InetAddress; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -/** - * Command to remove external IP Addresses from HostResources identified by text file listing - * resource ids, one per line. - * - *

Written for b/23757755 so we can clean up records with IP addresses that should always be - * resolved by hostname. - * - *

The JSON file should contain a list of objects each of which has a "roid" attribute. - */ -@Parameters(separators = " =", commandDescription = "Remove all IP Addresses.") -public class RemoveIpAddressCommand extends MutatingEppToolCommand { - public static String registrarId = "CharlestonRoad"; - - @Parameter(names = "--roids_file", - description = "Text file containing a list of HostResource roids to remove", - required = true, - validateWith = PathParameter.InputFile.class) - private Path roidsFilePath; - - @Override - protected void initMutatingEppToolCommand() throws Exception { - List roids = Files.readAllLines(roidsFilePath, UTF_8); - - for (String roid : roids) { - // Look up the HostResource from its roid. - Optional host = - transactIfJpaTm(() -> tm().loadByKeyIfPresent(VKey.create(HostResource.class, roid))); - if (!host.isPresent()) { - System.err.printf("Record for %s not found.\n", roid); - continue; - } - - ArrayList ipAddresses = new ArrayList<>(); - for (InetAddress address : host.get().getInetAddresses()) { - SoyMapData dataMap = new SoyMapData( - "address", address.getHostAddress(), - "version", address instanceof Inet6Address ? "v6" : "v4"); - ipAddresses.add(dataMap); - } - - // Build and execute the EPP command. - setSoyTemplate( - RemoveIpAddressSoyInfo.getInstance(), RemoveIpAddressSoyInfo.REMOVE_IP_ADDRESS); - addSoyRecord( - registrarId, - new SoyMapData( - "name", host.get().getHostName(), - "ipAddresses", ipAddresses, - "requestedByRegistrar", registrarId)); - } - } -} diff --git a/core/src/main/java/google/registry/tools/javascrap/ResaveAllTldsCommand.java b/core/src/main/java/google/registry/tools/javascrap/ResaveAllTldsCommand.java deleted file mode 100644 index 907afbcba..000000000 --- a/core/src/main/java/google/registry/tools/javascrap/ResaveAllTldsCommand.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools.javascrap; - -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; - -import com.beust.jcommander.Parameters; -import google.registry.model.tld.Registry; -import google.registry.tools.CommandWithRemoteApi; - -/** Scrap command to resave all Registry entities. */ -@Parameters(commandDescription = "Resave all TLDs") -public class ResaveAllTldsCommand implements CommandWithRemoteApi { - @Override - public void run() throws Exception { - tm().transact(() -> tm().putAll(tm().loadAllOf(Registry.class))); - } -} diff --git a/core/src/test/java/google/registry/tools/javascrap/BackfillRegistryLocksCommandTest.java b/core/src/test/java/google/registry/tools/javascrap/BackfillRegistryLocksCommandTest.java deleted file mode 100644 index d7a394ddf..000000000 --- a/core/src/test/java/google/registry/tools/javascrap/BackfillRegistryLocksCommandTest.java +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools.javascrap; - -import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth8.assertThat; -import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.DatabaseHelper.persistActiveDomain; -import static google.registry.testing.DatabaseHelper.persistDeletedDomain; -import static google.registry.testing.DatabaseHelper.persistNewRegistrar; -import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.SqlHelper.getMostRecentRegistryLockByRepoId; -import static google.registry.testing.SqlHelper.getMostRecentVerifiedRegistryLockByRepoId; -import static google.registry.testing.SqlHelper.getRegistryLocksByRegistrarId; -import static google.registry.testing.SqlHelper.saveRegistryLock; -import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.truth.Truth8; -import google.registry.model.domain.DomainBase; -import google.registry.model.domain.DomainHistory; -import google.registry.model.domain.RegistryLock; -import google.registry.model.registrar.Registrar; -import google.registry.model.reporting.HistoryEntry; -import google.registry.testing.DeterministicStringGenerator; -import google.registry.testing.DualDatabaseTest; -import google.registry.testing.TestOfyAndSql; -import google.registry.tools.CommandTestCase; -import google.registry.util.StringGenerator.Alphabets; -import java.util.Optional; -import org.joda.time.DateTime; -import org.joda.time.Duration; -import org.junit.jupiter.api.BeforeEach; - -/** Unit tests for {@link BackfillRegistryLocksCommand}. */ -@DualDatabaseTest -class BackfillRegistryLocksCommandTest extends CommandTestCase { - - @BeforeEach - void beforeEach() { - persistNewRegistrar("adminreg", "Admin Registrar", Registrar.Type.REAL, 693L); - createTld("tld"); - command.registryAdminClientId = "adminreg"; - command.clock = fakeClock; - command.stringGenerator = new DeterministicStringGenerator(Alphabets.BASE_58); - } - - @TestOfyAndSql - void testSimpleBackfill() throws Exception { - DomainBase domain = persistLockedDomain("example.tld"); - Truth8.assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId())).isEmpty(); - - runCommandForced("--domain_roids", domain.getRepoId()); - - Optional lockOptional = getMostRecentRegistryLockByRepoId(domain.getRepoId()); - Truth8.assertThat(lockOptional).isPresent(); - Truth8.assertThat(lockOptional.get().getLockCompletionTime()).isPresent(); - } - - @TestOfyAndSql - void testBackfill_onlyLockedDomains() throws Exception { - DomainBase neverLockedDomain = persistActiveDomain("neverlocked.tld"); - DomainBase previouslyLockedDomain = persistLockedDomain("unlocked.tld"); - persistResource(previouslyLockedDomain.asBuilder().setStatusValues(ImmutableSet.of()).build()); - DomainBase lockedDomain = persistLockedDomain("locked.tld"); - - runCommandForced( - "--domain_roids", - String.format( - "%s,%s,%s", - neverLockedDomain.getRepoId(), - previouslyLockedDomain.getRepoId(), - lockedDomain.getRepoId())); - - ImmutableList locks = getRegistryLocksByRegistrarId("adminreg"); - assertThat(locks).hasSize(1); - assertThat(Iterables.getOnlyElement(locks).getDomainName()).isEqualTo("locked.tld"); - } - - @TestOfyAndSql - void testBackfill_skipsDeletedDomains() throws Exception { - DomainBase domain = persistDeletedDomain("example.tld", fakeClock.nowUtc()); - persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build()); - fakeClock.advanceBy(Duration.standardSeconds(1)); - runCommandForced("--domain_roids", domain.getRepoId()); - Truth8.assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId())).isEmpty(); - } - - @TestOfyAndSql - void testBackfill_skipsDomains_ifLockAlreadyExists() throws Exception { - DomainBase domain = persistLockedDomain("example.tld"); - - RegistryLock previousLock = - saveRegistryLock( - new RegistryLock.Builder() - .isSuperuser(true) - .setRegistrarId("adminreg") - .setRepoId(domain.getRepoId()) - .setDomainName(domain.getDomainName()) - .setLockCompletionTime(fakeClock.nowUtc()) - .setVerificationCode(command.stringGenerator.createString(32)) - .build()); - - fakeClock.advanceBy(Duration.standardDays(1)); - runCommandForced("--domain_roids", domain.getRepoId()); - - assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId()).get().getLockCompletionTime()) - .isEqualTo(previousLock.getLockCompletionTime()); - } - - @TestOfyAndSql - void testBackfill_usesUrsTime_ifExists() throws Exception { - DateTime ursTime = fakeClock.nowUtc(); - DomainBase ursDomain = persistLockedDomain("urs.tld"); - persistResource( - new DomainHistory.Builder() - .setBySuperuser(true) - .setRegistrarId("adminreg") - .setModificationTime(ursTime) - .setDomain(ursDomain) - .setReason("Uniform Rapid Suspension") - .setType(HistoryEntry.Type.DOMAIN_UPDATE) - .setRequestedByRegistrar(false) - .build()); - DomainBase nonUrsDomain = persistLockedDomain("nonurs.tld"); - persistResource( - new DomainHistory.Builder() - .setBySuperuser(true) - .setRegistrarId("adminreg") - .setDomain(nonUrsDomain) - .setType(HistoryEntry.Type.DOMAIN_UPDATE) - .setRequestedByRegistrar(false) - .setModificationTime(ursTime) - .build()); - - fakeClock.advanceBy(Duration.standardDays(10)); - runCommandForced( - "--domain_roids", String.format("%s,%s", ursDomain.getRepoId(), nonUrsDomain.getRepoId())); - - RegistryLock ursLock = getMostRecentVerifiedRegistryLockByRepoId(ursDomain.getRepoId()).get(); - assertThat(ursLock.getLockCompletionTime()).hasValue(ursTime); - RegistryLock nonUrsLock = - getMostRecentVerifiedRegistryLockByRepoId(nonUrsDomain.getRepoId()).get(); - assertThat(nonUrsLock.getLockCompletionTime()).hasValue(fakeClock.nowUtc()); - } - - @TestOfyAndSql - void testFailure_mustProvideDomainRoids() { - assertThat(assertThrows(IllegalArgumentException.class, this::runCommandForced)) - .hasMessageThat() - .isEqualTo("Must provide non-empty domain_roids argument"); - } - - private static DomainBase persistLockedDomain(String domainName) { - DomainBase domain = persistActiveDomain(domainName); - return persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build()); - } -} diff --git a/core/src/test/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommandTest.java b/core/src/test/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommandTest.java deleted file mode 100644 index 4d6e39807..000000000 --- a/core/src/test/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommandTest.java +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools.javascrap; - -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects; -import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches; -import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.DatabaseHelper.deleteResource; -import static google.registry.testing.DatabaseHelper.insertInDb; -import static google.registry.testing.DatabaseHelper.newDomainBase; -import static google.registry.testing.DatabaseHelper.persistActiveDomain; -import static google.registry.testing.DatabaseHelper.persistResource; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import google.registry.model.domain.DomainBase; -import google.registry.model.reporting.Spec11ThreatMatch; -import google.registry.model.reporting.Spec11ThreatMatch.ThreatType; -import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser; -import google.registry.testing.DualDatabaseTest; -import google.registry.testing.TestOfyAndSql; -import google.registry.tools.CommandTestCase; -import java.io.IOException; -import org.joda.time.DateTime; -import org.joda.time.LocalDate; -import org.junit.jupiter.api.BeforeEach; - -/** Tests for {@link BackfillSpec11ThreatMatchesCommand}. */ -@DualDatabaseTest -public class BackfillSpec11ThreatMatchesCommandTest - extends CommandTestCase { - - private static final LocalDate CURRENT_DATE = DateTime.parse("2020-11-22").toLocalDate(); - private final Spec11RegistrarThreatMatchesParser threatMatchesParser = - mock(Spec11RegistrarThreatMatchesParser.class); - - private DomainBase domainA; - - @BeforeEach - void beforeEach() throws Exception { - createTld("com"); - domainA = persistActiveDomain("a.com"); - persistActiveDomain("b.com"); - persistActiveDomain("c.com"); - fakeClock.setTo(CURRENT_DATE.toDateTimeAtStartOfDay()); - command.threatMatchesParser = threatMatchesParser; - command.clock = fakeClock; - when(threatMatchesParser.getRegistrarThreatMatches(any(LocalDate.class))) - .thenReturn(ImmutableSet.of()); - } - - @TestOfyAndSql - void testSuccess_singleFile() throws Exception { - when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE)) - .thenReturn(sampleThreatMatches()); - runCommandForced(); - assertInStdout("Backfill Spec11 results from 692 files?"); - assertInStdout("Successfully parsed through 692 files with 3 threats."); - verifyExactlyThreeEntriesInDbFromLastDay(); - } - - @TestOfyAndSql - void testSuccess_sameDomain_multipleDays() throws Exception { - // If the same domains show up on multiple days, there should be multiple entries for them - when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE)) - .thenReturn(sampleThreatMatches()); - when(threatMatchesParser.getRegistrarThreatMatches(LocalDate.parse("2019-01-01"))) - .thenReturn(sampleThreatMatches()); - runCommandForced(); - assertInStdout("Backfill Spec11 results from 692 files?"); - assertInStdout("Successfully parsed through 692 files with 6 threats."); - jpaTm() - .transact( - () -> { - ImmutableList threatMatches = - jpaTm().loadAllOf(Spec11ThreatMatch.class); - assertThat(threatMatches).hasSize(6); - assertThat( - threatMatches.stream() - .map(Spec11ThreatMatch::getDomainName) - .collect(toImmutableSet())) - .containsExactly("a.com", "b.com", "c.com"); - assertThat( - threatMatches.stream() - .map(Spec11ThreatMatch::getCheckDate) - .collect(toImmutableSet())) - .containsExactly(CURRENT_DATE, LocalDate.parse("2019-01-01")); - }); - } - - @TestOfyAndSql - void testSuccess_empty() throws Exception { - runCommandForced(); - assertInStdout("Backfill Spec11 results from 692 files?"); - assertInStdout("Successfully parsed through 692 files with 0 threats."); - } - - @TestOfyAndSql - void testSuccess_sameDayTwice() throws Exception { - when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE)) - .thenReturn(sampleThreatMatches()); - runCommandForced(); - runCommandForced(); - verifyExactlyThreeEntriesInDbFromLastDay(); - } - - @TestOfyAndSql - void testSuccess_threeDomainsForDomainName() throws Exception { - // We should use the repo ID from the proper DomainBase object at the scan's point in time. - // First, domain was created at START_OF_TIME and deleted one year ago - DateTime now = fakeClock.nowUtc(); - domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build()); - - // Next, domain was created six months ago and deleted two months ago - DomainBase secondSave = - persistResource( - newDomainBase("a.com") - .asBuilder() - .setCreationTimeForTest(now.minusMonths(6)) - .setDeletionTime(now.minusMonths(2)) - .build()); - - // Lastly, domain was created one month ago and is still valid - DomainBase thirdSave = - persistResource( - newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build()); - - // If the scan result was from three months ago, we should use the second save - when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3))) - .thenReturn(sampleThreatMatches()); - runCommandForced(); - String threatMatchRepoId = - jpaTm() - .transact( - () -> - jpaTm().loadAllOf(Spec11ThreatMatch.class).stream() - .filter((match) -> match.getDomainName().equals("a.com")) - .findFirst() - .get() - .getDomainRepoId()); - assertThat(threatMatchRepoId).isNotEqualTo(domainA.getRepoId()); - assertThat(threatMatchRepoId).isEqualTo(secondSave.getRepoId()); - assertThat(threatMatchRepoId).isNotEqualTo(thirdSave.getRepoId()); - } - - @TestOfyAndSql - void testSuccess_skipsExistingDatesWithoutOverwrite() throws Exception { - when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE)) - .thenReturn(sampleThreatMatches()); - Spec11ThreatMatch previous = - new Spec11ThreatMatch.Builder() - .setCheckDate(CURRENT_DATE) - .setDomainName("previous.tld") - .setDomainRepoId("1-DOMAIN") - .setRegistrarId("TheRegistrar") - .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE)) - .build(); - insertInDb(previous); - - runCommandForced(); - ImmutableList threatMatches = - jpaTm().transact(() -> jpaTm().loadAllOf(Spec11ThreatMatch.class)); - assertAboutImmutableObjects() - .that(Iterables.getOnlyElement(threatMatches)) - .isEqualExceptFields(previous, "id"); - } - - @TestOfyAndSql - void testSuccess_overwritesExistingDatesWhenSpecified() throws Exception { - when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE)) - .thenReturn(sampleThreatMatches()); - Spec11ThreatMatch previous = - new Spec11ThreatMatch.Builder() - .setCheckDate(CURRENT_DATE) - .setDomainName("previous.tld") - .setDomainRepoId("1-DOMAIN") - .setRegistrarId("TheRegistrar") - .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE)) - .build(); - insertInDb(previous); - - runCommandForced("--overwrite_existing_dates"); - verifyExactlyThreeEntriesInDbFromLastDay(); - } - - @TestOfyAndSql - void testFailure_oneFileFails() throws Exception { - // If there are any exceptions, we should fail loud and fast - when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE)) - .thenReturn(sampleThreatMatches()); - when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE.minusDays(1))) - .thenThrow(new IOException("hi")); - RuntimeException runtimeException = - assertThrows(RuntimeException.class, this::runCommandForced); - assertThat(runtimeException.getCause().getClass()).isEqualTo(IOException.class); - assertThat(runtimeException).hasCauseThat().hasMessageThat().isEqualTo("hi"); - assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(Spec11ThreatMatch.class))).isEmpty(); - } - - @TestOfyAndSql - void testFailure_noDomainForDomainName() throws Exception { - deleteResource(domainA); - when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE)) - .thenReturn(sampleThreatMatches()); - assertThat(assertThrows(IllegalStateException.class, this::runCommandForced)) - .hasMessageThat() - .isEqualTo("Domain name a.com had no associated DomainBase objects."); - } - - @TestOfyAndSql - void testFailure_noDomainAtTimeOfScan() throws Exception { - // If the domain existed at some point(s) in time but not the time of the scan, fail. - // First, domain was created at START_OF_TIME and deleted one year ago - DateTime now = fakeClock.nowUtc(); - domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build()); - - // Second, domain was created one month ago and is still valid - persistResource( - newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build()); - - // If we have a result for this domain from 3 months ago when it didn't exist, fail. - when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3))) - .thenReturn(sampleThreatMatches()); - assertThat(assertThrows(IllegalStateException.class, this::runCommandForced)) - .hasMessageThat() - .isEqualTo("Could not find a DomainBase valid for a.com on day 2020-08-22."); - } - - private void verifyExactlyThreeEntriesInDbFromLastDay() { - jpaTm() - .transact( - () -> { - ImmutableList threatMatches = - jpaTm().loadAllOf(Spec11ThreatMatch.class); - assertThat(threatMatches) - .comparingElementsUsing(immutableObjectCorrespondence("id", "domainRepoId")) - .containsExactly( - expectedThreatMatch("TheRegistrar", "a.com"), - expectedThreatMatch("NewRegistrar", "b.com"), - expectedThreatMatch("NewRegistrar", "c.com")); - }); - } - - private Spec11ThreatMatch expectedThreatMatch(String registrarId, String domainName) { - return new Spec11ThreatMatch.Builder() - .setDomainRepoId("ignored") - .setDomainName(domainName) - .setRegistrarId(registrarId) - .setCheckDate(CURRENT_DATE) - .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE)) - .build(); - } -}