zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-07-21 18:26:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

Remove usage of the AppEngine remote API (#1858)

Browse source 
This is only used for contacting Datastore. With the removal of:
1. All standard usages of Datastore
2. Usage of Datastore for allocation of object IDs
3. Usage of Datastore for GAE user IDs

we can remove the remote API without affecting functionality.

This also allows us to just use SQL for every command since it's lazily
supplied. This simplifies the SQL setup and means that we remove a
possible situation where we forget the SQL setup.
This commit is contained in:
gbrodman 2022-12-05 13:23:18 -05:00 • committed by GitHub
parent 7b0b104616
commit 1e2b17fff7
55 changed files with 99 additions and 440 deletions
Download patch file Download diff file Expand all files Collapse all files

2
core/src/main/java/google/registry/tools/AckPollMessagesCommand.java
View file

@ -53,7 +53,7 @@ import javax.inject.Inject;
* event time is in the past), same as through EPP.
*/
@Parameters(separators = " =", commandDescription = "Acknowledge one-time poll messages.")
final class AckPollMessagesCommand implements CommandWithRemoteApi {
final class AckPollMessagesCommand implements Command {
@Parameter(
names = {"-c", "--client"},

23
core/src/main/java/google/registry/tools/CommandWithRemoteApi.java
View file

@ -1,23 +0,0 @@
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
/**
* Marker interface for commands that use the remote api.
*
* <p>Just implementing this is sufficient to use the remote api; {@link RegistryTool} will install
* it as needed.
*/
public interface CommandWithRemoteApi extends Command {}

2
core/src/main/java/google/registry/tools/CountDomainsCommand.java
View file

@ -28,7 +28,7 @@ import org.joda.time.DateTime;
/** Command to show the count of active domains on a given TLD. */
@Parameters(separators = " =", commandDescription = "Show count of domains on TLD")
final class CountDomainsCommand implements CommandWithRemoteApi {
final class CountDomainsCommand implements Command {
@Parameter(
names = {"-t", "--tld", "--tlds"},

3
core/src/main/java/google/registry/tools/CreateDomainCommand.java
View file

@ -33,8 +33,7 @@ import org.joda.time.DateTime;
/** A command to create a new domain via EPP. */
@Parameters(separators = " =", commandDescription = "Create a new domain via EPP.")
final class CreateDomainCommand extends CreateOrUpdateDomainCommand
implements CommandWithRemoteApi {
final class CreateDomainCommand extends CreateOrUpdateDomainCommand {
@Parameter(
names = "--period",

3
core/src/main/java/google/registry/tools/CreateOrUpdatePremiumListCommand.java
View file

@ -27,8 +27,7 @@ import org.joda.money.CurrencyUnit;
* Base class for specification of command line parameters common to creating and updating premium
* lists.
*/
abstract class CreateOrUpdatePremiumListCommand extends ConfirmingCommand
implements CommandWithRemoteApi {
abstract class CreateOrUpdatePremiumListCommand extends ConfirmingCommand {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
protected List<String> inputData;

3
core/src/main/java/google/registry/tools/CreateOrUpdateReservedListCommand.java
View file

@ -26,8 +26,7 @@ import javax.annotation.Nullable;
* Base class for specification of command line parameters common to creating and updating reserved
* lists.
*/
public abstract class CreateOrUpdateReservedListCommand extends ConfirmingCommand
implements CommandWithRemoteApi {
public abstract class CreateOrUpdateReservedListCommand extends ConfirmingCommand {
static final FluentLogger logger = FluentLogger.forEnclosingClass();

2
core/src/main/java/google/registry/tools/CreateRegistrarCommand.java
View file

@ -40,7 +40,7 @@ import javax.annotation.Nullable;
/** Command to create a Registrar. */
@Parameters(separators = " =", commandDescription = "Create new registrar account(s)")
final class CreateRegistrarCommand extends CreateOrUpdateRegistrarCommand
implements CommandWithConnection, CommandWithRemoteApi {
implements CommandWithConnection {
private static final ImmutableSet<RegistryToolEnvironment> ENVIRONMENTS_ALLOWING_GROUP_CREATION =
ImmutableSet.of(PRODUCTION, SANDBOX, UNITTEST);

2
core/src/main/java/google/registry/tools/CreateRegistrarGroupsCommand.java
View file

@ -30,7 +30,7 @@ import java.util.List;
/** Command to create groups in Google Groups for all contact types for a registrar. */
@Parameters(separators = " =", commandDescription = "Create groups for a registrar.")
public class CreateRegistrarGroupsCommand extends ConfirmingCommand
implements CommandWithConnection, CommandWithRemoteApi {
implements CommandWithConnection {
@Parameter(
description = "Client identifier(s) of the registrar(s) to create groups for",

2
core/src/main/java/google/registry/tools/DeletePremiumListCommand.java
View file

@ -29,7 +29,7 @@ import javax.annotation.Nullable;
* in use on a tld.
*/
@Parameters(separators = " =", commandDescription = "Delete a PremiumList.")
final class DeletePremiumListCommand extends ConfirmingCommand implements CommandWithRemoteApi {
final class DeletePremiumListCommand extends ConfirmingCommand {
@Nullable PremiumList premiumList;

2
core/src/main/java/google/registry/tools/DeleteReservedListCommand.java
View file

@ -28,7 +28,7 @@ import google.registry.model.tld.label.ReservedListDao;
* reserved list is currently in use on a tld.
*/
@Parameters(separators = " =", commandDescription = "Deletes a ReservedList from the database.")
final class DeleteReservedListCommand extends ConfirmingCommand implements CommandWithRemoteApi {
final class DeleteReservedListCommand extends ConfirmingCommand {
@Parameter(
names = {"-n", "--name"},

2
core/src/main/java/google/registry/tools/DeleteTldCommand.java
View file

@ -31,7 +31,7 @@ import google.registry.persistence.transaction.QueryComposer.Comparator;
* <p>This command will fail if any domains are currently registered on the TLD.
*/
@Parameters(separators = " =", commandDescription = "Delete a TLD from Datastore.")
final class DeleteTldCommand extends ConfirmingCommand implements CommandWithRemoteApi {
final class DeleteTldCommand extends ConfirmingCommand {
private Registry registry;

2
core/src/main/java/google/registry/tools/EncryptEscrowDepositCommand.java
View file

@ -26,7 +26,7 @@ import javax.inject.Inject;
/** Command to encrypt an escrow deposit. */
@Parameters(separators = " =", commandDescription = "Encrypt an escrow deposit")
class EncryptEscrowDepositCommand implements CommandWithRemoteApi {
class EncryptEscrowDepositCommand implements Command {
@Parameter(
names = {"-t", "--tld"},

3
core/src/main/java/google/registry/tools/EppToolCommand.java
View file

@ -47,8 +47,7 @@ import java.util.Map;
import java.util.Objects;
/** A command to execute an epp command. */
abstract class EppToolCommand extends ConfirmingCommand
implements CommandWithConnection, CommandWithRemoteApi {
abstract class EppToolCommand extends ConfirmingCommand implements CommandWithConnection {
@Parameter(
names = {"-u", "--superuser"},

2
core/src/main/java/google/registry/tools/GenerateAllocationTokensCommand.java
View file

@ -69,7 +69,7 @@ import org.joda.time.DateTime;
"Generates and persists the given number of AllocationTokens, "
+ "printing each token to stdout.")
@NonFinalForTesting
class GenerateAllocationTokensCommand implements CommandWithRemoteApi {
class GenerateAllocationTokensCommand implements Command {
@Parameter(
names = {"--tokens"},

2
core/src/main/java/google/registry/tools/GenerateDnsReportCommand.java
View file

@ -42,7 +42,7 @@ import org.json.simple.JSONValue;
/** Command to generate a report of all DNS data. */
@Parameters(separators = " =", commandDescription = "Generate report of all DNS data in a TLD.")
final class GenerateDnsReportCommand implements CommandWithRemoteApi {
final class GenerateDnsReportCommand implements Command {
@Parameter(
names = {"-t", "--tld"},

2
core/src/main/java/google/registry/tools/GenerateEscrowDepositCommand.java
View file

@ -43,7 +43,7 @@ import org.joda.time.DateTime;
* be stored in the specified manual subdirectory of the GCS RDE bucket.
*/
@Parameters(separators = " =", commandDescription = "Generate an XML escrow deposit.")
final class GenerateEscrowDepositCommand implements CommandWithRemoteApi {
final class GenerateEscrowDepositCommand implements Command {
@Parameter(
names = {"-t", "--tld"},

2
core/src/main/java/google/registry/tools/GenerateLordnCommand.java
View file

@ -33,7 +33,7 @@ import org.joda.time.DateTime;
/** Command to generate a LORDN CSV file for an entire TLD. */
@Parameters(separators = " =", commandDescription = "Generate LORDN CSV file")
final class GenerateLordnCommand implements CommandWithRemoteApi {
final class GenerateLordnCommand implements Command {
@Parameter(
names = {"-t", "--tld"},

2
core/src/main/java/google/registry/tools/GenerateZoneFilesCommand.java
View file

@ -30,7 +30,7 @@ import org.joda.time.DateTime;
/** Command to generate zone files. */
@Parameters(separators = " =", commandDescription = "Generate zone files")
final class GenerateZoneFilesCommand implements CommandWithConnection, CommandWithRemoteApi {
final class GenerateZoneFilesCommand implements CommandWithConnection {
@Parameter(
description = "One or more TLDs to generate zone files for",

2
core/src/main/java/google/registry/tools/GetAllocationTokenCommand.java
View file

@ -32,7 +32,7 @@ import java.util.Optional;
/** Command to show allocation tokens. */
@Parameters(separators = " =", commandDescription = "Show allocation token(s)")
final class GetAllocationTokenCommand implements CommandWithRemoteApi {
final class GetAllocationTokenCommand implements Command {
@Parameter(
description = "Allocation token(s)",

2
core/src/main/java/google/registry/tools/GetClaimsListCommand.java
View file

@ -34,7 +34,7 @@ import java.nio.file.Paths;
* currently storing in SQL.
*/
@Parameters(separators = " =", commandDescription = "Download the current claims list")
final class GetClaimsListCommand implements CommandWithRemoteApi {
final class GetClaimsListCommand implements Command {
@Parameter(
names = {"-o", "--output"},

2
core/src/main/java/google/registry/tools/GetDatabaseMigrationStateCommand.java
View file

@ -23,7 +23,7 @@ import google.registry.model.common.TimedTransitionProperty;
/** A command to check the current Registry 3.0 migration state of the database. */
@DeleteAfterMigration
@Parameters(separators = " =", commandDescription = "Check current Registry 3.0 migration state")
public class GetDatabaseMigrationStateCommand implements CommandWithRemoteApi {
public class GetDatabaseMigrationStateCommand implements Command {
@Override
public void run() throws Exception {

2
core/src/main/java/google/registry/tools/GetEppResourceCommand.java
View file

@ -26,7 +26,7 @@ import org.joda.time.DateTime;
/** Abstract command to print one or more resources to stdout. */
@Parameters(separators = " =")
abstract class GetEppResourceCommand implements CommandWithRemoteApi {
abstract class GetEppResourceCommand implements Command {
@Parameter(
names = "--read_timestamp",

2
core/src/main/java/google/registry/tools/GetHistoryEntriesCommand.java
View file

@ -34,7 +34,7 @@ import org.joda.time.DateTime;
@Parameters(
separators = " =",
commandDescription = "Show history entries that occurred in a given time range")
final class GetHistoryEntriesCommand implements CommandWithRemoteApi {
final class GetHistoryEntriesCommand implements Command {
@Parameter(
names = {"-a", "--after"},

7
core/src/main/java/google/registry/tools/GetKeyringSecretCommand.java
View file

@ -30,10 +30,9 @@ import org.bouncycastle.openpgp.PGPKeyPair;
/** Retrieves ASCII-armored secrets from the active {@link Keyring} implementation. */
@Parameters(
separators = " =",
commandDescription = "Retrieves the value of a secret from the keyring."
)
final class GetKeyringSecretCommand implements CommandWithRemoteApi {
separators = " =",
commandDescription = "Retrieves the value of a secret from the keyring.")
final class GetKeyringSecretCommand implements Command {
@Inject Keyring keyring;

2
core/src/main/java/google/registry/tools/GetPremiumListCommand.java
View file

@ -26,7 +26,7 @@ import java.util.stream.Collectors;
/** Retrieves and prints one or more premium lists. */
@Parameters(separators = " =", commandDescription = "Show one or more premium lists")
public class GetPremiumListCommand implements CommandWithRemoteApi {
public class GetPremiumListCommand implements Command {
@Parameter(description = "Name(s) of the premium list(s) to retrieve", required = true)
private List<String> mainParameters;

2
core/src/main/java/google/registry/tools/GetRegistrarCommand.java
View file

@ -23,7 +23,7 @@ import java.util.List;
/** Command to show a registrar record. */
@Parameters(separators = " =", commandDescription = "Show registrar record(s)")
final class GetRegistrarCommand implements CommandWithRemoteApi {
final class GetRegistrarCommand implements Command {
@Parameter(
description = "Client identifier of the registrar account(s)",

2
core/src/main/java/google/registry/tools/GetReservedListCommand.java
View file

@ -24,7 +24,7 @@ import java.util.stream.Collectors;
/** Retrieves and prints one or more reserved lists. */
@Parameters(separators = " =", commandDescription = "Show one or more reserved lists")
public class GetReservedListCommand implements CommandWithRemoteApi {
public class GetReservedListCommand implements Command {
@Parameter(
names = {"-n", "--name"},

2
core/src/main/java/google/registry/tools/GetTldCommand.java
View file

@ -23,7 +23,7 @@ import java.util.List;
/** Command to show a TLD record. */
@Parameters(separators = " =", commandDescription = "Show TLD record(s)")
final class GetTldCommand implements CommandWithRemoteApi {
final class GetTldCommand implements Command {
@Parameter(
description = "TLD(s) to show",

2
core/src/main/java/google/registry/tools/GhostrydeCommand.java
View file

@ -37,7 +37,7 @@ import org.bouncycastle.openpgp.PGPPublicKey;
/** Command to encrypt/decrypt {@code .ghostryde} files. */
@Parameters(separators = " =", commandDescription = "Encrypt/decrypt a ghostryde file.")
final class GhostrydeCommand implements CommandWithRemoteApi {
final class GhostrydeCommand implements Command {
@Parameter(
names = {"-e", "--encrypt"},

2
core/src/main/java/google/registry/tools/ListCursorsCommand.java
View file

@ -32,7 +32,7 @@ import java.util.Optional;
/** Lists {@link Cursor} timestamps used by locking rolling cursor tasks, like in RDE. */
@Parameters(separators = " =", commandDescription = "Lists cursor timestamps used by LRC tasks")
final class ListCursorsCommand implements CommandWithRemoteApi {
final class ListCursorsCommand implements Command {
@Parameter(names = "--type", description = "Which cursor to list.", required = true)
private CursorType cursorType;

2
core/src/main/java/google/registry/tools/ListObjectsCommand.java
View file

@ -33,7 +33,7 @@ import org.json.simple.JSONValue;
*
* <p>The formatting is done on the server side; this class just dumps the results to the screen.
*/
abstract class ListObjectsCommand implements CommandWithConnection, CommandWithRemoteApi {
abstract class ListObjectsCommand implements CommandWithConnection {
@Nullable
@Parameter(

3
core/src/main/java/google/registry/tools/LoadTestCommand.java
View file

@ -24,8 +24,7 @@ import google.registry.model.tld.Registries;
/** Command to initiate a load-test. */
@Parameters(separators = " =", commandDescription = "Run a load test.")
class LoadTestCommand extends ConfirmingCommand
implements CommandWithConnection, CommandWithRemoteApi {
class LoadTestCommand extends ConfirmingCommand implements CommandWithConnection {
// This is a mostly arbitrary value, roughly two and a half hours. It served as a generous
// timespan for initial backup/restore testing, but has no other special significance.

3
core/src/main/java/google/registry/tools/LockOrUnlockDomainCommand.java
View file

@ -35,8 +35,7 @@ import java.util.List;
import javax.inject.Inject;
/** Shared base class for commands to registry lock or unlock a domain via EPP. */
public abstract class LockOrUnlockDomainCommand extends ConfirmingCommand
implements CommandWithRemoteApi {
public abstract class LockOrUnlockDomainCommand extends ConfirmingCommand {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();

2
core/src/main/java/google/registry/tools/MutatingCommand.java
View file

@ -41,7 +41,7 @@ import java.util.Set;
import javax.annotation.Nullable;
/** A {@link ConfirmingCommand} that changes objects in Datastore. */
public abstract class MutatingCommand extends ConfirmingCommand implements CommandWithRemoteApi {
public abstract class MutatingCommand extends ConfirmingCommand {
/**
* A mutation of a specific entity, represented by an old and a new version of the entity. Storing

2
core/src/main/java/google/registry/tools/PendingEscrowCommand.java
View file

@ -25,7 +25,7 @@ import javax.inject.Inject;
/** Command to show what escrow deposits are pending generation on the server. */
@Parameters(separators = " =", commandDescription = "List pending RDE/BRDA deposits.")
final class PendingEscrowCommand implements CommandWithRemoteApi {
final class PendingEscrowCommand implements Command {
private static final Ordering<PendingDeposit> SORTER =
new Ordering<PendingDeposit>() {

68
core/src/main/java/google/registry/tools/RegistryCli.java
View file

@ -15,25 +15,21 @@
package google.registry.tools;
import static com.google.common.base.Preconditions.checkState;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.tools.Injector.injectReflectively;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import com.google.appengine.tools.remoteapi.RemoteApiInstaller;
import com.google.appengine.tools.remoteapi.RemoteApiOptions;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import google.registry.config.RegistryConfig;
import google.registry.persistence.transaction.JpaTransactionManager;
import google.registry.persistence.transaction.TransactionManagerFactory;
import google.registry.tools.AuthModule.LoginRequiredException;
import google.registry.tools.params.ParameterFactory;
import java.io.ByteArrayInputStream;
import java.net.URL;
import java.security.Security;
import java.util.Map;
import java.util.Optional;
@ -42,7 +38,7 @@ import org.postgresql.util.PSQLException;
/** Container class to create and run remote commands against a Datastore instance. */
@Parameters(separators = " =", commandDescription = "Command-line interface to the registry")
final class RegistryCli implements AutoCloseable, CommandRunner {
final class RegistryCli implements CommandRunner {
// The environment parameter is parsed twice: once here, and once with {@link
// RegistryToolEnvironment#parseFromArgs} in the {@link RegistryTool#main} function.
@ -78,9 +74,8 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
RegistryToolComponent component;
// These are created lazily on first use.
// This is created lazily on first use.
private ServiceConnection connection;
private RemoteApiInstaller installer;
// The "shell" command should only exist on first use - so that we can't run "shell" inside
// "shell".
@ -206,24 +201,6 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
}
}
@Override
public void close() {
if (installer != null) {
try {
installer.uninstall();
installer = null;
} catch (IllegalArgumentException e) {
// There is no point throwing the error if the API is already uninstalled, which is most
// likely caused by something wrong when installing the API. That something (e. g. no
// credential found) must have already thrown an error message earlier (e. g. must run
// "nomulus login" first). This error message here is non-actionable.
if (!e.getMessage().equals("remote API is already uninstalled")) {
throw e;
}
}
}
}
private ServiceConnection getConnection() {
// Get the App Engine connection, advise the user if they are not currently logged in..
if (connection == null) {
@ -239,37 +216,14 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
((CommandWithConnection) command).setConnection(getConnection());
}
// CommandWithRemoteApis need to have the remote api installed to work.
if (command instanceof CommandWithRemoteApi) {
if (installer == null) {
installer = new RemoteApiInstaller();
RemoteApiOptions options = new RemoteApiOptions();
options.server(
getConnection().getServer().getHost(), getPort(getConnection().getServer()));
if (RegistryConfig.areServersLocal()) {
// Use dev credentials for localhost.
options.useDevelopmentServerCredential();
} else {
RemoteApiOptionsUtil.useGoogleCredentialStream(
options, new ByteArrayInputStream(component.googleCredentialJson().getBytes(UTF_8)));
}
installer.install(options);
// Enable Cloud SQL for command that needs remote API as they will very likely use
// Cloud SQL after the database migration. Note that the DB password is stored in Datastore
// and it is already initialized above.
TransactionManagerFactory.setJpaTm(
() -> component.nomulusToolJpaTransactionManager().get());
TransactionManagerFactory.setReplicaJpaTm(
() -> component.nomulusToolReplicaJpaTransactionManager().get());
}
}
// Reset the JPA transaction manager after every command to avoid a situation where a test can
// interfere with other tests
JpaTransactionManager cachedJpaTm = jpaTm();
TransactionManagerFactory.setJpaTm(() -> component.nomulusToolJpaTransactionManager().get());
TransactionManagerFactory.setReplicaJpaTm(
() -> component.nomulusToolReplicaJpaTransactionManager().get());
command.run();
}
private int getPort(URL url) {
return url.getPort() == -1 ? url.getDefaultPort() : url.getPort();
TransactionManagerFactory.setJpaTm(() -> cachedJpaTm);
}
void setEnvironment(RegistryToolEnvironment environment) {

7
core/src/main/java/google/registry/tools/RegistryTool.java
View file

@ -17,7 +17,6 @@ package google.registry.tools;
import com.google.common.collect.ImmutableMap;
import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
import google.registry.tools.javascrap.CreateCancellationsForOneTimesCommand;
import google.registry.tools.javascrap.CreateSyntheticDomainHistoriesCommand;
/** Container class to create and run remote commands against a Datastore instance. */
public final class RegistryTool {
@ -48,7 +47,6 @@ public final class RegistryTool {
.put("create_registrar", CreateRegistrarCommand.class)
.put("create_registrar_groups", CreateRegistrarGroupsCommand.class)
.put("create_reserved_list", CreateReservedListCommand.class)
.put("create_synthetic_domain_histories", CreateSyntheticDomainHistoriesCommand.class)
.put("create_tld", CreateTldCommand.class)
.put("curl", CurlCommand.class)
.put("delete_allocation_tokens", DeleteAllocationTokensCommand.class)
@ -123,8 +121,7 @@ public final class RegistryTool {
public static void main(String[] args) throws Exception {
RegistryToolEnvironment.parseFromArgs(args).setup();
try (RegistryCli cli = new RegistryCli("nomulus", COMMAND_MAP)) {
cli.run(args);
}
RegistryCli cli = new RegistryCli("nomulus", COMMAND_MAP);
cli.run(args);
}
}

3
core/src/main/java/google/registry/tools/RegistryToolComponent.java
View file

@ -42,7 +42,6 @@ import google.registry.request.Modules.UserServiceModule;
import google.registry.tools.AuthModule.LocalCredentialModule;
import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
import google.registry.tools.javascrap.CreateCancellationsForOneTimesCommand;
import google.registry.tools.javascrap.CreateSyntheticDomainHistoriesCommand;
import google.registry.util.UtilsModule;
import google.registry.whois.NonCachingWhoisModule;
import javax.annotation.Nullable;
@ -106,8 +105,6 @@ interface RegistryToolComponent {
void inject(CreateRegistrarCommand command);
void inject(CreateSyntheticDomainHistoriesCommand command);
void inject(CreateTldCommand command);
void inject(EncryptEscrowDepositCommand command);

44
core/src/main/java/google/registry/tools/RemoteApiOptionsUtil.java
View file

@ -1,44 +0,0 @@
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import static com.google.common.base.Preconditions.checkState;
import com.google.appengine.tools.remoteapi.RemoteApiOptions;
import java.io.InputStream;
import java.lang.reflect.Method;
/**
* Provides a method to access {@link RemoteApiOptions#useGoogleCredentialStream(InputStream)},
* which is a package private method.
*
* <p>This is obviously a hack, but until that method is exposed, we have to do this to set up the
* {@link RemoteApiOptions} with a JSON representing a user credential.
*/
public class RemoteApiOptionsUtil {
public static RemoteApiOptions useGoogleCredentialStream(
RemoteApiOptions options, InputStream stream) throws Exception {
Method method =
options.getClass().getDeclaredMethod("useGoogleCredentialStream", InputStream.class);
checkState(
!method.isAccessible(),
"RemoteApiOptoins#useGoogleCredentialStream(InputStream) is accessible."
+ " Stop using RemoteApiOptionsUtil.");
method.setAccessible(true);
method.invoke(options, stream);
method.setAccessible(false);
return options;
}
}

2
core/src/main/java/google/registry/tools/SendEscrowReportToIcannCommand.java
View file

@ -26,7 +26,7 @@ import javax.inject.Inject;
/** Command to send ICANN notification that an escrow deposit was uploaded. */
@Parameters(separators = " =", commandDescription = "Send an ICANN report of an uploaded deposit.")
final class SendEscrowReportToIcannCommand implements CommandWithRemoteApi {
final class SendEscrowReportToIcannCommand implements Command {
@Parameter(
description = "One or more foo-report.xml files.",

3
core/src/main/java/google/registry/tools/SetDatabaseMigrationStateCommand.java
View file

@ -30,8 +30,7 @@ import org.joda.time.DateTime;
@Parameters(
separators = " =",
commandDescription = "Set the current database migration state schedule.")
public class SetDatabaseMigrationStateCommand extends ConfirmingCommand
implements CommandWithRemoteApi {
public class SetDatabaseMigrationStateCommand extends ConfirmingCommand {
private static final String WARNING_MESSAGE =
"Attempting to change the schedule with an effect that would take place within the next 10 "

2
core/src/main/java/google/registry/tools/SetNumInstancesCommand.java
View file

@ -39,7 +39,7 @@ import javax.inject.Inject;
commandDescription =
"Set the number of instances for a given service and version. "
+ "Note that this command only works for manual scaling service.")
final class SetNumInstancesCommand implements CommandWithRemoteApi {
final class SetNumInstancesCommand implements Command {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();

2
core/src/main/java/google/registry/tools/SetupOteCommand.java
View file

@ -35,7 +35,7 @@ import javax.inject.Named;
/** Composite command to set up OT&E TLDs and accounts. */
@Parameters(separators = " =", commandDescription = "Set up OT&E TLDs and registrars")
final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemoteApi {
final class SetupOteCommand extends ConfirmingCommand {
private static final int PASSWORD_LENGTH = 16;

2
core/src/main/java/google/registry/tools/UnrenewDomainCommand.java
View file

@ -58,7 +58,7 @@ import org.joda.time.DateTime;
*/
@Parameters(separators = " =", commandDescription = "Unrenew a domain.")
@NonFinalForTesting
class UnrenewDomainCommand extends ConfirmingCommand implements CommandWithRemoteApi {
class UnrenewDomainCommand extends ConfirmingCommand {
@Parameter(
names = {"-p", "--period"},

2
core/src/main/java/google/registry/tools/UpdateCursorsCommand.java
View file

@ -29,7 +29,7 @@ import org.joda.time.DateTime;
/** Modifies {@link Cursor} timestamps used by locking rolling cursor tasks, like in RDE. */
@Parameters(separators = " =", commandDescription = "Modifies cursor timestamps used by LRC tasks")
final class UpdateCursorsCommand extends ConfirmingCommand implements CommandWithRemoteApi {
final class UpdateCursorsCommand extends ConfirmingCommand implements Command {
@Parameter(description = "TLDs on which to operate. Omit for global cursors.")
private List<String> tlds;

2
core/src/main/java/google/registry/tools/UpdateKeyringSecretCommand.java
View file

@ -31,7 +31,7 @@ import javax.inject.Inject;
* Command to set and update ASCII-armored secret from the active {@code Keyring} implementation.
*/
@Parameters(separators = " =", commandDescription = "Update values of secret in the keyring.")
final class UpdateKeyringSecretCommand implements CommandWithRemoteApi {
final class UpdateKeyringSecretCommand implements Command {
@Inject SecretManagerKeyringUpdater secretManagerKeyringUpdater;

3
core/src/main/java/google/registry/tools/UpdateOrDeleteAllocationTokensCommand.java
View file

@ -26,8 +26,7 @@ import google.registry.persistence.VKey;
import java.util.List;
/** Shared base class for commands to update or delete allocation tokens. */
abstract class UpdateOrDeleteAllocationTokensCommand extends ConfirmingCommand
implements CommandWithRemoteApi {
abstract class UpdateOrDeleteAllocationTokensCommand extends ConfirmingCommand {
@Parameter(
names = {"-p", "--prefix"},

2
core/src/main/java/google/registry/tools/UploadClaimsListCommand.java
View file

@ -31,7 +31,7 @@ import java.util.List;
/** A command to upload a {@link ClaimsList}. */
@Parameters(separators = " =", commandDescription = "Manually upload a new claims list file")
final class UploadClaimsListCommand extends ConfirmingCommand implements CommandWithRemoteApi {
final class UploadClaimsListCommand extends ConfirmingCommand {
@Parameter(description = "Claims list filename")
private List<String> mainParameters = new ArrayList<>();

2
core/src/main/java/google/registry/tools/ValidateLoginCredentialsCommand.java
View file

@ -35,7 +35,7 @@ import javax.inject.Inject;
/** A command to test registrar login credentials. */
@Parameters(separators = " =", commandDescription = "Test registrar login credentials")
final class ValidateLoginCredentialsCommand implements CommandWithRemoteApi {
final class ValidateLoginCredentialsCommand implements Command {
@Parameter(
names = {"-c", "--client"},

2
core/src/main/java/google/registry/tools/VerifyOteCommand.java
View file

@ -45,7 +45,7 @@ import java.util.Objects;
@Parameters(
separators = " =",
commandDescription = "Verify passage of OT&E for specified (or all) registrars")
final class VerifyOteCommand implements CommandWithConnection, CommandWithRemoteApi {
final class VerifyOteCommand implements CommandWithConnection {
@Parameter(
description = "List of registrar names to check; must be the same names as the ones used "

2
core/src/main/java/google/registry/tools/WhoisQueryCommand.java
View file

@ -23,7 +23,7 @@ import javax.inject.Inject;
/** Command to execute a WHOIS query. */
@Parameters(separators = " =", commandDescription = "Manually perform a WHOIS query")
final class WhoisQueryCommand implements CommandWithRemoteApi {
final class WhoisQueryCommand implements Command {
@Parameter(
description = "WHOIS query string",

4
core/src/main/java/google/registry/tools/javascrap/CreateCancellationsForOneTimesCommand.java
View file

@ -24,7 +24,6 @@ import google.registry.model.billing.BillingEvent.Cancellation;
import google.registry.model.billing.BillingEvent.OneTime;
import google.registry.persistence.VKey;
import google.registry.persistence.transaction.QueryComposer.Comparator;
import google.registry.tools.CommandWithRemoteApi;
import google.registry.tools.ConfirmingCommand;
import google.registry.tools.params.LongParameter;
import java.util.List;
@ -37,8 +36,7 @@ import java.util.List;
* refunds after the fact.
*/
@Parameters(separators = " =", commandDescription = "Manually create Cancellations for OneTimes.")
public class CreateCancellationsForOneTimesCommand extends ConfirmingCommand
implements CommandWithRemoteApi {
public class CreateCancellationsForOneTimesCommand extends ConfirmingCommand {
@Parameter(
description = "Space-delimited billing event ID(s) to cancel",

207
core/src/main/java/google/registry/tools/javascrap/CreateSyntheticDomainHistoriesCommand.java
View file

@ -1,207 +0,0 @@
// Copyright 2022 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools.javascrap;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.beust.jcommander.Parameters;
import com.google.appengine.tools.remoteapi.RemoteApiInstaller;
import com.google.appengine.tools.remoteapi.RemoteApiOptions;
import com.google.common.collect.ImmutableSet;
import com.google.common.flogger.FluentLogger;
import google.registry.config.CredentialModule;
import google.registry.config.RegistryConfig;
import google.registry.config.RegistryConfig.Config;
import google.registry.model.domain.Domain;
import google.registry.model.reporting.HistoryEntry;
import google.registry.persistence.VKey;
import google.registry.tools.CommandWithConnection;
import google.registry.tools.CommandWithRemoteApi;
import google.registry.tools.ConfirmingCommand;
import google.registry.tools.RemoteApiOptionsUtil;
import google.registry.tools.ServiceConnection;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import javax.inject.Inject;
import org.joda.time.DateTime;
/**
* Command that creates an additional synthetic history object for domains.
*
* <p>This is created to fix the issue identified in b/248112997. After b/245940594, there were some
* domains where the most recent history object did not represent the state of the domain as it
* exists in the world. Because RDE loads only from DomainHistory objects, this means that RDE was
* producing wrong data. This command mitigates that issue by creating synthetic history events for
* every domain that was not deleted as of the start of the bad {@link
* google.registry.beam.resave.ResaveAllEppResourcesPipeline} -- then, we can guarantee that this
* new history object represents the state of the domain as far as we know.
*
* <p>A previous run of this command (in pipeline form) attempted to do this and succeeded in most
* cases. Unfortunately, that pipeline had an issue where it used self-allocated IDs for some of the
* dependent objects (e.g. {@link google.registry.model.domain.secdns.DomainDsDataHistory}). As a
* result, we want to run this again as a command using Datastore-allocated IDs to re-create
* synthetic history objects for any domain whose last history object is one of the
* potentially-incorrect synthetic objects.
*
* <p>We further restrict the domains to domains whose latest history object is before October 4.
* This is an arbitrary date that is suitably far after the previous incorrect run of this synthetic
* history pipeline, with the purpose of making future runs of this command idempotent (in case the
* command fails, we can just run it again and again).
*/
@Parameters(
separators = " =",
commandDescription = "Create synthetic domain history objects to fix RDE.")
public class CreateSyntheticDomainHistoriesCommand extends ConfirmingCommand
implements CommandWithRemoteApi, CommandWithConnection {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final String HISTORY_REASON =
"Create synthetic domain histories to fix RDE for b/248112997";
private static final DateTime BAD_PIPELINE_END_TIME = DateTime.parse("2022-09-10T12:00:00.000Z");
private static final DateTime NEW_SYNTHETIC_ROUND_START =
DateTime.parse("2022-10-04T00:00:00.000Z");
private static final ExecutorService executor = Executors.newFixedThreadPool(20);
private static final AtomicInteger numDomainsProcessed = new AtomicInteger();
private ServiceConnection connection;
@Inject
@Config("registryAdminClientId")
String registryAdminRegistrarId;
@Inject @CredentialModule.LocalCredentialJson String localCredentialJson;
private final ThreadLocal<RemoteApiInstaller> installerThreadLocal =
ThreadLocal.withInitial(this::createInstaller);
private ImmutableSet<String> domainRepoIds;
@Override
protected String prompt() {
jpaTm()
.transact(
() -> {
domainRepoIds =
jpaTm()
.query(
"SELECT dh.domainRepoId FROM DomainHistory dh JOIN Tld t ON t.tldStr ="
+ " dh.domainBase.tld WHERE t.tldType = 'REAL' AND dh.type ="
+ " 'SYNTHETIC' AND dh.modificationTime > :badPipelineEndTime AND"
+ " dh.modificationTime < :newSyntheticRoundStart AND"
+ " (dh.domainRepoId, dh.modificationTime) IN (SELECT domainRepoId,"
+ " MAX(modificationTime) FROM DomainHistory GROUP BY domainRepoId)",
String.class)
.setParameter("badPipelineEndTime", BAD_PIPELINE_END_TIME)
.setParameter("newSyntheticRoundStart", NEW_SYNTHETIC_ROUND_START)
.getResultStream()
.collect(toImmutableSet());
});
return String.format(
"Attempt to create synthetic history entries for %d domains?", domainRepoIds.size());
}
@Override
protected String execute() throws Exception {
List<Future<?>> futures = new ArrayList<>();
for (String domainRepoId : domainRepoIds) {
futures.add(
executor.submit(
() -> {
// Make sure the remote API is installed for ID generation
installerThreadLocal.get();
jpaTm()
.transact(
() -> {
Domain domain =
jpaTm().loadByKey(VKey.create(Domain.class, domainRepoId));
jpaTm()
.put(
HistoryEntry.createBuilderForResource(domain)
.setRegistrarId(registryAdminRegistrarId)
.setBySuperuser(true)
.setRequestedByRegistrar(false)
.setModificationTime(jpaTm().getTransactionTime())
.setReason(HISTORY_REASON)
.setType(HistoryEntry.Type.SYNTHETIC)
.build());
});
int numProcessed = numDomainsProcessed.incrementAndGet();
if (numProcessed % 1000 == 0) {
System.out.printf("Saved histories for %d domains%n", numProcessed);
}
return null;
}));
}
for (Future<?> future : futures) {
try {
future.get();
} catch (Exception e) {
logger.atSevere().withCause(e).log("Error");
}
}
executor.shutdown();
executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
return String.format("Saved entries for %d domains", numDomainsProcessed.get());
}
@Override
public void setConnection(ServiceConnection connection) {
this.connection = connection;
}
/**
* Installs the remote API so that the worker threads can use Datastore for ID generation.
*
* <p>Lifted from the RegistryCli class
*/
private RemoteApiInstaller createInstaller() {
RemoteApiInstaller installer = new RemoteApiInstaller();
RemoteApiOptions options = new RemoteApiOptions();
options.server(connection.getServer().getHost(), getPort(connection.getServer()));
if (RegistryConfig.areServersLocal()) {
// Use dev credentials for localhost.
options.useDevelopmentServerCredential();
} else {
try {
RemoteApiOptionsUtil.useGoogleCredentialStream(
options, new ByteArrayInputStream(localCredentialJson.getBytes(UTF_8)));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
try {
installer.install(options);
} catch (IOException e) {
throw new RuntimeException(e);
}
return installer;
}
private static int getPort(URL url) {
return url.getPort() == -1 ? url.getDefaultPort() : url.getPort();
}
}

5
core/src/nonprod/java/google/registry/tools/DevTool.java
View file

@ -33,8 +33,7 @@ public class DevTool {
public static void main(String[] args) throws Exception {
RegistryToolEnvironment.parseFromArgs(args).setup();
try (RegistryCli cli = new RegistryCli("devtool", COMMAND_MAP)) {
cli.run(args);
}
RegistryCli cli = new RegistryCli("devtool", COMMAND_MAP);
cli.run(args);
}
}

73
core/src/test/java/google/registry/tools/ShellCommandTest.java
View file

@ -145,40 +145,29 @@ class ShellCommandTest {
shellCommand.run();
}
static class MockCli implements CommandRunner {
public ArrayList<ImmutableList<String>> calls = new ArrayList<>();
@Override
public void run(String[] args) {
calls.add(ImmutableList.copyOf(args));
}
}
@Test
void testMultipleCommandInvocations() throws Exception {
try (RegistryCli cli =
new RegistryCli("unittest", ImmutableMap.of("test_command", TestCommand.class))) {
RegistryToolEnvironment.UNITTEST.setup(systemPropertyExtension);
cli.setEnvironment(RegistryToolEnvironment.UNITTEST);
cli.run(new String[] {"test_command", "-x", "xval", "arg1", "arg2"});
cli.run(new String[] {"test_command", "-x", "otherxval", "arg3"});
cli.run(new String[] {"test_command"});
assertThat(TestCommand.commandInvocations)
.containsExactly(
ImmutableList.of("xval", "arg1", "arg2"),
ImmutableList.of("otherxval", "arg3"),
ImmutableList.of("default value"));
}
RegistryCli cli =
new RegistryCli("unittest", ImmutableMap.of("test_command", TestCommand.class));
RegistryToolEnvironment.UNITTEST.setup(systemPropertyExtension);
cli.setEnvironment(RegistryToolEnvironment.UNITTEST);
cli.run(new String[] {"test_command", "-x", "xval", "arg1", "arg2"});
cli.run(new String[] {"test_command", "-x", "otherxval", "arg3"});
cli.run(new String[] {"test_command"});
assertThat(TestCommand.commandInvocations)
.containsExactly(
ImmutableList.of("xval", "arg1", "arg2"),
ImmutableList.of("otherxval", "arg3"),
ImmutableList.of("default value"));
}
@Test
void testNonExistentCommand() {
try (RegistryCli cli =
new RegistryCli("unittest", ImmutableMap.of("test_command", TestCommand.class))) {
RegistryCli cli =
new RegistryCli("unittest", ImmutableMap.of("test_command", TestCommand.class));
cli.setEnvironment(RegistryToolEnvironment.UNITTEST);
assertThrows(MissingCommandException.class, () -> cli.run(new String[] {"bad_command"}));
}
cli.setEnvironment(RegistryToolEnvironment.UNITTEST);
assertThrows(MissingCommandException.class, () -> cli.run(new String[] {"bad_command"}));
}
private void performJCommanderCompletorTest(
@ -341,12 +330,22 @@ class ShellCommandTest {
System.setIn(new ByteArrayInputStream("command1\n".getBytes(UTF_8)));
}
static class MockCli implements CommandRunner {
public ArrayList<ImmutableList<String>> calls = new ArrayList<>();
@Override
public void run(String[] args) {
calls.add(ImmutableList.copyOf(args));
}
}
@Parameters(commandDescription = "Test command")
static class TestCommand implements Command {
enum OrgType {
PRIVATE,
PUBLIC
}
// List for recording command invocations by run().
//
// This has to be static because it gets populated by multiple TestCommand instances, which are
// created in RegistryCli by using reflection to call the constructor.
static final List<List<String>> commandInvocations = new ArrayList<>();
@Parameter(
names = {"-x", "--xparam"},
@ -357,13 +356,6 @@ class ShellCommandTest {
names = {"--xorg"},
description = "test organization")
OrgType orgType = OrgType.PRIVATE;
// List for recording command invocations by run().
//
// This has to be static because it gets populated by multiple TestCommand instances, which are
// created in RegistryCli by using reflection to call the constructor.
static final List<List<String>> commandInvocations = new ArrayList<>();
@Parameter(description = "normal argument")
List<String> args;
@ -378,6 +370,11 @@ class ShellCommandTest {
}
commandInvocations.add(callRecord.build());
}
enum OrgType {
PRIVATE,
PUBLIC
}
}
@Parameters(commandDescription = "Another test command")