From f9ef170a017404f02c6e1297ac7aa94322e5be2a Mon Sep 17 00:00:00 2001 From: gbrodman Date: Fri, 7 Jan 2022 13:21:22 -0500 Subject: [PATCH] Allow usage of a read-only Postgres replica (#1470) * Allow usage of a read-only Postgres replica This adds the Dagger provider code for both the regular and the BEAM environments, which are similar but not quite the same. In addition, this demonstrates usage of the replica DB in the RdePipeline. I tested this on alpha with a modified version of the RdePipeline that attempts to write some dummy values to the database and it failed with the expected message that one cannot write to a replica. --- .../common/RegistryPipelineComponent.java | 8 ++ .../RegistryPipelineWorkerInitializer.java | 4 + .../registry/config/RegistryConfig.java | 13 +++- .../config/RegistryConfigSettings.java | 1 + .../registry/config/files/default-config.yaml | 1 + .../persistence/PersistenceComponent.java | 4 + .../persistence/PersistenceModule.java | 76 ++++++++++++++----- .../google/registry/rde/RdeStagingAction.java | 4 + .../registry/beam/rde_pipeline_metadata.json | 6 ++ 9 files changed, 95 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java b/core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java index 12a425cd3..cec1beef5 100644 --- a/core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java +++ b/core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java @@ -23,6 +23,7 @@ import google.registry.config.RegistryConfig.ConfigModule; import google.registry.persistence.PersistenceModule; import google.registry.persistence.PersistenceModule.BeamBulkQueryJpaTm; import google.registry.persistence.PersistenceModule.BeamJpaTm; +import google.registry.persistence.PersistenceModule.BeamReadOnlyReplicaJpaTm; import google.registry.persistence.PersistenceModule.TransactionIsolationLevel; import google.registry.persistence.transaction.JpaTransactionManager; import google.registry.privileges.secretmanager.SecretManagerModule; @@ -59,6 +60,13 @@ public interface RegistryPipelineComponent { @BeamBulkQueryJpaTm Lazy getBulkQueryJpaTransactionManager(); + /** + * A {@link JpaTransactionManager} that uses the Postgres read-only replica if configured (uses + * the standard DB otherwise). + */ + @BeamReadOnlyReplicaJpaTm + Lazy getReadOnlyReplicaJpaTransactionManager(); + @Component.Builder interface Builder { diff --git a/core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java b/core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java index a5e01ef95..f4d13e903 100644 --- a/core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java +++ b/core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java @@ -56,6 +56,10 @@ public class RegistryPipelineWorkerInitializer implements JvmInitializer { case BULK_QUERY: transactionManagerLazy = registryPipelineComponent.getBulkQueryJpaTransactionManager(); break; + case READ_ONLY_REPLICA: + transactionManagerLazy = + registryPipelineComponent.getReadOnlyReplicaJpaTransactionManager(); + break; case REGULAR: default: transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager(); diff --git a/core/src/main/java/google/registry/config/RegistryConfig.java b/core/src/main/java/google/registry/config/RegistryConfig.java index fb14077b2..5c6d602d9 100644 --- a/core/src/main/java/google/registry/config/RegistryConfig.java +++ b/core/src/main/java/google/registry/config/RegistryConfig.java @@ -392,19 +392,26 @@ public final class RegistryConfig { @Provides @Config("cloudSqlJdbcUrl") - public static String providesCloudSqlJdbcUrl(RegistryConfigSettings config) { + public static String provideCloudSqlJdbcUrl(RegistryConfigSettings config) { return config.cloudSql.jdbcUrl; } @Provides @Config("cloudSqlInstanceConnectionName") - public static String providesCloudSqlInstanceConnectionName(RegistryConfigSettings config) { + public static String provideCloudSqlInstanceConnectionName(RegistryConfigSettings config) { return config.cloudSql.instanceConnectionName; } + @Provides + @Config("cloudSqlReplicaInstanceConnectionName") + public static Optional provideCloudSqlReplicaInstanceConnectionName( + RegistryConfigSettings config) { + return Optional.ofNullable(config.cloudSql.replicaInstanceConnectionName); + } + @Provides @Config("cloudSqlDbInstanceName") - public static String providesCloudSqlDbInstance(RegistryConfigSettings config) { + public static String provideCloudSqlDbInstance(RegistryConfigSettings config) { // Format of instanceConnectionName: project-id:region:instance-name int lastColonIndex = config.cloudSql.instanceConnectionName.lastIndexOf(':'); return config.cloudSql.instanceConnectionName.substring(lastColonIndex + 1); diff --git a/core/src/main/java/google/registry/config/RegistryConfigSettings.java b/core/src/main/java/google/registry/config/RegistryConfigSettings.java index 84b4a674c..f8ad85d29 100644 --- a/core/src/main/java/google/registry/config/RegistryConfigSettings.java +++ b/core/src/main/java/google/registry/config/RegistryConfigSettings.java @@ -128,6 +128,7 @@ public class RegistryConfigSettings { // TODO(05012021): remove username field after it is removed from all yaml files. public String username; public String instanceConnectionName; + public String replicaInstanceConnectionName; } /** Configuration for Apache Beam (Cloud Dataflow). */ diff --git a/core/src/main/java/google/registry/config/files/default-config.yaml b/core/src/main/java/google/registry/config/files/default-config.yaml index 2bca9c5f1..5372e6cad 100644 --- a/core/src/main/java/google/registry/config/files/default-config.yaml +++ b/core/src/main/java/google/registry/config/files/default-config.yaml @@ -231,6 +231,7 @@ cloudSql: jdbcUrl: jdbc:postgresql://localhost # This name is used by Cloud SQL when connecting to the database. instanceConnectionName: project-id:region:instance-id + replicaInstanceConnectionName: null cloudDns: # Set both properties to null in Production. diff --git a/core/src/main/java/google/registry/persistence/PersistenceComponent.java b/core/src/main/java/google/registry/persistence/PersistenceComponent.java index cbe6cea9a..b918815c2 100644 --- a/core/src/main/java/google/registry/persistence/PersistenceComponent.java +++ b/core/src/main/java/google/registry/persistence/PersistenceComponent.java @@ -19,6 +19,7 @@ import google.registry.config.CredentialModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.keyring.kms.KmsModule; import google.registry.persistence.PersistenceModule.AppEngineJpaTm; +import google.registry.persistence.PersistenceModule.ReadOnlyReplicaJpaTm; import google.registry.persistence.transaction.JpaTransactionManager; import google.registry.privileges.secretmanager.SecretManagerModule; import google.registry.util.UtilsModule; @@ -40,4 +41,7 @@ public interface PersistenceComponent { @AppEngineJpaTm JpaTransactionManager appEngineJpaTransactionManager(); + + @ReadOnlyReplicaJpaTm + JpaTransactionManager readOnlyReplicaJpaTransactionManager(); } diff --git a/core/src/main/java/google/registry/persistence/PersistenceModule.java b/core/src/main/java/google/registry/persistence/PersistenceModule.java index 6c6c4132e..0d1bc658e 100644 --- a/core/src/main/java/google/registry/persistence/PersistenceModule.java +++ b/core/src/main/java/google/registry/persistence/PersistenceModule.java @@ -122,8 +122,11 @@ public abstract class PersistenceModule { @Config("cloudSqlJdbcUrl") String jdbcUrl, @Config("cloudSqlInstanceConnectionName") String instanceConnectionName, @DefaultHibernateConfigs ImmutableMap defaultConfigs) { - return createPartialSqlConfigs( - jdbcUrl, instanceConnectionName, defaultConfigs, Optional.empty()); + HashMap overrides = Maps.newHashMap(defaultConfigs); + overrides.put(Environment.URL, jdbcUrl); + overrides.put(HIKARI_DS_SOCKET_FACTORY, "com.google.cloud.sql.postgres.SocketFactory"); + overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, instanceConnectionName); + return ImmutableMap.copyOf(overrides); } /** @@ -184,22 +187,6 @@ public abstract class PersistenceModule { return ImmutableMap.copyOf(overrides); } - @VisibleForTesting - static ImmutableMap createPartialSqlConfigs( - String jdbcUrl, - String instanceConnectionName, - ImmutableMap defaultConfigs, - Optional> isolationOverride) { - HashMap overrides = Maps.newHashMap(defaultConfigs); - overrides.put(Environment.URL, jdbcUrl); - overrides.put(HIKARI_DS_SOCKET_FACTORY, "com.google.cloud.sql.postgres.SocketFactory"); - overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, instanceConnectionName); - isolationOverride - .map(Provider::get) - .ifPresent(override -> overrides.put(Environment.ISOLATION, override.name())); - return ImmutableMap.copyOf(overrides); - } - /** * Provides a {@link Supplier} of single-use JDBC {@link Connection connections} that can manage * the database DDL schema. @@ -280,6 +267,36 @@ public abstract class PersistenceModule { return new JpaTransactionManagerImpl(create(overrides), clock); } + @Provides + @Singleton + @ReadOnlyReplicaJpaTm + static JpaTransactionManager provideReadOnlyReplicaJpaTm( + SqlCredentialStore credentialStore, + @PartialCloudSqlConfigs ImmutableMap cloudSqlConfigs, + @Config("cloudSqlReplicaInstanceConnectionName") + Optional replicaInstanceConnectionName, + Clock clock) { + HashMap overrides = Maps.newHashMap(cloudSqlConfigs); + setSqlCredential(credentialStore, new RobotUser(RobotId.NOMULUS), overrides); + replicaInstanceConnectionName.ifPresent( + name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name)); + return new JpaTransactionManagerImpl(create(overrides), clock); + } + + @Provides + @Singleton + @BeamReadOnlyReplicaJpaTm + static JpaTransactionManager provideBeamReadOnlyReplicaJpaTm( + @BeamPipelineCloudSqlConfigs ImmutableMap beamCloudSqlConfigs, + @Config("cloudSqlReplicaInstanceConnectionName") + Optional replicaInstanceConnectionName, + Clock clock) { + HashMap overrides = Maps.newHashMap(beamCloudSqlConfigs); + replicaInstanceConnectionName.ifPresent( + name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name)); + return new JpaTransactionManagerImpl(create(overrides), clock); + } + /** Constructs the {@link EntityManagerFactory} instance. */ @VisibleForTesting static EntityManagerFactory create( @@ -357,7 +374,12 @@ public abstract class PersistenceModule { * The {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities. Please * see {@link google.registry.model.bulkquery.BulkQueryEntities} for more information. */ - BULK_QUERY + BULK_QUERY, + /** + * The {@link JpaTransactionManager} that uses the read-only Postgres replica if configured, or + * the standard DB if not. + */ + READ_ONLY_REPLICA } /** Dagger qualifier for JDBC {@link Connection} with schema management privilege. */ @@ -383,6 +405,22 @@ public abstract class PersistenceModule { @Documented public @interface BeamBulkQueryJpaTm {} + /** + * Dagger qualifier for {@link JpaTransactionManager} used inside BEAM pipelines that uses the + * read-only Postgres replica if one is configured (otherwise it uses the standard DB). + */ + @Qualifier + @Documented + public @interface BeamReadOnlyReplicaJpaTm {} + + /** + * Dagger qualifier for {@link JpaTransactionManager} that uses the read-only Postgres replica if + * one is configured (otherwise it uses the standard DB). + */ + @Qualifier + @Documented + public @interface ReadOnlyReplicaJpaTm {} + /** Dagger qualifier for {@link JpaTransactionManager} used for Nomulus tool. */ @Qualifier @Documented diff --git a/core/src/main/java/google/registry/rde/RdeStagingAction.java b/core/src/main/java/google/registry/rde/RdeStagingAction.java index bae879cfb..f4047e395 100644 --- a/core/src/main/java/google/registry/rde/RdeStagingAction.java +++ b/core/src/main/java/google/registry/rde/RdeStagingAction.java @@ -56,6 +56,7 @@ import google.registry.model.host.HostResource; import google.registry.model.index.EppResourceIndex; import google.registry.model.rde.RdeMode; import google.registry.model.registrar.Registrar; +import google.registry.persistence.PersistenceModule.JpaTransactionManagerType; import google.registry.request.Action; import google.registry.request.HttpException.BadRequestException; import google.registry.request.Parameter; @@ -340,6 +341,9 @@ public final class RdeStagingAction implements Runnable { .encode(stagingKeyBytes)) .put("registryEnvironment", RegistryEnvironment.get().name()) .put("workerMachineType", machineType) + .put( + "jpaTransactionManagerType", + JpaTransactionManagerType.READ_ONLY_REPLICA.toString()) // TODO (jianglai): Investigate turning off public IPs (for which // there is a quota) in order to increase the total number of // workers allowed (also under quota). diff --git a/core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json b/core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json index 9f0fc7de9..410ff5e7d 100644 --- a/core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json +++ b/core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json @@ -11,6 +11,12 @@ "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$" ] }, + { + "name": "jpaTransactionManagerType", + "label": "The type of JPA transaction manager to use", + "helpText": "The standard SQL instance or a read-only replica may be used", + "regexes": ["^REGULAR|READ_ONLY_REPLICA$"] + }, { "name": "pendings", "label": "The pendings deposits to generate.",