Allow usage of a read-only Postgres replica (#1470)

* Allow usage of a read-only Postgres replica

This adds the Dagger provider code for both the regular and the BEAM
environments, which are similar but not quite the same.

In addition, this demonstrates usage of the replica DB in the
RdePipeline. I tested this on alpha with a modified version of the
RdePipeline that attempts to write some dummy values to the database and
it failed with the expected message that one cannot write to a replica.
This commit is contained in:
gbrodman 2022-01-07 13:21:22 -05:00 committed by GitHub
parent 15a59eda09
commit f9ef170a01
9 changed files with 95 additions and 22 deletions

View file

@ -23,6 +23,7 @@ import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.persistence.PersistenceModule;
import google.registry.persistence.PersistenceModule.BeamBulkQueryJpaTm;
import google.registry.persistence.PersistenceModule.BeamJpaTm;
import google.registry.persistence.PersistenceModule.BeamReadOnlyReplicaJpaTm;
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
import google.registry.persistence.transaction.JpaTransactionManager;
import google.registry.privileges.secretmanager.SecretManagerModule;
@ -59,6 +60,13 @@ public interface RegistryPipelineComponent {
@BeamBulkQueryJpaTm
Lazy<JpaTransactionManager> getBulkQueryJpaTransactionManager();
/**
* A {@link JpaTransactionManager} that uses the Postgres read-only replica if configured (uses
* the standard DB otherwise).
*/
@BeamReadOnlyReplicaJpaTm
Lazy<JpaTransactionManager> getReadOnlyReplicaJpaTransactionManager();
@Component.Builder
interface Builder {

View file

@ -56,6 +56,10 @@ public class RegistryPipelineWorkerInitializer implements JvmInitializer {
case BULK_QUERY:
transactionManagerLazy = registryPipelineComponent.getBulkQueryJpaTransactionManager();
break;
case READ_ONLY_REPLICA:
transactionManagerLazy =
registryPipelineComponent.getReadOnlyReplicaJpaTransactionManager();
break;
case REGULAR:
default:
transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();

View file

@ -392,19 +392,26 @@ public final class RegistryConfig {
@Provides
@Config("cloudSqlJdbcUrl")
public static String providesCloudSqlJdbcUrl(RegistryConfigSettings config) {
public static String provideCloudSqlJdbcUrl(RegistryConfigSettings config) {
return config.cloudSql.jdbcUrl;
}
@Provides
@Config("cloudSqlInstanceConnectionName")
public static String providesCloudSqlInstanceConnectionName(RegistryConfigSettings config) {
public static String provideCloudSqlInstanceConnectionName(RegistryConfigSettings config) {
return config.cloudSql.instanceConnectionName;
}
@Provides
@Config("cloudSqlReplicaInstanceConnectionName")
public static Optional<String> provideCloudSqlReplicaInstanceConnectionName(
RegistryConfigSettings config) {
return Optional.ofNullable(config.cloudSql.replicaInstanceConnectionName);
}
@Provides
@Config("cloudSqlDbInstanceName")
public static String providesCloudSqlDbInstance(RegistryConfigSettings config) {
public static String provideCloudSqlDbInstance(RegistryConfigSettings config) {
// Format of instanceConnectionName: project-id:region:instance-name
int lastColonIndex = config.cloudSql.instanceConnectionName.lastIndexOf(':');
return config.cloudSql.instanceConnectionName.substring(lastColonIndex + 1);

View file

@ -128,6 +128,7 @@ public class RegistryConfigSettings {
// TODO(05012021): remove username field after it is removed from all yaml files.
public String username;
public String instanceConnectionName;
public String replicaInstanceConnectionName;
}
/** Configuration for Apache Beam (Cloud Dataflow). */

View file

@ -231,6 +231,7 @@ cloudSql:
jdbcUrl: jdbc:postgresql://localhost
# This name is used by Cloud SQL when connecting to the database.
instanceConnectionName: project-id:region:instance-id
replicaInstanceConnectionName: null
cloudDns:
# Set both properties to null in Production.

View file

@ -19,6 +19,7 @@ import google.registry.config.CredentialModule;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.keyring.kms.KmsModule;
import google.registry.persistence.PersistenceModule.AppEngineJpaTm;
import google.registry.persistence.PersistenceModule.ReadOnlyReplicaJpaTm;
import google.registry.persistence.transaction.JpaTransactionManager;
import google.registry.privileges.secretmanager.SecretManagerModule;
import google.registry.util.UtilsModule;
@ -40,4 +41,7 @@ public interface PersistenceComponent {
@AppEngineJpaTm
JpaTransactionManager appEngineJpaTransactionManager();
@ReadOnlyReplicaJpaTm
JpaTransactionManager readOnlyReplicaJpaTransactionManager();
}

View file

@ -122,8 +122,11 @@ public abstract class PersistenceModule {
@Config("cloudSqlJdbcUrl") String jdbcUrl,
@Config("cloudSqlInstanceConnectionName") String instanceConnectionName,
@DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs) {
return createPartialSqlConfigs(
jdbcUrl, instanceConnectionName, defaultConfigs, Optional.empty());
HashMap<String, String> overrides = Maps.newHashMap(defaultConfigs);
overrides.put(Environment.URL, jdbcUrl);
overrides.put(HIKARI_DS_SOCKET_FACTORY, "com.google.cloud.sql.postgres.SocketFactory");
overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, instanceConnectionName);
return ImmutableMap.copyOf(overrides);
}
/**
@ -184,22 +187,6 @@ public abstract class PersistenceModule {
return ImmutableMap.copyOf(overrides);
}
@VisibleForTesting
static ImmutableMap<String, String> createPartialSqlConfigs(
String jdbcUrl,
String instanceConnectionName,
ImmutableMap<String, String> defaultConfigs,
Optional<Provider<TransactionIsolationLevel>> isolationOverride) {
HashMap<String, String> overrides = Maps.newHashMap(defaultConfigs);
overrides.put(Environment.URL, jdbcUrl);
overrides.put(HIKARI_DS_SOCKET_FACTORY, "com.google.cloud.sql.postgres.SocketFactory");
overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, instanceConnectionName);
isolationOverride
.map(Provider::get)
.ifPresent(override -> overrides.put(Environment.ISOLATION, override.name()));
return ImmutableMap.copyOf(overrides);
}
/**
* Provides a {@link Supplier} of single-use JDBC {@link Connection connections} that can manage
* the database DDL schema.
@ -280,6 +267,36 @@ public abstract class PersistenceModule {
return new JpaTransactionManagerImpl(create(overrides), clock);
}
@Provides
@Singleton
@ReadOnlyReplicaJpaTm
static JpaTransactionManager provideReadOnlyReplicaJpaTm(
SqlCredentialStore credentialStore,
@PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
@Config("cloudSqlReplicaInstanceConnectionName")
Optional<String> replicaInstanceConnectionName,
Clock clock) {
HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
setSqlCredential(credentialStore, new RobotUser(RobotId.NOMULUS), overrides);
replicaInstanceConnectionName.ifPresent(
name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
return new JpaTransactionManagerImpl(create(overrides), clock);
}
@Provides
@Singleton
@BeamReadOnlyReplicaJpaTm
static JpaTransactionManager provideBeamReadOnlyReplicaJpaTm(
@BeamPipelineCloudSqlConfigs ImmutableMap<String, String> beamCloudSqlConfigs,
@Config("cloudSqlReplicaInstanceConnectionName")
Optional<String> replicaInstanceConnectionName,
Clock clock) {
HashMap<String, String> overrides = Maps.newHashMap(beamCloudSqlConfigs);
replicaInstanceConnectionName.ifPresent(
name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
return new JpaTransactionManagerImpl(create(overrides), clock);
}
/** Constructs the {@link EntityManagerFactory} instance. */
@VisibleForTesting
static EntityManagerFactory create(
@ -357,7 +374,12 @@ public abstract class PersistenceModule {
* The {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities. Please
* see {@link google.registry.model.bulkquery.BulkQueryEntities} for more information.
*/
BULK_QUERY
BULK_QUERY,
/**
* The {@link JpaTransactionManager} that uses the read-only Postgres replica if configured, or
* the standard DB if not.
*/
READ_ONLY_REPLICA
}
/** Dagger qualifier for JDBC {@link Connection} with schema management privilege. */
@ -383,6 +405,22 @@ public abstract class PersistenceModule {
@Documented
public @interface BeamBulkQueryJpaTm {}
/**
* Dagger qualifier for {@link JpaTransactionManager} used inside BEAM pipelines that uses the
* read-only Postgres replica if one is configured (otherwise it uses the standard DB).
*/
@Qualifier
@Documented
public @interface BeamReadOnlyReplicaJpaTm {}
/**
* Dagger qualifier for {@link JpaTransactionManager} that uses the read-only Postgres replica if
* one is configured (otherwise it uses the standard DB).
*/
@Qualifier
@Documented
public @interface ReadOnlyReplicaJpaTm {}
/** Dagger qualifier for {@link JpaTransactionManager} used for Nomulus tool. */
@Qualifier
@Documented

View file

@ -56,6 +56,7 @@ import google.registry.model.host.HostResource;
import google.registry.model.index.EppResourceIndex;
import google.registry.model.rde.RdeMode;
import google.registry.model.registrar.Registrar;
import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
import google.registry.request.Action;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.Parameter;
@ -340,6 +341,9 @@ public final class RdeStagingAction implements Runnable {
.encode(stagingKeyBytes))
.put("registryEnvironment", RegistryEnvironment.get().name())
.put("workerMachineType", machineType)
.put(
"jpaTransactionManagerType",
JpaTransactionManagerType.READ_ONLY_REPLICA.toString())
// TODO (jianglai): Investigate turning off public IPs (for which
// there is a quota) in order to increase the total number of
// workers allowed (also under quota).

View file

@ -11,6 +11,12 @@
"^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
]
},
{
"name": "jpaTransactionManagerType",
"label": "The type of JPA transaction manager to use",
"helpText": "The standard SQL instance or a read-only replica may be used",
"regexes": ["^REGULAR|READ_ONLY_REPLICA$"]
},
{
"name": "pendings",
"label": "The pendings deposits to generate.",