Use a read-only replica SQL instance in RdapDomainSearchAction (#1495)

We can use it more places later but this can serve as a template. We
should inject the connection to the read-only replica (only created
once) to the constructor of the action, then use that instead of the
regular transaction manager.

We add a transaction manager that simulates the read-only-replica
behavior for testing purposes as well.

In addition, we set the transaction isolation level to READ COMMITTED
for this transaction manager (this is fine since we're never writing to
it). Postgres requires this for replica SQL access (it fails if we try
to use SERIALIZABLE) transactions. We didn't see this with the pipelines
before since those already had transaction isolation level overrides
This commit is contained in:
gbrodman 2022-01-20 20:39:07 +00:00 committed by GitHub
parent 7e506f4d90
commit c8636f6c90
5 changed files with 432 additions and 125 deletions

View file

@ -30,6 +30,7 @@ import google.registry.keyring.api.KeyModule;
import google.registry.keyring.kms.KmsModule;
import google.registry.module.pubapi.PubApiRequestComponent.PubApiRequestComponentModule;
import google.registry.monitoring.whitebox.StackdriverModule;
import google.registry.persistence.PersistenceModule;
import google.registry.privileges.secretmanager.SecretManagerModule;
import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.NetHttpTransportModule;
@ -56,6 +57,7 @@ import javax.inject.Singleton;
KeyringModule.class,
KmsModule.class,
NetHttpTransportModule.class,
PersistenceModule.class,
PubApiRequestComponentModule.class,
SecretManagerModule.class,
ServerTridProviderModule.class,

View file

@ -277,6 +277,8 @@ public abstract class PersistenceModule {
setSqlCredential(credentialStore, new RobotUser(RobotId.NOMULUS), overrides);
replicaInstanceConnectionName.ifPresent(
name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
overrides.put(
Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_READ_COMMITTED.name());
return new JpaTransactionManagerImpl(create(overrides), clock);
}
@ -291,6 +293,8 @@ public abstract class PersistenceModule {
HashMap<String, String> overrides = Maps.newHashMap(beamCloudSqlConfigs);
replicaInstanceConnectionName.ifPresent(
name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
overrides.put(
Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_READ_COMMITTED.name());
return new JpaTransactionManagerImpl(create(overrides), clock);
}

View file

@ -18,7 +18,6 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static google.registry.model.EppResourceUtils.loadByForeignKey;
import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.request.Action.Method.GET;
import static google.registry.request.Action.Method.HEAD;
@ -38,8 +37,10 @@ import com.google.common.primitives.Booleans;
import com.googlecode.objectify.cmd.Query;
import google.registry.model.domain.DomainBase;
import google.registry.model.host.HostResource;
import google.registry.persistence.PersistenceModule.ReadOnlyReplicaJpaTm;
import google.registry.persistence.VKey;
import google.registry.persistence.transaction.CriteriaQueryBuilder;
import google.registry.persistence.transaction.JpaTransactionManager;
import google.registry.rdap.RdapJsonFormatter.OutputDataType;
import google.registry.rdap.RdapMetrics.EndpointType;
import google.registry.rdap.RdapMetrics.SearchType;
@ -91,7 +92,11 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
@Inject @Parameter("name") Optional<String> nameParam;
@Inject @Parameter("nsLdhName") Optional<String> nsLdhNameParam;
@Inject @Parameter("nsIp") Optional<String> nsIpParam;
@Inject public RdapDomainSearchAction() {
@Inject @ReadOnlyReplicaJpaTm JpaTransactionManager readOnlyJpaTm;
@Inject
public RdapDomainSearchAction() {
super("domain search", EndpointType.DOMAINS);
}
@ -223,32 +228,31 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
resultSet = getMatchingResources(query, true, querySizeLimit);
} else {
resultSet =
jpaTm()
.transact(
() -> {
CriteriaBuilder criteriaBuilder =
jpaTm().getEntityManager().getCriteriaBuilder();
CriteriaQueryBuilder<DomainBase> queryBuilder =
CriteriaQueryBuilder.create(DomainBase.class)
.where(
"fullyQualifiedDomainName",
criteriaBuilder::like,
String.format("%s%%", partialStringQuery.getInitialString()))
.orderByAsc("fullyQualifiedDomainName");
if (cursorString.isPresent()) {
queryBuilder =
queryBuilder.where(
"fullyQualifiedDomainName",
criteriaBuilder::greaterThan,
cursorString.get());
}
if (partialStringQuery.getSuffix() != null) {
queryBuilder =
queryBuilder.where(
"tld", criteriaBuilder::equal, partialStringQuery.getSuffix());
}
return getMatchingResourcesSql(queryBuilder, true, querySizeLimit);
});
readOnlyJpaTm.transact(
() -> {
CriteriaBuilder criteriaBuilder =
readOnlyJpaTm.getEntityManager().getCriteriaBuilder();
CriteriaQueryBuilder<DomainBase> queryBuilder =
CriteriaQueryBuilder.create(DomainBase.class)
.where(
"fullyQualifiedDomainName",
criteriaBuilder::like,
String.format("%s%%", partialStringQuery.getInitialString()))
.orderByAsc("fullyQualifiedDomainName");
if (cursorString.isPresent()) {
queryBuilder =
queryBuilder.where(
"fullyQualifiedDomainName",
criteriaBuilder::greaterThan,
cursorString.get());
}
if (partialStringQuery.getSuffix() != null) {
queryBuilder =
queryBuilder.where(
"tld", criteriaBuilder::equal, partialStringQuery.getSuffix());
}
return getMatchingResourcesSql(queryBuilder, true, querySizeLimit);
});
}
return makeSearchResults(resultSet);
}
@ -270,20 +274,19 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
resultSet = getMatchingResources(query, true, querySizeLimit);
} else {
resultSet =
jpaTm()
.transact(
() -> {
CriteriaQueryBuilder<DomainBase> builder =
queryItemsSql(
DomainBase.class,
"tld",
tld,
Optional.of("fullyQualifiedDomainName"),
cursorString,
DeletedItemHandling.INCLUDE)
.orderByAsc("fullyQualifiedDomainName");
return getMatchingResourcesSql(builder, true, querySizeLimit);
});
readOnlyJpaTm.transact(
() -> {
CriteriaQueryBuilder<DomainBase> builder =
queryItemsSql(
DomainBase.class,
"tld",
tld,
Optional.of("fullyQualifiedDomainName"),
cursorString,
DeletedItemHandling.INCLUDE)
.orderByAsc("fullyQualifiedDomainName");
return getMatchingResourcesSql(builder, true, querySizeLimit);
});
}
return makeSearchResults(resultSet);
}
@ -354,28 +357,28 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
.map(VKey::from)
.collect(toImmutableSet());
} else {
return jpaTm()
.transact(
() -> {
CriteriaQueryBuilder<HostResource> builder =
queryItemsSql(
HostResource.class,
"fullyQualifiedHostName",
partialStringQuery,
Optional.empty(),
DeletedItemHandling.EXCLUDE);
if (desiredRegistrar.isPresent()) {
builder =
builder.where(
"currentSponsorClientId",
jpaTm().getEntityManager().getCriteriaBuilder()::equal,
desiredRegistrar.get());
}
return getMatchingResourcesSql(builder, true, maxNameserversInFirstStage)
.resources().stream()
.map(HostResource::createVKey)
.collect(toImmutableSet());
});
return readOnlyJpaTm.transact(
() -> {
CriteriaQueryBuilder<HostResource> builder =
queryItemsSql(
HostResource.class,
"fullyQualifiedHostName",
partialStringQuery,
Optional.empty(),
DeletedItemHandling.EXCLUDE);
if (desiredRegistrar.isPresent()) {
builder =
builder.where(
"currentSponsorClientId",
readOnlyJpaTm.getEntityManager().getCriteriaBuilder()::equal,
desiredRegistrar.get());
}
return getMatchingResourcesSql(builder, true, maxNameserversInFirstStage)
.resources()
.stream()
.map(HostResource::createVKey)
.collect(toImmutableSet());
});
}
}
@ -509,21 +512,20 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
parameters.put("desiredRegistrar", desiredRegistrar.get());
}
hostKeys =
jpaTm()
.transact(
() -> {
javax.persistence.Query query =
jpaTm()
.getEntityManager()
.createNativeQuery(queryBuilder.toString())
.setMaxResults(maxNameserversInFirstStage);
parameters.build().forEach(query::setParameter);
@SuppressWarnings("unchecked")
Stream<String> resultStream = query.getResultStream();
return resultStream
.map(repoId -> VKey.create(HostResource.class, repoId))
.collect(toImmutableSet());
});
readOnlyJpaTm.transact(
() -> {
javax.persistence.Query query =
readOnlyJpaTm
.getEntityManager()
.createNativeQuery(queryBuilder.toString())
.setMaxResults(maxNameserversInFirstStage);
parameters.build().forEach(query::setParameter);
@SuppressWarnings("unchecked")
Stream<String> resultStream = query.getResultStream();
return resultStream
.map(repoId -> VKey.create(HostResource.class, repoId))
.collect(toImmutableSet());
});
}
return searchByNameserverRefs(hostKeys);
}
@ -568,39 +570,38 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
}
stream.forEach(domainSetBuilder::add);
} else {
jpaTm()
.transact(
() -> {
for (VKey<HostResource> hostKey : hostKeys) {
CriteriaQueryBuilder<DomainBase> queryBuilder =
CriteriaQueryBuilder.create(DomainBase.class)
.whereFieldContains("nsHosts", hostKey)
.orderByAsc("fullyQualifiedDomainName");
CriteriaBuilder criteriaBuilder =
jpaTm().getEntityManager().getCriteriaBuilder();
if (!shouldIncludeDeleted()) {
queryBuilder =
queryBuilder.where(
"deletionTime", criteriaBuilder::greaterThan, getRequestTime());
}
if (cursorString.isPresent()) {
queryBuilder =
queryBuilder.where(
"fullyQualifiedDomainName",
criteriaBuilder::greaterThan,
cursorString.get());
}
jpaTm()
.criteriaQuery(queryBuilder.build())
.getResultStream()
.filter(this::isAuthorized)
.forEach(
(domain) -> {
Hibernate.initialize(domain.getDsData());
domainSetBuilder.add(domain);
});
}
});
readOnlyJpaTm.transact(
() -> {
for (VKey<HostResource> hostKey : hostKeys) {
CriteriaQueryBuilder<DomainBase> queryBuilder =
CriteriaQueryBuilder.create(DomainBase.class)
.whereFieldContains("nsHosts", hostKey)
.orderByAsc("fullyQualifiedDomainName");
CriteriaBuilder criteriaBuilder =
readOnlyJpaTm.getEntityManager().getCriteriaBuilder();
if (!shouldIncludeDeleted()) {
queryBuilder =
queryBuilder.where(
"deletionTime", criteriaBuilder::greaterThan, getRequestTime());
}
if (cursorString.isPresent()) {
queryBuilder =
queryBuilder.where(
"fullyQualifiedDomainName",
criteriaBuilder::greaterThan,
cursorString.get());
}
readOnlyJpaTm
.criteriaQuery(queryBuilder.build())
.getResultStream()
.filter(this::isAuthorized)
.forEach(
(domain) -> {
Hibernate.initialize(domain.getDsData());
domainSetBuilder.add(domain);
});
}
});
}
}
List<DomainBase> domains = domainSetBuilder.build().asList();

View file

@ -0,0 +1,292 @@
// Copyright 2022 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.persistence.transaction;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import google.registry.model.ImmutableObject;
import google.registry.persistence.VKey;
import java.util.Optional;
import java.util.function.Supplier;
import java.util.stream.Stream;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.CriteriaQuery;
import org.joda.time.DateTime;
/**
* A {@link JpaTransactionManager} that simulates a read-only replica SQL instance.
*
* <p>We accomplish this by delegating all calls to the standard transaction manager except for
* calls that start transactions. For these, we create a transaction like normal but set it to READ
* ONLY mode before doing any work. This is similar to how the read-only Postgres replica works; it
* treats all transactions as read-only transactions.
*/
public class ReplicaSimulatingJpaTransactionManager implements JpaTransactionManager {
private final JpaTransactionManager delegate;
public ReplicaSimulatingJpaTransactionManager(JpaTransactionManager delegate) {
this.delegate = delegate;
}
public void teardown() {
delegate.teardown();
}
public EntityManager getStandaloneEntityManager() {
return delegate.getStandaloneEntityManager();
}
public EntityManager getEntityManager() {
return delegate.getEntityManager();
}
public JpaTransactionManager setDatabaseSnapshot(String snapshotId) {
return delegate.setDatabaseSnapshot(snapshotId);
}
public <T> TypedQuery<T> query(String sqlString, Class<T> resultClass) {
return delegate.query(sqlString, resultClass);
}
public <T> TypedQuery<T> criteriaQuery(CriteriaQuery<T> criteriaQuery) {
return delegate.criteriaQuery(criteriaQuery);
}
public Query query(String sqlString) {
return delegate.query(sqlString);
}
public boolean inTransaction() {
return delegate.inTransaction();
}
public void assertInTransaction() {
delegate.assertInTransaction();
}
public <T> T transact(Supplier<T> work) {
return delegate.transact(
() -> {
delegate.getEntityManager().createQuery("SET TRANSACTION READ ONLY").executeUpdate();
return work.get();
});
}
public <T> T transactWithoutBackup(Supplier<T> work) {
return transact(work);
}
public <T> T transactNoRetry(Supplier<T> work) {
return transact(work);
}
public void transact(Runnable work) {
transact(
() -> {
work.run();
return null;
});
}
public void transactNoRetry(Runnable work) {
transact(work);
}
public <T> T transactNew(Supplier<T> work) {
return transact(work);
}
public void transactNew(Runnable work) {
transact(work);
}
public <T> T transactNewReadOnly(Supplier<T> work) {
return transact(work);
}
public void transactNewReadOnly(Runnable work) {
transact(work);
}
public <T> T doTransactionless(Supplier<T> work) {
return delegate.doTransactionless(work);
}
public DateTime getTransactionTime() {
return delegate.getTransactionTime();
}
public void insert(Object entity) {
delegate.insert(entity);
}
public void insertAll(ImmutableCollection<?> entities) {
delegate.insertAll(entities);
}
public void insertAll(ImmutableObject... entities) {
delegate.insertAll(entities);
}
public void insertWithoutBackup(ImmutableObject entity) {
delegate.insertWithoutBackup(entity);
}
public void insertAllWithoutBackup(ImmutableCollection<?> entities) {
delegate.insertAllWithoutBackup(entities);
}
public void put(Object entity) {
delegate.put(entity);
}
public void putAll(ImmutableObject... entities) {
delegate.putAll(entities);
}
public void putAll(ImmutableCollection<?> entities) {
delegate.putAll(entities);
}
public void putWithoutBackup(ImmutableObject entity) {
delegate.putWithoutBackup(entity);
}
public void putAllWithoutBackup(ImmutableCollection<?> entities) {
delegate.putAllWithoutBackup(entities);
}
public void update(Object entity) {
delegate.update(entity);
}
public void updateAll(ImmutableCollection<?> entities) {
delegate.updateAll(entities);
}
public void updateAll(ImmutableObject... entities) {
delegate.updateAll(entities);
}
public void updateWithoutBackup(ImmutableObject entity) {
delegate.updateWithoutBackup(entity);
}
public void updateAllWithoutBackup(ImmutableCollection<?> entities) {
delegate.updateAllWithoutBackup(entities);
}
public <T> boolean exists(VKey<T> key) {
return delegate.exists(key);
}
public boolean exists(Object entity) {
return delegate.exists(entity);
}
public <T> Optional<T> loadByKeyIfPresent(VKey<T> key) {
return delegate.loadByKeyIfPresent(key);
}
public <T> ImmutableMap<VKey<? extends T>, T> loadByKeysIfPresent(
Iterable<? extends VKey<? extends T>> vKeys) {
return delegate.loadByKeysIfPresent(vKeys);
}
public <T> ImmutableList<T> loadByEntitiesIfPresent(Iterable<T> entities) {
return delegate.loadByEntitiesIfPresent(entities);
}
public <T> T loadByKey(VKey<T> key) {
return delegate.loadByKey(key);
}
public <T> ImmutableMap<VKey<? extends T>, T> loadByKeys(
Iterable<? extends VKey<? extends T>> vKeys) {
return delegate.loadByKeys(vKeys);
}
public <T> T loadByEntity(T entity) {
return delegate.loadByEntity(entity);
}
public <T> ImmutableList<T> loadByEntities(Iterable<T> entities) {
return delegate.loadByEntities(entities);
}
public <T> ImmutableList<T> loadAllOf(Class<T> clazz) {
return delegate.loadAllOf(clazz);
}
public <T> Stream<T> loadAllOfStream(Class<T> clazz) {
return delegate.loadAllOfStream(clazz);
}
public <T> Optional<T> loadSingleton(Class<T> clazz) {
return delegate.loadSingleton(clazz);
}
public void delete(VKey<?> key) {
delegate.delete(key);
}
public void delete(Iterable<? extends VKey<?>> vKeys) {
delegate.delete(vKeys);
}
public <T> T delete(T entity) {
return delegate.delete(entity);
}
public void deleteWithoutBackup(VKey<?> key) {
delegate.deleteWithoutBackup(key);
}
public void deleteWithoutBackup(Iterable<? extends VKey<?>> keys) {
delegate.deleteWithoutBackup(keys);
}
public void deleteWithoutBackup(Object entity) {
delegate.deleteWithoutBackup(entity);
}
public <T> QueryComposer<T> createQueryComposer(Class<T> entity) {
return delegate.createQueryComposer(entity);
}
public void clearSessionCache() {
delegate.clearSessionCache();
}
public boolean isOfy() {
return delegate.isOfy();
}
public void putIgnoringReadOnlyWithoutBackup(Object entity) {
delegate.putIgnoringReadOnlyWithoutBackup(entity);
}
public void deleteIgnoringReadOnlyWithoutBackup(VKey<?> key) {
delegate.deleteIgnoringReadOnlyWithoutBackup(key);
}
public <T> void assertDelete(VKey<T> key) {
delegate.assertDelete(key);
}
}

View file

@ -15,6 +15,7 @@
package google.registry.rdap;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.rdap.RdapTestHelper.assertThat;
import static google.registry.rdap.RdapTestHelper.parseJsonObject;
import static google.registry.request.Action.Method.POST;
@ -45,6 +46,7 @@ import google.registry.model.registrar.Registrar;
import google.registry.model.reporting.HistoryEntry;
import google.registry.model.tld.Registry;
import google.registry.persistence.VKey;
import google.registry.persistence.transaction.ReplicaSimulatingJpaTransactionManager;
import google.registry.rdap.RdapMetrics.EndpointType;
import google.registry.rdap.RdapMetrics.SearchType;
import google.registry.rdap.RdapMetrics.WildcardType;
@ -93,39 +95,27 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
}
private JsonObject generateActualJson(RequestType requestType, String paramValue, String cursor) {
action.requestPath = actionPath;
action.requestMethod = POST;
String requestTypeParam = null;
String requestTypeParam;
switch (requestType) {
case NAME:
action.nameParam = Optional.of(paramValue);
action.nsLdhNameParam = Optional.empty();
action.nsIpParam = Optional.empty();
requestTypeParam = "name";
break;
case NS_LDH_NAME:
action.nameParam = Optional.empty();
action.nsLdhNameParam = Optional.of(paramValue);
action.nsIpParam = Optional.empty();
requestTypeParam = "nsLdhName";
break;
case NS_IP:
action.nameParam = Optional.empty();
action.nsLdhNameParam = Optional.empty();
action.nsIpParam = Optional.of(paramValue);
requestTypeParam = "nsIp";
break;
default:
action.nameParam = Optional.empty();
action.nsLdhNameParam = Optional.empty();
action.nsIpParam = Optional.empty();
requestTypeParam = "";
break;
}
if (paramValue != null) {
if (cursor == null) {
action.parameterMap = ImmutableListMultimap.of(requestTypeParam, paramValue);
action.cursorTokenParam = Optional.empty();
} else {
action.parameterMap =
ImmutableListMultimap.of(requestTypeParam, paramValue, "cursor", cursor);
@ -381,6 +371,12 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
clock.nowUtc()));
action.requestMethod = POST;
action.nameParam = Optional.empty();
action.nsLdhNameParam = Optional.empty();
action.nsIpParam = Optional.empty();
action.cursorTokenParam = Optional.empty();
action.requestPath = actionPath;
action.readOnlyJpaTm = jpaTm();
}
private JsonObject generateExpectedJsonForTwoDomainsNsReply() {
@ -728,6 +724,18 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
verifyMetrics(SearchType.BY_DOMAIN_NAME, Optional.of(1L));
}
@TestSqlOnly
void testDomainMatch_readOnlyReplica() {
login("evilregistrar");
rememberWildcardType("cat.lol");
action.readOnlyJpaTm = new ReplicaSimulatingJpaTransactionManager(jpaTm());
action.nameParam = Optional.of("cat.lol");
action.parameterMap = ImmutableListMultimap.of("name", "cat.lol");
action.run();
assertThat(response.getPayload()).contains("Yes Virginia <script>");
assertThat(response.getStatus()).isEqualTo(200);
}
@TestOfyAndSql
void testDomainMatch_foundWithUpperCase() {
login("evilregistrar");