zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-06-22 20:30:46 +02:00
Code Issues Projects Releases Packages Wiki Activity

Remove unused BeamJpaExtension and related classes (#1102)

Browse source 
* Remove unused BeamJpaExtension and related classes

* Remove unused qualifiers
This commit is contained in:
Lai Jiang 2021-04-22 10:02:18 -04:00 • committed by GitHub
parent 0c9fd57d40
commit b600faf08e
8 changed files with 0 additions and 541 deletions
Download patch file Download diff file Expand all files Collapse all files

201
core/src/main/java/google/registry/beam/initsql/BeamJpaModule.java
View file

@ -1,201 +0,0 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.beam.initsql;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
import com.google.common.base.Splitter;
import dagger.Component;
import dagger.Lazy;
import dagger.Module;
import dagger.Provides;
import google.registry.config.CredentialModule;
import google.registry.config.RegistryConfig.Config;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.keyring.kms.KmsModule;
import google.registry.persistence.PersistenceModule;
import google.registry.persistence.PersistenceModule.JdbcJpaTm;
import google.registry.persistence.PersistenceModule.SocketFactoryJpaTm;
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
import google.registry.persistence.transaction.JpaTransactionManager;
import google.registry.privileges.secretmanager.SecretManagerModule;
import google.registry.util.UtilsModule;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.channels.Channels;
import java.nio.charset.StandardCharsets;
import java.util.List;
import javax.annotation.Nullable;
import javax.inject.Singleton;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.fs.ResourceId;
/**
* Provides bindings for {@link JpaTransactionManager} to Cloud SQL.
*
* <p>This module is intended for use in BEAM pipelines, and uses a BEAM utility to access GCS like
* a regular file system.
*/
@Module
public class BeamJpaModule {
private static final String GCS_SCHEME = "gs://";
@Nullable private final String sqlAccessInfoFile;
@Nullable private final String cloudKmsProjectId;
@Nullable private final TransactionIsolationLevel isolationOverride;
/**
* Constructs a new instance of {@link BeamJpaModule}.
*
* <p>Note: it is an unfortunately necessary antipattern to check for the validity of
* sqlAccessInfoFile in {@link #provideCloudSqlAccessInfo} rather than in the constructor.
* Unfortunately, this is a restriction imposed upon us by Dagger. Specifically, because we use
* this in at least one 1 {@link google.registry.tools.RegistryTool} command(s), it must be
* instantiated in {@code google.registry.tools.RegistryToolComponent} for all possible commands;
* Dagger doesn't permit it to ever be null. For the vast majority of commands, it will never be
* used (so a null credential file path is fine in those cases).
*
* @param sqlAccessInfoFile the path to a Cloud SQL credential file. This must refer to either a
* real encrypted file on GCS as returned by {@link
* BackupPaths#getCloudSQLCredentialFilePatterns} or an unencrypted file on local filesystem
* with credentials to a test database.
* @param cloudKmsProjectId the GCP project where the credential decryption key can be found
* @param isolationOverride the desired Transaction Isolation level for all JDBC connections
*/
public BeamJpaModule(
@Nullable String sqlAccessInfoFile,
@Nullable String cloudKmsProjectId,
@Nullable TransactionIsolationLevel isolationOverride) {
this.sqlAccessInfoFile = sqlAccessInfoFile;
this.cloudKmsProjectId = cloudKmsProjectId;
this.isolationOverride = isolationOverride;
}
public BeamJpaModule(@Nullable String sqlAccessInfoFile, @Nullable String cloudKmsProjectId) {
this(sqlAccessInfoFile, cloudKmsProjectId, null);
}
/** Returns true if the credential file is on GCS (and therefore expected to be encrypted). */
private boolean isCloudSqlCredential() {
return sqlAccessInfoFile.startsWith(GCS_SCHEME);
}
@Provides
@Singleton
SqlAccessInfo provideCloudSqlAccessInfo(Lazy<CloudSqlCredentialDecryptor> lazyDecryptor) {
checkArgument(!isNullOrEmpty(sqlAccessInfoFile), "Null or empty credentialFilePath");
String line = readOnlyLineFromCredentialFile();
if (isCloudSqlCredential()) {
line = lazyDecryptor.get().decrypt(line);
}
// See ./BackupPaths.java for explanation of the line format.
List<String> parts = Splitter.on(' ').splitToList(line.trim());
checkState(parts.size() == 3, "Expecting three phrases in %s", line);
if (isCloudSqlCredential()) {
return SqlAccessInfo.createCloudSqlAccessInfo(parts.get(0), parts.get(1), parts.get(2));
} else {
return SqlAccessInfo.createLocalSqlAccessInfo(parts.get(0), parts.get(1), parts.get(2));
}
}
String readOnlyLineFromCredentialFile() {
try {
ResourceId resourceId = FileSystems.matchSingleFileSpec(sqlAccessInfoFile).resourceId();
try (BufferedReader reader =
new BufferedReader(
new InputStreamReader(
Channels.newInputStream(FileSystems.open(resourceId)), StandardCharsets.UTF_8))) {
return reader.readLine();
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Provides
@Config("beamCloudSqlJdbcUrl")
String provideJdbcUrl(SqlAccessInfo sqlAccessInfo) {
return sqlAccessInfo.jdbcUrl();
}
@Provides
@Config("beamCloudSqlInstanceConnectionName")
String provideSqlInstanceName(SqlAccessInfo sqlAccessInfo) {
return sqlAccessInfo
.cloudSqlInstanceName()
.orElseThrow(() -> new IllegalStateException("Cloud SQL not provisioned."));
}
@Provides
@Config("beamCloudSqlUsername")
String provideSqlUsername(SqlAccessInfo sqlAccessInfo) {
return sqlAccessInfo.user();
}
@Provides
@Config("beamCloudSqlPassword")
String provideSqlPassword(SqlAccessInfo sqlAccessInfo) {
return sqlAccessInfo.password();
}
@Provides
@Config("beamCloudKmsProjectId")
String kmsProjectId() {
return cloudKmsProjectId;
}
@Provides
@Config("beamCloudKmsKeyRing")
static String keyRingName() {
return "nomulus-tool-keyring";
}
@Provides
@Config("beamIsolationOverride")
@Nullable
TransactionIsolationLevel providesIsolationOverride() {
return isolationOverride;
}
@Provides
@Config("beamHibernateHikariMaximumPoolSize")
static int getBeamHibernateHikariMaximumPoolSize() {
// TODO(weiminyu): make this configurable. Should be equal to number of cores.
return 4;
}
@Singleton
@Component(
modules = {
ConfigModule.class,
CredentialModule.class,
BeamJpaModule.class,
KmsModule.class,
PersistenceModule.class,
SecretManagerModule.class,
UtilsModule.class
})
public interface JpaTransactionManagerComponent {
@SocketFactoryJpaTm
JpaTransactionManager cloudSqlJpaTransactionManager();
@JdbcJpaTm
JpaTransactionManager localDbJpaTransactionManager();
}
}

60
core/src/main/java/google/registry/beam/initsql/JpaSupplierFactory.java
View file

@ -1,60 +0,0 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.beam.initsql;
import google.registry.beam.initsql.BeamJpaModule.JpaTransactionManagerComponent;
import google.registry.beam.initsql.Transforms.SerializableSupplier;
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
import google.registry.persistence.transaction.JpaTransactionManager;
import javax.annotation.Nullable;
import org.apache.beam.sdk.transforms.SerializableFunction;
public class JpaSupplierFactory implements SerializableSupplier<JpaTransactionManager> {
private static final long serialVersionUID = 1L;
private final String credentialFileUrl;
@Nullable private final String cloudKmsProjectId;
private final SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager>
jpaGetter;
@Nullable private final TransactionIsolationLevel isolationLevelOverride;
public JpaSupplierFactory(
String credentialFileUrl,
@Nullable String cloudKmsProjectId,
SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager> jpaGetter) {
this(credentialFileUrl, cloudKmsProjectId, jpaGetter, null);
}
public JpaSupplierFactory(
String credentialFileUrl,
@Nullable String cloudKmsProjectId,
SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager> jpaGetter,
@Nullable TransactionIsolationLevel isolationLevelOverride) {
this.credentialFileUrl = credentialFileUrl;
this.cloudKmsProjectId = cloudKmsProjectId;
this.jpaGetter = jpaGetter;
this.isolationLevelOverride = isolationLevelOverride;
}
@Override
public JpaTransactionManager get() {
return jpaGetter.apply(
DaggerBeamJpaModule_JpaTransactionManagerComponent.builder()
.beamJpaModule(
new BeamJpaModule(credentialFileUrl, cloudKmsProjectId, isolationLevelOverride))
.build());
}
}

67
core/src/main/java/google/registry/persistence/PersistenceModule.java
View file

@ -47,7 +47,6 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.function.Supplier;
@ -260,55 +259,6 @@ public abstract class PersistenceModule {
return new JpaTransactionManagerImpl(create(overrides), clock);
}
@Provides
@Singleton
@SocketFactoryJpaTm
static JpaTransactionManager provideSocketFactoryJpaTm(
SqlCredentialStore credentialStore,
@Config("beamCloudSqlUsername") String username,
@Config("beamCloudSqlPassword") String password,
@Config("beamHibernateHikariMaximumPoolSize") int hikariMaximumPoolSize,
@BeamPipelineCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
Clock clock) {
HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(hikariMaximumPoolSize));
overrides.put(Environment.USER, username);
overrides.put(Environment.PASS, password);
// TODO(b/175700623): consider assigning different logins to pipelines
// TODO(b/179839014): Make SqlCredentialStore injectable in BEAM
// Note: the logs below appear in the pipeline's Worker logs, not the Job log.
try {
SqlCredential credential = credentialStore.getCredential(new RobotUser(RobotId.NOMULUS));
if (!Objects.equals(username, credential.login())) {
logger.atWarning().log(
"Wrong username for nomulus. Expecting %s, found %s.", username, credential.login());
} else if (!Objects.equals(password, credential.password())) {
logger.atWarning().log("Wrong password for nomulus.");
} else {
logger.atWarning().log("Credentials in the kerying and the secret manager match.");
}
} catch (Exception e) {
logger.atWarning().withCause(e).log("Failed to get SQL credential from Secret Manager.");
}
return new JpaTransactionManagerImpl(create(overrides), clock);
}
@Provides
@Singleton
@JdbcJpaTm
static JpaTransactionManager provideLocalJpaTm(
@Config("beamCloudSqlJdbcUrl") String jdbcUrl,
@Config("beamCloudSqlUsername") String username,
@Config("beamCloudSqlPassword") String password,
@DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs,
Clock clock) {
HashMap<String, String> overrides = Maps.newHashMap(defaultConfigs);
overrides.put(Environment.URL, jdbcUrl);
overrides.put(Environment.USER, username);
overrides.put(Environment.PASS, password);
return new JpaTransactionManagerImpl(create(overrides), clock);
}
/** Constructs the {@link EntityManagerFactory} instance. */
@VisibleForTesting
static EntityManagerFactory create(
@ -399,23 +349,6 @@ public abstract class PersistenceModule {
@Documented
public @interface NomulusToolJpaTm {}
/**
* Dagger qualifier for {@link JpaTransactionManager} that accesses Cloud SQL using socket
* factory. This is meant for applications not running on AppEngine, therefore without access to a
* {@link google.registry.keyring.api.Keyring}.
*/
@Qualifier
@Documented
public @interface SocketFactoryJpaTm {}
/**
* Dagger qualifier for {@link JpaTransactionManager} backed by plain JDBC connections. This is
* mainly used by tests.
*/
@Qualifier
@Documented
public @interface JdbcJpaTm {}
/** Dagger qualifier for the partial Cloud SQL configs. */
@Qualifier
@Documented

4
core/src/main/java/google/registry/tools/RegistryToolComponent.java
View file

@ -18,7 +18,6 @@ import dagger.BindsInstance;
import dagger.Component;
import dagger.Lazy;
import google.registry.batch.BatchModule;
import google.registry.beam.initsql.BeamJpaModule;
import google.registry.bigquery.BigqueryModule;
import google.registry.config.CredentialModule.LocalCredentialJson;
import google.registry.config.RegistryConfig.Config;
@ -60,7 +59,6 @@ import javax.inject.Singleton;
AppEngineAdminApiModule.class,
AuthModule.class,
BatchModule.class,
BeamJpaModule.class,
BigqueryModule.class,
ConfigModule.class,
CloudDnsWriterModule.class,
@ -191,8 +189,6 @@ interface RegistryToolComponent {
@BindsInstance
Builder sqlAccessInfoFile(@Nullable @Config("sqlAccessInfoFile") String sqlAccessInfoFile);
Builder beamJpaModule(BeamJpaModule beamJpaModule);
RegistryToolComponent build();
}
}

72
core/src/test/java/google/registry/beam/initsql/BeamJpaExtension.java
View file

@ -1,72 +0,0 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.beam.initsql;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.function.Supplier;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.testcontainers.containers.JdbcDatabaseContainer;
/**
* Helpers for setting up {@link BeamJpaModule} in tests.
*
* <p>This extension is often used with a Database container and/or temporary file folder. User must
* make sure that all dependent extensions are set up before this extension, e.g., by assigning
* {@link org.junit.jupiter.api.Order orders}.
*/
public final class BeamJpaExtension implements BeforeEachCallback, AfterEachCallback, Serializable {
private final transient JdbcDatabaseContainer<?> database;
private final transient Supplier<Path> credentialPathSupplier;
private transient BeamJpaModule beamJpaModule;
private File credentialFile;
public BeamJpaExtension(Supplier<Path> credentialPathSupplier, JdbcDatabaseContainer database) {
this.database = database;
this.credentialPathSupplier = credentialPathSupplier;
}
public File getCredentialFile() {
return credentialFile;
}
public BeamJpaModule getBeamJpaModule() {
if (beamJpaModule != null) {
return beamJpaModule;
}
return beamJpaModule = new BeamJpaModule(credentialFile.getAbsolutePath(), null);
}
@Override
public void beforeEach(ExtensionContext context) throws IOException {
credentialFile = Files.createFile(credentialPathSupplier.get()).toFile();
new PrintStream(credentialFile)
.printf("%s %s %s", database.getJdbcUrl(), database.getUsername(), database.getPassword())
.close();
}
@Override
public void afterEach(ExtensionContext context) {
credentialFile.delete();
}
}

94
core/src/test/java/google/registry/beam/initsql/BeamJpaModuleTest.java
View file

@ -1,94 +0,0 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.beam.initsql;
import static com.google.common.truth.Truth.assertThat;
import google.registry.persistence.NomulusPostgreSql;
import google.registry.persistence.transaction.JpaTransactionManager;
import google.registry.testing.DatastoreEntityExtension;
import java.nio.file.Path;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.junit.jupiter.api.io.TempDir;
import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
/** Unit tests for {@link BeamJpaModule}. */
@Testcontainers
class BeamJpaModuleTest {
@RegisterExtension
final DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension();
@Container
final PostgreSQLContainer database = new PostgreSQLContainer(NomulusPostgreSql.getDockerTag());
@SuppressWarnings("WeakerAccess")
@TempDir
Path tmpDir;
@RegisterExtension
@Order(Order.DEFAULT + 1)
final BeamJpaExtension beamJpaExtension =
new BeamJpaExtension(() -> tmpDir.resolve("credential.dat"), database);
@Test
void getJpaTransactionManager_local() {
JpaTransactionManager jpa =
DaggerBeamJpaModule_JpaTransactionManagerComponent.builder()
.beamJpaModule(beamJpaExtension.getBeamJpaModule())
.build()
.localDbJpaTransactionManager();
assertThat(
jpa.transact(
() -> jpa.getEntityManager().createNativeQuery("select 1").getSingleResult()))
.isEqualTo(1);
}
/**
* Integration test with a GCP project, only run when the 'test.gcp_integration.env' property is
* defined. Otherwise this test is ignored. This is meant to be run from a developer's desktop,
* with auth already set up by gcloud.
*
* <p>Example: {@code gradlew test -P test.gcp_integration.env=alpha}.
*
* <p>See <a href="../../../../../../../../java_common.gradle">java_common.gradle</a> for more
* information.
*/
@Test
@EnabledIfSystemProperty(named = "test.gcp_integration.env", matches = "\\S+")
void getJpaTransactionManager_cloudSql_authRequired() {
String environmentName = System.getProperty("test.gcp_integration.env");
FileSystems.setDefaultPipelineOptions(PipelineOptionsFactory.create());
JpaTransactionManager jpa =
DaggerBeamJpaModule_JpaTransactionManagerComponent.builder()
.beamJpaModule(
new BeamJpaModule(
BackupPaths.getCloudSQLCredentialFilePatterns(environmentName).get(0),
String.format("domain-registry-%s", environmentName)))
.build()
.cloudSqlJpaTransactionManager();
assertThat(
jpa.transact(
() -> jpa.getEntityManager().createNativeQuery("select 1").getSingleResult()))
.isEqualTo(1);
}
}

6
core/src/test/java/google/registry/beam/initsql/InitSqlPipelineTest.java
View file

@ -117,12 +117,6 @@ class InitSqlPipelineTest {
final transient JpaIntegrationTestExtension database =
new JpaTestRules.Builder().withClock(fakeClock).buildIntegrationTestRule();
// Must not be transient!
@RegisterExtension
@Order(Order.DEFAULT + 1)
final BeamJpaExtension beamJpaExtension =
new BeamJpaExtension(() -> tmpDir.resolve("credential.dat"), database.getDatabase());
private File exportRootDir;
private File exportDir;
private File commitLogDir;

37
core/src/test/java/google/registry/persistence/PersistenceModuleTest.java
View file

@ -18,7 +18,6 @@ import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import dagger.Component;
import google.registry.beam.initsql.BeamJpaModule;
import google.registry.config.CredentialModule;
import google.registry.config.RegistryConfig.Config;
import google.registry.config.RegistryConfig.ConfigModule;
@ -85,45 +84,9 @@ class PersistenceModuleTest {
.isEqualTo(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE.name());
}
@Test
void beamIsolation_default() {
Optional<Provider<TransactionIsolationLevel>> injected =
DaggerPersistenceModuleTest_BeamConfigTestComponent.builder()
.beamJpaModule(new BeamJpaModule(null, null))
.build()
.getIsolationOverride();
assertThat(injected).isNotNull();
assertThat(injected.get().get()).isNull();
assertThat(
PersistenceModule.provideBeamPipelineCloudSqlConfigs(
"", "", PersistenceModule.provideDefaultDatabaseConfigs(), injected)
.get(Environment.ISOLATION))
.isEqualTo(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE.name());
}
@Test
void beamIsolation_override() {
Optional<Provider<TransactionIsolationLevel>> injected =
DaggerPersistenceModuleTest_BeamConfigTestComponent.builder()
.beamJpaModule(
new BeamJpaModule(
null, null, TransactionIsolationLevel.TRANSACTION_READ_UNCOMMITTED))
.build()
.getIsolationOverride();
assertThat(injected).isNotNull();
assertThat(injected.get().get())
.isEqualTo(TransactionIsolationLevel.TRANSACTION_READ_UNCOMMITTED);
assertThat(
PersistenceModule.provideBeamPipelineCloudSqlConfigs(
"", "", PersistenceModule.provideDefaultDatabaseConfigs(), injected)
.get(Environment.ISOLATION))
.isEqualTo(TransactionIsolationLevel.TRANSACTION_READ_UNCOMMITTED.name());
}
@Singleton
@Component(
modules = {
BeamJpaModule.class,
ConfigModule.class,
CredentialModule.class,
KmsModule.class,