Implement dump_golden_schema command in devtool (#467)

* Implement dump_golden_schema command in devtool

Add a dump_golden_schema command so that we can generate the golden schema
in-place without having to do the test -> fail -> copy -> test dance.

Refactor the SQL container functionality from GenerateSqlCommand.  There is
some duplication of code between the dump command and SchemaTest which should
be dealt with in a subsequent PR.

* Reformatted and changes in response to review

* Fix getDockerTag() usage

* Fix "leaked resource"
This commit is contained in:
Michael Muller 2020-02-03 13:25:27 -05:00 committed by GitHub
parent 2c3e7c98ce
commit b8b2f85e25
55 changed files with 1020 additions and 703 deletions

View file

@ -26,7 +26,9 @@ public class DevTool {
* any invocations in scripts (e.g. PDT, ICANN reporting).
*/
public static final ImmutableMap<String, Class<? extends Command>> COMMAND_MAP =
ImmutableMap.of("generate_sql_schema", GenerateSqlSchemaCommand.class);
ImmutableMap.of(
"dump_golden_schema", DumpGoldenSchemaCommand.class,
"generate_sql_schema", GenerateSqlSchemaCommand.class);
public static void main(String[] args) throws Exception {
RegistryToolEnvironment.parseFromArgs(args).setup();

View file

@ -0,0 +1,91 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Path;
import org.flywaydb.core.Flyway;
import org.testcontainers.containers.BindMode;
import org.testcontainers.containers.Container;
/**
* Generates a schema for JPA annotated classes using Hibernate.
*
* <p>Note that this isn't complete yet, as all of the persistent classes have not yet been
* converted. After converting a class, a call to "addAnnotatedClass()" for the new class must be
* added to the code below.
*/
@Parameters(separators = " =", commandDescription = "Dump golden schema.")
public class DumpGoldenSchemaCommand extends PostgresqlCommand {
// The mount point in the container.
private static final String CONTAINER_MOUNT_POINT = "/tmp/pg_dump.out";
@Parameter(
names = {"--output", "-o"},
description = "Output file",
required = true)
Path output;
@Override
void runCommand() throws IOException, InterruptedException {
Flyway flyway =
Flyway.configure()
.locations("sql/flyway")
.dataSource(
postgresContainer.getJdbcUrl(),
postgresContainer.getUsername(),
postgresContainer.getPassword())
.load();
flyway.migrate();
String userName = postgresContainer.getUsername();
String databaseName = postgresContainer.getDatabaseName();
Container.ExecResult result =
postgresContainer.execInContainer(getSchemaDumpCommand(userName, databaseName));
if (result.getExitCode() != 0) {
throw new RuntimeException(result.toString());
}
}
@Override
protected void onContainerCreate() throws IOException {
// open the output file for write so we can mount it.
new FileOutputStream(output.toFile()).close();
postgresContainer.withFileSystemBind(
output.toString(), CONTAINER_MOUNT_POINT, BindMode.READ_WRITE);
}
private static String[] getSchemaDumpCommand(String username, String dbName) {
return new String[] {
"pg_dump",
"-h",
"localhost",
"-U",
username,
"-f",
CONTAINER_MOUNT_POINT,
"--schema-only",
"--no-owner",
"--no-privileges",
"--exclude-table",
"flyway_schema_history",
dbName
};
}
}

View file

@ -0,0 +1,128 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import com.beust.jcommander.Parameter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.flogger.FluentLogger;
import google.registry.persistence.NomulusPostgreSql;
import org.testcontainers.containers.PostgreSQLContainer;
/** Base class for commands that need a PostgreSQL database. */
public abstract class PostgresqlCommand implements Command {
static final FluentLogger logger = FluentLogger.forEnclosingClass();
protected static final String DB_NAME = "postgres";
protected static final String DB_USERNAME = "postgres";
protected static final String DB_PASSWORD = "domain-registry";
@VisibleForTesting
public static final String DB_OPTIONS_CLASH =
"Database host and port may not be specified along with the option to start a "
+ "PostgreSQL container.";
@VisibleForTesting public static final int POSTGRESQL_PORT = 5432;
protected PostgreSQLContainer postgresContainer = null;
@Parameter(
names = {"-s", "--start_postgresql"},
description = "If specified, start PostgreSQL in a Docker container.")
boolean startPostgresql = false;
@Parameter(
names = {"-a", "--db_host"},
description = "Database host name.")
String databaseHost;
@Parameter(
names = {"-p", "--db_port"},
description = "Database port number. This defaults to the PostgreSQL default port.")
Integer databasePort;
/**
* Starts the database if appropriate.
*
* <p>Returns true if the database was successfully initialized, false if not.
*/
private boolean initializeDatabase() {
// Start PostgreSQL if requested.
if (startPostgresql) {
// Complain if the user has also specified either --db_host or --db_port.
if (databaseHost != null || databasePort != null) {
System.err.println(DB_OPTIONS_CLASH);
// TODO: it would be nice to exit(1) here, but this breaks testability.
return false;
}
// Start the container and store the address information.
postgresContainer =
new PostgreSQLContainer(NomulusPostgreSql.getDockerTag())
.withDatabaseName(DB_NAME)
.withUsername(DB_USERNAME)
.withPassword(DB_PASSWORD);
try {
onContainerCreate();
} catch (Exception e) {
logger.atSevere().withCause(e).log("Error in container callback hook.");
return false;
}
postgresContainer.start();
databaseHost = postgresContainer.getContainerIpAddress();
databasePort = postgresContainer.getMappedPort(POSTGRESQL_PORT);
} else if (databaseHost == null) {
System.err.println(
"You must specify either --start_postgresql to start a PostgreSQL database in a\n"
+ "docker instance, or specify --db_host (and, optionally, --db_port) to identify\n"
+ "the location of a running instance. To start a long-lived instance (suitable\n"
+ "for running this command multiple times) run this:\n\n"
+ " docker run --rm --name some-postgres -e POSTGRES_PASSWORD=domain-registry \\\n"
+ " -d "
+ NomulusPostgreSql.getDockerTag()
+ "\n\nCopy the container id output from the command, then run:\n\n"
+ " docker inspect <container-id> | grep IPAddress\n\n"
+ "To obtain the value for --db-host.\n");
// TODO(mmuller): need exit(1), see above.
return false;
}
// use the default port if non has been defined.
if (databasePort == null) {
databasePort = POSTGRESQL_PORT;
}
return true;
}
@Override
public void run() throws Exception {
if (!initializeDatabase()) {
return;
}
try {
runCommand();
} finally {
if (postgresContainer != null) {
postgresContainer.stop();
}
}
}
/** Called after the container has been created but before it has been started. */
protected void onContainerCreate() throws Exception {}
/** Command to be run while the database is running. */
abstract void runCommand() throws Exception;
}

View file

@ -0,0 +1,39 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import static com.google.common.truth.Truth.assertThat;
import java.io.File;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class DumpGoldenSchemaCommandTest extends CommandTestCase<DumpGoldenSchemaCommand> {
@Rule public TemporaryFolder tmp = new TemporaryFolder();
public DumpGoldenSchemaCommandTest() {}
@Test
public void testSchemaGeneration() throws Exception {
runCommand(
"--output=" + tmp.getRoot() + File.separatorChar + "golden.sql", "--start_postgresql");
assertThat(new File(tmp.getRoot(), "golden.sql").length()).isGreaterThan(1);
}
}