diff --git a/config/nom_build.py b/config/nom_build.py index 8bb0584ab..76c4cf909 100644 --- a/config/nom_build.py +++ b/config/nom_build.py @@ -19,6 +19,7 @@ import argparse import attr import io import os +import shutil import subprocess import sys from typing import List, Union @@ -58,6 +59,21 @@ PROPERTIES_HEADER = """\ org.gradle.jvmargs=-Xmx1024m """ +# Help text to be displayed (in addition to the synopsis and flag help, which +# are displayed automatically). +HELP_TEXT = """\ +A wrapper around the gradle build that provides the following features: + +- Converts properties into flags to guard against property name spelling errors + and to provide help descriptions for all properties. +- Provides pseudo-commands (with the ":nom:" prefix) that encapsulate common + actions that are difficult to implement in gradle. + +Pseudo-commands: + :nom:generate_golden_file - regenerates the golden file from the current + set of flyway files. +""" + # Define all of our special gradle properties here. PROPERTIES = [ Property('mavenUrl', @@ -256,8 +272,42 @@ def get_root() -> str: return cur_dir -def main(args): - parser = argparse.ArgumentParser('nom_build') +class Abort(Exception): + """Raised to terminate the process with a non-zero error code. + + Parameters are ignored. + """ + + +def do_pseudo_task(task: str) -> None: + root = get_root() + if task == ':nom:generate_golden_file': + if not subprocess.call([f'{root}/gradlew', ':db:test']): + print('\033[33mWARNING:\033[0m Golden schema appears to be ' + 'up-to-date. If you are making schema changes, be sure to ' + 'add a flyway file for them.') + return + print('\033[33mWARNING:\033[0m Ignore the above failure, it is ' + 'expected.') + + # Copy the new schema into place. + shutil.copy(f'{root}/db/build/resources/test/testcontainer/' + 'mount/dump.txt', + f'{root}/db/src/main/resources/sql/schema/' + 'nomulus.golden.sql') + + if subprocess.call([f'{root}/gradlew', ':db:test']): + print('\033[31mERROR:\033[0m Golden file test failed after ' + 'copying schema. Please check your flyway files.') + raise Abort() + else: + print(f'\033[31mERROR:\033[0m Unknown task {task}') + raise Abort() + + +def main(args) -> int: + parser = argparse.ArgumentParser('nom_build', description=HELP_TEXT, + formatter_class=argparse.RawTextHelpFormatter) for prop in PROPERTIES: parser.add_argument('--' + prop.name, default=prop.default, help=prop.desc) @@ -296,7 +346,7 @@ def main(args): if args.generate_gradle_properties: with open(f'{root}/gradle.properties', 'w') as dst: dst.write(gradle_properties) - return + return 0 # Verify that the gradle properties file is what we expect it to be. with open(f'{root}/gradle.properties') as src: @@ -321,12 +371,39 @@ def main(args): if flag.has_arg: gradle_command.append(arg_val) + # See if there are any special ":nom:" pseudo-tasks specified. + got_non_pseudo_tasks = False + for arg in args.non_flag_args[1:]: + if arg.startswith(':nom:'): + if got_non_pseudo_tasks: + # We can't currently deal with the situation of gradle tasks + # before pseudo-tasks. This could be implemented by invoking + # gradle for only the set of gradle tasks before the pseudo + # task, but that's overkill for now. + print(f'\033[31mERROR:\033[0m Pseudo task ({arg}) must be ' + 'specified prior to all actual gradle tasks. Aborting.') + return 1 + do_pseudo_task(arg) + else: + got_non_pseudo_tasks = True + non_flag_args = [ + arg for arg in args.non_flag_args[1:] if not arg.startswith(':nom:')] + + if not non_flag_args: + if not got_non_pseudo_tasks: + print('\033[33mWARNING:\033[0m No tasks specified. Not ' + 'doing anything') + return 0 + # Add the non-flag args (we exclude the first, which is the command name # itself) and run. - gradle_command.extend(args.non_flag_args[1:]) - subprocess.call(gradle_command) + gradle_command.extend(non_flag_args) + return subprocess.call(gradle_command) if __name__ == '__main__': - main(sys.argv) + try: + sys.exit(main(sys.argv)) + except Abort as ex: + sys.exit(1) diff --git a/config/nom_build_test.py b/config/nom_build_test.py index 9745cfe7c..495e53011 100644 --- a/config/nom_build_test.py +++ b/config/nom_build_test.py @@ -14,6 +14,7 @@ import io import os +import shutil import unittest from unittest import mock import nom_build @@ -67,6 +68,7 @@ class MyTest(unittest.TestCase): mock.patch.object(nom_build, 'print', self.print_fake).start()) self.call_mock = mock.patch.object(subprocess, 'call').start() + self.copy_mock = mock.patch.object(shutil, 'copy').start() self.file_contents = { # Prefil with the actual file contents. @@ -92,17 +94,32 @@ class MyTest(unittest.TestCase): def test_no_args(self): nom_build.main(['nom_build']) - self.assertEqual(self.printed, []) - self.call_mock.assert_called_with([GRADLEW]) + self.assertEqual(self.printed, + ['\x1b[33mWARNING:\x1b[0m No tasks specified. Not ' + 'doing anything']) def test_property_calls(self): - nom_build.main(['nom_build', '--testFilter=foo']) - self.call_mock.assert_called_with([GRADLEW, '-P', 'testFilter=foo']) + nom_build.main(['nom_build', 'task-name', '--testFilter=foo']) + self.call_mock.assert_called_with([GRADLEW, '-P', 'testFilter=foo', + 'task-name']) def test_gradle_flags(self): - nom_build.main(['nom_build', '-d', '-b', 'foo']) + nom_build.main(['nom_build', 'task-name', '-d', '-b', 'foo']) self.call_mock.assert_called_with([GRADLEW, '--build-file', 'foo', - '--debug']) + '--debug', 'task-name']) + + def test_generate_golden_file(self): + self.call_mock.side_effect = [1, 0] + nom_build.main(['nom_build', ':nom:generate_golden_file']) + self.call_mock.assert_has_calls([ + mock.call([GRADLEW, ':db:test']), + mock.call([GRADLEW, ':db:test']) + ]) + + def test_generate_golden_file_nofail(self): + self.call_mock.return_value = 0 + nom_build.main(['nom_build', ':nom:generate_golden_file']) + self.call_mock.assert_has_calls([mock.call([GRADLEW, ':db:test'])]) unittest.main() diff --git a/db/README.md b/db/README.md index 944979fb2..297b9fd2d 100644 --- a/db/README.md +++ b/db/README.md @@ -34,9 +34,7 @@ Below are the steps to submit a schema change: 2. Run the `devTool generate_sql_schema` command to generate a new version of `db-schema.sql.generated`. The full command line to do this is: - `./gradlew devTool --args="-e localhost generate_sql_schema - --start_postgresql -o - /path/to/nomulus/db/src/main/resources/sql/schema/db-schema.sql.generated"` + `./nom_build generateSqlSchema` 3. Write an incremental DDL script that changes the existing schema to your new one. The generated SQL file from the previous step should help. New create @@ -49,14 +47,18 @@ Below are the steps to submit a schema change: following the existing scripts in that folder. Note the double underscore in the naming pattern. -4. Run the `:db:test` task from the Gradle root project. The SchemaTest will - fail because the new schema does not match the golden file. +4. Run `./nom_build :nom:generate_golden_file`. This is a pseudo-task + implemented in the `nom_build` script that does the following: + - Runs the `:db:test` task from the Gradle root project. The SchemaTest + will fail because the new schema does not match the golden file. -5. Copy `db/build/resources/test/testcontainer/mount/dump.txt` to the golden - file `db/src/main/resources/sql/schema/nomulus.golden.sql`. Diff it against - the old version and verify that all changes are expected. + - Copies `db/build/resources/test/testcontainer/mount/dump.txt` to the golden + file `db/src/main/resources/sql/schema/nomulus.golden.sql`. -6. Re-run the `:db:test` task. This time all tests should pass. + - Re-runs the `:db:test` task. This time all tests should pass. + + You'll want to have a look at the diffs in the golden schema to verify + that all changes are intentional. Relevant files (under db/src/main/resources/sql/schema/): @@ -97,7 +99,7 @@ gcloud builds submit --config=release/cloudbuild-schema-deploy.yaml \ --substitutions=TAG_NAME=${SCHEMA_TAG},_ENV=${SQL_ENV} \ --project domain-registry-dev # Verify by checking Flyway Schema History: -./gradlew :db:flywayInfo -PdbServer=${SQL_ENV} +./nom_build :db:flywayInfo --dbServer=${SQL_ENV} ``` #### Glass Breaking @@ -135,9 +137,9 @@ test instance. E.g., ```shell # Deploy to a local instance at standard port as the super user. -gradlew :db:flywayMigrate -PdbServer=192.168.9.2 -PdbPassword=domain-registry +./nom_build :db:flywayMigrate --dbServer=192.168.9.2 --dbPassword=domain-registry # Full specification of all parameters -gradlew :db:flywayMigrate -PdbServer=192.168.9.2:5432 -PdbUser=postgres \ - -PdbPassword=domain-registry +./nom_build :db:flywayMigrate --dbServer=192.168.9.2:5432 --dbUser=postgres \ + --dbPassword=domain-registry ```