Add the :nom:generate_golden_schema pseudo-task (#718)

Add a "pseudo-task" in nom_build to do the three step process of generating
the golden schema.  In the course of this, add support for pseudo-tasks in
general, improve the database directory readme and make nom_build not call
gradlew if there are no tasks.

config/nom_build.py

@@ -19,6 +19,7 @@ import argparse
 import attr
 import io
 import os
+import shutil
 import subprocess
 import sys
 from typing import List, Union
@@ -58,6 +59,21 @@ PROPERTIES_HEADER = """\
 org.gradle.jvmargs=-Xmx1024m
 """
 
+# Help text to be displayed (in addition to the synopsis and flag help, which
+# are displayed automatically).
+HELP_TEXT = """\
+A wrapper around the gradle build that provides the following features:
+
+-   Converts properties into flags to guard against property name spelling errors
+    and to provide help descriptions for all properties.
+-   Provides pseudo-commands (with the ":nom:" prefix) that encapsulate common
+    actions that are difficult to implement in gradle.
+
+Pseudo-commands:
+    :nom:generate_golden_file - regenerates the golden file from the current
+        set of flyway files.
+"""
+
 # Define all of our special gradle properties here.
 PROPERTIES = [
     Property('mavenUrl',
@@ -256,8 +272,42 @@ def get_root() -> str:
     return cur_dir
 
 
-def main(args):
+class Abort(Exception):
-    parser = argparse.ArgumentParser('nom_build')
+    """Raised to terminate the process with a non-zero error code.
+
+    Parameters are ignored.
+    """
+
+
+def do_pseudo_task(task: str) -> None:
+    root = get_root()
+    if task == ':nom:generate_golden_file':
+        if not subprocess.call([f'{root}/gradlew', ':db:test']):
+            print('\033[33mWARNING:\033[0m Golden schema appears to be '
+                  'up-to-date.  If you are making schema changes,  be sure to '
+                  'add a flyway file for them.')
+            return
+        print('\033[33mWARNING:\033[0m Ignore the above failure, it is '
+              'expected.')
+
+        # Copy the new schema into place.
+        shutil.copy(f'{root}/db/build/resources/test/testcontainer/'
+                    'mount/dump.txt',
+                    f'{root}/db/src/main/resources/sql/schema/'
+                    'nomulus.golden.sql')
+
+        if subprocess.call([f'{root}/gradlew', ':db:test']):
+            print('\033[31mERROR:\033[0m Golden file test failed after '
+                  'copying schema.  Please check your flyway files.')
+            raise Abort()
+    else:
+        print(f'\033[31mERROR:\033[0m Unknown task {task}')
+        raise Abort()
+
+
+def main(args) -> int:
+    parser = argparse.ArgumentParser('nom_build', description=HELP_TEXT,
+                                     formatter_class=argparse.RawTextHelpFormatter)
     for prop in PROPERTIES:
         parser.add_argument('--' + prop.name, default=prop.default,
                             help=prop.desc)
@@ -296,7 +346,7 @@ def main(args):
     if args.generate_gradle_properties:
         with open(f'{root}/gradle.properties', 'w') as dst:
             dst.write(gradle_properties)
-        return
+        return 0
 
     # Verify that the gradle properties file is what we expect it to be.
     with open(f'{root}/gradle.properties') as src:
@@ -321,12 +371,39 @@ def main(args):
             if flag.has_arg:
                 gradle_command.append(arg_val)
 
+    # See if there are any special ":nom:" pseudo-tasks specified.
+    got_non_pseudo_tasks = False
+    for arg in args.non_flag_args[1:]:
+        if arg.startswith(':nom:'):
+            if got_non_pseudo_tasks:
+                # We can't currently deal with the situation of gradle tasks
+                # before pseudo-tasks.  This could be implemented by invoking
+                # gradle for only the set of gradle tasks before the pseudo
+                # task, but that's overkill for now.
+                print(f'\033[31mERROR:\033[0m Pseudo task ({arg}) must be '
+                      'specified prior to all actual gradle tasks.  Aborting.')
+                return 1
+            do_pseudo_task(arg)
+        else:
+            got_non_pseudo_tasks = True
+    non_flag_args = [
+        arg for arg in args.non_flag_args[1:] if not arg.startswith(':nom:')]
+
+    if not non_flag_args:
+        if not got_non_pseudo_tasks:
+            print('\033[33mWARNING:\033[0m No tasks specified.  Not '
+                  'doing anything')
+        return 0
+
     # Add the non-flag args (we exclude the first, which is the command name
     # itself) and run.
-    gradle_command.extend(args.non_flag_args[1:])
+    gradle_command.extend(non_flag_args)
-    subprocess.call(gradle_command)
+    return subprocess.call(gradle_command)
 
 
 if __name__ == '__main__':
-    main(sys.argv)
+    try:
+        sys.exit(main(sys.argv))
+    except Abort as ex:
+        sys.exit(1)
 

config/nom_build_test.py

@@ -14,6 +14,7 @@
 
 import io
 import os
+import shutil
 import unittest
 from unittest import mock
 import nom_build
@@ -67,6 +68,7 @@ class MyTest(unittest.TestCase):
             mock.patch.object(nom_build, 'print', self.print_fake).start())
 
         self.call_mock = mock.patch.object(subprocess, 'call').start()
+        self.copy_mock = mock.patch.object(shutil, 'copy').start()
 
         self.file_contents = {
             # Prefil with the actual file contents.
@@ -92,17 +94,32 @@ class MyTest(unittest.TestCase):
 
     def test_no_args(self):
         nom_build.main(['nom_build'])
-        self.assertEqual(self.printed, [])
+        self.assertEqual(self.printed,
-        self.call_mock.assert_called_with([GRADLEW])
+                         ['\x1b[33mWARNING:\x1b[0m No tasks specified.  Not '
+                         'doing anything'])
 
     def test_property_calls(self):
-        nom_build.main(['nom_build', '--testFilter=foo'])
+        nom_build.main(['nom_build', 'task-name', '--testFilter=foo'])
-        self.call_mock.assert_called_with([GRADLEW, '-P', 'testFilter=foo'])
+        self.call_mock.assert_called_with([GRADLEW, '-P', 'testFilter=foo',
+                                          'task-name'])
 
     def test_gradle_flags(self):
-        nom_build.main(['nom_build', '-d', '-b', 'foo'])
+        nom_build.main(['nom_build', 'task-name', '-d', '-b', 'foo'])
         self.call_mock.assert_called_with([GRADLEW, '--build-file', 'foo',
-                                          '--debug'])
+                                          '--debug', 'task-name'])
+
+    def test_generate_golden_file(self):
+        self.call_mock.side_effect = [1, 0]
+        nom_build.main(['nom_build', ':nom:generate_golden_file'])
+        self.call_mock.assert_has_calls([
+            mock.call([GRADLEW, ':db:test']),
+            mock.call([GRADLEW, ':db:test'])
+        ])
+
+    def test_generate_golden_file_nofail(self):
+        self.call_mock.return_value = 0
+        nom_build.main(['nom_build', ':nom:generate_golden_file'])
+        self.call_mock.assert_has_calls([mock.call([GRADLEW, ':db:test'])])
 
 unittest.main()
 

db/README.md

@@ -34,9 +34,7 @@ Below are the steps to submit a schema change:
 2.  Run the `devTool generate_sql_schema` command to generate a new version of
     `db-schema.sql.generated`. The full command line to do this is:
 
-    `./gradlew devTool --args="-e localhost generate_sql_schema
+    `./nom_build generateSqlSchema`
-    --start_postgresql -o
-    /path/to/nomulus/db/src/main/resources/sql/schema/db-schema.sql.generated"`
 
 3.  Write an incremental DDL script that changes the existing schema to your new
     one. The generated SQL file from the previous step should help. New create
@@ -49,14 +47,18 @@ Below are the steps to submit a schema change:
     following the existing scripts in that folder. Note the double underscore in
     the naming pattern.
 
-4.  Run the `:db:test` task from the Gradle root project. The SchemaTest will
+4.  Run `./nom_build :nom:generate_golden_file`.  This is a pseudo-task
-    fail because the new schema does not match the golden file.
+    implemented in the `nom_build` script that does the following:
+    -   Runs the `:db:test` task from the Gradle root project. The SchemaTest
+        will fail because the new schema does not match the golden file.
 
-5.  Copy `db/build/resources/test/testcontainer/mount/dump.txt` to the golden
+    -   Copies `db/build/resources/test/testcontainer/mount/dump.txt` to the golden
-    file `db/src/main/resources/sql/schema/nomulus.golden.sql`. Diff it against
+        file `db/src/main/resources/sql/schema/nomulus.golden.sql`.
-    the old version and verify that all changes are expected.
 
-6.  Re-run the `:db:test` task. This time all tests should pass.
+    -   Re-runs the `:db:test` task. This time all tests should pass.
+
+    You'll want to have a look at the diffs in the golden schema to verify
+    that all changes are intentional.
 
 Relevant files (under db/src/main/resources/sql/schema/):
 
@@ -97,7 +99,7 @@ gcloud builds submit --config=release/cloudbuild-schema-deploy.yaml \
     --substitutions=TAG_NAME=${SCHEMA_TAG},_ENV=${SQL_ENV} \
     --project domain-registry-dev
 # Verify by checking Flyway Schema History:
-./gradlew :db:flywayInfo -PdbServer=${SQL_ENV}
+./nom_build :db:flywayInfo --dbServer=${SQL_ENV}
 ```
 
 #### Glass Breaking
@@ -135,9 +137,9 @@ test instance. E.g.,
 
 ```shell
 # Deploy to a local instance at standard port as the super user.
-gradlew :db:flywayMigrate -PdbServer=192.168.9.2 -PdbPassword=domain-registry
+./nom_build :db:flywayMigrate --dbServer=192.168.9.2 --dbPassword=domain-registry
 
 # Full specification of all parameters
-gradlew :db:flywayMigrate -PdbServer=192.168.9.2:5432 -PdbUser=postgres \
+./nom_build :db:flywayMigrate --dbServer=192.168.9.2:5432 --dbUser=postgres \
-    -PdbPassword=domain-registry
+    --dbPassword=domain-registry
 ```