diff --git a/config/nom_build.py b/config/nom_build.py new file mode 100644 index 000000000..2ce896ddb --- /dev/null +++ b/config/nom_build.py @@ -0,0 +1,327 @@ +# Copyright 2020 The Nomulus Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script to generate dr-build and the properties file. +""" + +import argparse +import attr +import io +import os +import subprocess +import sys +from typing import List, Union + + +@attr.s(auto_attribs=True) +class Property: + name : str = '' + desc : str = '' + default : str = '' + constraints : type = str + + def validate(self, value: str): + """Verify that "value" is appropriate for the property.""" + if type is bool: + if value not in ('true', 'false'): + raise ValidationError('value of {self.name} must be "true" or ' + '"false".') + +@attr.s(auto_attribs=True) +class GradleFlag: + flags : Union[str, List[str]] + desc : str + has_arg : bool = False + + +PROPERTIES_HEADER = """\ +# This file defines properties used by the gradle build. It must be kept in +# sync with config/nom_build.py. +# +# To regenerate, run config/nom_build.py --generate-gradle-properties +# +# To view property descriptions (which are command line flags for +# nom_build), run config/nom_build.py --help. +# +# DO NOT EDIT THIS FILE BY HAND +org.gradle.jvmargs=-Xmx1024m +""" + +# Define all of our special gradle properties here. +PROPERTIES = [ + Property('mavenUrl', + 'URL to use for the main maven repository (defaults to maven ' + 'central). This can be http(s) or a "gcs" repo.'), + Property('pluginsUrl', + 'URL to use for the gradle plugins repository (defaults to maven ' + 'central, see also mavenUrl'), + Property('uploaderDestination', + 'Location to upload test reports to. Normally this should be a ' + 'GCS url (see also uploaderCredentialsFile)'), + Property('uploaderCredentialsFile', + 'json credentials file to use to upload test reports.'), + Property('uploaderMultithreadedUpload', + 'Whether to enable multithread upload.'), + Property('verboseTestOutput', + 'If true, show all test output in near-realtime.', + 'false', + bool), + Property('flowDocsFile', + 'Output filename for the flowDocsTool command.'), + Property('enableDependencyLocking', + 'Enables dependency locking.', + 'true', + bool), + Property('enableCrossReferencing', + 'generate metadata during java compile (used for kythe source ' + 'reference generation).', + 'false'), + Property('testFilter', + 'Comma separated list of test patterns, if specified run only ' + 'these.'), + Property('environment', 'GAE Environment for deployment and staging.'), + + # Cloud SQL properties + Property('dbServer', + 'A registry environment name (e.g., "alpha") or a host[:port] ' + 'string'), + Property('dbName', + 'Database name to use in connection.', + 'postgres'), + Property('dbUser', 'Database user name for use in connection'), + Property('dbPassword', 'Database password for use in connection'), + + Property('publish_repo', + 'Maven repository that hosts the Cloud SQL schema jar and the ' + 'registry server test jars. Such jars are needed for ' + 'server/schema integration tests. Please refer to integration project for more ' + 'information.'), + Property('schema_version', + 'The nomulus version tag of the schema for use in a database' + 'integration test.'), + Property('nomulus_version', + 'The version of nomulus to test against in a database ' + 'integration test.'), +] + +GRADLE_FLAGS = [ + GradleFlag(['-a', '--no-rebuild'], + 'Do not rebuild project dependencies.'), + GradleFlag(['-b', '--build-file'], 'Specify the build file.', True), + GradleFlag(['--build-cache'], + 'Enables the Gradle build cache. Gradle will try to reuse ' + 'outputs from previous builds.'), + GradleFlag(['-c', '--settings-file'], 'Specify the settings file.', True), + GradleFlag(['--configure-on-demand'], + 'Configure necessary projects only. Gradle will attempt to ' + 'reduce configuration time for large multi-project builds. ' + '[incubating]'), + GradleFlag(['--console'], + 'Specifies which type of console output to generate. Values ' + "are 'plain', 'auto' (default), 'rich' or 'verbose'.", + True), + GradleFlag(['--continue'], 'Continue task execution after a task failure.'), + GradleFlag(['-D', '--system-prop'], + 'Set system property of the JVM (e.g. -Dmyprop=myvalue).', + True), + GradleFlag(['-d', '--debug'], + 'Log in debug mode (includes normal stacktrace).'), + GradleFlag(['--daemon'], + 'Uses the Gradle Daemon to run the build. Starts the Daemon ' + 'if not running.'), + GradleFlag(['--foreground'], 'Starts the Gradle Daemon in the foreground.'), + GradleFlag(['-g', '--gradle-user-home'], + 'Specifies the gradle user home directory.', + True), + GradleFlag(['-I', '--init-script'], 'Specify an initialization script.', + True), + GradleFlag(['-i', '--info'], 'Set log level to info.'), + GradleFlag(['--include-build'], + 'Include the specified build in the composite.', + True), + GradleFlag(['-m', '--dry-run'], + 'Run the builds with all task actions disabled.'), + GradleFlag(['--max-workers'], + 'Configure the number of concurrent workers Gradle is ' + 'allowed to use.', + True), + GradleFlag(['--no-build-cache'], 'Disables the Gradle build cache.'), + GradleFlag(['--no-configure-on-demand'], + 'Disables the use of configuration on demand. [incubating]'), + GradleFlag(['--no-daemon'], + 'Do not use the Gradle daemon to run the build. Useful ' + 'occasionally if you have configured Gradle to always run ' + 'with the daemon by default.'), + GradleFlag(['--no-parallel'], + 'Disables parallel execution to build projects.'), + GradleFlag(['--no-scan'], + 'Disables the creation of a build scan. For more information ' + 'about build scans, please visit ' + 'https://gradle.com/build-scans.'), + GradleFlag(['--offline'], + 'Execute the build without accessing network resources.'), + GradleFlag(['-P', '--project-prop'], + 'Set project property for the build script (e.g. ' + '-Pmyprop=myvalue).', + True), + GradleFlag(['-p', '--project-dir'], + 'Specifies the start directory for Gradle. Defaults to ' + 'current directory.'), + GradleFlag(['--parallel'], + 'Build projects in parallel. Gradle will attempt to ' + 'determine the optimal number of executor threads to use.'), + GradleFlag(['--priority'], + 'Specifies the scheduling priority for the Gradle daemon and ' + "all processes launched by it. Values are 'normal' (default) " + "or 'low' [incubating]", + True), + GradleFlag(['--profile'], + 'Profile build execution time and generates a report in the ' + '/reports/profile directory.'), + GradleFlag(['--project-cache-dir'], + 'Specify the project-specific cache directory. Defaults to ' + '.gradle in the root project directory.', + True), + GradleFlag(['-q', '--quiet'], 'Log errors only.'), + GradleFlag(['--refresh-dependencies'], 'Refresh the state of dependencies.'), + GradleFlag(['--rerun-tasks'], 'Ignore previously cached task results.'), + GradleFlag(['-S', '--full-stacktrace'], + 'Print out the full (very verbose) stacktrace for all ' + 'exceptions.'), + GradleFlag(['-s', '--stacktrace'], + 'Print out the stacktrace for all exceptions.'), + GradleFlag(['--scan'], + 'Creates a build scan. Gradle will emit a warning if the ' + 'build scan plugin has not been applied. ' + '(https://gradle.com/build-scans)'), + GradleFlag(['--status'], + 'Shows status of running and recently stopped Gradle ' + 'Daemon(s).'), + GradleFlag(['--stop'], 'Stops the Gradle Daemon if it is running.'), + GradleFlag(['-t', '--continuous'], + 'Enables continuous build. Gradle does not exit and will ' + 're-execute tasks when task file inputs change.'), + GradleFlag(['--update-locks'], + 'Perform a partial update of the dependency lock, letting ' + 'passed in module notations change version. [incubating]'), + GradleFlag(['-v', '--version'], 'Print version info.'), + GradleFlag(['-w', '--warn'], 'Set log level to warn.'), + GradleFlag(['--warning-mode'], + 'Specifies which mode of warnings to generate. Values are ' + "'all', 'fail', 'summary'(default) or 'none'", + True), + GradleFlag(['--write-locks'], + 'Persists dependency resolution for locked configurations, ' + 'ignoring existing locking information if it exists ' + '[incubating]'), + GradleFlag(['-x', '--exclude-task'], + 'Specify a task to be excluded from execution.', + True), +] +def generate_gradle_properties() -> str: + """Returns the expected contents of gradle.properties.""" + out = io.StringIO() + out.write(PROPERTIES_HEADER) + + for prop in PROPERTIES: + out.write(f'{prop.name}={prop.default}\n') + + return out.getvalue() + + +def get_root() -> str: + """Returns the root of the nomulus build tree.""" + cur_dir = os.getcwd() + if not os.path.exists(os.path.join(cur_dir, '.git')) or \ + not os.path.exists(os.path.join(cur_dir, 'core')) or \ + not os.path.exists(os.path.join(cur_dir, 'gradle.properties')): + raise Exception('You must run this script from the root directory') + return cur_dir + + +def main(args): + parser = argparse.ArgumentParser('nom_build') + for prop in PROPERTIES: + parser.add_argument('--' + prop.name, default=prop.default, + help=prop.desc) + + # Add Gradle flags. We set 'dest' to the first flag to get a name that is + # predictable for getattr (even though it will have a leading '-' and thus + # we can't use normal python attribute syntax to get it). + for flag in GRADLE_FLAGS: + if flag.has_arg: + parser.add_argument(*flag.flags, dest=flag.flags[0], + help=flag.desc) + else: + parser.add_argument(*flag.flags, dest=flag.flags[0], + help=flag.desc, + action='store_true') + + # Add a flag to regenerate the gradle properties file. + parser.add_argument('--generate-gradle-properties', + help='Regenerate the gradle.properties file. This ' + 'file must be regenerated when changes are made to ' + 'config/nom_build.py, and should not be updated by ' + 'hand.', + action='store_true') + + # Consume the remaining non-flag arguments. + parser.add_argument('non_flag_args', nargs='*') + + # Parse command line arguments. Note that this exits the program and + # prints usage if either of the help options (-h, --help) are specified. + args = parser.parse_args(args) + + gradle_properties = generate_gradle_properties() + root = get_root() + + # If we're regenerating properties, do so and exit. + if args.generate_gradle_properties: + with open(f'{root}/gradle.properties', 'w') as dst: + dst.write(gradle_properties) + return + + # Verify that the gradle properties file is what we expect it to be. + with open(f'{root}/gradle.properties') as src: + if src.read() != gradle_properties: + print('\033[33mWARNING:\033[0m Gradle properties out of sync ' + 'with nom_build. Run with --generate-gradle-properties ' + 'to regenerate.') + + # Add properties to the gradle argument list. + gradle_command = [f'{root}/gradlew'] + for prop in PROPERTIES: + arg_val = getattr(args, prop.name) + if arg_val != prop.default: + prop.validate(arg_val) + gradle_command.extend(['-P', f'{prop.name}={arg_val}']) + + # Add Gradle flags to the gradle argument list. + for flag in GRADLE_FLAGS: + arg_val = getattr(args, flag.flags[0]) + if arg_val: + gradle_command.append(flag.flags[-1]) + if flag.has_arg: + gradle_command.append(arg_val) + + # Add the non-flag args (we exclude the first, which is the command name + # itself) and run. + gradle_command.extend(args.non_flag_args[1:]) + subprocess.call(gradle_command) + + +if __name__ == '__main__': + main(sys.argv) + diff --git a/config/nom_build_test.py b/config/nom_build_test.py new file mode 100644 index 000000000..9745cfe7c --- /dev/null +++ b/config/nom_build_test.py @@ -0,0 +1,109 @@ +# Copyright 2020 The Nomulus Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import os +import unittest +from unittest import mock +import nom_build +import subprocess + +FAKE_PROPERTIES = [ + nom_build.Property('foo', 'help text'), + nom_build.Property('bar', 'more text', 'true', bool), +] + +FAKE_PROP_CONTENTS = nom_build.PROPERTIES_HEADER + 'foo=\nbar=true\n' +PROPERTIES_FILENAME = '/tmp/rootdir/gradle.properties' +GRADLEW = '/tmp/rootdir/gradlew' + + +class FileFake(io.StringIO): + """File fake that writes file contents to the dictionary on close.""" + def __init__(self, contents_dict, filename): + self.dict = contents_dict + self.filename = filename + super(FileFake, self).__init__() + + def close(self): + self.dict[self.filename] = self.getvalue() + super(FileFake, self).close() + + +class MyTest(unittest.TestCase): + + def open_fake(self, filename, action='r'): + if action == 'r': + return io.StringIO(self.file_contents.get(filename, '')) + elif action == 'w': + result = self.file_contents[filename] = ( + FileFake(self.file_contents, filename)) + return result + else: + raise Exception(f'Unexpected action {action}') + + def print_fake(self, data): + self.printed.append(data) + + def setUp(self): + self.addCleanup(mock.patch.stopall) + self.exists_mock = mock.patch.object(os.path, 'exists').start() + self.getcwd_mock = mock.patch.object(os, 'getcwd').start() + self.getcwd_mock.return_value = '/tmp/rootdir' + self.open_mock = ( + mock.patch.object(nom_build, 'open', self.open_fake).start()) + self.print_mock = ( + mock.patch.object(nom_build, 'print', self.print_fake).start()) + + self.call_mock = mock.patch.object(subprocess, 'call').start() + + self.file_contents = { + # Prefil with the actual file contents. + PROPERTIES_FILENAME: nom_build.generate_gradle_properties() + } + self.printed = [] + + @mock.patch.object(nom_build, 'PROPERTIES', FAKE_PROPERTIES) + def test_property_generation(self): + self.assertEqual(nom_build.generate_gradle_properties(), + FAKE_PROP_CONTENTS) + + @mock.patch.object(nom_build, 'PROPERTIES', FAKE_PROPERTIES) + def test_property_file_write(self): + nom_build.main(['nom_build', '--generate-gradle-properties']) + self.assertEqual(self.file_contents[PROPERTIES_FILENAME], + FAKE_PROP_CONTENTS) + + def test_property_file_incorrect(self): + self.file_contents[PROPERTIES_FILENAME] = 'bad contents' + nom_build.main(['nom_build']) + self.assertIn('', self.printed[0]) + + def test_no_args(self): + nom_build.main(['nom_build']) + self.assertEqual(self.printed, []) + self.call_mock.assert_called_with([GRADLEW]) + + def test_property_calls(self): + nom_build.main(['nom_build', '--testFilter=foo']) + self.call_mock.assert_called_with([GRADLEW, '-P', 'testFilter=foo']) + + def test_gradle_flags(self): + nom_build.main(['nom_build', '-d', '-b', 'foo']) + self.call_mock.assert_called_with([GRADLEW, '--build-file', 'foo', + '--debug']) + +unittest.main() + + diff --git a/gradle.properties b/gradle.properties index c5833626d..6400bcb31 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,12 @@ +# This file defines properties used by the gradle build. It must be kept in +# sync with config/nom_build.py. +# +# To regenerate, run config/nom_build.py --generate-gradle-properties +# +# To view property descriptions (which are command line flags for +# nom_build), run config/nom_build.py --help. +# +# DO NOT EDIT THIS FILE BY HAND org.gradle.jvmargs=-Xmx1024m mavenUrl= pluginsUrl= @@ -8,25 +17,12 @@ verboseTestOutput=false flowDocsFile= enableDependencyLocking=true enableCrossReferencing=false - -# Comma separated list of test patterns, if specified run only these. testFilter= - -# GAE Environment for deployment and staging. environment= - -# Cloud SQL properties - -# A registry environment name (e.g., 'alpha') or a host[:port] string dbServer= dbName=postgres dbUser= dbPassword= - -# Maven repository that hosts the Cloud SQL schema jar and the registry -# server test jars. Such jars are needed for server/schema integration tests. -# Please refer to integration project -# for more information. publish_repo= schema_version= nomulus_version= diff --git a/nom_build b/nom_build new file mode 100755 index 000000000..ae69b4e28 --- /dev/null +++ b/nom_build @@ -0,0 +1,5 @@ +#!/bin/sh +# Wrapper for nom_build.py. +cd $(dirname $0) +python3 ./config/nom_build.py "$@" +exit $?