From 7fd7828cd81473b220a6cf2dc3d07e5c92dd3d54 Mon Sep 17 00:00:00 2001 From: Michael Muller Date: Mon, 18 Oct 2021 08:10:09 -0400 Subject: [PATCH] Fix problems with the format tasks (#1390) * Fix problems with the format tasks The format check is using python2, and if "python" doesn't exist on the path (or isn't python 2, or there is any other error in the python code or in the shell script...) the format check just succeeds. This change: - Refactors out the gradle code that finds a python3 executable and use it to get the python executable to be used for the format check. - Upgrades google-java-format-diff.py to python3 and removes #! line. - Fixes shell script to ensure that failures are propagated. - Suppresses error output when checking for python commands. Tested: - verified that python errors cause the build to fail - verified that introducing a bad format diff causes check to fail - verified that javaIncrementalFormatDryRun shows the diffs that would be introduced. - verified that javaIncrementalFormatApply reformats a file. - verified that well formatted code passes the format check. - verified that an invalid or missing PYTHON env var causes google-java-format-git-diff.sh to fail with the appropriate error. * Fix presubmit issues Omit the format presubmit when not in a git repo and remove unused "string" import. --- build.gradle | 62 +++++++++++++--------- java-format/google-java-format-diff.py | 8 ++- java-format/google-java-format-git-diff.sh | 28 ++++++++-- 3 files changed, 63 insertions(+), 35 deletions(-) diff --git a/build.gradle b/build.gradle index 3c17d6f20..e13f65690 100644 --- a/build.gradle +++ b/build.gradle @@ -200,11 +200,28 @@ rootProject.ext { pyver = { exe -> try { ext.execInBash( - exe + " -c 'import sys; print(sys.hexversion)'", "/") as Integer + exe + " -c 'import sys; print(sys.hexversion)' 2>/dev/null", + "/") as Integer } catch (org.gradle.process.internal.ExecException e) { return -1; } } + + // Return the path to a usable python3 executable. + getPythonExecutable = { + // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we + // know we'd like at least 3.6, but 3.7.3 is the latest that ships with + // Debian so it seems like that should be available anywhere). + def MIN_PY_VER = 0x3070300 + if (pyver('python') >= MIN_PY_VER) { + return 'python' + } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) { + return '/usr/bin/python3' + } else { + throw new GradleException("No usable Python version found (build " + + "requires at least python 3.7.3)"); + } + } } task runPresubmits(type: Exec) { @@ -212,18 +229,7 @@ task runPresubmits(type: Exec) { args('config/presubmits.py') doFirst { - // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we - // know we'd like at least 3.6, but 3.7.3 is the latest that ships with - // Debian so it seems like that should be available anywhere). - def MIN_PY_VER = 0x3070300 - if (pyver('python') >= MIN_PY_VER) { - executable 'python' - } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) { - executable '/usr/bin/python3' - } else { - throw new GradleException("No usable Python version found (build " + - "requires at least python 3.7.3)"); - } + executable getPythonExecutable() } } @@ -438,9 +444,10 @@ rootProject.ext { ? "${rootDir}/.." : rootDir def formatDiffScript = "${scriptDir}/google-java-format-git-diff.sh" + def pythonExe = getPythonExecutable() return ext.execInBash( - "${formatDiffScript} ${action}", "${workingDir}") + "PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}") } } @@ -448,18 +455,23 @@ rootProject.ext { // Note that this task checks modified Java files in the entire repository. task javaIncrementalFormatCheck { doLast { - def checkResult = invokeJavaDiffFormatScript("check") - if (checkResult == 'true') { - throw new IllegalStateException( - "Some Java files need to be reformatted. You may use the " - + "'javaIncrementalFormatDryRun' task to review\n " - + "the changes, or the 'javaIncrementalFormatApply' task " - + "to reformat.") - } else if (checkResult != 'false') { - throw new RuntimeException( - "Failed to invoke format check script:\n" + checkResult) + // We can only do this in a git tree. + if (new File("${rootDir}/.git").exists()) { + def checkResult = invokeJavaDiffFormatScript("check") + if (checkResult == 'true') { + throw new IllegalStateException( + "Some Java files need to be reformatted. You may use the " + + "'javaIncrementalFormatDryRun' task to review\n " + + "the changes, or the 'javaIncrementalFormatApply' task " + + "to reformat.") + } else if (checkResult != 'false') { + throw new RuntimeException( + "Failed to invoke format check script:\n" + checkResult) + } + println("Incremental Java format check ok.") + } else { + println("Omitting format check: not in a git directory.") } - println("Incremental Java format check ok.") } } diff --git a/java-format/google-java-format-diff.py b/java-format/google-java-format-diff.py index 7cd540447..ebb3558db 100755 --- a/java-format/google-java-format-diff.py +++ b/java-format/google-java-format-diff.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python2.7 # #===- google-java-format-diff.py - google-java-format Diff Reformatter -----===# # @@ -24,7 +23,6 @@ For perforce users: import argparse import difflib import re -import string import subprocess import io import sys @@ -99,7 +97,7 @@ def main(): base_command = [binary] # Reformat files containing changes in place. - for filename, lines in lines_by_file.iteritems(): + for filename, lines in lines_by_file.items(): if args.i and args.verbose: print('Formatting ' + filename) command = base_command[:] @@ -120,11 +118,11 @@ def main(): if not args.i: with open(filename) as f: code = f.readlines() - formatted_code = io.BytesIO(stdout).readlines() + formatted_code = io.StringIO(stdout.decode()).readlines() diff = difflib.unified_diff(code, formatted_code, filename, filename, '(before formatting)', '(after formatting)') - diff_string = string.join(diff, '') + diff_string = ''.join(diff) if len(diff_string) > 0: sys.stdout.write(diff_string) diff --git a/java-format/google-java-format-git-diff.sh b/java-format/google-java-format-git-diff.sh index 366646626..d5d861db8 100755 --- a/java-format/google-java-format-git-diff.sh +++ b/java-format/google-java-format-git-diff.sh @@ -42,6 +42,16 @@ where: SCRIPT_DIR="$(realpath $(dirname $0))" JAR_NAME="google-java-format-1.8-all-deps.jar" +# Make sure we have a valid python interpreter. +if [ -z "$PYTHON" ]; then + echo "You must specify the name of a python3 interpreter in the PYTHON" \ + "environment variable." + exit 1 +elif ! "$PYTHON" -c ''; then + echo "Invalid python interpreter: $PYTHON" + exit 1 +fi + # Locate the java binary. if [ -n "$JAVA_HOME" ]; then JAVA_BIN="$JAVA_HOME/bin/java" @@ -69,10 +79,14 @@ function runGoogleJavaFormatAgainstDiffs() { shift git diff -U0 "$forkPoint" | \ - ${SCRIPT_DIR}/google-java-format-diff.py \ - --java-binary "$JAVA_BIN" \ - --google-java-format-jar "${SCRIPT_DIR}/${JAR_NAME}" \ - -p1 "$@" | tee gjf.out + "${PYTHON}" "${SCRIPT_DIR}/google-java-format-diff.py" \ + --java-binary "$JAVA_BIN" \ + --google-java-format-jar "${SCRIPT_DIR}/${JAR_NAME}" \ + -p1 "$@" | \ + tee gjf.out + + # If any of the commands in the last pipe failed, return false. + [[ ! "${PIPESTATUS[@]}" =~ [^0\ ] ]] } # Show the file names in a diff preceeded by a message. @@ -96,7 +110,11 @@ function callGoogleJavaFormatDiff() { local callResult case "$1" in "check") - local output=$(runGoogleJavaFormatAgainstDiffs "$forkPoint") + # We need to do explicit checks for an error and "exit 1" if there was + # one here (though not elsewhere), "set -e" doesn't catch this case, + # it's not clear why. + local output + output=$(runGoogleJavaFormatAgainstDiffs "$forkPoint") || exit 1 echo "$output" | showFileNames "\033[1mNeeds formatting: " callResult=$(echo -n "$output" | wc -l) ;;