Fix problems with the format tasks (#1390)

* Fix problems with the format tasks

The format check is using python2, and if "python" doesn't exist on the path
(or isn't python 2, or there is any other error in the python code or in the
shell script...) the format check just succeeds.

This change:
- Refactors out the gradle code that finds a python3 executable and use it
  to get the python executable to be used for the format check.
- Upgrades google-java-format-diff.py to python3 and removes #! line.
- Fixes shell script to ensure that failures are propagated.
- Suppresses error output when checking for python commands.

Tested:
- verified that python errors cause the build to fail
- verified that introducing a bad format diff causes check to fail
- verified that javaIncrementalFormatDryRun shows the diffs that would be
  introduced.
- verified that javaIncrementalFormatApply reformats a file.
- verified that well formatted code passes the format check.
- verified that an invalid or missing PYTHON env var causes
  google-java-format-git-diff.sh to fail with the appropriate error.

* Fix presubmit issues

Omit the format presubmit when not in a git repo and remove unused "string"
import.

build.gradle

@@ -200,11 +200,28 @@ rootProject.ext {
   pyver = { exe ->
     try {
       ext.execInBash(
-          exe + " -c 'import sys; print(sys.hexversion)'", "/") as Integer
+          exe + " -c 'import sys; print(sys.hexversion)' 2>/dev/null",
+          "/") as Integer
     } catch (org.gradle.process.internal.ExecException e) {
       return -1;
     }
   }
+
+  // Return the path to a usable python3 executable.
+  getPythonExecutable = {
+    // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we
+    // know we'd like at least 3.6, but 3.7.3 is the latest that ships with
+    // Debian so it seems like that should be available anywhere).
+    def MIN_PY_VER = 0x3070300
+    if (pyver('python') >= MIN_PY_VER) {
+      return 'python'
+    } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) {
+      return '/usr/bin/python3'
+    } else {
+      throw new GradleException("No usable Python version found (build " +
+                                "requires at least python 3.7.3)");
+    }
+  }
 }
 
 task runPresubmits(type: Exec) {
@@ -212,18 +229,7 @@ task runPresubmits(type: Exec) {
   args('config/presubmits.py')
 
   doFirst {
-    // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we
+    executable getPythonExecutable()
-    // know we'd like at least 3.6, but 3.7.3 is the latest that ships with
-    // Debian so it seems like that should be available anywhere).
-    def MIN_PY_VER = 0x3070300
-    if (pyver('python') >= MIN_PY_VER) {
-      executable 'python'
-    } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) {
-      executable '/usr/bin/python3'
-    } else {
-      throw new GradleException("No usable Python version found (build " +
-                                "requires at least python 3.7.3)");
-    }
   }
 }
 
@@ -438,9 +444,10 @@ rootProject.ext {
             ? "${rootDir}/.."
             : rootDir
     def formatDiffScript = "${scriptDir}/google-java-format-git-diff.sh"
+    def pythonExe = getPythonExecutable()
 
     return ext.execInBash(
-        "${formatDiffScript} ${action}", "${workingDir}")
+        "PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
   }
 }
 
@@ -448,6 +455,8 @@ rootProject.ext {
 // Note that this task checks modified Java files in the entire repository.
 task javaIncrementalFormatCheck {
   doLast {
+    // We can only do this in a git tree.
+    if (new File("${rootDir}/.git").exists()) {
       def checkResult = invokeJavaDiffFormatScript("check")
       if (checkResult == 'true') {
         throw new IllegalStateException(
@@ -460,6 +469,9 @@ task javaIncrementalFormatCheck {
             "Failed to invoke format check script:\n" + checkResult)
       }
       println("Incremental Java format check ok.")
+    } else {
+      println("Omitting format check: not in a git directory.")
+    }
   }
 }
 

java-format/google-java-format-diff.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2.7
 #
 #===- google-java-format-diff.py - google-java-format Diff Reformatter -----===#
 #
@@ -24,7 +23,6 @@ For perforce users:
 import argparse
 import difflib
 import re
-import string
 import subprocess
 import io
 import sys
@@ -99,7 +97,7 @@ def main():
     base_command = [binary]
 
   # Reformat files containing changes in place.
-  for filename, lines in lines_by_file.iteritems():
+  for filename, lines in lines_by_file.items():
     if args.i and args.verbose:
       print('Formatting ' + filename)
     command = base_command[:]
@@ -120,11 +118,11 @@ def main():
     if not args.i:
       with open(filename) as f:
         code = f.readlines()
-      formatted_code = io.BytesIO(stdout).readlines()
+      formatted_code = io.StringIO(stdout.decode()).readlines()
       diff = difflib.unified_diff(code, formatted_code,
                                   filename, filename,
                                   '(before formatting)', '(after formatting)')
-      diff_string = string.join(diff, '')
+      diff_string = ''.join(diff)
       if len(diff_string) > 0:
         sys.stdout.write(diff_string)
 

java-format/google-java-format-git-diff.sh

@@ -42,6 +42,16 @@ where:
 SCRIPT_DIR="$(realpath $(dirname $0))"
 JAR_NAME="google-java-format-1.8-all-deps.jar"
 
+# Make sure we have a valid python interpreter.
+if [ -z "$PYTHON" ]; then
+  echo "You must specify the name of a python3 interpreter in the PYTHON" \
+       "environment variable."
+  exit 1
+elif ! "$PYTHON" -c ''; then
+  echo "Invalid python interpreter: $PYTHON"
+  exit 1
+fi
+
 # Locate the java binary.
 if [ -n "$JAVA_HOME" ]; then
   JAVA_BIN="$JAVA_HOME/bin/java"
@@ -69,10 +79,14 @@ function runGoogleJavaFormatAgainstDiffs() {
   shift
 
   git diff -U0 "$forkPoint" | \
-      ${SCRIPT_DIR}/google-java-format-diff.py \
+      "${PYTHON}" "${SCRIPT_DIR}/google-java-format-diff.py" \
           --java-binary "$JAVA_BIN" \
           --google-java-format-jar "${SCRIPT_DIR}/${JAR_NAME}" \
-      -p1 "$@" | tee gjf.out
+          -p1 "$@" | \
+      tee gjf.out
+
+  # If any of the commands in the last pipe failed, return false.
+  [[ ! "${PIPESTATUS[@]}" =~ [^0\ ] ]]
 }
 
 # Show the file names in a diff preceeded by a message.
@@ -96,7 +110,11 @@ function callGoogleJavaFormatDiff() {
   local callResult
   case "$1" in
     "check")
-      local output=$(runGoogleJavaFormatAgainstDiffs "$forkPoint")
+      # We need to do explicit checks for an error and "exit 1" if there was
+      # one here (though not elsewhere), "set -e" doesn't catch this case,
+      # it's not clear why.
+      local output
+      output=$(runGoogleJavaFormatAgainstDiffs "$forkPoint") || exit 1
       echo "$output" | showFileNames "\033[1mNeeds formatting: "
       callResult=$(echo -n "$output" | wc -l)
       ;;