Add "libreadlines"-like behavior to the shell command

Using the jline open-source library.

We save the history between invocations to ~/.nomulus_history

We add some simple completions:
- first argument completes to command name
- all other arguments complete to the command parameters, or filename

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=191501023

java/google/registry/repositories.bzl

@@ -118,6 +118,7 @@ def domain_registry_repositories(
     omit_javax_xml_bind_jaxb_api=False,
     omit_javax_xml_soap_api=False,
     omit_javax_xml_ws_jaxws_api=False,
+    omit_jline=False,
     omit_joda_time=False,
     omit_junit=False,
     omit_org_apache_avro=False,
@@ -350,6 +351,8 @@ def domain_registry_repositories(
     javax_xml_soap_api()
   if not omit_javax_xml_ws_jaxws_api:
     javax_xml_ws_jaxws_api()
+  if not omit_jline:
+    jline()
   if not omit_joda_time:
     joda_time()
   if not omit_junit:
@@ -1906,6 +1909,18 @@ def javax_xml_ws_jaxws_api():
       ],
   )
 
+def jline():
+  java_import_external(
+      name = "jline",
+      licenses = ["notice"],  # BSD
+      jar_sha256 = "b0d884980fab1df2f948c568f576c365f3379dc8bc930272fa508843d1f3652b",
+      jar_urls = [
+          "http://maven.ibiblio.org/maven2/jline/jline/1.0/jline-1.0.jar",
+          "http://repo1.maven.org/maven2/jline/jline/1.0/jline-1.0.jar",
+      ],
+  )
+
+
 def joda_time():
   java_import_external(
       name = "joda_time",

java/google/registry/tools/BUILD

@@ -86,6 +86,7 @@ java_library(
         "@com_google_re2j",
         "@com_googlecode_json_simple",
         "@io_bazel_rules_closure//closure/templates",
+        "@jline",
         "@joda_time",
         "@org_bouncycastle_bcpg_jdk15on",
         "@org_bouncycastle_bcpkix_jdk15on",

java/google/registry/tools/RegistryCli.java

@@ -14,6 +14,9 @@
 
 package google.registry.tools;
 
+import com.google.appengine.tools.remoteapi.RemoteApiInstaller;
+import com.google.appengine.tools.remoteapi.RemoteApiOptions;
+
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.tools.Injector.injectReflectively;
 
@@ -22,8 +25,6 @@ import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
 import com.beust.jcommander.ParametersDelegate;
-import com.google.appengine.tools.remoteapi.RemoteApiInstaller;
-import com.google.appengine.tools.remoteapi.RemoteApiOptions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import google.registry.model.ofy.ObjectifyService;
@@ -61,6 +62,10 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
   private AppEngineConnection connection;
   private RemoteApiInstaller installer;
 
+  // The "shell" command should only exist on first use - so that we can't run "shell" inside
+  // "shell".
+  private boolean isFirstUse = true;
+
   Map<String, ? extends Class<? extends Command>> commands;
   String programName;
 
@@ -87,7 +92,12 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
     // Create the "help" and "shell" commands (these are special in that they don't have a default
     // constructor).
     jcommander.addCommand("help", new HelpCommand(jcommander));
-    jcommander.addCommand("shell", new ShellCommand(System.in, this));
+    ShellCommand shellCommand = null;
+    if (isFirstUse) {
+      shellCommand = new ShellCommand(this);
+      jcommander.addCommand("shell", shellCommand);
+      isFirstUse = false;
+    }
 
     // Create all command instances. It would be preferrable to do this in the constructor, but
     // JCommander mutates the command instances and doesn't reset them so we have to do it for every
@@ -101,6 +111,10 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
       throw new RuntimeException(e);
     }
 
+    if (shellCommand != null) {
+      shellCommand.buildCompletions(jcommander);
+    }
+
     try {
       jcommander.parse(args);
     } catch (ParameterException e) {
@@ -119,15 +133,18 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
     }
 
     if (showAllCommands) {
-      for (Map.Entry<String, ? extends Class<? extends Command>> entry : commands.entrySet()) {
-        System.out.println(entry.getKey());
-      }
+      commands.keySet().forEach(System.out::println);
       return;
     }
 
     checkState(RegistryToolEnvironment.get() == environment,
         "RegistryToolEnvironment argument pre-processing kludge failed.");
 
+    // We have to set the prompt here, because the environment wasn't set until this point
+    if (shellCommand != null) {
+      shellCommand.setPrompt(String.format("nom@%s > ", environment));
+    }
+
     // JCommander stores sub-commands as nested JCommander objects containing a list of user objects
     // to be populated.  Extract the subcommand by getting the JCommander wrapper and then
     // retrieving the first (and, by virtue of our usage, only) object from it.

java/google/registry/tools/ShellCommand.java

@@ -14,17 +14,30 @@
 
 package google.registry.tools;
 
+import static com.google.common.base.StandardSystemProperty.USER_HOME;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 
+import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameters;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.StreamTokenizer;
 import java.io.StringReader;
+import java.util.Arrays;
+import java.util.List;
+import jline.Completor;
+import jline.ConsoleReader;
+import jline.ConsoleReaderInputStream;
+import jline.FileNameCompletor;
+import jline.History;
 
 /**
  * Implements a tiny shell interpreter for the nomulus tool.
@@ -35,12 +48,52 @@ import java.io.StringReader;
 @Parameters(commandDescription = "Run an interactive shell")
 public class ShellCommand implements Command {
 
+  private static final String HISTORY_FILE = ".nomulus_history";
+
   private final CommandRunner runner;
   private final BufferedReader lineReader;
+  private final ConsoleReader consoleReader;
 
+  public ShellCommand(CommandRunner runner) throws IOException {
+    this.runner = runner;
+    InputStream in = System.in;
+    if (System.console() != null) {
+      consoleReader = new ConsoleReader();
+      // There are 104 different commands. We want the threshold to be more than that
+      consoleReader.setAutoprintThreshhold(200);
+      // Setting the prompt to a temporary value - will include the environment once that is set
+      consoleReader.setDefaultPrompt("nom@??? > ");
+      consoleReader.setHistory(new History(new File(USER_HOME.value(), HISTORY_FILE)));
+      in = new ConsoleReaderInputStream(consoleReader);
+    } else {
+      consoleReader = null;
+    }
+    this.lineReader = new BufferedReader(new InputStreamReader(in, US_ASCII));
+  }
+
+  @VisibleForTesting
   ShellCommand(InputStream in, CommandRunner runner) {
     this.runner = runner;
     this.lineReader = new BufferedReader(new InputStreamReader(in, US_ASCII));
+    this.consoleReader = null;
+  }
+
+  public ShellCommand setPrompt(String prompt) {
+    if (consoleReader != null) {
+      consoleReader.setDefaultPrompt(prompt);
+    }
+    return this;
+  }
+
+  public ShellCommand buildCompletions(JCommander jcommander) {
+    if (consoleReader != null) {
+      @SuppressWarnings("unchecked")
+      ImmutableList<Completor> completors = ImmutableList.copyOf(consoleReader.getCompletors());
+      completors
+          .forEach(consoleReader::removeCompletor);
+      consoleReader.addCompletor(new JCommanderCompletor(jcommander));
+    }
+    return this;
   }
 
   /** Run the shell until the user presses "Ctrl-D". */
@@ -55,12 +108,10 @@ public class ShellCommand implements Command {
         System.err.println("Got an exception:\n" + e);
       }
     }
+    System.err.println();
   }
 
   private String getLine() {
-    if (System.console() != null) {
-      System.err.print("nom> ");
-    }
     try {
       return lineReader.readLine();
     } catch (IOException e) {
@@ -90,4 +141,106 @@ public class ShellCommand implements Command {
 
     return resultBuilder.build().toArray(new String[0]);
   }
+
+  @VisibleForTesting
+  static class JCommanderCompletor implements Completor {
+
+    private final ImmutableSet<String> commands;
+    private final ImmutableSetMultimap<String, String> commandArguments;
+    private final FileNameCompletor filenameCompletor = new FileNameCompletor();
+
+    JCommanderCompletor(JCommander jcommander) {
+      commands = ImmutableSet.copyOf(jcommander.getCommands().keySet());
+      ImmutableSetMultimap.Builder<String, String> builder = new ImmutableSetMultimap.Builder<>();
+      jcommander
+          .getCommands()
+          .entrySet()
+          .forEach(
+              entry -> {
+                builder.putAll(
+                    entry.getKey(),
+                    entry
+                        .getValue()
+                        .getParameters()
+                        .stream()
+                        .flatMap(p -> Arrays.stream(p.getParameter().names()))
+                        .collect(toImmutableList()));
+              });
+      commandArguments = builder.build();
+    }
+
+    @Override
+    @SuppressWarnings({"unchecked", "rawtypes"})
+    public int complete(String buffer, int location, List completions) {
+      return completeInternal(buffer, location, completions);
+    }
+
+    /**
+     * Given a string, finds all the possible completions to the end of that string.
+     *
+     * @param buffer the command line.
+     * @param location the location in the command line we want to complete
+     * @param completions a list to fill with the completion results
+     * @return the number of character back from the location that are part of the completions
+     */
+    int completeInternal(String buffer, int location, List<String> completions) {
+      String truncatedBuffer = buffer.substring(0, location);
+      String[] parsedBuffer = parseCommand(truncatedBuffer);
+      int argumentIndex = parsedBuffer.length - 1;
+
+      if (argumentIndex < 0 || !truncatedBuffer.endsWith(parsedBuffer[argumentIndex])) {
+        argumentIndex += 1;
+      }
+      String argument = argumentIndex < parsedBuffer.length ? parsedBuffer[argumentIndex] : "";
+      int argumentStart = location - argument.length();
+
+      // Complete the first argument based on the jcommander commands
+      if (argumentIndex == 0) {
+        completions.addAll(getCommandCompletions(argument));
+        return argumentStart;
+      }
+      String commandName = parsedBuffer[0];
+
+      // For the "help" command, complete the second argument based on the jcommander commands, and
+      // the rest of the arguments fail to complete
+      if (commandName.equals("help")) {
+        if (argumentIndex >= 2) {
+          return argumentStart;
+        }
+        completions.addAll(getCommandCompletions(argument));
+        return argumentStart;
+      }
+
+      // For existing commands, complete based on the command arguments
+      if (argument.isEmpty() || argument.startsWith("-")) {
+        completions.addAll(getArgumentCompletions(commandName, argument));
+        return argumentStart;
+      }
+
+      // However, if it's obviously not an argument (starts with something that isn't "-"), default
+      // to a filename.
+      int offset = filenameCompletor.complete(argument, argument.length(), completions);
+      if (offset < 0) {
+        return argumentStart;
+      }
+      return argumentStart + offset;
+    }
+
+    private List<String> getCommandCompletions(String word) {
+      return commands
+          .stream()
+          .filter(s -> s.startsWith(word))
+          .map(s -> s + " ")
+          .collect(toImmutableList());
+    }
+
+    private List<String> getArgumentCompletions(String command, String word) {
+      return commandArguments.get(command)
+          .stream()
+          .filter(s -> s.startsWith(word))
+          .map(s -> s + " ")
+          .collect(toImmutableList());
+    }
+  }
+
 }

javatests/google/registry/tools/ShellCommandTest.java

@@ -19,11 +19,13 @@ import static google.registry.testing.JUnitBackports.assertThrows;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static org.mockito.Mockito.mock;
 
+import com.beust.jcommander.JCommander;
 import com.beust.jcommander.MissingCommandException;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import google.registry.tools.ShellCommand.JCommanderCompletor;
 import java.io.ByteArrayInputStream;
 import java.util.ArrayList;
 import java.util.List;
@@ -71,8 +73,8 @@ public class ShellCommandTest {
 
   @Test
   public void testMultipleCommandInvocations() throws Exception {
-    RegistryCli cli =
-        new RegistryCli("unittest", ImmutableMap.of("test_command", TestCommand.class));
+    try (RegistryCli cli =
+        new RegistryCli("unittest", ImmutableMap.of("test_command", TestCommand.class))) {
       RegistryToolEnvironment.UNITTEST.setup();
       cli.setEnvironment(RegistryToolEnvironment.UNITTEST);
       cli.run(new String[] {"test_command", "-x", "xval", "arg1", "arg2"});
@@ -84,15 +86,61 @@ public class ShellCommandTest {
               ImmutableList.of("otherxval", "arg3"),
               ImmutableList.of("default value"));
     }
+  }
 
   @Test
   public void testNonExistentCommand() throws Exception {
-    RegistryCli cli =
-        new RegistryCli("unittest", ImmutableMap.of("test_command", TestCommand.class));
+    try (RegistryCli cli =
+        new RegistryCli("unittest", ImmutableMap.of("test_command", TestCommand.class))) {
 
       cli.setEnvironment(RegistryToolEnvironment.UNITTEST);
       assertThrows(MissingCommandException.class, () -> cli.run(new String[] {"bad_command"}));
     }
+  }
+
+  private void performJCommanderCompletorTest(
+      String line,
+      int expectedBackMotion,
+      String... expectedCompletions) {
+    JCommander jcommander = new JCommander();
+    jcommander.setProgramName("test");
+    jcommander.addCommand("help", new HelpCommand(jcommander));
+    jcommander.addCommand("testCommand", new TestCommand());
+    jcommander.addCommand("testAnotherCommand", new TestAnotherCommand());
+    List<String> completions = new ArrayList<>();
+    assertThat(
+            line.length()
+                - new JCommanderCompletor(jcommander)
+                    .completeInternal(line, line.length(), completions))
+        .isEqualTo(expectedBackMotion);
+    assertThat(completions).containsExactlyElementsIn(expectedCompletions);
+  }
+
+  @Test
+  public void testCompletion_commands() throws Exception {
+    performJCommanderCompletorTest("", 0, "testCommand ", "testAnotherCommand ", "help ");
+    performJCommanderCompletorTest("n", 1);
+    performJCommanderCompletorTest("test", 4, "testCommand ", "testAnotherCommand ");
+    performJCommanderCompletorTest(" test", 4, "testCommand ", "testAnotherCommand ");
+    performJCommanderCompletorTest("testC", 5, "testCommand ");
+    performJCommanderCompletorTest("testA", 5, "testAnotherCommand ");
+  }
+
+  @Test
+  public void testCompletion_help() throws Exception {
+    performJCommanderCompletorTest("h", 1, "help ");
+    performJCommanderCompletorTest("help ", 0, "testCommand ", "testAnotherCommand ", "help ");
+    performJCommanderCompletorTest("help testC", 5, "testCommand ");
+    performJCommanderCompletorTest("help testCommand ", 0);
+  }
+
+  @Test
+  public void testCompletion_arguments() throws Exception {
+    performJCommanderCompletorTest("testCommand ", 0, "-x ", "--xparam ", "--xorg ");
+    performJCommanderCompletorTest("testCommand --wrong", 7);
+    performJCommanderCompletorTest("testCommand noise  --", 2, "--xparam ", "--xorg ");
+    performJCommanderCompletorTest("testAnotherCommand --o", 3);
+  }
 
   @Parameters(commandDescription = "Test command")
   static class TestCommand implements Command {
@@ -102,6 +150,12 @@ public class ShellCommandTest {
     )
     String xparam = "default value";
 
+    @Parameter(
+      names = {"--xorg"},
+      description = "test organization"
+    )
+    String xorg = "default value";
+
     // List for recording command invocations by run().
     //
     // This has to be static because it gets populated by multiple TestCommand instances, which are
@@ -123,4 +177,10 @@ public class ShellCommandTest {
       commandInvocations.add(callRecord.build());
     }
   }
+
+  @Parameters(commandDescription = "Another test command")
+  static class TestAnotherCommand implements Command {
+    @Override
+    public void run() {}
+  }
 }