Enable screenshot comparison in external build

This change actually enabled the screenshot comparison in the
visual regression tests. We used Docker to provision Chrome
and ChromeDriver to eliminate the discrepancy of environment
between local development and Travis CI.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=237811918

.travis.yml

@@ -41,9 +41,9 @@ cache:
     - $HOME/.gradle/caches/
     - $HOME/.gradle/wrapper/
 
-# Get Chrome binary for WebDriver tests
-addons:
-  chrome: stable
+# WebDriver tests need Chrome and ChromeDriver provisioned by the docker image
+services:
+  - docker
 
 env:
   # Disable fancy status information (looks bad on travis and exceeds logfile

gradle/core/build.gradle

@@ -6,6 +6,15 @@ plugins {
 // used for easy inspection.
 def generatedDir = "${project.buildDir}/generated/source/custom/main"
 def resourcesDir = "${project.buildDir}/resources/main"
+def screenshotsDir = "${project.buildDir}/screenshots"
+def screenshotsForGoldensDir = "${project.buildDir}/screenshots_for_goldens"
+def newGoldensDir = "${project.buildDir}/new_golden_images"
+def goldensDir =
+  "${javatestsDir}/google/registry/webdriver/goldens/chrome-linux"
+def chromeWebdriverServicePort = 4444
+// Url to the Chrome Webdriver service used by class ChromeWebDriverPlusScreenDiffer
+def chromeWebdriverServiceUrl =
+  "http://localhost:${chromeWebdriverServicePort}/wd/hub"
 
 // Tests that conflict with (mostly unidentified) members of the main test
 // suite. It is unclear if they are offenders (i.e., those that pollute global
@@ -575,33 +584,63 @@ task outcastTest(type: Test) {
   maxParallelForks 5
 }
 
-import org.apache.tools.ant.taskdefs.condition.Os
-import java.nio.file.Files
-import java.nio.file.Paths
-ext.getChromedriver = {
-    def chromedriverZipFileName
-    if (Os.isFamily(Os.FAMILY_UNIX)) {
-        chromedriverZipFileName = 'chromedriver_linux64.zip'
-    } else if (Os.isFamily(Os.FAMILY_MAC)) {
-        chromedriverZipFileName = 'chromedriver_mac64.zip'
-    } else if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-        chromedriverZipFileName = 'chromedriver_win32.zip'
-    } else {
-        throw new RuntimeException("Unsupported OS: ${OperatingSystem.getDisplayName()}")
-    }
-
-    def tempDir = Files.createTempDirectory("chromedriver-dir").toString()
-
-    ant.get(src: 'https://chromedriver.storage.googleapis.com/LATEST_RELEASE', dest: tempDir)
-    def latestReleaseVersion = Paths.get(tempDir, 'LATEST_RELEASE').readLines().get(0)
-    ant.get(src: "https://chromedriver.storage.googleapis.com/${latestReleaseVersion}/${chromedriverZipFileName}", dest: tempDir)
-    ant.unzip(src: "${tempDir}/${chromedriverZipFileName}", dest: tempDir)
-
-    def chromedriverFile = new File(tempDir, 'chromedriver')
-    chromedriverFile.setExecutable(true)
-
-    return chromedriverFile.getCanonicalPath()
+task dockerStopAtStart(type: Exec) {
+  ignoreExitValue true
+  commandLine 'docker', 'stop', 'chrome-plus-chromedriver'
 }
+task dockerStopAtEnd(type: Exec) {
+  ignoreExitValue true
+  commandLine 'docker', 'stop', 'chrome-plus-chromedriver'
+}
+
+task dockerRun(type: Exec) {
+  dependsOn dockerStopAtStart
+  def runCommand = []
+  runCommand << 'docker'
+  runCommand << 'run' << '--detach'
+  runCommand << '--name' << 'chrome-plus-chromedriver'
+  runCommand << '--publish'
+  runCommand << "${chromeWebdriverServicePort}:${chromeWebdriverServicePort}"
+  runCommand << '--volume' << '/dev/shm:/dev/shm'
+  runCommand << '--network' << 'host'
+  runCommand << '--rm'
+  runCommand << 'selenium/standalone-chrome:3.141.59-gold'
+  commandLine runCommand
+}
+
+task findGoldenImages(type: JavaExec) {
+  classpath = sourceSets.test.runtimeClasspath
+  main = 'google.registry.webdriver.GoldenImageFinder'
+
+  def arguments = []
+  arguments << "--screenshots_for_goldens_dir=${screenshotsForGoldensDir}"
+  arguments << "--new_goldens_dir=${newGoldensDir}"
+  arguments << "--existing_goldens_dir=${goldensDir}"
+  if (rootProject.findProperty("overrideExistingGoldens") == "true") {
+    arguments << "--override_existing_goldens=true"
+  }
+  args arguments
+}
+
+task generateGoldenImages(type: Test) {
+  dependsOn dockerRun
+  // Common exclude pattern. See README in parent directory for explanation.
+  exclude "**/*TestCase.*", "**/*TestSuite.*"
+  include "**/webdriver/*"
+
+  // Sets the maximum number of test executors that may exist at the same time.
+  maxParallelForks 5
+
+  systemProperty 'test.screenshot.dir', screenshotsForGoldensDir
+  systemProperty 'test.screenshot.runAllAttempts', 'true'
+  systemProperty 'test.screenshot.maxAttempts', '5'
+  systemProperty 'webdriver.chromeDriverServiceUrl', chromeWebdriverServiceUrl
+
+  doFirst {
+    new File(screenshotsForGoldensDir).deleteDir()
+  }
+}
+generateGoldenImages.finalizedBy(dockerStopAtEnd, findGoldenImages)
 
 test {
   // Common exclude pattern. See README in parent directory for explanation.
@@ -610,6 +649,8 @@ test {
   exclude outcastTestPatterns
   if (rootProject.hasProperty("excludeWebDriverTests")) {
       exclude "**/webdriver/*"
+  } else {
+      dependsOn dockerRun
   }
 
   // Run every test class in its own process.
@@ -620,11 +661,17 @@ test {
   // Sets the maximum number of test executors that may exist at the same time.
   maxParallelForks 5
 
-  // Set system property for Webdriver
-  systemProperty 'webdriver.chrome.driver', project.getChromedriver()
+  systemProperty 'webdriver.chromeDriverServiceUrl', chromeWebdriverServiceUrl
 
+  doFirst {
+    new File(screenshotsDir).deleteDir()
+  }
 }.dependsOn(fragileTest, outcastTest)
 
+if (!rootProject.hasProperty("excludeWebDriverTests")) {
+  test.finalizedBy(dockerStopAtEnd)
+}
+
 task nomulus(type: Jar) {
   manifest {
     attributes 'Main-Class': 'google.registry.tools.RegistryTool'

javatests/google/registry/webdriver/ActualScreenshot.java

@@ -26,7 +26,7 @@ import java.util.Arrays;
 import javax.imageio.ImageIO;
 import javax.imageio.ImageReader;
 
-/** An immutable class represents . */
+/** An immutable class represents a screenshot taken in a visual regression test. */
 public class ActualScreenshot {
   public static final String IMAGE_FORMAT = "png";
   private String imageKey;

javatests/google/registry/webdriver/ChromeWebDriverPlusScreenDiffer.java

@@ -15,11 +15,12 @@
 package google.registry.webdriver;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static google.registry.webdriver.RepeatableRunner.currentAttemptIndex;
 import static java.lang.Math.abs;
 import static java.util.stream.Collectors.joining;
 
 import com.google.auto.value.AutoValue;
-import com.google.common.base.Joiner;
 import com.google.common.collect.Lists;
 import com.google.common.flogger.FluentLogger;
 import java.awt.Color;
@@ -27,6 +28,8 @@ import java.awt.image.BufferedImage;
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
@@ -38,7 +41,6 @@ import org.openqa.selenium.TakesScreenshot;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.chrome.ChromeDriver;
-import org.openqa.selenium.chrome.ChromeDriverService;
 import org.openqa.selenium.chrome.ChromeOptions;
 import org.openqa.selenium.remote.RemoteWebDriver;
 
@@ -46,8 +48,10 @@ import org.openqa.selenium.remote.RemoteWebDriver;
 class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  private static final String SCREENSHOT_DIR = "build/screenshots";
-  private static final ChromeDriverService chromeDriverService = createChromeDriverService();
+  // Url address to the remote Webdriver service. In this case, we use
+  // Docker to provision the service and it is configured in core/build.gradle
+  private static final String CHROME_DRIVER_SERVICE_URL =
+      checkNotNull(System.getProperty("webdriver.chromeDriverServiceUrl"));
 
   private final WebDriver webDriver;
   private final String goldensPath;
@@ -55,6 +59,8 @@ class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer {
   private final int maxPixelDiff;
   private final List<ActualScreenshot> actualScreenshots;
 
+  private String screenshotDir = System.getProperty("test.screenshot.dir", "build/screenshots");
+
   public ChromeWebDriverPlusScreenDiffer(String goldensPath, int maxColorDiff, int maxPixelDiff) {
     this.webDriver = createChromeDriver();
     this.goldensPath = goldensPath;
@@ -66,21 +72,11 @@ class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer {
   private WebDriver createChromeDriver() {
     ChromeOptions chromeOptions = new ChromeOptions();
     chromeOptions.setHeadless(true);
-    // For Windows OS, this is required to let headless mode work properly
-    chromeOptions.addArguments("disable-gpu");
-    return new RemoteWebDriver(chromeDriverService.getUrl(), chromeOptions);
-  }
-
-  private static ChromeDriverService createChromeDriverService() {
-    ChromeDriverService chromeDriverService =
-        new ChromeDriverService.Builder().usingAnyFreePort().build();
     try {
-      chromeDriverService.start();
-    } catch (IOException e) {
-      throw new UncheckedIOException(e);
+      return new RemoteWebDriver(new URL(CHROME_DRIVER_SERVICE_URL), chromeOptions);
+    } catch (MalformedURLException e) {
+      throw new IllegalArgumentException(e);
     }
-    Runtime.getRuntime().addShutdownHook(new Thread(chromeDriverService::stop));
-    return chromeDriverService;
   }
 
   @AutoValue
@@ -128,10 +124,19 @@ class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer {
 
   @Override
   public void diffElement(String imageKey, WebElement element) {
+    ActualScreenshot elementImage =
+        takeScreenshot(imageKey)
+            .getSubimage(
+                element.getLocation().getX(),
+                element.getLocation().getY(),
+                element.getSize().getWidth(),
+                element.getSize().getHeight());
+    actualScreenshots.add(elementImage);
   }
 
   @Override
   public void diffPage(String imageKey) {
+    actualScreenshots.add(takeScreenshot(imageKey));
   }
 
   @Override
@@ -144,7 +149,7 @@ class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer {
                 result -> {
                   String imageName = result.actualScreenshot().getImageName();
                   String goldenImagePath = goldenImagePath(imageName);
-                  Path persistedScreenshot = persistScreenshot(result.actualScreenshot());
+                  Path persistedScreenshot = persistScreenshot(result);
 
                   if (result.isConsideredSimilar()) {
                     logger.atInfo().log(
@@ -237,14 +242,16 @@ class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer {
     return commonBuilder.build();
   }
 
-  private Path persistScreenshot(ActualScreenshot screenshot) {
-    File thisScreenshotDir = new File(SCREENSHOT_DIR, screenshot.getImageKey());
+  private Path persistScreenshot(ComparisonResult result) {
+    File thisScreenshotDir =
+        Paths.get(
+                screenshotDir,
+                "attempt_" + currentAttemptIndex,
+                result.isConsideredSimilar() ? "similar" : "different")
+            .toFile();
     thisScreenshotDir.mkdirs();
-    File thisScreenshotFile =
-        new File(
-            thisScreenshotDir,
-            Joiner.on(".").join(System.currentTimeMillis(), ActualScreenshot.IMAGE_FORMAT));
-    screenshot.writeTo(thisScreenshotFile);
+    File thisScreenshotFile = new File(thisScreenshotDir, result.actualScreenshot().getImageName());
+    result.actualScreenshot().writeTo(thisScreenshotFile);
     return thisScreenshotFile.toPath();
   }
 

javatests/google/registry/webdriver/GoldenImageFinder.java

@@ -0,0 +1,118 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.webdriver;
+
+import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
+import static com.google.common.hash.Hashing.sha256;
+import static com.google.common.io.MoreFiles.asByteSource;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.Maps;
+import com.google.common.io.MoreFiles;
+import google.registry.tools.params.ParameterFactory;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.stream.Stream;
+
+/** A tool to find the new golden image by selecting the screenshot which appears the most times. */
+@Parameters(
+    separators = " =",
+    commandDescription = "Find the new golden images from the given screenshots.")
+public class GoldenImageFinder {
+
+  @Parameter(
+      names = {"--screenshots_for_goldens_dir"},
+      description = "Directory to store screenshots generated as candidate of golden images.",
+      required = true)
+  private Path screenshotsForGoldensDir;
+
+  @Parameter(
+      names = {"--new_goldens_dir"},
+      description = "Directory to store the new golden images selected by this application.",
+      required = true)
+  private Path newGoldensDir;
+
+  @Parameter(
+      names = {"--existing_goldens_dir"},
+      description = "Directory to store the existing golden images.",
+      required = true)
+  private Path existingGoldensDir;
+
+  @Parameter(
+      names = {"--override_existing_goldens"},
+      description =
+          "If set to true, the new golden images are copied to override the existing ones.")
+  private boolean overrideExistingGoldens = false;
+
+  private void run() {
+    try (Stream<Path> allScreenshots =
+        Files.find(screenshotsForGoldensDir, 3, (file, attr) -> attr.isRegularFile())) {
+      if (newGoldensDir.toFile().isDirectory()) {
+        MoreFiles.deleteRecursively(newGoldensDir);
+      }
+      newGoldensDir.toFile().mkdirs();
+
+      allScreenshots
+          .collect(
+              toImmutableSetMultimap(
+                  imagePath -> imagePath.toFile().getName(), imagePath -> imagePath))
+          .asMap()
+          .forEach(
+              (imageFileName, screenshots) -> {
+                Map<String, Integer> occurrenceByHash = Maps.newHashMap();
+                int currNumOccurrence = 0;
+                Path currImagePath = null;
+                for (Path screenshot : screenshots) {
+                  String imageHash;
+                  try {
+                    imageHash = asByteSource(screenshot).hash(sha256()).toString();
+                  } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                  }
+                  int numOccurrence = occurrenceByHash.getOrDefault(imageHash, 0);
+                  occurrenceByHash.put(imageHash, numOccurrence + 1);
+                  if (occurrenceByHash.get(imageHash) > currNumOccurrence) {
+                    currNumOccurrence = occurrenceByHash.get(imageHash);
+                    currImagePath = screenshot;
+                  }
+                }
+                try {
+                  Files.copy(currImagePath, newGoldensDir.resolve(imageFileName));
+                  if (overrideExistingGoldens) {
+                    Files.copy(currImagePath, existingGoldensDir.resolve(imageFileName));
+                  }
+                } catch (IOException e) {
+                  throw new UncheckedIOException(e);
+                }
+              });
+
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  public static void main(String[] args) {
+    GoldenImageFinder finder = new GoldenImageFinder();
+    JCommander jCommander = new JCommander(finder);
+    jCommander.addConverterFactory(new ParameterFactory());
+    jCommander.parse(args);
+    finder.run();
+  }
+}

javatests/google/registry/webdriver/README.md

@@ -0,0 +1,70 @@
+# Webdriver Tests
+
+This is the home of the Nomulus webdriver tests. These tests run our user-facing
+HTML/JavaScript code in the context of a set of common browsers. See the[Webdriver documentation](https://www.seleniumhq.org/docs/03_webdriver.jsp)
+for more information on using webdriver.
+
+## My Screenshot Tests are Failing!
+
+1. Make sure [Docker](https://www.docker.com/) is installed
+  * Docker is used to provision the browser and the Webdriver service, so please
+    make sure Docker is installed and the `docker` command is availbe in the shell.
+2. Missing golden images
+  * If you added a new test using screenshot comparison, you have to generate
+    the golden image for that test in advance and copy it to
+    [goldens/](https://github.com/google/nomulus/tree/master/javatests/google/registry/webdriver/goldens)
+    folder. There
+    is an auxiliary Gradle build task to help with this, and here are some examples:
+    ```shell
+    # Generate golden images for all tests. All generated images are stored in
+    # ${projectRoot}/gradle/core/build/new_golden_images
+    $ ./gradlew :core:generateGoldenImages
+
+    # Generate the golden image for a certain test
+    $ ./gradlew :core:generateGoldenImages \
+      --tests "google.registry.webdriver.OteSetupConsoleScreenshotTest.get_owner_fails"
+    ```-webkit-transition
+3. Screenshot differs by X pixels
+  * If you made any change to the existing test and expected to affect the web page,
+    the screenshot should be different from the previously generated golden image.
+    In this case, you can just use the screenshot as the new golden image.
+  * If you didn't expect to have any change, you have to use your own justification
+    to mitigate it. For example, if the number of different pixels is very small,
+    like 1 piexel, it may be because the test is flaky and a retry can probably solve
+    it; also, if there is any change to the version of browser or Webdriver library,
+    it could affect the screenshot as well.
+
+## My Screenshot Tests are Timing Out!
+
+The screenshot tests display your HTML and generated Javascript in an actual
+browser. There can be multiple HTTP queries sent to the browser and the tests
+need to know when everything is finished loading before taking a screenshot.
+This is normally done by waiting for the browser page to contain a specific HTML
+element that isn't present until the end of the load and render. When the
+element shows up, it's safe to take a screenshot.
+
+If that element doesn't show up, the tests will eventually time out. This could
+be due to a difference that you've introduced but more likely it is due to a bug
+in your javascript. Unfortunately, Javascript errors aren't visible from the
+screenshot tests. This can make these problems very difficult to debug.
+
+Compounding this problem is the fact that we do have unit tests, but they run
+against the uncompiled javascript. The screenshot tests run against the closure
+compiled javascript, which means that they are susceptible to at least one new
+class of errors due to the name-mangling that happens during a compile. One
+thing to consider is that name-mangling breaks the use of object attributes
+accessed via strings: e.g. "foo['bar']" is a different attribute from "foo.bar"
+after name mangling - the second case gets mangled, the first does not.
+
+In any case, the following tools can be helpful in debugging these kinds of
+problems:
+
+-   The "webm" files. Along with the pngs in the output files, there are a set
+    of .webm files. These are movies taking of the browser during the test, and
+    will show all intermediate steps up to the timeout. Useful information can
+    be gleaned from them.
+-   Alerts. Javascript alerts (generated by the plain old "`alert()`" function)
+    show up right in the logfile, which is directly available from the shell.
+    You can instrument your code with alerts, but keep in mind that the driver
+    doesn't respond to alerts, so the first one you hit is the only one you'll
+    see.

javatests/google/registry/webdriver/RepeatableRunner.java

@@ -14,6 +14,8 @@
 
 package google.registry.webdriver;
 
+import static com.google.common.base.Preconditions.checkState;
+
 import com.google.common.flogger.FluentLogger;
 import org.junit.runners.BlockJUnit4ClassRunner;
 import org.junit.runners.model.FrameworkMethod;
@@ -23,15 +25,21 @@ import org.junit.runners.model.Statement;
 /**
  * A JUnit test runner which can retry each test up to 3 times.
  *
- * <p>This runner is for our visual regression to prevent flakes
- * during the run. The test is repeated multiple times until it
- * passes and it only fails if it failed 3 times.
+ * <p>This runner is for our visual regression to prevent flakes during the run. The test is
+ * repeated multiple times until it passes and it only fails if it failed 3 times.
  */
 public class RepeatableRunner extends BlockJUnit4ClassRunner {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  private static final int MAX_ATTEMPTS = 3;
+  private static final boolean RUN_ALL_ATTEMPTS =
+      Boolean.parseBoolean(System.getProperty("test.screenshot.runAllAttempts", "false"));
+
+  private static final int MAX_ATTEMPTS =
+      Integer.parseInt(System.getProperty("test.screenshot.maxAttempts", "3"));
+
+  // TODO(b/127984872): Find an elegant way to pass the index of attempt to the test
+  public static int currentAttemptIndex;
 
   /** Constructs a new instance of the default runner */
   public RepeatableRunner(Class<?> klass) throws InitializationError {
@@ -56,18 +64,37 @@ public class RepeatableRunner extends BlockJUnit4ClassRunner {
 
     @Override
     public void evaluate() throws Throwable {
-      for (int i = 1; i <= MAX_ATTEMPTS; i++) {
+      checkState(MAX_ATTEMPTS > 0);
+      int numSuccess = 0, numFailure = 0;
+      Throwable firstException = null;
+      for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+        currentAttemptIndex = attempt;
         try {
           statement.evaluate();
+          numSuccess++;
+          if (RUN_ALL_ATTEMPTS) {
+            logger.atInfo().log(
+                "[%s] Attempt %d of %d succeeded!\n", method.getName(), attempt, MAX_ATTEMPTS);
+            continue;
+          }
           return;
         } catch (Throwable e) {
-          logger.atSevere().log(
-              "[%s] Attempt %d of %d failed!\n", method.getName(), i, MAX_ATTEMPTS);
-          if (i == MAX_ATTEMPTS) {
-            throw e;
+          numFailure++;
+          if (firstException == null) {
+            firstException = e;
           }
+          logger.atWarning().log(
+              "[%s] Attempt %d of %d failed!\n", method.getName(), attempt, MAX_ATTEMPTS);
         }
       }
+      logger.atInfo().log(
+          "Test [%s] was executed %d times, %d attempts succeeded and %d attempts failed.",
+          method.getName(), numSuccess + numFailure, numSuccess, numFailure);
+      if (numSuccess == 0) {
+        logger.atSevere().log(
+            "[%s] didn't pass after all %d attempts failed!\n", method.getName(), MAX_ATTEMPTS);
+        throw firstException;
+      }
     }
   }
 }

javatests/google/registry/webdriver/WebDriverRule.java

@@ -66,7 +66,7 @@ public final class WebDriverRule extends ExternalResource
   private static final Dimension DEFAULT_WINDOW_SIZE = new Dimension(1200, 2000);
 
   private static final String GOLDENS_PATH =
-      getResource(WebDriverRule.class, "scuba_goldens/chrome-linux").getFile();
+      getResource(WebDriverRule.class, "goldens/chrome-linux").getFile();
 
   private WebDriver driver;
   private WebDriverPlusScreenDiffer webDriverPlusScreenDiffer;
@@ -99,10 +99,10 @@ public final class WebDriverRule extends ExternalResource
 
   @Override
   protected void after() {
-    webDriverPlusScreenDiffer.verifyAndQuit();
     try {
-      driver.quit();
+      webDriverPlusScreenDiffer.verifyAndQuit();
     } finally {
+      driver.quit();
       driver = null;
     }
   }