Implement screenshot comparison

This change added the implementation of screenshot comparison with
ChromeWebDriver which will be used in the open-source code base.

This change also set a default window size(1200x2000) for each screenshot
test. This is to make the size of screenshot deterministic to help
build the screenshot comparison tests.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=235539713

javatests/google/registry/webdriver/ActualScreenshot.java

@@ -0,0 +1,94 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.webdriver;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import javax.imageio.ImageIO;
+import javax.imageio.ImageReader;
+
+/** An immutable class represents . */
+public class ActualScreenshot {
+  public static final String IMAGE_FORMAT = "png";
+  private String imageKey;
+  private BufferedImage bufferedImage;
+
+  private ActualScreenshot(String imageKey, BufferedImage bufferedImage) {
+    this.imageKey = imageKey;
+    this.bufferedImage = bufferedImage;
+  }
+
+  /** Creates an ActualScreenshot from the given image format and byte array. */
+  public static ActualScreenshot create(String imageKey, byte[] imageBytes) {
+    checkNotNull(imageKey);
+    checkNotNull(imageBytes);
+    byte[] imageBytesClone = Arrays.copyOf(imageBytes, imageBytes.length);
+    ByteArrayInputStream imageInputStream = new ByteArrayInputStream(imageBytesClone);
+    ImageReader imageReader = ImageIO.getImageReadersByFormatName(IMAGE_FORMAT).next();
+    try {
+      imageReader.setInput(checkNotNull(ImageIO.createImageInputStream(imageInputStream)));
+      BufferedImage bufferedImage = checkNotNull(imageReader.read(0));
+      return new ActualScreenshot(imageKey, bufferedImage);
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  /** {@link BufferedImage#getSubimage(int, int, int, int)} */
+  public ActualScreenshot getSubimage(int x, int y, int w, int h) {
+    return new ActualScreenshot(imageKey, bufferedImage.getSubimage(x, y, w, h));
+  }
+
+  /** {@link BufferedImage#getWidth()} */
+  public int getWidth() {
+    return bufferedImage.getWidth();
+  }
+
+  /** {@link BufferedImage#getHeight()} */
+  public int getHeight() {
+    return bufferedImage.getHeight();
+  }
+
+  /** {@link BufferedImage#getRGB(int, int)} */
+  public int getRGB(int x, int y) {
+    return bufferedImage.getRGB(x, y);
+  }
+
+  /** Writes the underlying BufferedImage to the given file. */
+  public void writeTo(File file) {
+    try {
+      checkState(ImageIO.write(bufferedImage, IMAGE_FORMAT, checkNotNull(file)));
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  /** Returns the imageKey of the screenshot. */
+  public String getImageKey() {
+    return imageKey;
+  }
+
+  /** Returns the concat of imageKey and imageFormat. */
+  public String getImageName() {
+    return String.join(".", imageKey, IMAGE_FORMAT);
+  }
+}

javatests/google/registry/webdriver/ChromeWebDriverPlusScreenDiffer.java

@@ -14,24 +14,61 @@
 
 package google.registry.webdriver;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.lang.Math.abs;
+import static java.util.stream.Collectors.joining;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import com.google.common.flogger.FluentLogger;
+import java.awt.Color;
+import java.awt.image.BufferedImage;
+import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.IntStream;
+import javax.imageio.ImageIO;
+import org.openqa.selenium.OutputType;
+import org.openqa.selenium.TakesScreenshot;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.chrome.ChromeDriver;
 import org.openqa.selenium.chrome.ChromeDriverService;
 import org.openqa.selenium.chrome.ChromeOptions;
+import org.openqa.selenium.remote.RemoteWebDriver;
 
 /** Implementation of {@link WebDriverPlusScreenDiffer} that uses {@link ChromeDriver}. */
 class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer {
 
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final String SCREENSHOT_DIR = "build/screenshots";
   private static final ChromeDriverService chromeDriverService = createChromeDriverService();
-  private final WebDriver webDriver;
 
-  public ChromeWebDriverPlusScreenDiffer() {
+  private final WebDriver webDriver;
+  private final String goldensPath;
+  private final int maxColorDiff;
+  private final int maxPixelDiff;
+  private final List<ActualScreenshot> actualScreenshots;
+
+  public ChromeWebDriverPlusScreenDiffer(String goldensPath, int maxColorDiff, int maxPixelDiff) {
+    this.webDriver = createChromeDriver();
+    this.goldensPath = goldensPath;
+    this.maxColorDiff = maxColorDiff;
+    this.maxPixelDiff = maxPixelDiff;
+    this.actualScreenshots = Lists.newArrayList();
+  }
+
+  private WebDriver createChromeDriver() {
     ChromeOptions chromeOptions = new ChromeOptions();
     chromeOptions.setHeadless(true);
-    webDriver = new ChromeDriver(chromeDriverService, chromeOptions);
+    // For Windows OS, this is required to let headless mode work properly
+    chromeOptions.addArguments("disable-gpu");
+    return new RemoteWebDriver(chromeDriverService.getUrl(), chromeOptions);
   }
 
   private static ChromeDriverService createChromeDriverService() {
@@ -46,6 +83,44 @@ class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer {
     return chromeDriverService;
   }
 
+  @AutoValue
+  abstract static class ComparisonResult {
+    abstract ActualScreenshot actualScreenshot();
+
+    abstract boolean isConsideredSimilar();
+
+    abstract boolean isMissingGoldenImage();
+
+    abstract boolean isSizeDifferent();
+
+    abstract int numDiffPixels();
+
+    static Builder builder() {
+      return new AutoValue_ChromeWebDriverPlusScreenDiffer_ComparisonResult.Builder();
+    }
+
+    @AutoValue.Builder
+    abstract static class Builder {
+      abstract Builder setActualScreenshot(ActualScreenshot actualScreenshot);
+
+      abstract Builder setIsConsideredSimilar(boolean isConsideredSimilar);
+
+      abstract Builder setIsMissingGoldenImage(boolean isMissingGoldenImage);
+
+      abstract Builder setIsSizeDifferent(boolean isSizeDifferent);
+
+      abstract Builder setNumDiffPixels(int numDiffPixels);
+
+      abstract ComparisonResult build();
+    }
+  }
+
+  static class ScreenshotNotSimilarException extends RuntimeException {
+    public ScreenshotNotSimilarException(String message) {
+      super(message);
+    }
+  }
+
   @Override
   public WebDriver getWebDriver() {
     return webDriver;
@@ -53,19 +128,139 @@ class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer {
 
   @Override
   public void diffElement(String imageKey, WebElement element) {
-    // TODO(b/122650673): Add implementation for screenshots comparison. It is no-op
-    //   right now to prevent the test failure.
   }
 
   @Override
   public void diffPage(String imageKey) {
-    // TODO(b/122650673): Add implementation for screenshots comparison. It is no-op
-    //   right now to prevent the test failure.
   }
 
   @Override
   public void verifyAndQuit() {
-    // TODO(b/122650673): Add implementation for screenshots comparison. It is no-op
+    String errorMessage =
-    //   right now to prevent the test failure.
+        actualScreenshots
+            .parallelStream()
+            .map(this::compareScreenshots)
+            .map(
+                result -> {
+                  String imageName = result.actualScreenshot().getImageName();
+                  String goldenImagePath = goldenImagePath(imageName);
+                  Path persistedScreenshot = persistScreenshot(result.actualScreenshot());
+
+                  if (result.isConsideredSimilar()) {
+                    logger.atInfo().log(
+                        String.format(
+                            "Screenshot test for [%s] passed:\n"
+                                + "  - golden image location: %s\n"
+                                + "  - screenshot image location: %s",
+                            imageName, goldenImagePath, persistedScreenshot.toAbsolutePath()));
+                    return "";
+                  } else {
+                    String diffReason =
+                        String.format("it differed by %d pixels", result.numDiffPixels());
+                    if (result.isMissingGoldenImage()) {
+                      diffReason = "the golden image was missing";
+                    } else if (result.isSizeDifferent()) {
+                      diffReason = "the size of image was different";
+                    }
+
+                    return String.format(
+                        "Screenshot test for [%s] failed because %s:\n"
+                            + "  - golden image location: %s\n"
+                            + "  - screenshot image location: %s",
+                        imageName,
+                        diffReason,
+                        result.isMissingGoldenImage() ? "missing" : goldenImagePath(imageName),
+                        persistedScreenshot.toAbsolutePath());
+                  }
+                })
+            .filter(message -> !message.isEmpty())
+            .collect(joining("\n"));
+
+    if (!errorMessage.isEmpty()) {
+      errorMessage =
+          String.format(
+              "Following screenshot comparison comparisonInputs failed: \n%s", errorMessage);
+      logger.atSevere().log(errorMessage);
+      throw new ScreenshotNotSimilarException(errorMessage);
+    }
+  }
+
+  private ActualScreenshot takeScreenshot(String imageKey) {
+    checkArgument(webDriver instanceof TakesScreenshot);
+    TakesScreenshot takesScreenshot = (TakesScreenshot) webDriver;
+    return ActualScreenshot.create(imageKey, takesScreenshot.getScreenshotAs(OutputType.BYTES));
+  }
+
+  private ComparisonResult compareScreenshots(ActualScreenshot screenshot) {
+    int totalPixels = screenshot.getWidth() * screenshot.getHeight();
+    Optional<BufferedImage> maybeGoldenImage = loadGoldenImageByName(screenshot.getImageName());
+    ComparisonResult.Builder commonBuilder =
+        ComparisonResult.builder()
+            .setActualScreenshot(screenshot)
+            .setIsConsideredSimilar(false)
+            .setIsMissingGoldenImage(false)
+            .setIsSizeDifferent(false)
+            .setNumDiffPixels(totalPixels);
+
+    if (!maybeGoldenImage.isPresent()) {
+      return commonBuilder.setIsMissingGoldenImage(true).build();
+    }
+    BufferedImage goldenImage = maybeGoldenImage.get();
+
+    if ((screenshot.getWidth() != goldenImage.getWidth())
+        || (screenshot.getHeight() != goldenImage.getHeight())) {
+      return commonBuilder.setIsSizeDifferent(true).build();
+    }
+
+    int currPixelDiff = 0;
+    for (int x = 0; x < screenshot.getWidth(); x++) {
+      for (int y = 0; y < screenshot.getHeight(); y++) {
+        Color screenshotColor = new Color(screenshot.getRGB(x, y));
+        Color goldenImageColor = new Color(goldenImage.getRGB(x, y));
+        int currColorDiff =
+            IntStream.of(
+                    abs(screenshotColor.getRed() - goldenImageColor.getRed()),
+                    abs(screenshotColor.getGreen() - goldenImageColor.getGreen()),
+                    abs(screenshotColor.getBlue() - goldenImageColor.getBlue()))
+                .max()
+                .getAsInt();
+        if (currColorDiff > maxColorDiff) {
+          currPixelDiff++;
+        }
+      }
+    }
+    commonBuilder.setNumDiffPixels(currPixelDiff);
+    if (currPixelDiff <= maxPixelDiff) {
+      commonBuilder.setIsConsideredSimilar(true);
+    }
+
+    return commonBuilder.build();
+  }
+
+  private Path persistScreenshot(ActualScreenshot screenshot) {
+    File thisScreenshotDir = new File(SCREENSHOT_DIR, screenshot.getImageKey());
+    thisScreenshotDir.mkdirs();
+    File thisScreenshotFile =
+        new File(
+            thisScreenshotDir,
+            Joiner.on(".").join(System.currentTimeMillis(), ActualScreenshot.IMAGE_FORMAT));
+    screenshot.writeTo(thisScreenshotFile);
+    return thisScreenshotFile.toPath();
+  }
+
+  private String goldenImagePath(String imageName) {
+    return Paths.get(goldensPath, imageName).toString();
+  }
+
+  private Optional<BufferedImage> loadGoldenImageByName(String imageName) {
+    File imageFile = new File(goldenImagePath(imageName));
+    if (!imageFile.isFile()) {
+      return Optional.empty();
+    }
+    try {
+      return Optional.of(ImageIO.read(imageFile));
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
   }
 }

javatests/google/registry/webdriver/OteSetupConsoleScreenshotTest.java

@@ -24,12 +24,10 @@ import google.registry.server.RegistryTestServer;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
 import org.openqa.selenium.By;
-import org.openqa.selenium.Dimension;
 
 /** Registrar Console Screenshot Differ tests. */
-@RunWith(JUnit4.class)
+@RunWith(RepeatableRunner.class)
 public class OteSetupConsoleScreenshotTest {
 
   @Rule
@@ -46,7 +44,6 @@ public class OteSetupConsoleScreenshotTest {
 
   @Test
   public void get_owner_fails() throws Throwable {
-    driver.manage().window().setSize(new Dimension(1200, 2000));
     driver.get(server.getUrl("/registrar-ote-setup"));
     driver.waitForElement(By.tagName("h1"));
     driver.diffPage("unauthorized");
@@ -55,7 +52,6 @@ public class OteSetupConsoleScreenshotTest {
   @Test
   public void get_admin_succeeds() throws Throwable {
     server.setIsAdmin(true);
-    driver.manage().window().setSize(new Dimension(1200, 2000));
     driver.get(server.getUrl("/registrar-ote-setup"));
     driver.waitForElement(By.tagName("h1"));
     driver.diffPage("formEmpty");
@@ -71,7 +67,6 @@ public class OteSetupConsoleScreenshotTest {
   @Test
   public void get_admin_fails_badEmail() throws Throwable {
     server.setIsAdmin(true);
-    driver.manage().window().setSize(new Dimension(1200, 2000));
     driver.get(server.getUrl("/registrar-ote-setup"));
     driver.waitForElement(By.tagName("h1"));
     driver.findElement(By.id("clientId")).sendKeys("acmereg");

javatests/google/registry/webdriver/RegistrarConsoleScreenshotTest.java

@@ -30,12 +30,11 @@ import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
 import org.openqa.selenium.By;
 import org.openqa.selenium.Dimension;
 
 /** Registrar Console Screenshot Differ tests. */
-@RunWith(JUnit4.class)
+@RunWith(RepeatableRunner.class)
 public class RegistrarConsoleScreenshotTest {
 
   @Rule
@@ -56,7 +55,6 @@ public class RegistrarConsoleScreenshotTest {
 
   @Test
   public void index_owner() throws Throwable {
-    driver.manage().window().setSize(new Dimension(1200, 2000));
     driver.get(server.getUrl("/registrar"));
     driver.waitForElement(By.tagName("h1"));
     driver.diffPage("page");
@@ -66,7 +64,6 @@ public class RegistrarConsoleScreenshotTest {
   @Test
   public void index_adminAndOwner() throws Throwable {
     server.setIsAdmin(true);
-    driver.manage().window().setSize(new Dimension(1200, 2000));
     driver.get(server.getUrl("/registrar"));
     driver.waitForElement(By.tagName("h1"));
     driver.diffPage("page");
@@ -76,7 +73,6 @@ public class RegistrarConsoleScreenshotTest {
   @Test
   public void index_admin() throws Throwable {
     server.setIsAdmin(true);
-    driver.manage().window().setSize(new Dimension(1200, 2000));
     // To make sure we're only ADMIN (and not also "OWNER"), we switch to the NewRegistrar we
     // aren't in the contacts of
     driver.get(server.getUrl("/registrar?clientId=NewRegistrar"));

javatests/google/registry/webdriver/RegistrarCreateConsoleScreenshotTest.java

@@ -24,12 +24,10 @@ import google.registry.server.RegistryTestServer;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
 import org.openqa.selenium.By;
-import org.openqa.selenium.Dimension;
 
 /** Registrar Console Screenshot Differ tests. */
-@RunWith(JUnit4.class)
+@RunWith(RepeatableRunner.class)
 public class RegistrarCreateConsoleScreenshotTest {
 
   @Rule
@@ -46,7 +44,6 @@ public class RegistrarCreateConsoleScreenshotTest {
 
   @Test
   public void get_owner_fails() throws Throwable {
-    driver.manage().window().setSize(new Dimension(1200, 2000));
     driver.get(server.getUrl("/registrar-create"));
     driver.waitForElement(By.tagName("h1"));
     driver.diffPage("unauthorized");
@@ -55,7 +52,6 @@ public class RegistrarCreateConsoleScreenshotTest {
   @Test
   public void get_admin_succeeds() throws Throwable {
     server.setIsAdmin(true);
-    driver.manage().window().setSize(new Dimension(1200, 2000));
     driver.get(server.getUrl("/registrar-create"));
     driver.waitForElement(By.tagName("h1"));
     driver.diffPage("formEmpty");
@@ -84,7 +80,6 @@ public class RegistrarCreateConsoleScreenshotTest {
   @Test
   public void get_admin_fails_badEmail() throws Throwable {
     server.setIsAdmin(true);
-    driver.manage().window().setSize(new Dimension(1200, 2000));
     driver.get(server.getUrl("/registrar-create"));
     driver.waitForElement(By.tagName("h1"));
     driver.findElement(By.id("clientId")).sendKeys("my-name");

javatests/google/registry/webdriver/RepeatableRunner.java

@@ -0,0 +1,73 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.webdriver;
+
+import com.google.common.flogger.FluentLogger;
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.model.FrameworkMethod;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.Statement;
+
+/**
+ * A JUnit test runner which can retry each test up to 3 times.
+ *
+ * <p>This runner is for our visual regression to prevent flakes
+ * during the run. The test is repeated multiple times until it
+ * passes and it only fails if it failed 3 times.
+ */
+public class RepeatableRunner extends BlockJUnit4ClassRunner {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private static final int MAX_ATTEMPTS = 3;
+
+  /** Constructs a new instance of the default runner */
+  public RepeatableRunner(Class<?> klass) throws InitializationError {
+    super(klass);
+  }
+
+  @Override
+  protected Statement methodBlock(FrameworkMethod method) {
+    Statement statement = super.methodBlock(method);
+    return new RepeatableStatement(statement, method);
+  }
+
+  private static class RepeatableStatement extends Statement {
+
+    private Statement statement;
+    private FrameworkMethod method;
+
+    public RepeatableStatement(Statement statement, FrameworkMethod method) {
+      this.statement = statement;
+      this.method = method;
+    }
+
+    @Override
+    public void evaluate() throws Throwable {
+      for (int i = 1; i <= MAX_ATTEMPTS; i++) {
+        try {
+          statement.evaluate();
+          return;
+        } catch (Throwable e) {
+          logger.atSevere().log(
+              "[%s] Attempt %d of %d failed!\n", method.getName(), i, MAX_ATTEMPTS);
+          if (i == MAX_ATTEMPTS) {
+            throw e;
+          }
+        }
+      }
+    }
+  }
+}

javatests/google/registry/webdriver/WebDriverRule.java

@@ -14,6 +14,7 @@
 
 package google.registry.webdriver;
 
+import static com.google.common.io.Resources.getResource;
 import static java.util.stream.Collectors.joining;
 import static org.apache.commons.text.StringEscapeUtils.escapeEcmaScript;
 
@@ -29,6 +30,7 @@ import org.junit.runner.Description;
 import org.junit.runners.model.Statement;
 import org.openqa.selenium.By;
 import org.openqa.selenium.Capabilities;
+import org.openqa.selenium.Dimension;
 import org.openqa.selenium.HasCapabilities;
 import org.openqa.selenium.JavascriptExecutor;
 import org.openqa.selenium.OutputType;
@@ -59,6 +61,12 @@ public final class WebDriverRule extends ExternalResource
   // number of pixels that vary (usually on the corners of "rounded" buttons or similar)
   private static final int MAX_PIXEL_DIFF = 0;
 
+  // Default size of the browser window when taking screenshot. Having a fixed size of window can
+  // help make visual regression test deterministic.
+  private static final Dimension DEFAULT_WINDOW_SIZE = new Dimension(1200, 2000);
+
+  private static final String GOLDENS_PATH =
+      getResource(WebDriverRule.class, "scuba_goldens/chrome-linux").getFile();
 
   private WebDriver driver;
   private WebDriverPlusScreenDiffer webDriverPlusScreenDiffer;
@@ -82,20 +90,21 @@ public final class WebDriverRule extends ExternalResource
   @Override
   protected void before() {
     webDriverPlusScreenDiffer =
-        new ChromeWebDriverPlusScreenDiffer();
+        new ChromeWebDriverPlusScreenDiffer(GOLDENS_PATH, MAX_COLOR_DIFF, MAX_PIXEL_DIFF);
     driver = webDriverPlusScreenDiffer.getWebDriver();
     // non-zero timeout so findByElement will wait for the element to appear
     driver.manage().timeouts().implicitlyWait(5, TimeUnit.SECONDS);
+    driver.manage().window().setSize(DEFAULT_WINDOW_SIZE);
   }
 
   @Override
   protected void after() {
+    webDriverPlusScreenDiffer.verifyAndQuit();
     try {
       driver.quit();
     } finally {
       driver = null;
     }
-    webDriverPlusScreenDiffer.verifyAndQuit();
   }
 
   /** @see #get(String) */