diff --git a/javatests/google/registry/webdriver/ActualScreenshot.java b/javatests/google/registry/webdriver/ActualScreenshot.java index b83f0befb..e43cbbce4 100644 --- a/javatests/google/registry/webdriver/ActualScreenshot.java +++ b/javatests/google/registry/webdriver/ActualScreenshot.java @@ -31,14 +31,16 @@ public class ActualScreenshot { public static final String IMAGE_FORMAT = "png"; private String imageKey; private BufferedImage bufferedImage; + private int attempt; - private ActualScreenshot(String imageKey, BufferedImage bufferedImage) { + private ActualScreenshot(String imageKey, BufferedImage bufferedImage, int attempt) { this.imageKey = imageKey; this.bufferedImage = bufferedImage; + this.attempt = attempt; } /** Creates an ActualScreenshot from the given image format and byte array. */ - public static ActualScreenshot create(String imageKey, byte[] imageBytes) { + public static ActualScreenshot create(String imageKey, int attempt, byte[] imageBytes) { checkNotNull(imageKey); checkNotNull(imageBytes); byte[] imageBytesClone = Arrays.copyOf(imageBytes, imageBytes.length); @@ -47,7 +49,7 @@ public class ActualScreenshot { try { imageReader.setInput(checkNotNull(ImageIO.createImageInputStream(imageInputStream))); BufferedImage bufferedImage = checkNotNull(imageReader.read(0)); - return new ActualScreenshot(imageKey, bufferedImage); + return new ActualScreenshot(imageKey, bufferedImage, attempt); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -55,7 +57,7 @@ public class ActualScreenshot { /** {@link BufferedImage#getSubimage(int, int, int, int)} */ public ActualScreenshot getSubimage(int x, int y, int w, int h) { - return new ActualScreenshot(imageKey, bufferedImage.getSubimage(x, y, w, h)); + return new ActualScreenshot(imageKey, bufferedImage.getSubimage(x, y, w, h), attempt); } /** {@link BufferedImage#getWidth()} */ @@ -82,9 +84,9 @@ public class ActualScreenshot { } } - /** Returns the imageKey of the screenshot. */ - public String getImageKey() { - return imageKey; + /** Returns the attempt number of the test. */ + public int getAttempt() { + return attempt; } /** Returns the concat of imageKey and imageFormat. */ diff --git a/javatests/google/registry/webdriver/ChromeWebDriverPlusScreenDiffer.java b/javatests/google/registry/webdriver/ChromeWebDriverPlusScreenDiffer.java index 70135b305..9c8b2d102 100644 --- a/javatests/google/registry/webdriver/ChromeWebDriverPlusScreenDiffer.java +++ b/javatests/google/registry/webdriver/ChromeWebDriverPlusScreenDiffer.java @@ -16,7 +16,6 @@ package google.registry.webdriver; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static google.registry.webdriver.RepeatableRunner.currentAttemptIndex; import static java.lang.Math.abs; import static java.util.stream.Collectors.joining; @@ -123,9 +122,9 @@ class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer { } @Override - public void diffElement(String imageKey, WebElement element) { + public void diffElement(WebElement element, String imageKey, int attempt) { ActualScreenshot elementImage = - takeScreenshot(imageKey) + ActualScreenshot.create(imageKey, attempt, takeScreenshot()) .getSubimage( element.getLocation().getX(), element.getLocation().getY(), @@ -135,8 +134,8 @@ class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer { } @Override - public void diffPage(String imageKey) { - actualScreenshots.add(takeScreenshot(imageKey)); + public void diffPage(String imageKey, int attempt) { + actualScreenshots.add(ActualScreenshot.create(imageKey, attempt, takeScreenshot())); } @Override @@ -190,10 +189,10 @@ class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer { } } - private ActualScreenshot takeScreenshot(String imageKey) { + private byte[] takeScreenshot() { checkArgument(webDriver instanceof TakesScreenshot); TakesScreenshot takesScreenshot = (TakesScreenshot) webDriver; - return ActualScreenshot.create(imageKey, takesScreenshot.getScreenshotAs(OutputType.BYTES)); + return takesScreenshot.getScreenshotAs(OutputType.BYTES); } private ComparisonResult compareScreenshots(ActualScreenshot screenshot) { @@ -246,7 +245,7 @@ class ChromeWebDriverPlusScreenDiffer implements WebDriverPlusScreenDiffer { File thisScreenshotDir = Paths.get( screenshotDir, - "attempt_" + currentAttemptIndex, + "attempt_" + result.actualScreenshot().getAttempt(), result.isConsideredSimilar() ? "similar" : "different") .toFile(); thisScreenshotDir.mkdirs(); diff --git a/javatests/google/registry/webdriver/OteSetupConsoleScreenshotTest.java b/javatests/google/registry/webdriver/OteSetupConsoleScreenshotTest.java index 40c205d72..17cd0b09d 100644 --- a/javatests/google/registry/webdriver/OteSetupConsoleScreenshotTest.java +++ b/javatests/google/registry/webdriver/OteSetupConsoleScreenshotTest.java @@ -21,6 +21,7 @@ import com.googlecode.objectify.ObjectifyFilter; import google.registry.model.ofy.OfyFilter; import google.registry.module.frontend.FrontendServlet; import google.registry.server.RegistryTestServer; +import google.registry.webdriver.RepeatableRunner.AttemptNumber; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -40,7 +41,8 @@ public class OteSetupConsoleScreenshotTest { .setEmail("Marla.Singer@google.com") .build(); - @Rule public final WebDriverRule driver = new WebDriverRule(); + private final AttemptNumber attemptNumber = new AttemptNumber(); + @Rule public final WebDriverRule driver = new WebDriverRule(attemptNumber); @Test public void get_owner_fails() throws Throwable { diff --git a/javatests/google/registry/webdriver/RegistrarConsoleScreenshotTest.java b/javatests/google/registry/webdriver/RegistrarConsoleScreenshotTest.java index 8eb46bb93..21291eb47 100644 --- a/javatests/google/registry/webdriver/RegistrarConsoleScreenshotTest.java +++ b/javatests/google/registry/webdriver/RegistrarConsoleScreenshotTest.java @@ -27,6 +27,7 @@ import google.registry.model.registrar.Registrar.State; import google.registry.module.frontend.FrontendServlet; import google.registry.server.RegistryTestServer; import google.registry.testing.CertificateSamples; +import google.registry.webdriver.RepeatableRunner.AttemptNumber; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -50,7 +51,8 @@ public class RegistrarConsoleScreenshotTest { .setEmail("Marla.Singer@google.com") .build(); - @Rule public final WebDriverRule driver = new WebDriverRule(); + private final AttemptNumber attemptNumber = new AttemptNumber(); + @Rule public final WebDriverRule driver = new WebDriverRule(attemptNumber); @Test public void index_owner() throws Throwable { diff --git a/javatests/google/registry/webdriver/RegistrarConsoleWebTest.java b/javatests/google/registry/webdriver/RegistrarConsoleWebTest.java index fa667fb07..218a1108b 100644 --- a/javatests/google/registry/webdriver/RegistrarConsoleWebTest.java +++ b/javatests/google/registry/webdriver/RegistrarConsoleWebTest.java @@ -29,15 +29,15 @@ import google.registry.model.registrar.RegistrarContact; import google.registry.module.frontend.FrontendServlet; import google.registry.server.RegistryTestServer; import google.registry.testing.AppEngineRule; +import google.registry.webdriver.RepeatableRunner.AttemptNumber; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; import org.openqa.selenium.By; /** WebDriver tests for Registrar Console UI. */ -@RunWith(JUnit4.class) +@RunWith(RepeatableRunner.class) public class RegistrarConsoleWebTest { @Rule @@ -59,7 +59,8 @@ public class RegistrarConsoleWebTest { .setEmail("Marla.Singer@google.com") .build(); - @Rule public final WebDriverRule driver = new WebDriverRule(); + private final AttemptNumber attemptNumber = new AttemptNumber(); + @Rule public final WebDriverRule driver = new WebDriverRule(attemptNumber); @Rule public final Timeout deathClock = new Timeout(60000); diff --git a/javatests/google/registry/webdriver/RegistrarCreateConsoleScreenshotTest.java b/javatests/google/registry/webdriver/RegistrarCreateConsoleScreenshotTest.java index 896c24bf7..3cf973a2e 100644 --- a/javatests/google/registry/webdriver/RegistrarCreateConsoleScreenshotTest.java +++ b/javatests/google/registry/webdriver/RegistrarCreateConsoleScreenshotTest.java @@ -21,6 +21,7 @@ import com.googlecode.objectify.ObjectifyFilter; import google.registry.model.ofy.OfyFilter; import google.registry.module.frontend.FrontendServlet; import google.registry.server.RegistryTestServer; +import google.registry.webdriver.RepeatableRunner.AttemptNumber; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -40,7 +41,8 @@ public class RegistrarCreateConsoleScreenshotTest { .setEmail("Marla.Singer@google.com") .build(); - @Rule public final WebDriverRule driver = new WebDriverRule(); + private final AttemptNumber attemptNumber = new AttemptNumber(); + @Rule public final WebDriverRule driver = new WebDriverRule(attemptNumber); @Test public void get_owner_fails() throws Throwable { diff --git a/javatests/google/registry/webdriver/RepeatableRunner.java b/javatests/google/registry/webdriver/RepeatableRunner.java index 5eae90b05..76251ea92 100644 --- a/javatests/google/registry/webdriver/RepeatableRunner.java +++ b/javatests/google/registry/webdriver/RepeatableRunner.java @@ -17,6 +17,10 @@ package google.registry.webdriver; import static com.google.common.base.Preconditions.checkState; import com.google.common.flogger.FluentLogger; +import java.lang.reflect.Field; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.runners.BlockJUnit4ClassRunner; import org.junit.runners.model.FrameworkMethod; import org.junit.runners.model.InitializationError; @@ -25,6 +29,21 @@ import org.junit.runners.model.Statement; /** * A JUnit test runner which can retry each test up to 3 times. * + *
To use this runner, annotate the test class with {@link RepeatableRunner} and define a field + * with type of {@link AttemptNumber}: + * + *
+ * @RunWith(RepeatableRunner.class) + * public class RepeatableRunnerTest { + * private AttemptNumber attemptNumber = new AttemptNumber(); + * + * @Test + * public void test() { + * print(attemptNumber.get()); + * } + * } + *+ * *
This runner is for our visual regression to prevent flakes during the run. The test is
* repeated multiple times until it passes and it only fails if it failed 3 times.
*/
@@ -38,51 +57,102 @@ public class RepeatableRunner extends BlockJUnit4ClassRunner {
private static final int MAX_ATTEMPTS =
Integer.parseInt(System.getProperty("test.screenshot.maxAttempts", "3"));
- // TODO(b/127984872): Find an elegant way to pass the index of attempt to the test
- public static int currentAttemptIndex;
+ private final Field attemptNumberField;
+ private AttemptNumber lastAttemptNumber;
/** Constructs a new instance of the default runner */
- public RepeatableRunner(Class> klass) throws InitializationError {
- super(klass);
+ public RepeatableRunner(Class> clazz) throws InitializationError {
+ super(clazz);
+ attemptNumberField = getAttemptNumberField();
+ }
+
+ private Field getAttemptNumberField() {
+ List