diff --git a/core/build.gradle b/core/build.gradle
index 4fb98c5f8..d267571f6 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -97,6 +97,10 @@ sourceSets {
   }
 }
 
+processTestResources {
+  exclude '**/webdriver/*'
+}
+
 configurations {
   css
   jaxb
@@ -140,6 +144,8 @@ dependencies {
       "${rootDir}/third_party/objectify/v4_1/objectify-4.1.3.jar")
   testImplementation project(':third_party')
 
+  testRuntime files(sourceSets.test.resources.srcDirs)
+
   compile deps['com.beust:jcommander']
   compile deps['com.google.api-client:google-api-client']
   maybeRuntime deps['com.google.api-client:google-api-client-appengine']
diff --git a/core/src/test/java/google/registry/webdriver/RepeatableRunner.java b/core/src/test/java/google/registry/webdriver/RepeatableRunner.java
index 36c9feb8d..39b0f7c84 100644
--- a/core/src/test/java/google/registry/webdriver/RepeatableRunner.java
+++ b/core/src/test/java/google/registry/webdriver/RepeatableRunner.java
@@ -141,7 +141,15 @@ public class RepeatableRunner extends BlockJUnit4ClassRunner {
       int numSuccess = 0, numFailure = 0;
       Throwable lastException = null;
       for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
-        attemptNumber.set(attempt);
+        // attemptNumber would be null if any exception happens during the
+        // instantiation of test class object(including test rules). However,
+        // those exceptions would not be actually thrown until statement.evaluate()
+        // is invoked. So, we should just skip the setter and let the statement
+        // be evaluated. Then we should be able to find the stack trace of
+        // root cause in the test reports.
+        if (attemptNumber != null) {
+          attemptNumber.set(attempt);
+        }
         try {
           statement.evaluate();
           numSuccess++;