Add per-test output to verify_ote command in --summarize mode

Before this change the output looks like this:

registrar1      - Num actions:   93 - Reqs passed: 16/16 - Overall: PASS
registrar2      - Num actions:   47 - Reqs passed:  6/16 - Overall: FAIL

After this change the output looks like this:

registrar1      - # actions:   93 - Reqs: [----------------] 16/16 - Overall: PASS
registrar2      - # actions:   47 - Reqs: [...--.-...-...--]  6/16 - Overall: FAIL

The status of each test is displayed as a hyphen (passing) or a period (failing),
and the tests are always displayed in the same order so it's easier to get an overall
view of whether registrars are struggling with the same tests.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=190776935
This commit is contained in:
mcilwain 2018-03-28 08:36:44 -07:00 committed by jianglai
parent e1ad4d663c
commit 038a2679c7
2 changed files with 43 additions and 16 deletions

View file

@ -89,18 +89,23 @@ public class VerifyOteAction implements Runnable, JsonAction {
HistoryEntryStats historyEntryStats = HistoryEntryStats historyEntryStats =
new HistoryEntryStats().recordRegistrarHistory(registrarName); new HistoryEntryStats().recordRegistrarHistory(registrarName);
List<String> failureMessages = historyEntryStats.findFailures(); List<String> failureMessages = historyEntryStats.findFailures();
String passedFraction = int testsPassed = StatType.NUM_REQUIREMENTS - failureMessages.size();
String.format(
"%2d/%2d",
StatType.NUM_REQUIREMENTS - failureMessages.size(), StatType.NUM_REQUIREMENTS);
String status = failureMessages.isEmpty() ? "PASS" : "FAIL"; String status = failureMessages.isEmpty() ? "PASS" : "FAIL";
return summarize return summarize
? String.format( ? String.format(
"Num actions: %4d - Reqs passed: %s - Overall: %s", "# actions: %4d - Reqs: [%s] %2d/%2d - Overall: %s",
historyEntryStats.statCounts.size(), passedFraction, status) historyEntryStats.statCounts.size(),
historyEntryStats.toSummary(),
testsPassed,
StatType.NUM_REQUIREMENTS,
status)
: String.format( : String.format(
"%s\n%s\nRequirements passed: %s\nOverall OT&E status: %s\n", "%s\n%s\nRequirements passed: %2d/%2d\nOverall OT&E status: %s\n",
historyEntryStats, Joiner.on('\n').join(failureMessages), passedFraction, status); historyEntryStats,
Joiner.on('\n').join(failureMessages),
testsPassed,
StatType.NUM_REQUIREMENTS,
status);
} }
private static final Predicate<EppInput> HAS_CLAIMS_NOTICE = private static final Predicate<EppInput> HAS_CLAIMS_NOTICE =
@ -299,5 +304,15 @@ public class VerifyOteAction implements Runnable, JsonAction {
.collect(Collectors.joining("\n")), .collect(Collectors.joining("\n")),
statCounts.size()); statCounts.size());
} }
/** Returns a string showing the results of each test, one character per test. */
String toSummary() {
return EnumSet.allOf(StatType.class)
.stream()
.filter(statType -> statType.requirement > 0)
.sorted()
.map(statType -> (statCounts.count(statType) < statType.requirement) ? "." : "-")
.collect(Collectors.joining(""));
}
} }
} }

View file

@ -41,6 +41,8 @@ public class VerifyOteActionTest {
private final VerifyOteAction action = new VerifyOteAction(); private final VerifyOteAction action = new VerifyOteAction();
HistoryEntry hostDeleteHistoryEntry; HistoryEntry hostDeleteHistoryEntry;
HistoryEntry domainCreateHistoryEntry;
HistoryEntry domainRestoreHistoryEntry;
@Before @Before
public void init() throws Exception { public void init() throws Exception {
@ -50,7 +52,7 @@ public class VerifyOteActionTest {
.setType(Type.DOMAIN_CREATE) .setType(Type.DOMAIN_CREATE)
.setXmlBytes(ToolsTestData.loadBytes("domain_create_sunrise.xml").read()) .setXmlBytes(ToolsTestData.loadBytes("domain_create_sunrise.xml").read())
.build()); .build());
persistResource( domainCreateHistoryEntry = persistResource(
new HistoryEntry.Builder() new HistoryEntry.Builder()
.setClientId("blobio-1") .setClientId("blobio-1")
.setType(Type.DOMAIN_CREATE) .setType(Type.DOMAIN_CREATE)
@ -86,7 +88,7 @@ public class VerifyOteActionTest {
.setType(Type.DOMAIN_DELETE) .setType(Type.DOMAIN_DELETE)
.setXmlBytes(ToolsTestData.loadBytes("domain_delete.xml").read()) .setXmlBytes(ToolsTestData.loadBytes("domain_delete.xml").read())
.build()); .build());
persistResource( domainRestoreHistoryEntry = persistResource(
new HistoryEntry.Builder() new HistoryEntry.Builder()
.setClientId("blobio-1") .setClientId("blobio-1")
.setType(Type.DOMAIN_RESTORE) .setType(Type.DOMAIN_RESTORE)
@ -144,16 +146,26 @@ public class VerifyOteActionTest {
} }
@Test @Test
public void testSuccess_passSummarize() throws Exception { public void testSuccess_summarize_allPass() throws Exception {
Map<String, Object> response = Map<String, Object> response =
action.handleJsonRequest( action.handleJsonRequest(
ImmutableMap.of("summarize", "true", "registrars", ImmutableList.of("blobio"))); ImmutableMap.of("summarize", "true", "registrars", ImmutableList.of("blobio")));
assertThat(response)
for (Entry<String, Object> registrar : response.entrySet()) { .containsExactly(
assertThat(registrar.getKey()).matches("blobio"); "blobio", "# actions: 31 - Reqs: [----------------] 16/16 - Overall: PASS");
assertThat(registrar.getValue().toString()).containsMatch("Reqs passed: 16/16");
assertThat(registrar.getValue().toString()).containsMatch("Overall: PASS");
} }
@Test
public void testSuccess_summarize_someFailures() throws Exception {
deleteResource(hostDeleteHistoryEntry);
deleteResource(domainCreateHistoryEntry);
deleteResource(domainRestoreHistoryEntry);
Map<String, Object> response =
action.handleJsonRequest(
ImmutableMap.of("summarize", "true", "registrars", ImmutableList.of("blobio")));
assertThat(response)
.containsExactly(
"blobio", "# actions: 26 - Reqs: [-.-----.------.-] 13/16 - Overall: FAIL");
} }
@Test @Test