Add per-test output to verify_ote command in --summarize mode

Before this change the output looks like this:

registrar1      - Num actions:   93 - Reqs passed: 16/16 - Overall: PASS
registrar2      - Num actions:   47 - Reqs passed:  6/16 - Overall: FAIL

After this change the output looks like this:

registrar1      - # actions:   93 - Reqs: [----------------] 16/16 - Overall: PASS
registrar2      - # actions:   47 - Reqs: [...--.-...-...--]  6/16 - Overall: FAIL

The status of each test is displayed as a hyphen (passing) or a period (failing),
and the tests are always displayed in the same order so it's easier to get an overall
view of whether registrars are struggling with the same tests.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=190776935
This commit is contained in:
mcilwain 2018-03-28 08:36:44 -07:00 committed by jianglai
parent e1ad4d663c
commit 038a2679c7
2 changed files with 43 additions and 16 deletions

View file

@ -89,18 +89,23 @@ public class VerifyOteAction implements Runnable, JsonAction {
HistoryEntryStats historyEntryStats =
new HistoryEntryStats().recordRegistrarHistory(registrarName);
List<String> failureMessages = historyEntryStats.findFailures();
String passedFraction =
String.format(
"%2d/%2d",
StatType.NUM_REQUIREMENTS - failureMessages.size(), StatType.NUM_REQUIREMENTS);
int testsPassed = StatType.NUM_REQUIREMENTS - failureMessages.size();
String status = failureMessages.isEmpty() ? "PASS" : "FAIL";
return summarize
? String.format(
"Num actions: %4d - Reqs passed: %s - Overall: %s",
historyEntryStats.statCounts.size(), passedFraction, status)
"# actions: %4d - Reqs: [%s] %2d/%2d - Overall: %s",
historyEntryStats.statCounts.size(),
historyEntryStats.toSummary(),
testsPassed,
StatType.NUM_REQUIREMENTS,
status)
: String.format(
"%s\n%s\nRequirements passed: %s\nOverall OT&E status: %s\n",
historyEntryStats, Joiner.on('\n').join(failureMessages), passedFraction, status);
"%s\n%s\nRequirements passed: %2d/%2d\nOverall OT&E status: %s\n",
historyEntryStats,
Joiner.on('\n').join(failureMessages),
testsPassed,
StatType.NUM_REQUIREMENTS,
status);
}
private static final Predicate<EppInput> HAS_CLAIMS_NOTICE =
@ -299,5 +304,15 @@ public class VerifyOteAction implements Runnable, JsonAction {
.collect(Collectors.joining("\n")),
statCounts.size());
}
/** Returns a string showing the results of each test, one character per test. */
String toSummary() {
return EnumSet.allOf(StatType.class)
.stream()
.filter(statType -> statType.requirement > 0)
.sorted()
.map(statType -> (statCounts.count(statType) < statType.requirement) ? "." : "-")
.collect(Collectors.joining(""));
}
}
}

View file

@ -41,6 +41,8 @@ public class VerifyOteActionTest {
private final VerifyOteAction action = new VerifyOteAction();
HistoryEntry hostDeleteHistoryEntry;
HistoryEntry domainCreateHistoryEntry;
HistoryEntry domainRestoreHistoryEntry;
@Before
public void init() throws Exception {
@ -50,7 +52,7 @@ public class VerifyOteActionTest {
.setType(Type.DOMAIN_CREATE)
.setXmlBytes(ToolsTestData.loadBytes("domain_create_sunrise.xml").read())
.build());
persistResource(
domainCreateHistoryEntry = persistResource(
new HistoryEntry.Builder()
.setClientId("blobio-1")
.setType(Type.DOMAIN_CREATE)
@ -86,7 +88,7 @@ public class VerifyOteActionTest {
.setType(Type.DOMAIN_DELETE)
.setXmlBytes(ToolsTestData.loadBytes("domain_delete.xml").read())
.build());
persistResource(
domainRestoreHistoryEntry = persistResource(
new HistoryEntry.Builder()
.setClientId("blobio-1")
.setType(Type.DOMAIN_RESTORE)
@ -144,16 +146,26 @@ public class VerifyOteActionTest {
}
@Test
public void testSuccess_passSummarize() throws Exception {
public void testSuccess_summarize_allPass() throws Exception {
Map<String, Object> response =
action.handleJsonRequest(
ImmutableMap.of("summarize", "true", "registrars", ImmutableList.of("blobio")));
assertThat(response)
.containsExactly(
"blobio", "# actions: 31 - Reqs: [----------------] 16/16 - Overall: PASS");
}
for (Entry<String, Object> registrar : response.entrySet()) {
assertThat(registrar.getKey()).matches("blobio");
assertThat(registrar.getValue().toString()).containsMatch("Reqs passed: 16/16");
assertThat(registrar.getValue().toString()).containsMatch("Overall: PASS");
}
@Test
public void testSuccess_summarize_someFailures() throws Exception {
deleteResource(hostDeleteHistoryEntry);
deleteResource(domainCreateHistoryEntry);
deleteResource(domainRestoreHistoryEntry);
Map<String, Object> response =
action.handleJsonRequest(
ImmutableMap.of("summarize", "true", "registrars", ImmutableList.of("blobio")));
assertThat(response)
.containsExactly(
"blobio", "# actions: 26 - Reqs: [-.-----.------.-] 13/16 - Overall: FAIL");
}
@Test