From 038a2679c71fed45db8bb0c244d7ab1c2e0bff64 Mon Sep 17 00:00:00 2001 From: mcilwain Date: Wed, 28 Mar 2018 08:36:44 -0700 Subject: [PATCH] Add per-test output to verify_ote command in --summarize mode Before this change the output looks like this: registrar1 - Num actions: 93 - Reqs passed: 16/16 - Overall: PASS registrar2 - Num actions: 47 - Reqs passed: 6/16 - Overall: FAIL After this change the output looks like this: registrar1 - # actions: 93 - Reqs: [----------------] 16/16 - Overall: PASS registrar2 - # actions: 47 - Reqs: [...--.-...-...--] 6/16 - Overall: FAIL The status of each test is displayed as a hyphen (passing) or a period (failing), and the tests are always displayed in the same order so it's easier to get an overall view of whether registrars are struggling with the same tests. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=190776935 --- .../tools/server/VerifyOteAction.java | 31 ++++++++++++++----- .../tools/server/VerifyOteActionTest.java | 28 ++++++++++++----- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/java/google/registry/tools/server/VerifyOteAction.java b/java/google/registry/tools/server/VerifyOteAction.java index b49db2487..a1eb47aa7 100644 --- a/java/google/registry/tools/server/VerifyOteAction.java +++ b/java/google/registry/tools/server/VerifyOteAction.java @@ -89,18 +89,23 @@ public class VerifyOteAction implements Runnable, JsonAction { HistoryEntryStats historyEntryStats = new HistoryEntryStats().recordRegistrarHistory(registrarName); List failureMessages = historyEntryStats.findFailures(); - String passedFraction = - String.format( - "%2d/%2d", - StatType.NUM_REQUIREMENTS - failureMessages.size(), StatType.NUM_REQUIREMENTS); + int testsPassed = StatType.NUM_REQUIREMENTS - failureMessages.size(); String status = failureMessages.isEmpty() ? "PASS" : "FAIL"; return summarize ? String.format( - "Num actions: %4d - Reqs passed: %s - Overall: %s", - historyEntryStats.statCounts.size(), passedFraction, status) + "# actions: %4d - Reqs: [%s] %2d/%2d - Overall: %s", + historyEntryStats.statCounts.size(), + historyEntryStats.toSummary(), + testsPassed, + StatType.NUM_REQUIREMENTS, + status) : String.format( - "%s\n%s\nRequirements passed: %s\nOverall OT&E status: %s\n", - historyEntryStats, Joiner.on('\n').join(failureMessages), passedFraction, status); + "%s\n%s\nRequirements passed: %2d/%2d\nOverall OT&E status: %s\n", + historyEntryStats, + Joiner.on('\n').join(failureMessages), + testsPassed, + StatType.NUM_REQUIREMENTS, + status); } private static final Predicate HAS_CLAIMS_NOTICE = @@ -299,5 +304,15 @@ public class VerifyOteAction implements Runnable, JsonAction { .collect(Collectors.joining("\n")), statCounts.size()); } + + /** Returns a string showing the results of each test, one character per test. */ + String toSummary() { + return EnumSet.allOf(StatType.class) + .stream() + .filter(statType -> statType.requirement > 0) + .sorted() + .map(statType -> (statCounts.count(statType) < statType.requirement) ? "." : "-") + .collect(Collectors.joining("")); + } } } diff --git a/javatests/google/registry/tools/server/VerifyOteActionTest.java b/javatests/google/registry/tools/server/VerifyOteActionTest.java index d9b513629..ae57e2f00 100644 --- a/javatests/google/registry/tools/server/VerifyOteActionTest.java +++ b/javatests/google/registry/tools/server/VerifyOteActionTest.java @@ -41,6 +41,8 @@ public class VerifyOteActionTest { private final VerifyOteAction action = new VerifyOteAction(); HistoryEntry hostDeleteHistoryEntry; + HistoryEntry domainCreateHistoryEntry; + HistoryEntry domainRestoreHistoryEntry; @Before public void init() throws Exception { @@ -50,7 +52,7 @@ public class VerifyOteActionTest { .setType(Type.DOMAIN_CREATE) .setXmlBytes(ToolsTestData.loadBytes("domain_create_sunrise.xml").read()) .build()); - persistResource( + domainCreateHistoryEntry = persistResource( new HistoryEntry.Builder() .setClientId("blobio-1") .setType(Type.DOMAIN_CREATE) @@ -86,7 +88,7 @@ public class VerifyOteActionTest { .setType(Type.DOMAIN_DELETE) .setXmlBytes(ToolsTestData.loadBytes("domain_delete.xml").read()) .build()); - persistResource( + domainRestoreHistoryEntry = persistResource( new HistoryEntry.Builder() .setClientId("blobio-1") .setType(Type.DOMAIN_RESTORE) @@ -144,16 +146,26 @@ public class VerifyOteActionTest { } @Test - public void testSuccess_passSummarize() throws Exception { + public void testSuccess_summarize_allPass() throws Exception { Map response = action.handleJsonRequest( ImmutableMap.of("summarize", "true", "registrars", ImmutableList.of("blobio"))); + assertThat(response) + .containsExactly( + "blobio", "# actions: 31 - Reqs: [----------------] 16/16 - Overall: PASS"); + } - for (Entry registrar : response.entrySet()) { - assertThat(registrar.getKey()).matches("blobio"); - assertThat(registrar.getValue().toString()).containsMatch("Reqs passed: 16/16"); - assertThat(registrar.getValue().toString()).containsMatch("Overall: PASS"); - } + @Test + public void testSuccess_summarize_someFailures() throws Exception { + deleteResource(hostDeleteHistoryEntry); + deleteResource(domainCreateHistoryEntry); + deleteResource(domainRestoreHistoryEntry); + Map response = + action.handleJsonRequest( + ImmutableMap.of("summarize", "true", "registrars", ImmutableList.of("blobio"))); + assertThat(response) + .containsExactly( + "blobio", "# actions: 26 - Reqs: [-.-----.------.-] 13/16 - Overall: FAIL"); } @Test