From 7e2e13e37693366d2a3ff435eb5ea2dee9ea87e6 Mon Sep 17 00:00:00 2001 From: Javier Maestro Date: Thu, 16 May 2024 20:57:29 +0100 Subject: [PATCH] [test_stats] Add basic test stats Add some basic test stats to the test runner output(s): number of tests that pass/fail, % of success and number of assertions within each test that pass / fail. --- .../kotlin/org/pkl/cli/CliTestRunnerTest.kt | 20 +++---- .../org/pkl/core/runtime/TestResults.java | 11 +++- .../java/org/pkl/core/runtime/TestRunner.java | 2 + .../core/stdlib/test/report/JUnitReport.java | 36 ++++++++++-- .../core/stdlib/test/report/SimpleReport.java | 58 ++++++++++++++----- 5 files changed, 98 insertions(+), 29 deletions(-) diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt index 79435e939..e21d9fc66 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt @@ -62,8 +62,8 @@ class CliTestRunnerTest { assertThat(out.toString().stripFileAndLines(tempDir)) .isEqualTo( """ - module test - succeed ✅ + module test ✅ 100.0% tests pass [1 passed] + succeed ✅ 100.0% assertions pass (2 passed) """ .trimIndent() @@ -80,7 +80,7 @@ class CliTestRunnerTest { facts { ["fail"] { 4 == 9 - "foo" == "bar" + "foo" != "bar" } } """ @@ -96,10 +96,9 @@ class CliTestRunnerTest { assertThat(out.toString().stripFileAndLines(tempDir)) .isEqualTo( """ - module test - fail ❌ + module test ❌ 0.0% tests pass [0 passed, 1 failed] + fail ❌ 50.0% assertions pass (1 passed, 1 failed) 4 == 9 ❌ - "foo" == "bar" ❌ """ .trimIndent() @@ -118,7 +117,8 @@ class CliTestRunnerTest { 9 == trace(9) "foo" == "foo" } - ["fail"] { + ["bar"] { + "foo" == "foo" 5 == 9 } } @@ -135,9 +135,9 @@ class CliTestRunnerTest { .isEqualTo( """ - - - + + + 5 == 9 ❌ failures = new ArrayList<>(); private final List errors = new ArrayList<>(); private boolean isExampleWritten = false; @@ -108,6 +109,14 @@ public String getName() { return name; } + public int getTotalAsserts() { + return totalAsserts; + } + + public void countAssert() { + totalAsserts++; + } + public boolean isExampleWritten() { return isExampleWritten; } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java b/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java index 47da5a83e..28bccca2b 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java @@ -83,6 +83,8 @@ private void runFacts(VmTyped testModule, TestResults results) { var groupListing = (VmListing) groupValue; groupListing.forceAndIterateMemberValues( ((factIndex, factMember, factValue) -> { + result.countAssert(); + assert factValue instanceof Boolean; if (factValue == Boolean.FALSE) { result.addFailure( diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java index c5c10eeff..14175db3d 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java @@ -59,7 +59,7 @@ private ArrayList testCases(TestResults results) { var className = results.getModuleName(); var elements = new ArrayList(results.totalTests()); for (var res : results.getResults()) { - var attrs = buildAttributes("classname", className, "name", res.getName()); + var attrs = buildTestcaseAttributes(className, res); var failures = failures(res); failures.addAll(errors(res)); var element = buildXmlElement("testcase", attrs, failures.toArray(new VmDynamic[0])); @@ -131,13 +131,41 @@ private VmDynamic buildXmlElement( } private VmMapping buildRootAttributes(TestResults results) { + var total = results.totalTests(); + var failed = results.totalFailures(); + var passed = total - failed; + var pct_passed = total > 0 ? 100.0 * passed / total : 0.0; + return buildAttributes( "name", results.getModuleName(), "tests", - (long) results.totalTests(), - "failures", - (long) results.totalFailures()); + (long) total, + "pct_passed", + String.format("%.1f", pct_passed), + "passed", + (long) passed, + "failed", + (long) failed); + } + + private VmMapping buildTestcaseAttributes(String className, TestResult result) { + var total = result.getTotalAsserts(); + var failed = result.getFailures().size(); + var passed = total - failed; + var pct_passed = total > 0 ? 100.0 * passed / total : 0; + + return buildAttributes( + "classname", + className, + "name", + result.getName(), + "pct_passed", + String.format("%.1f", pct_passed), + "passed", + (long) passed, + "failed", + (long) failed); } private VmMapping buildAttributes(Object... attributes) { diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java index 82e788cef..9d3c304b0 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java @@ -30,9 +30,27 @@ public void report(TestResults results, Writer writer) throws IOException { var builder = new StringBuilder(); builder.append("module "); builder.append(results.getModuleName()); - builder.append(" (").append(results.getDisplayUri()).append(")\n"); + + var total = results.totalTests(); + var failed = results.totalFailures(); + var passed = total - failed; + var pct_passed = total > 0 ? 100.0 * passed / total : 0.0; + + if (results.failed()) { + builder.append(" ❌"); + var msg = + String.format(" %.1f%% tests pass [%d passed, %d failed]", pct_passed, passed, failed); + builder.append(msg); + } else { + builder.append(" ✅"); + var msg = String.format(" %.1f%% tests pass [%d passed]", pct_passed, passed); + builder.append(msg); + } + builder.append(" (").append(results.getDisplayUri()).append(")"); + builder.append("\n"); StringUtils.joinToStringBuilder( builder, results.getResults(), "\n", res -> reportResult(res, builder)); + // res -> reportResult(res, builder, results.totalTests(), results.totalFailures())); builder.append("\n"); writer.append(builder); } @@ -41,20 +59,32 @@ private void reportResult(TestResult result, StringBuilder builder) { builder.append(" ").append(result.getName()); if (result.isExampleWritten()) { builder.append(" ✍️"); - } else if (result.isSuccess()) { - builder.append(" ✅"); } else { - builder.append(" ❌\n"); - StringUtils.joinToStringBuilder( - builder, result.getFailures(), "\n", failure -> reportFailure(failure, builder)); - StringUtils.joinToStringBuilder( - builder, - result.getErrors(), - "\n", - error -> { - builder.append(" Error:\n"); - appendPadded(builder, error.getException().getMessage(), " "); - }); + var total = result.getTotalAsserts(); + var failed = result.getFailures().size(); + var passed = total - failed; + var pct_passed = total > 0 ? 100.0 * passed / total : 0; + + if (result.isSuccess()) { + builder.append(" ✅"); + builder.append(String.format(" %.1f%% assertions pass (%d passed)", pct_passed, passed)); + } else { + builder.append(" ❌"); + builder.append( + String.format( + " %.1f%% assertions pass (%d passed, %d failed)", pct_passed, passed, failed)); + builder.append("\n"); + StringUtils.joinToStringBuilder( + builder, result.getFailures(), "\n", failure -> reportFailure(failure, builder)); + StringUtils.joinToStringBuilder( + builder, + result.getErrors(), + "\n", + error -> { + builder.append(" Error:\n"); + appendPadded(builder, error.getException().getMessage(), " "); + }); + } } }