[test_stats] Add basic test stats

Add some basic test stats to the test runner output(s): number of tests that pass/fail, % of success and number of assertions within each test that pass / fail.
apple · May 17, 2024 · 7e2e13e · 7e2e13e
1 parent c1949a9
commit 7e2e13e
Show file tree

Hide file tree

Showing 5 changed files with 98 additions and 29 deletions.
diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt
@@ -62,8 +62,8 @@ class CliTestRunnerTest {
     assertThat(out.toString().stripFileAndLines(tempDir))
       .isEqualTo(
         """
-      module test
-        succeed ✅
+      module test ✅ 100.0% tests pass [1 passed]
+        succeed ✅ 100.0% assertions pass (2 passed)
       
     """
           .trimIndent()
@@ -80,7 +80,7 @@ class CliTestRunnerTest {
       facts {
         ["fail"] {
           4 == 9
-          "foo" == "bar"
+          "foo" != "bar"
         }
       }
     """
@@ -96,10 +96,9 @@ class CliTestRunnerTest {
     assertThat(out.toString().stripFileAndLines(tempDir))
       .isEqualTo(
         """
-      module test
-        fail ❌
+      module test ❌ 0.0% tests pass [0 passed, 1 failed]
+        fail ❌ 50.0% assertions pass (1 passed, 1 failed)
           4 == 9 ❌
-          "foo" == "bar" ❌
       
     """
           .trimIndent()
@@ -118,7 +117,8 @@ class CliTestRunnerTest {
           9 == trace(9)
           "foo" == "foo"
         }
-        ["fail"] {
+        ["bar"] {
+          "foo" == "foo"
           5 == 9
         }
       }
@@ -135,9 +135,9 @@ class CliTestRunnerTest {
       .isEqualTo(
         """
       <?xml version="1.0" encoding="UTF-8"?>
-      <testsuite name="test" tests="2" failures="1">
-          <testcase classname="test" name="foo"></testcase>
-          <testcase classname="test" name="fail">
+      <testsuite name="test" tests="2" pct_passed="50.0" passed="1" failed="1">
+          <testcase classname="test" name="foo" pct_passed="100.0" passed="2" failed="0"></testcase>
+          <testcase classname="test" name="bar" pct_passed="50.0" passed="1" failed="1">
               <failure message="Fact Failure">5 == 9 ❌</failure>
           </testcase>
           <system-err><![CDATA[9 = 9

diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java b/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java
@@ -65,7 +65,7 @@ public int totalTests() {
   public int totalFailures() {
     int total = 0;
     for (var res : results) {
-      total += res.getFailures().size();
+      if (res.isFailure()) total++;
     }
     return total;
   }
@@ -88,6 +88,7 @@ public void setErr(String err) {
   public static class TestResult {
 
     private final String name;
+    private int totalAsserts = 0;
     private final List<Failure> failures = new ArrayList<>();
     private final List<Error> errors = new ArrayList<>();
     private boolean isExampleWritten = false;
@@ -108,6 +109,14 @@ public String getName() {
       return name;
     }
 
+    public int getTotalAsserts() {
+      return totalAsserts;
+    }
+
+    public void countAssert() {
+      totalAsserts++;
+    }
+
     public boolean isExampleWritten() {
       return isExampleWritten;
     }

diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java b/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java
@@ -83,6 +83,8 @@ private void runFacts(VmTyped testModule, TestResults results) {
           var groupListing = (VmListing) groupValue;
           groupListing.forceAndIterateMemberValues(
               ((factIndex, factMember, factValue) -> {
+                result.countAssert();
+
                 assert factValue instanceof Boolean;
                 if (factValue == Boolean.FALSE) {
                   result.addFailure(

diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java
@@ -59,7 +59,7 @@ private ArrayList<VmDynamic> testCases(TestResults results) {
     var className = results.getModuleName();
     var elements = new ArrayList<VmDynamic>(results.totalTests());
     for (var res : results.getResults()) {
-      var attrs = buildAttributes("classname", className, "name", res.getName());
+      var attrs = buildTestcaseAttributes(className, res);
       var failures = failures(res);
       failures.addAll(errors(res));
       var element = buildXmlElement("testcase", attrs, failures.toArray(new VmDynamic[0]));
@@ -131,13 +131,41 @@ private VmDynamic buildXmlElement(
   }
 
   private VmMapping buildRootAttributes(TestResults results) {
+    var total = results.totalTests();
+    var failed = results.totalFailures();
+    var passed = total - failed;
+    var pct_passed = total > 0 ? 100.0 * passed / total : 0.0;
+
     return buildAttributes(
         "name",
         results.getModuleName(),
         "tests",
-        (long) results.totalTests(),
-        "failures",
-        (long) results.totalFailures());
+        (long) total,
+        "pct_passed",
+        String.format("%.1f", pct_passed),
+        "passed",
+        (long) passed,
+        "failed",
+        (long) failed);
+  }
+
+  private VmMapping buildTestcaseAttributes(String className, TestResult result) {
+    var total = result.getTotalAsserts();
+    var failed = result.getFailures().size();
+    var passed = total - failed;
+    var pct_passed = total > 0 ? 100.0 * passed / total : 0;
+
+    return buildAttributes(
+        "classname",
+        className,
+        "name",
+        result.getName(),
+        "pct_passed",
+        String.format("%.1f", pct_passed),
+        "passed",
+        (long) passed,
+        "failed",
+        (long) failed);
   }
 
   private VmMapping buildAttributes(Object... attributes) {

diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java
@@ -30,9 +30,27 @@ public void report(TestResults results, Writer writer) throws IOException {
     var builder = new StringBuilder();
     builder.append("module ");
     builder.append(results.getModuleName());
-    builder.append(" (").append(results.getDisplayUri()).append(")\n");
+
+    var total = results.totalTests();
+    var failed = results.totalFailures();
+    var passed = total - failed;
+    var pct_passed = total > 0 ? 100.0 * passed / total : 0.0;
+
+    if (results.failed()) {
+      builder.append(" ❌");
+      var msg =
+          String.format(" %.1f%% tests pass [%d passed, %d failed]", pct_passed, passed, failed);
+      builder.append(msg);
+    } else {
+      builder.append(" ✅");
+      var msg = String.format(" %.1f%% tests pass [%d passed]", pct_passed, passed);
+      builder.append(msg);
+    }
+    builder.append(" (").append(results.getDisplayUri()).append(")");
+    builder.append("\n");
     StringUtils.joinToStringBuilder(
         builder, results.getResults(), "\n", res -> reportResult(res, builder));
+    // res -> reportResult(res, builder, results.totalTests(), results.totalFailures()));
     builder.append("\n");
     writer.append(builder);
   }
@@ -41,20 +59,32 @@ private void reportResult(TestResult result, StringBuilder builder) {
     builder.append("  ").append(result.getName());
     if (result.isExampleWritten()) {
       builder.append(" ✍️");
-    } else if (result.isSuccess()) {
-      builder.append(" ✅");
     } else {
-      builder.append(" ❌\n");
-      StringUtils.joinToStringBuilder(
-          builder, result.getFailures(), "\n", failure -> reportFailure(failure, builder));
-      StringUtils.joinToStringBuilder(
-          builder,
-          result.getErrors(),
-          "\n",
-          error -> {
-            builder.append("    Error:\n");
-            appendPadded(builder, error.getException().getMessage(), "        ");
-          });
+      var total = result.getTotalAsserts();
+      var failed = result.getFailures().size();
+      var passed = total - failed;
+      var pct_passed = total > 0 ? 100.0 * passed / total : 0;
+
+      if (result.isSuccess()) {
+        builder.append(" ✅");
+        builder.append(String.format(" %.1f%% assertions pass (%d passed)", pct_passed, passed));
+      } else {
+        builder.append(" ❌");
+        builder.append(
+            String.format(
+                " %.1f%% assertions pass (%d passed, %d failed)", pct_passed, passed, failed));
+        builder.append("\n");
+        StringUtils.joinToStringBuilder(
+            builder, result.getFailures(), "\n", failure -> reportFailure(failure, builder));
+        StringUtils.joinToStringBuilder(
+            builder,
+            result.getErrors(),
+            "\n",
+            error -> {
+              builder.append("    Error:\n");
+              appendPadded(builder, error.getException().getMessage(), "        ");
+            });
+      }
     }
   }