From 8e0aa9448543e200d6a97eb332d03da54fb53d90 Mon Sep 17 00:00:00 2001 From: Javier Maestro Date: Thu, 16 May 2024 20:57:29 +0100 Subject: [PATCH] Improve testing with stats and errors per test section * TestRunner: separated the running of each "section" of a test into its own try-catch so that we can evaluate each section separately even if another section failed (e.g. evaluate all examples even if one test in "facts" throws an exception). I've added tests that exercise this "new feature". * TestResults: refactored it a bit so that TestResults now has the three sections and the old TestResults becomes TestSectionResults. Also, TestSectionResults now has only *one* error, since that's what we get when there's an exception evaluating the Pkl code. * SimpleReport: * Separated reporting of each test section so that it's easy to see which facts or which examples fail. * Added a "stats line" to the module and section levels that reports how many tests / assertions pass/fail. Note that, when reporting errors, there's no stats for the section since the error is all the info we get. Similarly, when examples fail, there's no stats because each example is converted into its own test so there are no failed assertions to report, it's all-or-nothing. * For the examples converted to tests, when multiple examples share the same name I've also added a counter ("# 1" and so on) so that it's easier to identify which one is failing. * JUnitReport: fixed the reporting of failures as it wasn't consistent with "tests": the tests were the number of tests that we run but "failures" were the number of assertions that failed. --- .../kotlin/org/pkl/cli/CliTestRunnerTest.kt | 22 +- .../org/pkl/core/runtime/TestResults.java | 335 ++++++++++-------- .../java/org/pkl/core/runtime/TestRunner.java | 48 ++- .../core/stdlib/test/report/JUnitReport.java | 76 ++-- .../core/stdlib/test/report/SimpleReport.java | 96 +++-- .../kotlin/org/pkl/core/EvaluateTestsTest.kt | 133 ++++++- .../test/kotlin/org/pkl/gradle/TestsTest.kt | 332 ++++++++++++----- 7 files changed, 711 insertions(+), 331 deletions(-) diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt index 79435e939..71d478dc4 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt @@ -62,8 +62,9 @@ class CliTestRunnerTest { assertThat(out.toString().stripFileAndLines(tempDir)) .isEqualTo( """ - module test - succeed ✅ + module test ✅ 100.0% pass [1 passed] + facts ✅ 100.0% pass [1 passed] + succeed ✅ 100.0% pass [2 passed] """ .trimIndent() @@ -80,7 +81,7 @@ class CliTestRunnerTest { facts { ["fail"] { 4 == 9 - "foo" == "bar" + "foo" != "bar" } } """ @@ -96,10 +97,10 @@ class CliTestRunnerTest { assertThat(out.toString().stripFileAndLines(tempDir)) .isEqualTo( """ - module test - fail ❌ - 4 == 9 ❌ - "foo" == "bar" ❌ + module test ❌ 0.0% pass [0 passed, 1 failed] + facts ❌ 0.0% pass [0 passed, 1 failed] + fail ❌ 50.0% pass [1 passed, 1 failed] + 4 == 9 ❌ """ .trimIndent() @@ -118,7 +119,8 @@ class CliTestRunnerTest { 9 == trace(9) "foo" == "foo" } - ["fail"] { + ["bar"] { + "foo" == "foo" 5 == 9 } } @@ -136,8 +138,8 @@ class CliTestRunnerTest { """ - - + + 5 == 9 ❌ results = new ArrayList<>(); + public final String moduleName; + public final String displayUri; + public final TestSectionResults module = new TestSectionResults(TestSection.MODULE); + public final TestSectionResults facts = new TestSectionResults(TestSection.FACTS); + public final TestSectionResults examples = new TestSectionResults(TestSection.EXAMPLES); private String err = ""; - public TestResults(String module, String displayUri) { - this.module = module; + public TestResults(String moduleName, String displayUri) { + this.moduleName = moduleName; this.displayUri = displayUri; } - public String getModuleName() { - return module; - } - - public String getDisplayUri() { - return displayUri; - } - - public List getResults() { - return Collections.unmodifiableList(results); - } - - public TestResult newResult(String name) { - var result = new TestResult(name); - results.add(result); - return result; - } - - public void newResult(String name, Failure failure) { - var result = new TestResult(name); - result.addFailure(failure); - results.add(result); - } - public int totalTests() { - return results.size(); + return module.totalTests() + facts.totalTests() + examples.totalTests(); } public int totalFailures() { - int total = 0; - for (var res : results) { - total += res.getFailures().size(); - } - return total; + return module.totalFailures() + facts.totalFailures() + examples.totalFailures(); } public boolean failed() { - for (var res : results) { - if (res.isFailure()) return true; - } - return false; + return module.failed() || facts.failed() || examples.failed(); } public String getErr() { @@ -85,147 +56,225 @@ public void setErr(String err) { this.err = err; } - public static class TestResult { - - private final String name; - private final List failures = new ArrayList<>(); - private final List errors = new ArrayList<>(); - private boolean isExampleWritten = false; + public static class TestSectionResults { + public final TestSection name; + private final List results = new ArrayList<>(); + private Error error; - public TestResult(String name) { + public TestSectionResults(TestSection name) { this.name = name; } - public boolean isSuccess() { - return failures.isEmpty() && errors.isEmpty(); + public void setError(Error error) { + this.error = error; } - boolean isFailure() { - return !isSuccess(); + public Error getError() { + return error; } - public String getName() { - return name; + public boolean hasError() { + return error != null; } - public boolean isExampleWritten() { - return isExampleWritten; + public List getResults() { + return Collections.unmodifiableList(results); } - public void setExampleWritten(boolean exampleWritten) { - isExampleWritten = exampleWritten; + public TestResult newResult(String name) { + var result = new TestResult(name, this.name == TestSection.EXAMPLES); + results.add(result); + return result; } - public List getFailures() { - return Collections.unmodifiableList(failures); + public void newResult(String name, Failure failure) { + var result = new TestResult(name, this.name == TestSection.EXAMPLES); + result.addFailure(failure); + results.add(result); } - public void addFailure(Failure description) { - failures.add(description); + public int totalTests() { + var total = results.size(); + return (hasError() ? ++total : total); } - public List getErrors() { - return Collections.unmodifiableList(errors); + public int totalFailures() { + int total = 0; + for (var res : results) { + if (res.isFailure()) total++; + } + return (hasError() ? ++total : total); } - public void addError(Error err) { - errors.add(err); + public boolean failed() { + if (hasError()) return true; + + for (var res : results) { + if (res.isFailure()) return true; + } + return false; } - } - public static class Failure { + public static class TestResult { + public final String name; + private int totalAsserts = 0; + private final List failures = new ArrayList<>(); + public final boolean isExample; + private boolean isExampleWritten = false; - private final String kind; - private final String rendered; + public TestResult(String name, boolean isExample) { + this.name = name; + this.isExample = isExample; + } - private Failure(String kind, String rendered) { - this.kind = kind; - this.rendered = rendered; - } + public boolean isSuccess() { + return failures.isEmpty(); + } - public String getKind() { - return kind; - } + public boolean isFailure() { + return !isSuccess(); + } - public String getRendered() { - return rendered; - } + public boolean isExampleWritten() { + return isExampleWritten; + } - public static Failure buildFactFailure(SourceSection sourceSection, String description) { - return new Failure( - "Fact Failure", sourceSection.getCharacters() + " ❌ (" + description + ")"); - } + public void setExampleWritten(boolean exampleWritten) { + isExampleWritten = exampleWritten; + } - public static Failure buildExampleLengthMismatchFailure( - String location, String property, int expectedLength, int actualLength) { - String builder = - "(" - + location - + ")\n" - + "Output mismatch: Expected \"" - + property - + "\" to contain " - + expectedLength - + " examples, but found " - + actualLength; - return new Failure("Output Mismatch (Length)", builder); - } + public int totalAsserts() { + return totalAsserts; + } - public static Failure buildExamplePropertyMismatchFailure( - String location, String property, boolean isMissingInExpected) { - var builder = new StringBuilder(); - builder - .append("(") - .append(location) - .append(")\n") - .append("Output mismatch: \"") - .append(property); - if (isMissingInExpected) { - builder.append("\" exists in actual but not in expected output"); - } else { - builder.append("\" exists in expected but not in actual output"); - } - return new Failure("Output Mismatch", builder.toString()); - } + public void countAssert() { + totalAsserts++; + } - public static Failure buildExampleFailure( - String location, - String expectedLocation, - String expectedValue, - String actualLocation, - String actualValue) { - String builder = - "(" - + location - + ")\n" - + "Expected: (" - + expectedLocation - + ")\n" - + expectedValue - + "\nActual: (" - + actualLocation - + ")\n" - + actualValue; - return new Failure("Example Failure", builder); + public List getFailures() { + return Collections.unmodifiableList(failures); + } + + public void addFailure(Failure description) { + failures.add(description); + } } - } - public static class Error { + public static class Failure { + + private final String kind; + private final String rendered; + + private Failure(String kind, String rendered) { + this.kind = kind; + this.rendered = rendered; + } + + public String getKind() { + return kind; + } + + public String getRendered() { + return rendered; + } + + public static Failure buildFactFailure(SourceSection sourceSection, String description) { + return new Failure( + "Fact Failure", sourceSection.getCharacters() + " ❌ (" + description + ")"); + } + + public static Failure buildExampleLengthMismatchFailure( + String location, String property, int expectedLength, int actualLength) { + String builder = + "(" + + location + + ")\n" + + "Output mismatch: Expected \"" + + property + + "\" to contain " + + expectedLength + + " examples, but found " + + actualLength; + return new Failure("Output Mismatch (Length)", builder); + } - private final String message; - private final PklException exception; + public static Failure buildExamplePropertyMismatchFailure( + String location, String property, boolean isMissingInExpected) { + var builder = new StringBuilder(); + builder + .append("(") + .append(location) + .append(")\n") + .append("Output mismatch: \"") + .append(property); + if (isMissingInExpected) { + builder.append("\" exists in actual but not in expected output"); + } else { + builder.append("\" exists in expected but not in actual output"); + } + return new Failure("Output Mismatch", builder.toString()); + } - public Error(String message, PklException exception) { - this.message = message; - this.exception = exception; + public static Failure buildExampleFailure( + String location, + String expectedLocation, + String expectedValue, + String actualLocation, + String actualValue) { + String builder = + "(" + + location + + ")\n" + + "Expected: (" + + expectedLocation + + ")\n" + + expectedValue + + "\nActual: (" + + actualLocation + + ")\n" + + actualValue; + return new Failure("Example Failure", builder); + } } - public String getMessage() { - return message; + public static class Error { + + private final String message; + private final PklException exception; + + public Error(String message, PklException exception) { + this.message = message; + this.exception = exception; + } + + public String getMessage() { + return message; + } + + public Exception getException() { + return exception; + } + + public String getRendered() { + return exception.getMessage(); + } } - public Exception getException() { - return exception; + public enum TestSection { + MODULE("module"), + FACTS("facts"), + EXAMPLES("examples"); + + private final String name; + + TestSection(final String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } } } } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java b/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java index 47da5a83e..e42c72e51 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java @@ -23,8 +23,9 @@ import org.pkl.core.StackFrameTransformer; import org.pkl.core.ast.member.ObjectMember; import org.pkl.core.module.ModuleKeys; -import org.pkl.core.runtime.TestResults.Error; -import org.pkl.core.runtime.TestResults.Failure; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.Error; +import org.pkl.core.runtime.TestResults.TestSectionResults.Failure; import org.pkl.core.stdlib.PklConverter; import org.pkl.core.stdlib.base.PcfRenderer; import org.pkl.core.util.EconomicMaps; @@ -51,12 +52,25 @@ public TestResults run(VmTyped testModule) { try { checkAmendsPklTest(testModule); - runFacts(testModule, results); - runExamples(testModule, info, results); } catch (VmException v) { - var meta = results.newResult(info.getModuleName()); - meta.addError(new Error(v.getMessage(), v.toPklException(stackFrameTransformer))); + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.module.setError(error); } + + try { + runFacts(testModule, results.facts); + } catch (VmException v) { + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.facts.setError(error); + } + + try { + runExamples(testModule, info, results.examples); + } catch (VmException v) { + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.examples.setError(error); + } + results.setErr(logger.getLogs()); return results; } @@ -72,7 +86,7 @@ private void checkAmendsPklTest(VmTyped value) { } } - private void runFacts(VmTyped testModule, TestResults results) { + private void runFacts(VmTyped testModule, TestSectionResults results) { var facts = VmUtils.readMember(testModule, Identifier.FACTS); if (facts instanceof VmNull) return; @@ -83,6 +97,8 @@ private void runFacts(VmTyped testModule, TestResults results) { var groupListing = (VmListing) groupValue; groupListing.forceAndIterateMemberValues( ((factIndex, factMember, factValue) -> { + result.countAssert(); + assert factValue instanceof Boolean; if (factValue == Boolean.FALSE) { result.addFailure( @@ -95,7 +111,7 @@ private void runFacts(VmTyped testModule, TestResults results) { }); } - private void runExamples(VmTyped testModule, ModuleInfo info, TestResults results) { + private void runExamples(VmTyped testModule, ModuleInfo info, TestSectionResults results) { var examples = VmUtils.readMember(testModule, Identifier.EXAMPLES); if (examples instanceof VmNull) return; @@ -138,7 +154,10 @@ private void runExamples(VmTyped testModule, ModuleInfo info, TestResults result } private void doRunAndValidateExamples( - VmMapping examples, Path expectedOutputFile, Path actualOutputFile, TestResults results) { + VmMapping examples, + Path expectedOutputFile, + Path actualOutputFile, + TestSectionResults results) { var expectedExampleOutputs = loadExampleOutputs(expectedOutputFile); var actualExampleOutputs = new MutableReference(null); var allGroupsSucceeded = new MutableBoolean(true); @@ -202,8 +221,11 @@ private void doRunAndValidateExamples( .build(); } + var exampleName = + group.getLength() == 1 ? testName : testName + " #" + exampleIndex; + results.newResult( - testName, + exampleName, Failure.buildExampleFailure( getDisplayUri(exampleMember), getDisplayUri(expectedMember), @@ -244,10 +266,12 @@ private void doRunAndValidateExamples( } } - private void doRunAndWriteExamples(VmMapping examples, Path outputFile, TestResults results) { + private void doRunAndWriteExamples( + VmMapping examples, Path outputFile, TestSectionResults results) { examples.forceAndIterateMemberValues( (groupKey, groupMember, groupValue) -> { - results.newResult(String.valueOf(groupKey)).setExampleWritten(true); + var example = results.newResult(String.valueOf(groupKey)); + example.setExampleWritten(true); return true; }); writeExampleOutputs(outputFile, examples); diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java index c5c10eeff..ca8378c72 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java @@ -24,7 +24,9 @@ import org.pkl.core.runtime.BaseModule; import org.pkl.core.runtime.Identifier; import org.pkl.core.runtime.TestResults; -import org.pkl.core.runtime.TestResults.TestResult; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.Error; +import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult; import org.pkl.core.runtime.VmDynamic; import org.pkl.core.runtime.VmMapping; import org.pkl.core.runtime.VmTyped; @@ -41,27 +43,46 @@ public void report(TestResults results, Writer writer) throws IOException { writer.append(renderXML(" ", "1.0", buildSuite(results))); } - private VmDynamic buildSuite(TestResults res) { - var testCases = testCases(res); - if (!res.getErr().isBlank()) { + private VmDynamic buildSuite(TestResults results) { + var testCases = testCases(results.moduleName, results.facts); + testCases.addAll(testCases(results.moduleName, results.examples)); + + if (!results.getErr().isBlank()) { var err = buildXmlElement( "system-err", VmMapping.empty(), - members -> members.put("body", syntheticElement(makeCdata(res.getErr())))); + members -> members.put("body", syntheticElement(makeCdata(results.getErr())))); testCases.add(err); } - return buildXmlElement( - "testsuite", buildRootAttributes(res), testCases.toArray(new VmDynamic[0])); + + var attrs = + buildAttributes( + "name", results.moduleName, + "tests", (long) results.totalTests(), + "failures", (long) results.totalFailures()); + + return buildXmlElement("testsuite", attrs, testCases.toArray(new VmDynamic[0])); } - private ArrayList testCases(TestResults results) { - var className = results.getModuleName(); - var elements = new ArrayList(results.totalTests()); - for (var res : results.getResults()) { - var attrs = buildAttributes("classname", className, "name", res.getName()); + private ArrayList testCases(String moduleName, TestSectionResults testSectionResults) { + var elements = new ArrayList(testSectionResults.totalTests()); + + if (testSectionResults.hasError()) { + var error = error(testSectionResults.getError()); + + var attrs = + buildAttributes("classname", moduleName + "." + testSectionResults.name, "name", "error"); + var element = buildXmlElement("testcase", attrs, error.toArray(new VmDynamic[0])); + + elements.add(element); + } + + for (var res : testSectionResults.getResults()) { + var attrs = + buildAttributes( + "classname", moduleName + "." + testSectionResults.name, "name", res.name); var failures = failures(res); - failures.addAll(errors(res)); var element = buildXmlElement("testcase", attrs, failures.toArray(new VmDynamic[0])); elements.add(element); } @@ -83,19 +104,14 @@ private ArrayList failures(TestResult res) { return list; } - private ArrayList errors(TestResult res) { + private ArrayList error(Error error) { var list = new ArrayList(); - long i = 0; - for (var error : res.getErrors()) { - var attrs = buildAttributes("message", error.getMessage()); - long element = i++; - list.add( - buildXmlElement( - "error", - attrs, - members -> - members.put(element, syntheticElement(error.getException().getMessage())))); - } + var attrs = buildAttributes("message", error.getMessage()); + list.add( + buildXmlElement( + "error", + attrs, + members -> members.put(1, syntheticElement("\n" + error.getRendered())))); return list; } @@ -130,16 +146,6 @@ private VmDynamic buildXmlElement( members.size() - 4); } - private VmMapping buildRootAttributes(TestResults results) { - return buildAttributes( - "name", - results.getModuleName(), - "tests", - (long) results.totalTests(), - "failures", - (long) results.totalFailures()); - } - private VmMapping buildAttributes(Object... attributes) { EconomicMap attrs = EconomicMaps.create(attributes.length); for (int i = 0; i < attributes.length; i += 2) { diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java index 82e788cef..ec94a1f62 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java @@ -19,8 +19,8 @@ import java.io.Writer; import java.util.stream.Collectors; import org.pkl.core.runtime.TestResults; -import org.pkl.core.runtime.TestResults.Failure; -import org.pkl.core.runtime.TestResults.TestResult; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult; import org.pkl.core.util.StringUtils; public final class SimpleReport implements TestReport { @@ -28,38 +28,69 @@ public final class SimpleReport implements TestReport { @Override public void report(TestResults results, Writer writer) throws IOException { var builder = new StringBuilder(); - builder.append("module "); - builder.append(results.getModuleName()); - builder.append(" (").append(results.getDisplayUri()).append(")\n"); - StringUtils.joinToStringBuilder( - builder, results.getResults(), "\n", res -> reportResult(res, builder)); + + builder.append("module ").append(results.moduleName); + + builder.append(results.failed() ? " ❌" : " ✅"); + + var statsLine = makeStatsLine(results.totalTests(), results.totalFailures(), results.failed()); + builder.append(statsLine); + + builder.append(" (").append(results.displayUri).append(")"); builder.append("\n"); + + reportResults(results.facts, builder); + reportResults(results.examples, builder); + writer.append(builder); } + private void reportResults(TestSectionResults section, StringBuilder builder) { + if (!section.getResults().isEmpty()) { + builder.append(" ").append(section.name); + + builder.append(section.failed() ? " ❌" : " ✅"); + + var statsLine = + makeStatsLine(section.totalTests(), section.totalFailures(), section.failed()); + builder.append(statsLine); + + builder.append("\n"); + + StringUtils.joinToStringBuilder( + builder, section.getResults(), "\n", res -> reportResult(res, builder)); + builder.append("\n"); + } else if (section.hasError()) { + builder.append(" ").append(section.name).append(" ❌\n"); + var error = "Error:\n" + section.getError().getRendered(); + appendPadded(builder, error, " "); + builder.append("\n"); + } + } + private void reportResult(TestResult result, StringBuilder builder) { - builder.append(" ").append(result.getName()); + builder.append(" ").append(result.name); + if (result.isExampleWritten()) { builder.append(" ✍️"); - } else if (result.isSuccess()) { - builder.append(" ✅"); } else { - builder.append(" ❌\n"); - StringUtils.joinToStringBuilder( - builder, result.getFailures(), "\n", failure -> reportFailure(failure, builder)); - StringUtils.joinToStringBuilder( - builder, - result.getErrors(), - "\n", - error -> { - builder.append(" Error:\n"); - appendPadded(builder, error.getException().getMessage(), " "); - }); - } - } + builder.append(result.isFailure() ? " ❌" : " ✅"); + + if (!result.isExample) { + var statsLine = + makeStatsLine(result.totalAsserts(), result.getFailures().size(), result.isFailure()); + builder.append(statsLine); + } - public static void reportFailure(Failure failure, StringBuilder builder) { - appendPadded(builder, failure.getRendered(), " "); + if (result.isFailure()) { + builder.append("\n"); + StringUtils.joinToStringBuilder( + builder, + result.getFailures(), + "\n", + failure -> appendPadded(builder, failure.getRendered(), " ")); + } + } } private static void appendPadded(StringBuilder builder, String lines, String padding) { @@ -69,4 +100,19 @@ private static void appendPadded(StringBuilder builder, String lines, String pad "\n", str -> builder.append(padding).append(str)); } + + private String makeStatsLine(int total, int failed, boolean isFailed) { + var passed = total - failed; + var pct_passed = total > 0 ? 100.0 * passed / total : 0.0; + + String line; + + if (isFailed) { + line = String.format(" %.1f%% pass [%d passed, %d failed]", pct_passed, passed, failed); + } else { + line = String.format(" %.1f%% pass [%d passed]", pct_passed, passed); + } + + return line; + } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt index 15f99d6b6..3c65a91a6 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt @@ -54,7 +54,7 @@ class EvaluateTestsTest { assertThat(results.displayUri).isEqualTo("repl:text") assertThat(results.totalTests()).isEqualTo(1) assertThat(results.failed()).isFalse - assertThat(results.results[0].name).isEqualTo("should pass") + assertThat(results.facts.results[0].name).isEqualTo("should pass") assertThat(results.err.isBlank()).isTrue } @@ -79,12 +79,13 @@ class EvaluateTestsTest { ) assertThat(results.totalTests()).isEqualTo(1) - assertThat(results.totalFailures()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) assertThat(results.failed()).isTrue - val res = results.results[0] + val res = results.facts.results[0] assertThat(res.name).isEqualTo("should fail") - assertThat(res.errors).isEmpty() + assertThat(results.facts.hasError()).isFalse + assertThat(res.failures.size).isEqualTo(2) val fail1 = res.failures[0] assertThat(fail1.rendered).isEqualTo("1 == 2 ❌ (repl:text)") @@ -114,15 +115,14 @@ class EvaluateTestsTest { ) assertThat(results.totalTests()).isEqualTo(1) - assertThat(results.totalFailures()).isEqualTo(0) + assertThat(results.totalFailures()).isEqualTo(1) assertThat(results.failed()).isTrue - val res = results.results[0] - assertThat(res.name).isEqualTo("text") - assertThat(res.failures).isEmpty() - assertThat(res.errors.size).isEqualTo(1) + val res = results.facts + assertThat(res.results).isEmpty() + assertThat(res.hasError()).isTrue - val error = res.errors[0] + val error = res.error assertThat(error.message).isEqualTo("got an error") assertThat(error.exception.message) .isEqualTo( @@ -183,7 +183,114 @@ class EvaluateTestsTest { assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") assertThat(results.totalTests()).isEqualTo(1) assertThat(results.failed()).isFalse - assertThat(results.results[0].name).isEqualTo("user") + assertThat(results.examples.results[0].name).isEqualTo("user") + } + + @Test + fun `test fact failures with successful example`(@TempDir tempDir: Path) { + val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl") + Files.writeString( + file, + """ + amends "pkl:test" + + facts { + ["should fail"] { + 1 == 2 + "foo" == "bar" + } + } + + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + Files.writeString( + createExpected(file), + """ + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + val results = evaluator.evaluateTest(path(file), false) + assertThat(results.moduleName).startsWith("example") + assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") + assertThat(results.totalTests()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) + assertThat(results.failed()).isTrue + + assertThat(results.facts.results[0].name).isEqualTo("should fail") + assertThat(results.facts.results[0].failures.size).isEqualTo(2) + assertThat(results.examples.results[0].name).isEqualTo("user") + } + + @Test + fun `test fact error with successful example`(@TempDir tempDir: Path) { + val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl") + Files.writeString( + file, + """ + amends "pkl:test" + + facts { + ["should fail"] { + throw("exception") + } + } + + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + Files.writeString( + createExpected(file), + """ + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + val results = evaluator.evaluateTest(path(file), false) + assertThat(results.moduleName).startsWith("example") + assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") + assertThat(results.totalTests()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) + assertThat(results.failed()).isTrue + + assertThat(results.facts.results).isEmpty() + assertThat(results.facts.hasError()).isTrue + assertThat(results.examples.results[0].name).isEqualTo("user") } @Test @@ -228,9 +335,9 @@ class EvaluateTestsTest { assertThat(results.failed()).isTrue assertThat(results.totalFailures()).isEqualTo(1) - val res = results.results[0] + val res = results.examples.results[0] assertThat(res.name).isEqualTo("user") - assertThat(res.errors.isEmpty()).isTrue + assertFalse(results.examples.hasError()) val fail1 = res.failures[0] assertThat(fail1.rendered.stripFileAndLines(tempDir)) diff --git a/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt b/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt index b5a77a954..25fac027e 100644 --- a/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt +++ b/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt @@ -68,24 +68,30 @@ class TestsTest : AbstractTest() { .trimIndent() ) - val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines() + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") - assertThat(output) - .contains( + assertThat(output.trimStart()) + .startsWith( """ - module test (file:///file, line x) - test ❌ + > Task :evalTest FAILED + module test ❌ 0.0% pass [0 passed, 1 failed] (file:///file, line x) + facts ❌ Error: - –– Pkl Error –– - exception - - 9 | throw("exception") - ^^^^^^^^^^^^^^^^^^ - at test#facts["error"][#1] (file:///file, line x) - - 3 | facts { - ^^^^^^^ - at test#facts (file:///file, line x) + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + + 3 | facts { + ^^^^^^^ + at test#facts (file:///file, line x) """ .trimIndent() ) @@ -98,42 +104,51 @@ class TestsTest : AbstractTest() { writeBuildFile() - val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines() + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") assertThat(output.trimStart()) - .contains( + .startsWith( """ - module test (file:///file, line x) - sum numbers ✅ - divide numbers ✅ - fail ❌ - 4 == 9 ❌ (file:///file, line x) - "foo" == "bar" ❌ (file:///file, line x) - user 0 ✅ - user 1 ❌ - (file:///file, line x) - Expected: (file:///file, line x) - new { - name = "Pigeon" - age = 40 - } - Actual: (file:///file, line x) - new { - name = "Pigeon" - age = 41 - } - user 1 ❌ - (file:///file, line x) - Expected: (file:///file, line x) - new { - name = "Parrot" - age = 35 - } - Actual: (file:///file, line x) - new { - name = "Welma" - age = 35 - } + > Task :evalTest FAILED + pkl: TRACE: 8 = 8 (file:///file, line x) + module test ❌ 50.0% pass [3 passed, 3 failed] (file:///file, line x) + facts ❌ 66.7% pass [2 passed, 1 failed] + sum numbers ✅ 100.0% pass [2 passed] + divide numbers ✅ 100.0% pass [2 passed] + fail ❌ 0.0% pass [0 passed, 2 failed] + 4 == 9 ❌ (file:///file, line x) + "foo" == "bar" ❌ (file:///file, line x) + examples ❌ 33.3% pass [1 passed, 2 failed] + user 0 ✅ + user 1 #0 ❌ + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Pigeon" + age = 40 + } + Actual: (file:///file, line x) + new { + name = "Pigeon" + age = 41 + } + user 1 #1 ❌ + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } """ .trimIndent() ) @@ -141,28 +156,7 @@ class TestsTest : AbstractTest() { @Test fun `overwrite expected examples`() { - writePklFile( - additionalExamples = - """ - ["user 0"] { - new { - name = "Cool" - age = 11 - } - } - ["user 1"] { - new { - name = "Pigeon" - age = 41 - } - new { - name = "Welma" - age = 35 - } - } - """ - .trimIndent() - ) + writePklFile(additionalExamples = examples) writeFile("test.pkl-expected.pcf", bigTestExpected) writeBuildFile("overwrite = true") @@ -173,6 +167,77 @@ class TestsTest : AbstractTest() { assertThat(output).contains("user 1 ✍️") } + @Test + fun `full example with error`() { + writeBuildFile() + + writePklFile( + additionalFacts = + """ + ["error"] { + throw("exception") + } + """ + .trimIndent(), + additionalExamples = examples + ) + writeFile("test.pkl-expected.pcf", bigTestExpected) + + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") + + assertThat(output.trimStart()) + .startsWith( + """ + > Task :evalTest FAILED + module test ❌ 25.0% pass [1 passed, 3 failed] (file:///file, line x) + facts ❌ + Error: + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + + 3 | facts { + ^^^^^^^ + at test#facts (file:///file, line x) + examples ❌ 33.3% pass [1 passed, 2 failed] + user 0 ✅ + user 1 #0 ❌ + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Pigeon" + age = 40 + } + Actual: (file:///file, line x) + new { + name = "Pigeon" + age = 41 + } + user 1 #1 ❌ + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } + """ + .trimIndent() + ) + } + @Test fun `JUnit reports`() { val pklFile = writePklFile(contents = bigTest) @@ -189,15 +254,15 @@ class TestsTest : AbstractTest() { .isEqualTo( """ - - - - + + + + 4 == 9 ❌ (file:///file, line x) "foo" == "bar" ❌ (file:///file, line x) - - + + (file:///file, line x) Expected: (file:///file, line x) new { @@ -210,7 +275,7 @@ class TestsTest : AbstractTest() { age = 41 } - + (file:///file, line x) Expected: (file:///file, line x) new { @@ -232,6 +297,102 @@ class TestsTest : AbstractTest() { ) } + @Test + fun `JUnit reports with error`() { + val pklFile = + writePklFile( + additionalFacts = + """ + ["error"] { + throw("exception") + } + """ + .trimIndent(), + additionalExamples = examples + ) + writeFile("test.pkl-expected.pcf", bigTestExpected) + + writeBuildFile("junitReportsDir = file('${pklFile.parent.toNormalizedPathString()}/build')") + + runTask("evalTest", expectFailure = true) + + val outputFile = testProjectDir.resolve("build/test.xml") + val report = outputFile.readText().stripFilesAndLines() + + assertThat(report) + .isEqualTo( + """ + + + + + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + + 3 | facts { + ^^^^^^^ + at test#facts (file:///file, line x) + + + + + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Pigeon" + age = 40 + } + Actual: (file:///file, line x) + new { + name = "Pigeon" + age = 41 + } + + + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } + + + + """ + .trimIndent() + ) + } + + private val examples = + """ + ["user 0"] { + new { + name = "Cool" + age = 11 + } + } + ["user 1"] { + new { + name = "Pigeon" + age = 41 + } + new { + name = "Welma" + age = 35 + } + } + """ + .trimIndent() + private val bigTest = """ amends "pkl:test" @@ -254,22 +415,7 @@ class TestsTest : AbstractTest() { } examples { - ["user 0"] { - new { - name = "Cool" - age = 11 - } - } - ["user 1"] { - new { - name = "Pigeon" - age = 41 - } - new { - name = "Welma" - age = 35 - } - } + $examples } """ .trimIndent()