improve report

src/pytest_xflaky/plugin.py

@@ -5,6 +5,7 @@
 import sys
 import uuid
 from dataclasses import dataclass
+from pathlib import Path
 
 import pytest
 from pytest_jsonreport.plugin import pytest_configure as jsonreport_pytest_configure
@@ -26,25 +27,32 @@ def __eq__(self, other):
 
 
 @dataclass
-class FlakyTest:
+class MaybeFlakyTest:
     test: Test
     ok: int
     failed: int
+    min_failures: bool
+
+    def is_flaky(self):
+        return self.ok > 0 and self.failed >= self.min_failures
 
 
 class Plugin:
     def __init__(self, config):
         self.config = config
+        self.flaky_report = None
 
-        self.check_jsonreport()
         self.make_reports_dir()
 
         if self.config.option.xflaky_report:
             self.generate_report()
+        else:
+            self.check_jsonreport()
 
         report_file = self.config.option.json_report_file
-        directory = self.config.option.xflaky_reports_directory
-        self.new_report_file = f"{uuid.uuid4()}-{os.path.basename(report_file)}"
+        directory = Path(self.config.option.xflaky_reports_directory)
+        filename = f"{uuid.uuid4()}-{os.path.basename(report_file)}"
+        self.new_report_file = str(directory / filename)
 
     def check_jsonreport(self):
         if not self.config.pluginmanager.hasplugin("pytest_jsonreport"):
@@ -70,19 +78,35 @@ def generate_report(self):
             min_failures=self.config.option.xflaky_min_failures,
         )
 
-        flaky_tests, total_tests = finder.run()
-        if flaky_tests:
-            self.print_report(flaky_tests)
-            pytest.exit(f"Flaky tests found: {flaky_tests}/{total_tests}", returncode=1)
+        tests, flaky = finder.run()
+
+        self.print_report(tests, flaky)
 
-        pytest.exit("No flaky tests found", returncode=0)
+        if flaky > 0:
+            pytest.exit("Flaky tests were found", returncode=1)
+        else:
+            pytest.exit("No flaky tests found", returncode=0)
 
-    def print_report(self, flaky_tests):
-        for flaky_test in flaky_tests:
+    def print_report(self, tests: list[MaybeFlakyTest], flaky: int):
+        print("FAILED TESTS:")
+        self.flaky_report = []
+        failed_tests = [test for test in tests if test.failed > 0]
+        sorted_tests = sorted(failed_tests, key=lambda t: t.is_flaky())
+        for maybe_flaky_test in sorted_tests:
+            label = " FLAKY" if maybe_flaky_test.is_flaky() else ""
             print(
-                f"Flaky test found: {flaky_test.test} (ok: {flaky_test.ok}, failed: {flaky_test.failed}"
+                f"{maybe_flaky_test.test} (ok: {maybe_flaky_test.ok}, failed: {maybe_flaky_test.failed}){label}"
             )
 
+        failures = len(failed_tests)
+        succeeds = len(tests) - failures
+        runs = failures + succeeds
+
+        print("-")
+        print(
+            f"Flaky tests result (tests: {len(tests)}, runs: {runs}, succeeds: {succeeds}, failures: {failures}, flaky: {flaky})",
+        )
+
     def pytest_terminal_summary(self, terminalreporter):
         terminalreporter.write_sep("-", "XFLAKY report")
         terminalreporter.write_line(f"Report file copied to {self.new_report_file}")
@@ -93,29 +117,24 @@ def __init__(self, *, directory: str, min_failures: int):
         self.directory = directory
         self.min_failures = min_failures
 
-    def run(self) -> list[FlakyTest]:
-        ok = {}
-        failures = {}
-        total = 0
+    def run(self) -> list[MaybeFlakyTest]:
+        cache = {}
         for test, failure in self.collect_tests():
-            total += 1
+            cache.setdefault(
+                test,
+                MaybeFlakyTest(
+                    test=test, ok=0, failed=0, min_failures=self.min_failures
+                ),
+            )
+
             if failure:
-                failures.setdefault(test, 0)
-                failures[test] += 1
+                cache[test].failed += 1
             else:
-                ok.setdefault(test, 0)
-                ok[test] += 1
-
-        flaky_tests = []
-        for test in failures:
-            if failures[test] > self.min_failures:
-                ok_ = ok.get(test, 0)
-                if ok_ > 0:
-                    flaky_tests.append(
-                        FlakyTest(test=test, ok=ok_, failed=failures[test])
-                    )
-
-        return flaky_tests, total
+                cache[test].ok += 1
+
+        tests = list(cache.values())
+        flaky_total = sum(1 for test in tests if test.is_flaky())
+        return tests, flaky_total
 
     def collect_tests(self):
         for f in os.listdir(self.directory):