From d1c0a4e35d727cc11f6dcb695b66a79c12ecbc8f Mon Sep 17 00:00:00 2001 From: Max Nusspickel Date: Wed, 20 Sep 2023 21:41:46 +0100 Subject: [PATCH 1/9] Adds tests for example files --- vayesta/tests/test_examples.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 vayesta/tests/test_examples.py diff --git a/vayesta/tests/test_examples.py b/vayesta/tests/test_examples.py new file mode 100644 index 000000000..e491ad82b --- /dev/null +++ b/vayesta/tests/test_examples.py @@ -0,0 +1,21 @@ +import sys +from pathlib import Path +import importlib + +import pytest + + +example_path = Path(__file__).parent.parent.parent / 'examples' +examples_files = list(example_path.glob('**/*.py')) + + +@pytest.fixture(params=examples_files[:4], ids=lambda x: Path(x).name) +def example_file(request): + return request.param + + +def test_example(example_file): + spec = importlib.util.spec_from_file_location(example_file.name, str(example_file)) + example = importlib.util.module_from_spec(spec) + sys.modules["module.name"] = example + spec.loader.exec_module(example) From 0c6a4af5c2084a3c4ac82bfdf167dd52f05f1534 Mon Sep 17 00:00:00 2001 From: Max Nusspickel Date: Wed, 20 Sep 2023 21:44:01 +0100 Subject: [PATCH 2/9] Fixes test_examples.py --- vayesta/tests/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vayesta/tests/test_examples.py b/vayesta/tests/test_examples.py index e491ad82b..5f3664fc3 100644 --- a/vayesta/tests/test_examples.py +++ b/vayesta/tests/test_examples.py @@ -9,7 +9,7 @@ examples_files = list(example_path.glob('**/*.py')) -@pytest.fixture(params=examples_files[:4], ids=lambda x: Path(x).name) +@pytest.fixture(params=examples_files, ids=lambda x: Path(x).name) def example_file(request): return request.param From efda54cd1456aed5783297262801d5f696ea872c Mon Sep 17 00:00:00 2001 From: Max Nusspickel Date: Wed, 20 Sep 2023 21:45:54 +0100 Subject: [PATCH 3/9] Removes unnecessary Path in test_examples.py --- vayesta/tests/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vayesta/tests/test_examples.py b/vayesta/tests/test_examples.py index 5f3664fc3..21bb38993 100644 --- a/vayesta/tests/test_examples.py +++ b/vayesta/tests/test_examples.py @@ -9,7 +9,7 @@ examples_files = list(example_path.glob('**/*.py')) -@pytest.fixture(params=examples_files, ids=lambda x: Path(x).name) +@pytest.fixture(params=examples_files, ids=lambda x: x.name) def example_file(request): return request.param From c7ca2de2f0cc746eb26e35e5b0127937f3918785 Mon Sep 17 00:00:00 2001 From: Max Nusspickel Date: Thu, 21 Sep 2023 22:01:13 +0100 Subject: [PATCH 4/9] Running tests in parallel --- .github/workflows/ci.yaml | 2 +- .github/workflows/run_tests.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7dc7d83b1..b7546444b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,7 +38,7 @@ jobs: python -m pip install -e .[dmet,ebcc] --user - name: Run unit tests run: | - python -m pip install pytest pytest-cov --user + python -m pip install pytest pytest-cov pytest-xdist --user python .github/workflows/run_tests.py - name: Upload to codecov uses: codecov/codecov-action@v3 diff --git a/.github/workflows/run_tests.py b/.github/workflows/run_tests.py index ef87a2887..56eb98b80 100644 --- a/.github/workflows/run_tests.py +++ b/.github/workflows/run_tests.py @@ -8,6 +8,7 @@ args = [ "vayesta/tests", "--cov=vayesta", + "-n auto", ] if len(sys.argv) > 1 and sys.argv[1] == "--with-veryslow": From 47853f9af2880c62bd4936d6e5d22758b08be393 Mon Sep 17 00:00:00 2001 From: Max Nusspickel Date: Thu, 21 Sep 2023 22:04:48 +0100 Subject: [PATCH 5/9] Adds pytest-timeout --- .github/workflows/ci.yaml | 2 +- vayesta/tests/test_examples.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b7546444b..d0f441946 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,7 +38,7 @@ jobs: python -m pip install -e .[dmet,ebcc] --user - name: Run unit tests run: | - python -m pip install pytest pytest-cov pytest-xdist --user + python -m pip install pytest pytest-cov pytest-xdist pytest-timeout --user python .github/workflows/run_tests.py - name: Upload to codecov uses: codecov/codecov-action@v3 diff --git a/vayesta/tests/test_examples.py b/vayesta/tests/test_examples.py index 21bb38993..afe5fd92a 100644 --- a/vayesta/tests/test_examples.py +++ b/vayesta/tests/test_examples.py @@ -14,6 +14,7 @@ def example_file(request): return request.param +@pytest.mark.timeout(300) def test_example(example_file): spec = importlib.util.spec_from_file_location(example_file.name, str(example_file)) example = importlib.util.module_from_spec(spec) From 3adc0212ef171faff481db42e5c6fd07e38b07a8 Mon Sep 17 00:00:00 2001 From: Max Nusspickel Date: Thu, 21 Sep 2023 22:11:30 +0100 Subject: [PATCH 6/9] Report example test times --- vayesta/tests/test_examples.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/vayesta/tests/test_examples.py b/vayesta/tests/test_examples.py index afe5fd92a..49d8b88ce 100644 --- a/vayesta/tests/test_examples.py +++ b/vayesta/tests/test_examples.py @@ -1,12 +1,14 @@ import sys from pathlib import Path import importlib +from time import perf_counter import pytest example_path = Path(__file__).parent.parent.parent / 'examples' examples_files = list(example_path.glob('**/*.py')) +timings = {} @pytest.fixture(params=examples_files, ids=lambda x: x.name) @@ -14,9 +16,19 @@ def example_file(request): return request.param +@pytest.fixture(scope='module', autouse=True) +def report_timings() -> None: + yield + for name, time in timings.items(): + print(f"Time for {name:20s}: {time:.1f} s") + + @pytest.mark.timeout(300) def test_example(example_file): spec = importlib.util.spec_from_file_location(example_file.name, str(example_file)) example = importlib.util.module_from_spec(spec) sys.modules["module.name"] = example + t_start = perf_counter() spec.loader.exec_module(example) + timings[example_file.name] = perf_counter() - t_start + From 0b730e07828dc1acf7de764a68092e2238c15c22 Mon Sep 17 00:00:00 2001 From: Max Nusspickel Date: Thu, 21 Sep 2023 22:57:37 +0100 Subject: [PATCH 7/9] Try again with equal sign --- .github/workflows/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.py b/.github/workflows/run_tests.py index 56eb98b80..a6ed3af73 100644 --- a/.github/workflows/run_tests.py +++ b/.github/workflows/run_tests.py @@ -8,7 +8,7 @@ args = [ "vayesta/tests", "--cov=vayesta", - "-n auto", + "-n=auto", ] if len(sys.argv) > 1 and sys.argv[1] == "--with-veryslow": From 7e93ab15e7a9efbda3491007c60cea14739eddb3 Mon Sep 17 00:00:00 2001 From: Max Nusspickel Date: Fri, 22 Sep 2023 18:57:04 +0100 Subject: [PATCH 8/9] Better test names --- .github/workflows/run_tests.py | 3 ++- vayesta/tests/test_examples.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_tests.py b/.github/workflows/run_tests.py index a6ed3af73..d5ac179ed 100644 --- a/.github/workflows/run_tests.py +++ b/.github/workflows/run_tests.py @@ -8,7 +8,8 @@ args = [ "vayesta/tests", "--cov=vayesta", - "-n=auto", + "-n=auto", # Multi-processing with pytest-xdist + "-s", # Show stdout output ] if len(sys.argv) > 1 and sys.argv[1] == "--with-veryslow": diff --git a/vayesta/tests/test_examples.py b/vayesta/tests/test_examples.py index 49d8b88ce..eecacdcf4 100644 --- a/vayesta/tests/test_examples.py +++ b/vayesta/tests/test_examples.py @@ -11,7 +11,7 @@ timings = {} -@pytest.fixture(params=examples_files, ids=lambda x: x.name) +@pytest.fixture(params=examples_files, ids=lambda x: '/'.join([x.parent.name, x.name])) def example_file(request): return request.param @@ -30,5 +30,5 @@ def test_example(example_file): sys.modules["module.name"] = example t_start = perf_counter() spec.loader.exec_module(example) - timings[example_file.name] = perf_counter() - t_start + timings['/'.join([example_file.parent.name, example_file.name])] = perf_counter() - t_start From 59f598182db54edde0afe390f9fa02779b212192 Mon Sep 17 00:00:00 2001 From: Max Nusspickel Date: Sat, 23 Sep 2023 14:19:29 +0100 Subject: [PATCH 9/9] Remove module registry, examples/test_examples.py --- examples/run_examples.py | 28 ---------------------------- vayesta/tests/test_examples.py | 2 -- 2 files changed, 30 deletions(-) delete mode 100644 examples/run_examples.py diff --git a/examples/run_examples.py b/examples/run_examples.py deleted file mode 100644 index 30739db10..000000000 --- a/examples/run_examples.py +++ /dev/null @@ -1,28 +0,0 @@ -import os -import sys -import subprocess - -if len(sys.argv) > 1: - directory = sys.argv[1] -else: - directory = os.path.abspath(os.path.dirname(__file__)) - -examples = os.popen("find . | grep \.py$").readlines() - -assert len(examples) > 0 - -N = len(examples) -errs = [] -for eg in examples[1:]: - print("Running %s" % eg) - errno = subprocess.call("python " + eg[:-1] + " -q", shell=True) - if errno != 0: - print("\033[91mException in %s \033[0m" % eg) - errs.append(eg) - -if len(errs) == 0: - print("\033[92mAll examples passed \033[0m") -else: - print("\033[91m Exceptions found: %d/%d examples failed \033[0m" % (len(errs), len(examples))) - for eg in errs: - print("Execption in %s" % eg) diff --git a/vayesta/tests/test_examples.py b/vayesta/tests/test_examples.py index eecacdcf4..2c06cc0cd 100644 --- a/vayesta/tests/test_examples.py +++ b/vayesta/tests/test_examples.py @@ -27,8 +27,6 @@ def report_timings() -> None: def test_example(example_file): spec = importlib.util.spec_from_file_location(example_file.name, str(example_file)) example = importlib.util.module_from_spec(spec) - sys.modules["module.name"] = example t_start = perf_counter() spec.loader.exec_module(example) timings['/'.join([example_file.parent.name, example_file.name])] = perf_counter() - t_start -