From 8b1565aea2b7a09fc64f7d01323506d49df54a1c Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Fri, 15 Mar 2024 20:05:27 +0000 Subject: [PATCH 01/12] update unit tests and existing architecture to use updated framework --- pyproject.toml | 7 +- tests/sparseml/export/test_export_data_new.py | 123 ++++++++++++++++++ .../generation_configs/tiny_stories.yaml | 4 + .../transformers/test_generation_export.py | 62 +++++++++ tests/testing_utils.py | 61 +++++++++ 5 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 tests/sparseml/export/test_export_data_new.py create mode 100644 tests/sparseml/export/transformers/generation_configs/tiny_stories.yaml create mode 100644 tests/sparseml/export/transformers/test_generation_export.py create mode 100644 tests/testing_utils.py diff --git a/pyproject.toml b/pyproject.toml index af9fccdb6f5..92c2c01c3f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,4 +3,9 @@ line-length = 88 target-version = ['py36'] [tool.pytest.ini_options] -tmp_path_retention_policy = "none" \ No newline at end of file +tmp_path_retention_policy = "none" +markers = [ + "integration: integration tests", + "unit: unit tests", + "custom: custom integration tests", +] \ No newline at end of file diff --git a/tests/sparseml/export/test_export_data_new.py b/tests/sparseml/export/test_export_data_new.py new file mode 100644 index 00000000000..b4e116ec07a --- /dev/null +++ b/tests/sparseml/export/test_export_data_new.py @@ -0,0 +1,123 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tarfile +import unittest +from enum import Enum +from pathlib import Path + +import pytest + +from parameterized import parameterized +from sparseml.export.export_data import create_data_samples, export_data_sample +from tests.sparseml.export.utils import get_dummy_dataset +from tests.testing_utils import requires_torch + + +@requires_torch +@pytest.mark.unit +class ExportDataTransformersUnitTest(unittest.TestCase): + def setUp(self): + import torch + from torch.utils.data import DataLoader + + class Identity(torch.nn.Module): + def __init__(self): + super().__init__() + self.dummy_param = torch.nn.Parameter(torch.empty(0)) + self.device = self.dummy_param.device + + def forward(self, input_ids, attention_mask): + return dict(input_ids=input_ids, attention_mask=attention_mask) + + self.identity_model = Identity() + self.data_loader = DataLoader(get_dummy_dataset("transformers"), batch_size=1) + + @parameterized.expand( + [[0, True], [0, False], [1, True], [1, False], [5, True], [5, False]] + ) + def test_create_data_samples(self, num_samples, model): + import torch + + model = self.identity_model.to("cpu") if model else None + + inputs, outputs, labels = create_data_samples( + data_loader=self.data_loader, num_samples=num_samples, model=model + ) + target_input = next(iter(self.data_loader)) + target_output = target_input + + assert len(inputs) == num_samples + for input in inputs: + for key, value in input.items(): + assert torch.equal(value.unsqueeze(0), target_input[key]) + assert labels == [] + if model is not None: + assert len(outputs) == num_samples + for output in outputs: + for key, value in output.items(): + assert torch.equal(value, target_output[key][0]) + + def tearDown(self): + pass + + +@requires_torch +@pytest.mark.unit +class ExportGenericDataUnitTest(unittest.TestCase): + def setUp(self): + import torch + + class LabelNames(Enum): + basename = "sample-dummies" + filename = "dummy" + + num_samples = 5 + batch_size = 3 + self.samples = [ + torch.randn(batch_size, 3, 224, 224) for _ in range(num_samples) + ] + self.names = LabelNames + self.tmp_path = Path("tmp") + self.tmp_path.mkdir(exist_ok=True) + + @parameterized.expand([[True], [False]]) + def test_export_data_sample(self, as_tar): + export_data_sample( + samples=self.samples, + names=self.names, + target_path=self.tmp_path, + as_tar=as_tar, + ) + + dir_name = self.names.basename.value + dir_name_tar = self.names.basename.value + ".tar.gz" + + if as_tar: + with tarfile.open(os.path.join(self.tmp_path, dir_name_tar)) as tar: + tar.extractall(path=self.tmp_path) + + assert ( + set(os.listdir(self.tmp_path)) == {dir_name} + if not as_tar + else {dir_name, dir_name_tar} + ) + assert set(os.listdir(os.path.join(self.tmp_path, "sample-dummies"))) == { + f"dummy-000{i}.npz" for i in range(len(self.samples)) + } + + def tearDown(self): + shutil.rmtree(self.tmp_path) diff --git a/tests/sparseml/export/transformers/generation_configs/tiny_stories.yaml b/tests/sparseml/export/transformers/generation_configs/tiny_stories.yaml new file mode 100644 index 00000000000..94b633db534 --- /dev/null +++ b/tests/sparseml/export/transformers/generation_configs/tiny_stories.yaml @@ -0,0 +1,4 @@ +cadence: "commit" +test_type: "sanity" +stub: "roneneldan/TinyStories-1M" +task: text-generation \ No newline at end of file diff --git a/tests/sparseml/export/transformers/test_generation_export.py b/tests/sparseml/export/transformers/test_generation_export.py new file mode 100644 index 00000000000..2fd3afaf084 --- /dev/null +++ b/tests/sparseml/export/transformers/test_generation_export.py @@ -0,0 +1,62 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import shutil +import unittest +from pathlib import Path +from typing import Dict + +import pytest + +from huggingface_hub import snapshot_download +from parameterized import parameterized_class +from sparseml import export +from tests.testing_utils import parse_params, requires_torch + + +CONFIGS_DIRECTORY = "tests/sparseml/export/transformers/generation_configs" + + +@pytest.mark.integration +@requires_torch +@parameterized_class(parse_params(CONFIGS_DIRECTORY)) +class TestGenerationExportIntegration(unittest.TestCase): + stub = None + task = None + + def setUp(self): + self.tmp_path = Path("tmp") + self.tmp_path.mkdir(exist_ok=True) + + model_path = self.tmp_path / "model" + self.target_path = self.tmp_path / "target" + self.source_path = snapshot_download(self.stub, local_dir=model_path) + + def test_export_with_external_data(self): + export( + source_path=self.source_path, + target_path=self.target_path, + task=self.task, + save_with_external_data=True, + ) + assert (self.target_path / "deployment" / "model.onnx").exists() + assert (self.target_path / "deployment" / "model.data").exists() + + def tearDown(self): + shutil.rmtree(self.tmp_path) + + +@pytest.mark.custom +class TestGenerationExportIntegrationCustom: + pass diff --git a/tests/testing_utils.py b/tests/testing_utils.py new file mode 100644 index 00000000000..943a921df38 --- /dev/null +++ b/tests/testing_utils.py @@ -0,0 +1,61 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import unittest +from typing import Any, Dict, List + +import yaml + + +# TODO: probably makes sense to move this type of function to a more central place, +# which can be used by __init__.py as well +def is_torch_available(): + try: + import torch + + return True + # Update + except ImportError: + return False + + +def requires_torch(test_case): + return unittest.skipUnless(is_torch_available(), "test requires PyTorch")(test_case) + + +def requires_gpu(test_case): + return unittest.skipUnless(False, "test requires GPU")(test_case) + + +def parse_params(configs_directory: str) -> List[Dict[str, Any]]: + # parses the config file provided + assert os.path.isdir( + configs_directory + ), f"Config_directory {configs_directory} is not a directory" + + config_dicts = [] + for file in os.listdir(configs_directory): + if file.endswith(".yaml") or file.endswith(".yml"): + config_path = os.path.join(configs_directory, file) + # reads the yaml file + with open(config_path, "r") as f: + config = yaml.safe_load(f) + + config.pop("cadence") + config.pop("test_type") + config_dicts.append(config) + print(config_dicts) + return config_dicts From 66297835596125cd39d8bc3bd47463ba13b832b0 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Fri, 15 Mar 2024 20:15:36 +0000 Subject: [PATCH 02/12] small update --- tests/sparseml/export/test_export_data_new.py | 1 + .../export/transformers/test_generation_export.py | 11 ++++++++--- tests/testing_utils.py | 7 ++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/sparseml/export/test_export_data_new.py b/tests/sparseml/export/test_export_data_new.py index b4e116ec07a..a4ceba7dfa2 100644 --- a/tests/sparseml/export/test_export_data_new.py +++ b/tests/sparseml/export/test_export_data_new.py @@ -27,6 +27,7 @@ from tests.testing_utils import requires_torch +# TODO: use assert equal @requires_torch @pytest.mark.unit class ExportDataTransformersUnitTest(unittest.TestCase): diff --git a/tests/sparseml/export/transformers/test_generation_export.py b/tests/sparseml/export/transformers/test_generation_export.py index 2fd3afaf084..6d200310852 100644 --- a/tests/sparseml/export/transformers/test_generation_export.py +++ b/tests/sparseml/export/transformers/test_generation_export.py @@ -22,14 +22,17 @@ from huggingface_hub import snapshot_download from parameterized import parameterized_class from sparseml import export -from tests.testing_utils import parse_params, requires_torch +from tests.testing_utils import parse_custom, parse_params, requires_torch CONFIGS_DIRECTORY = "tests/sparseml/export/transformers/generation_configs" +CUSTOM_CONFIGS_DIRECTORY = ( + "tests/sparseml/export/transformers/generation_configs/custom_configs" +) -@pytest.mark.integration @requires_torch +@pytest.mark.integration @parameterized_class(parse_params(CONFIGS_DIRECTORY)) class TestGenerationExportIntegration(unittest.TestCase): stub = None @@ -58,5 +61,7 @@ def tearDown(self): @pytest.mark.custom +@parameterized_class(parse_custom(CUSTOM_CONFIGS_DIRECTORY)) class TestGenerationExportIntegrationCustom: - pass + def test_custom_script(self): + pass diff --git a/tests/testing_utils.py b/tests/testing_utils.py index 943a921df38..5c7a081159c 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -54,8 +54,9 @@ def parse_params(configs_directory: str) -> List[Dict[str, Any]]: with open(config_path, "r") as f: config = yaml.safe_load(f) - config.pop("cadence") - config.pop("test_type") config_dicts.append(config) - print(config_dicts) return config_dicts + + +def parse_custom(configs_directory: str): + pass From 6dc171042b43c11e1b8d72fae4a859b61ab7da38 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Mon, 18 Mar 2024 21:46:03 +0000 Subject: [PATCH 03/12] updat custom integration teseting, adding support for custom sripts and classes --- tests/custom_test.py | 53 +++++++++++++++++ tests/data.py | 27 +++++++++ tests/sparseml/export/test_export_data_new.py | 3 +- .../generation_configs/custom_class/run.py | 20 +++++++ .../generation_configs/custom_class/run.yaml | 3 + .../custom_script/test_python_script.py | 20 +++++++ .../custom_script/test_python_script.yaml | 3 + .../transformers/test_generation_export.py | 33 +++++++---- tests/testing_utils.py | 57 ++++++++++++++----- 9 files changed, 192 insertions(+), 27 deletions(-) create mode 100644 tests/custom_test.py create mode 100644 tests/data.py create mode 100644 tests/sparseml/export/transformers/generation_configs/custom_class/run.py create mode 100644 tests/sparseml/export/transformers/generation_configs/custom_class/run.yaml create mode 100644 tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.py create mode 100644 tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.yaml diff --git a/tests/custom_test.py b/tests/custom_test.py new file mode 100644 index 00000000000..20d6c6dc4f7 --- /dev/null +++ b/tests/custom_test.py @@ -0,0 +1,53 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import runpy +import unittest +from typing import Optional + +import pytest + +from tests.data import CustomTestConfig + + +class CustomTestCase(unittest.TestCase): + ... + + +# TODO: consider breaking this up into two classes, similar to non-custom +# integration tests. Could then make use of parameterize_class instead +@pytest.mark.custom +class CustomIntegrationTest(unittest.TestCase): + custom_scripts_directory: str = None + custom_class_directory: str = None + + def test_custom_scripts(self, config: Optional[CustomTestConfig] = None): + if config is None: + self.skipTest("No custom scripts found. Testing test") + script_path = f"{self.custom_scripts_directory}/{config.script_path}" + runpy.run_path(script_path) + + def test_custom_class(self, config: Optional[CustomTestConfig] = None): + if config is None: + self.skipTest("No custom class found. Testing test") + loader = unittest.TestLoader() + tests = loader.discover(self.custom_class_directory, pattern=config.script_path) + testRunner = unittest.runner.TextTestRunner() + output = testRunner.run(tests) + for out in output.errors: + raise Exception(output[-1]) + + for out in output.failures: + assert False + assert True diff --git a/tests/data.py b/tests/data.py new file mode 100644 index 00000000000..594d6f36a93 --- /dev/null +++ b/tests/data.py @@ -0,0 +1,27 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import List, Union + + +@dataclass +class TestConfig: + test_type: str + cadence: Union[str, List] + + +@dataclass +class CustomTestConfig(TestConfig): + script_path: str diff --git a/tests/sparseml/export/test_export_data_new.py b/tests/sparseml/export/test_export_data_new.py index a4ceba7dfa2..1a7e24bc747 100644 --- a/tests/sparseml/export/test_export_data_new.py +++ b/tests/sparseml/export/test_export_data_new.py @@ -27,7 +27,6 @@ from tests.testing_utils import requires_torch -# TODO: use assert equal @requires_torch @pytest.mark.unit class ExportDataTransformersUnitTest(unittest.TestCase): @@ -61,7 +60,7 @@ def test_create_data_samples(self, num_samples, model): target_input = next(iter(self.data_loader)) target_output = target_input - assert len(inputs) == num_samples + self.assertEqual(len(inputs), num_samples) for input in inputs: for key, value in input.items(): assert torch.equal(value.unsqueeze(0), target_input[key]) diff --git a/tests/sparseml/export/transformers/generation_configs/custom_class/run.py b/tests/sparseml/export/transformers/generation_configs/custom_class/run.py new file mode 100644 index 00000000000..94317941888 --- /dev/null +++ b/tests/sparseml/export/transformers/generation_configs/custom_class/run.py @@ -0,0 +1,20 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tests.custom_test import CustomTestCase + + +class MyTests(CustomTestCase): + def test_something_else(self): + assert 1 == 1 diff --git a/tests/sparseml/export/transformers/generation_configs/custom_class/run.yaml b/tests/sparseml/export/transformers/generation_configs/custom_class/run.yaml new file mode 100644 index 00000000000..e1c368f2b20 --- /dev/null +++ b/tests/sparseml/export/transformers/generation_configs/custom_class/run.yaml @@ -0,0 +1,3 @@ +cadence: "commit" +test_type: "sanity" +script_path: "run.py" \ No newline at end of file diff --git a/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.py b/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.py new file mode 100644 index 00000000000..31ba8c36327 --- /dev/null +++ b/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.py @@ -0,0 +1,20 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def something(): + assert 1 == 0 + + +something() diff --git a/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.yaml b/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.yaml new file mode 100644 index 00000000000..69f52181259 --- /dev/null +++ b/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.yaml @@ -0,0 +1,3 @@ +cadence: "commit" +test_type: "sanity" +script_path: "test_python_script.py" \ No newline at end of file diff --git a/tests/sparseml/export/transformers/test_generation_export.py b/tests/sparseml/export/transformers/test_generation_export.py index 6d200310852..b322309b75b 100644 --- a/tests/sparseml/export/transformers/test_generation_export.py +++ b/tests/sparseml/export/transformers/test_generation_export.py @@ -15,23 +15,21 @@ import shutil import unittest from pathlib import Path -from typing import Dict +from typing import Optional import pytest from huggingface_hub import snapshot_download -from parameterized import parameterized_class +from parameterized import parameterized, parameterized_class from sparseml import export -from tests.testing_utils import parse_custom, parse_params, requires_torch +from tests.custom_test import CustomIntegrationTest +from tests.data import CustomTestConfig +from tests.testing_utils import parse_params CONFIGS_DIRECTORY = "tests/sparseml/export/transformers/generation_configs" -CUSTOM_CONFIGS_DIRECTORY = ( - "tests/sparseml/export/transformers/generation_configs/custom_configs" -) -@requires_torch @pytest.mark.integration @parameterized_class(parse_params(CONFIGS_DIRECTORY)) class TestGenerationExportIntegration(unittest.TestCase): @@ -60,8 +58,19 @@ def tearDown(self): shutil.rmtree(self.tmp_path) -@pytest.mark.custom -@parameterized_class(parse_custom(CUSTOM_CONFIGS_DIRECTORY)) -class TestGenerationExportIntegrationCustom: - def test_custom_script(self): - pass +class TestGenerationExportIntegrationCustom(CustomIntegrationTest): + custom_scripts_directory = ( + "tests/sparseml/export/transformers/generation_configs/custom_script" + ) + custom_class_directory = ( + "tests/sparseml/export/transformers/generation_configs/custom_class" + ) + + # TODO: make enum + @parameterized.expand([parse_params(custom_scripts_directory, type="custom")]) + def test_custom_scripts(self, config: Optional[CustomTestConfig] = None): + super().test_custom_scripts(config) + + @parameterized.expand([parse_params(custom_class_directory, type="custom")]) + def test_custom_class(self, config: Optional[CustomTestConfig] = None): + super().test_custom_class(config) diff --git a/tests/testing_utils.py b/tests/testing_utils.py index 5c7a081159c..94e375a8093 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -12,22 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import dataclasses import logging import os import unittest -from typing import Any, Dict, List +from typing import List, Optional, Union import yaml +from tests.data import CustomTestConfig, TestConfig + # TODO: probably makes sense to move this type of function to a more central place, # which can be used by __init__.py as well def is_torch_available(): try: - import torch + import torch # noqa: F401 return True - # Update except ImportError: return False @@ -40,7 +42,28 @@ def requires_gpu(test_case): return unittest.skipUnless(False, "test requires GPU")(test_case) -def parse_params(configs_directory: str) -> List[Dict[str, Any]]: +def _load_yaml(configs_directory, file): + if file.endswith(".yaml") or file.endswith(".yml"): + config_path = os.path.join(configs_directory, file) + # reads the yaml file + with open(config_path, "r") as f: + config = yaml.safe_load(f) + return config + return None + + +def _validate_test_config(config: dict): + for f in dataclasses.fields(TestConfig): + if f.name not in config: + return False + return True + + +# Set cadence in the config. The environment must set if nightly, weekly or commit +# tests are running +def parse_params( + configs_directory: str, type: Optional[str] = None +) -> List[Union[dict, CustomTestConfig]]: # parses the config file provided assert os.path.isdir( configs_directory @@ -48,15 +71,23 @@ def parse_params(configs_directory: str) -> List[Dict[str, Any]]: config_dicts = [] for file in os.listdir(configs_directory): - if file.endswith(".yaml") or file.endswith(".yml"): - config_path = os.path.join(configs_directory, file) - # reads the yaml file - with open(config_path, "r") as f: - config = yaml.safe_load(f) + config = _load_yaml(configs_directory, file) + if not config: + continue + + cadence = os.environ.get("CADENCE", "commit") + expected_cadence = config.get("cadence") + if not isinstance(expected_cadence, list): + expected_cadence = [expected_cadence] + if cadence in expected_cadence: + if type == "custom": + config = CustomTestConfig(**config) + else: + _validate_test_config(config) config_dicts.append(config) + else: + logging.info( + f"Skipping testing model: {file} " f"for cadence: {config['cadence']}" + ) return config_dicts - - -def parse_custom(configs_directory: str): - pass From c5f65bc239f2522ece1c094d19e6eda4c6b775e1 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Mon, 18 Mar 2024 21:57:50 +0000 Subject: [PATCH 04/12] add docstring --- tests/custom_test.py | 32 ++++++++++++++++++++++++++++++++ tests/data.py | 5 +++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/tests/custom_test.py b/tests/custom_test.py index 20d6c6dc4f7..f104bdeb47a 100644 --- a/tests/custom_test.py +++ b/tests/custom_test.py @@ -22,6 +22,12 @@ class CustomTestCase(unittest.TestCase): + """ + CustomTestCase class. All custom test classes written should inherit from this + class. They will be subsequently tested in the test_custom_class function defined + within the CustomIntegrationTest. + """ + ... @@ -29,16 +35,42 @@ class CustomTestCase(unittest.TestCase): # integration tests. Could then make use of parameterize_class instead @pytest.mark.custom class CustomIntegrationTest(unittest.TestCase): + """ + Base Class for all custom integration tests. + """ + custom_scripts_directory: str = None custom_class_directory: str = None def test_custom_scripts(self, config: Optional[CustomTestConfig] = None): + """ + This test case will run all custom python scripts that reside in the directory + defined by custom_scripts_directory. For each custom python script, there + should be a corresponding yaml file which consists of the values defined by + the dataclass CustomTestConfig, including the field script_path which is + populated with the name of the python script. The test will fail if any + of the defined assertions in the script fail + + :param config: config defined by the CustomTestConfig dataclass + + """ if config is None: self.skipTest("No custom scripts found. Testing test") script_path = f"{self.custom_scripts_directory}/{config.script_path}" runpy.run_path(script_path) def test_custom_class(self, config: Optional[CustomTestConfig] = None): + """ + This test case will run all custom test classes that reside in the directory + defined by custom_class_directory. For each custom test class, there + should be a corresponding yaml file which consists of the values defined by + the dataclass CustomTestConfig, including the field script_path which is + populated with the name of the python script. The test will fail if any + of the defined tests in the custom class fail. + + :param config: config defined by the CustomTestConfig dataclass + + """ if config is None: self.skipTest("No custom class found. Testing test") loader = unittest.TestLoader() diff --git a/tests/data.py b/tests/data.py index 594d6f36a93..e9a840a1818 100644 --- a/tests/data.py +++ b/tests/data.py @@ -16,10 +16,11 @@ from typing import List, Union +# TODO: add enums @dataclass class TestConfig: - test_type: str - cadence: Union[str, List] + test_type: str # sanity, regression, smoke + cadence: Union[str, List] # weekly, nightly, commit @dataclass From 35bc5acd575a2648e85e947fc94049783992caf5 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Mon, 18 Mar 2024 22:18:20 +0000 Subject: [PATCH 05/12] add dummy smoketest --- pyproject.toml | 1 + tests/sparseml/export/test_export_data_new.py | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 92c2c01c3f7..b767bd1af1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,4 +8,5 @@ markers = [ "integration: integration tests", "unit: unit tests", "custom: custom integration tests", + "smoke: smoke tests" ] \ No newline at end of file diff --git a/tests/sparseml/export/test_export_data_new.py b/tests/sparseml/export/test_export_data_new.py index 1a7e24bc747..f4c68279a71 100644 --- a/tests/sparseml/export/test_export_data_new.py +++ b/tests/sparseml/export/test_export_data_new.py @@ -121,3 +121,26 @@ def test_export_data_sample(self, as_tar): def tearDown(self): shutil.rmtree(self.tmp_path) + + +@pytest.mark.smoke +@requires_torch +class ExportDataDummySmokeTest(unittest.TestCase): + def setUp(self): + import torch + + self.samples = [torch.randn(1, 3, 224, 224) for _ in range(2)] + + class LabelNames(Enum): + basename = "sample-dummies" + filename = "dummy" + + self.names = LabelNames + + @parameterized.expand([["some_path"], [Path("some_path")]]) + def test_export_runs(self, target_path): + Path(target_path).mkdir(exist_ok=True) + export_data_sample( + samples=self.samples, names=self.names, target_path=target_path + ) + shutil.rmtree(target_path) From ef3924c1c53a114cac406b69e35bb01e4740c570 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Tue, 19 Mar 2024 13:42:23 +0000 Subject: [PATCH 06/12] clean-up --- pyproject.toml | 2 +- setup.py | 1 + tests/data.py | 3 +++ tests/sparseml/export/test_export_data_new.py | 7 +++++++ .../transformers/generation_configs/custom_class/run.py | 1 + .../transformers/generation_configs/custom_class/run.yaml | 2 +- .../custom_script/test_python_script.py | 8 ++++---- .../custom_script/test_python_script.yaml | 2 +- .../transformers/generation_configs/tiny_stories.yaml | 2 +- .../export/transformers/test_generation_export.py | 8 +++++++- tests/testing_utils.py | 6 +++++- 11 files changed, 32 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b767bd1af1a..37850262b95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,4 +9,4 @@ markers = [ "unit: unit tests", "custom: custom integration tests", "smoke: smoke tests" -] \ No newline at end of file +] diff --git a/setup.py b/setup.py index ed8f609a991..49d590b8f7d 100644 --- a/setup.py +++ b/setup.py @@ -114,6 +114,7 @@ "tensorboard>=1.0,<2.9", "tensorboardX>=1.0", "evaluate>=0.4.1", + "parameterized", ] _docs_deps = [ diff --git a/tests/data.py b/tests/data.py index e9a840a1818..da92ea558dc 100644 --- a/tests/data.py +++ b/tests/data.py @@ -17,6 +17,9 @@ # TODO: add enums +# test type as decorators? + + @dataclass class TestConfig: test_type: str # sanity, regression, smoke diff --git a/tests/sparseml/export/test_export_data_new.py b/tests/sparseml/export/test_export_data_new.py index f4c68279a71..bcad74ff4fa 100644 --- a/tests/sparseml/export/test_export_data_new.py +++ b/tests/sparseml/export/test_export_data_new.py @@ -27,6 +27,10 @@ from tests.testing_utils import requires_torch +# NOTE: These tests are equivalent to the tests in test_export_data, updated to use +# the new framework + + @requires_torch @pytest.mark.unit class ExportDataTransformersUnitTest(unittest.TestCase): @@ -123,6 +127,9 @@ def tearDown(self): shutil.rmtree(self.tmp_path) +# NOTE: Dummy smoke test + + @pytest.mark.smoke @requires_torch class ExportDataDummySmokeTest(unittest.TestCase): diff --git a/tests/sparseml/export/transformers/generation_configs/custom_class/run.py b/tests/sparseml/export/transformers/generation_configs/custom_class/run.py index 94317941888..3b7a612f413 100644 --- a/tests/sparseml/export/transformers/generation_configs/custom_class/run.py +++ b/tests/sparseml/export/transformers/generation_configs/custom_class/run.py @@ -15,6 +15,7 @@ from tests.custom_test import CustomTestCase +# Example custom class for testing class MyTests(CustomTestCase): def test_something_else(self): assert 1 == 1 diff --git a/tests/sparseml/export/transformers/generation_configs/custom_class/run.yaml b/tests/sparseml/export/transformers/generation_configs/custom_class/run.yaml index e1c368f2b20..b53e8b2ec09 100644 --- a/tests/sparseml/export/transformers/generation_configs/custom_class/run.yaml +++ b/tests/sparseml/export/transformers/generation_configs/custom_class/run.yaml @@ -1,3 +1,3 @@ cadence: "commit" test_type: "sanity" -script_path: "run.py" \ No newline at end of file +script_path: "run.py" diff --git a/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.py b/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.py index 31ba8c36327..284365da6f5 100644 --- a/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.py +++ b/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Example custom script for testing +def do_something(): + assert 1 == 1 -def something(): - assert 1 == 0 - -something() +do_something() diff --git a/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.yaml b/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.yaml index 69f52181259..c4e976c8e39 100644 --- a/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.yaml +++ b/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.yaml @@ -1,3 +1,3 @@ cadence: "commit" test_type: "sanity" -script_path: "test_python_script.py" \ No newline at end of file +script_path: "test_python_script.py" diff --git a/tests/sparseml/export/transformers/generation_configs/tiny_stories.yaml b/tests/sparseml/export/transformers/generation_configs/tiny_stories.yaml index 94b633db534..9ba62a3cced 100644 --- a/tests/sparseml/export/transformers/generation_configs/tiny_stories.yaml +++ b/tests/sparseml/export/transformers/generation_configs/tiny_stories.yaml @@ -1,4 +1,4 @@ cadence: "commit" test_type: "sanity" stub: "roneneldan/TinyStories-1M" -task: text-generation \ No newline at end of file +task: text-generation diff --git a/tests/sparseml/export/transformers/test_generation_export.py b/tests/sparseml/export/transformers/test_generation_export.py index b322309b75b..7bfd2d633bd 100644 --- a/tests/sparseml/export/transformers/test_generation_export.py +++ b/tests/sparseml/export/transformers/test_generation_export.py @@ -29,6 +29,9 @@ CONFIGS_DIRECTORY = "tests/sparseml/export/transformers/generation_configs" +# NOTE: this integration test class has the same integration test written in +# test_geneeration_export, updated to use the new framework + @pytest.mark.integration @parameterized_class(parse_params(CONFIGS_DIRECTORY)) @@ -59,6 +62,10 @@ def tearDown(self): class TestGenerationExportIntegrationCustom(CustomIntegrationTest): + """ + Integration test class which uses the base CustomIntegrationTest class. + """ + custom_scripts_directory = ( "tests/sparseml/export/transformers/generation_configs/custom_script" ) @@ -66,7 +73,6 @@ class TestGenerationExportIntegrationCustom(CustomIntegrationTest): "tests/sparseml/export/transformers/generation_configs/custom_class" ) - # TODO: make enum @parameterized.expand([parse_params(custom_scripts_directory, type="custom")]) def test_custom_scripts(self, config: Optional[CustomTestConfig] = None): super().test_custom_scripts(config) diff --git a/tests/testing_utils.py b/tests/testing_utils.py index 94e375a8093..a38215028e8 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -34,12 +34,16 @@ def is_torch_available(): return False +def is_gpu_avaialble(): + return False + + def requires_torch(test_case): return unittest.skipUnless(is_torch_available(), "test requires PyTorch")(test_case) def requires_gpu(test_case): - return unittest.skipUnless(False, "test requires GPU")(test_case) + return unittest.skipUnless(is_gpu_avaialble(), "test requires GPU")(test_case) def _load_yaml(configs_directory, file): From 5c27f8b8314f63cd14016fd7b921c02694a63566 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Tue, 19 Mar 2024 14:19:25 +0000 Subject: [PATCH 07/12] typo fix --- tests/sparseml/export/transformers/test_generation_export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sparseml/export/transformers/test_generation_export.py b/tests/sparseml/export/transformers/test_generation_export.py index 7bfd2d633bd..97c0ede25f7 100644 --- a/tests/sparseml/export/transformers/test_generation_export.py +++ b/tests/sparseml/export/transformers/test_generation_export.py @@ -73,10 +73,10 @@ class TestGenerationExportIntegrationCustom(CustomIntegrationTest): "tests/sparseml/export/transformers/generation_configs/custom_class" ) - @parameterized.expand([parse_params(custom_scripts_directory, type="custom")]) + @parameterized.expand(parse_params(custom_scripts_directory, type="custom")) def test_custom_scripts(self, config: Optional[CustomTestConfig] = None): super().test_custom_scripts(config) - @parameterized.expand([parse_params(custom_class_directory, type="custom")]) + @parameterized.expand(parse_params(custom_class_directory, type="custom")) def test_custom_class(self, config: Optional[CustomTestConfig] = None): super().test_custom_class(config) From e7b36ec14cc3dd98404fb64e71495a2f1bb85da2 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Wed, 20 Mar 2024 18:33:10 +0000 Subject: [PATCH 08/12] move examples to examples folder; add doc explaining updated framework --- tests/custom_test.py | 3 - tests/docs/testing_framework.md | 65 +++++++++++++++++++ tests/examples/__init__.py | 13 ++++ .../generation_configs/custom_class/run.py | 0 .../generation_configs/custom_class/run.yaml | 0 .../custom_script/test_python_script.py | 0 .../custom_script/test_python_script.yaml | 0 tests/examples/test_integration_custom.py | 40 ++++++++++++ .../transformers/test_generation_export.py | 26 +------- 9 files changed, 119 insertions(+), 28 deletions(-) create mode 100644 tests/docs/testing_framework.md create mode 100644 tests/examples/__init__.py rename tests/{sparseml/export/transformers => examples}/generation_configs/custom_class/run.py (100%) rename tests/{sparseml/export/transformers => examples}/generation_configs/custom_class/run.yaml (100%) rename tests/{sparseml/export/transformers => examples}/generation_configs/custom_script/test_python_script.py (100%) rename tests/{sparseml/export/transformers => examples}/generation_configs/custom_script/test_python_script.yaml (100%) create mode 100644 tests/examples/test_integration_custom.py diff --git a/tests/custom_test.py b/tests/custom_test.py index f104bdeb47a..3fe6f15653b 100644 --- a/tests/custom_test.py +++ b/tests/custom_test.py @@ -16,8 +16,6 @@ import unittest from typing import Optional -import pytest - from tests.data import CustomTestConfig @@ -33,7 +31,6 @@ class CustomTestCase(unittest.TestCase): # TODO: consider breaking this up into two classes, similar to non-custom # integration tests. Could then make use of parameterize_class instead -@pytest.mark.custom class CustomIntegrationTest(unittest.TestCase): """ Base Class for all custom integration tests. diff --git a/tests/docs/testing_framework.md b/tests/docs/testing_framework.md new file mode 100644 index 00000000000..952e5c56c84 --- /dev/null +++ b/tests/docs/testing_framework.md @@ -0,0 +1,65 @@ +# An Updated Testing Framework + +Below is a summary of the testing framework proposed for sparseml. + +## Existing Tests + +### Integration Tests + +Existing integration tests are rewritten such that all values relevant to the particular +test case are read from a config file, as opposed to hardcoded values in the test case +itself or through overloaded pytest fixtures. Each config file should include one +combination of relevant parameters that are needed to be tested for that particular +integration test. Each config file must at least have the values defined by the +`TestConfig` dataclass found under `tests/data`. These values include the `cadence` +(weekly, commit, or nightly) and the `test_type` (sanity, smoke, or regression) for the +particular test case. While the `test_type` is currently using a config value, we can +expand it to use pytest markers instead. An example of this updated approach can be +found in the export test case, `test_generation_export.py` + +### Unit Tests + +Unit tests are not changed significantly however, can be adapted to use the +`unittest.TestCase` base class. While this is not necessary to be used, it does +seem like `unittest` provides overall greater readability compared to normal pytest +tests. There is also the case where we can use both pytest and unittest for our test +cases. This is not uncommon and also what transformers currently does. An example of +an updated test can be in the `test_export_data_new.py` test file. A note about using +`unittest` is that it requires us to install the `parameterized` package for +decorating test cases. + +## Custom Testing + +For the purpose of custom integration testing, two new workflows are now enabled + +1. **Custom Script Testing**: Users can test their custom python script which is not +required to follow any specific structure. All asserts in the script will be validated +2. **Custom Testing Class**: For slightly more structure, users can write their own +testing class. The only requirement is that this testing class inherits from the base +class `CustomTestCase` which can be found under `tests/custom_test`. + +To enable custom integration testing for any of the cases above, a test class must be +written which inherits from `CustomIntegrationTest` under tests/custom_test. Within this +class, two paths can be defined: `custom_scripts_directory` which points to the +directory containing all the custom scripts which are to be tested and +`custom_class_directory` which points to the directory containing all the custom test +classes. + +Similar to the non-custom integration testing, each custom integration test script or +test class must include a .yaml file which includes 3 values +(defined by the `CustomTestConfig` dataclass found under `tests/data`): +`test_type` which indicates if the test is a sanity, smoke or regression test, +`cadence`: which dictates how often the test case runs (on commit, weekly, or nightly), +and the `script_path` which lists the name of the custom testing script or custom test +class within the directory. + +An example of an implementation of the `CustomIntegrationTest` can be found under +`tests/examples` + +## Additional markers and decorators + +- New markers are added in to mark tests as `unit`, `integration`, `smoke`, and `custom` +tests allowing us to run a subset of the tests when needed +- Two new decorators are added in to check for package and compute requirements. If +the requirements are not met, the test is skipped. Currently, `requires_torch` and +`requires_gpu` are added in and can be found under `testing_utils.py` \ No newline at end of file diff --git a/tests/examples/__init__.py b/tests/examples/__init__.py new file mode 100644 index 00000000000..0c44f887a47 --- /dev/null +++ b/tests/examples/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/sparseml/export/transformers/generation_configs/custom_class/run.py b/tests/examples/generation_configs/custom_class/run.py similarity index 100% rename from tests/sparseml/export/transformers/generation_configs/custom_class/run.py rename to tests/examples/generation_configs/custom_class/run.py diff --git a/tests/sparseml/export/transformers/generation_configs/custom_class/run.yaml b/tests/examples/generation_configs/custom_class/run.yaml similarity index 100% rename from tests/sparseml/export/transformers/generation_configs/custom_class/run.yaml rename to tests/examples/generation_configs/custom_class/run.yaml diff --git a/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.py b/tests/examples/generation_configs/custom_script/test_python_script.py similarity index 100% rename from tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.py rename to tests/examples/generation_configs/custom_script/test_python_script.py diff --git a/tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.yaml b/tests/examples/generation_configs/custom_script/test_python_script.yaml similarity index 100% rename from tests/sparseml/export/transformers/generation_configs/custom_script/test_python_script.yaml rename to tests/examples/generation_configs/custom_script/test_python_script.yaml diff --git a/tests/examples/test_integration_custom.py b/tests/examples/test_integration_custom.py new file mode 100644 index 00000000000..9986227b647 --- /dev/null +++ b/tests/examples/test_integration_custom.py @@ -0,0 +1,40 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import pytest + +from parameterized import parameterized +from tests.custom_test import CustomIntegrationTest +from tests.data import CustomTestConfig +from tests.testing_utils import parse_params + + +@pytest.mark.custom +class TestExampleIntegrationCustom(CustomIntegrationTest): + """ + Integration test class which uses the base CustomIntegrationTest class. + """ + + custom_scripts_directory = "tests/examples/generation_configs/custom_script" + custom_class_directory = "tests/examples/generation_configs/custom_class" + + @parameterized.expand(parse_params(custom_scripts_directory, type="custom")) + def test_custom_scripts(self, config: Optional[CustomTestConfig] = None): + super().test_custom_scripts(config) + + @parameterized.expand(parse_params(custom_class_directory, type="custom")) + def test_custom_class(self, config: Optional[CustomTestConfig] = None): + super().test_custom_class(config) diff --git a/tests/sparseml/export/transformers/test_generation_export.py b/tests/sparseml/export/transformers/test_generation_export.py index 97c0ede25f7..f92778382da 100644 --- a/tests/sparseml/export/transformers/test_generation_export.py +++ b/tests/sparseml/export/transformers/test_generation_export.py @@ -15,15 +15,12 @@ import shutil import unittest from pathlib import Path -from typing import Optional import pytest from huggingface_hub import snapshot_download -from parameterized import parameterized, parameterized_class +from parameterized import parameterized_class from sparseml import export -from tests.custom_test import CustomIntegrationTest -from tests.data import CustomTestConfig from tests.testing_utils import parse_params @@ -59,24 +56,3 @@ def test_export_with_external_data(self): def tearDown(self): shutil.rmtree(self.tmp_path) - - -class TestGenerationExportIntegrationCustom(CustomIntegrationTest): - """ - Integration test class which uses the base CustomIntegrationTest class. - """ - - custom_scripts_directory = ( - "tests/sparseml/export/transformers/generation_configs/custom_script" - ) - custom_class_directory = ( - "tests/sparseml/export/transformers/generation_configs/custom_class" - ) - - @parameterized.expand(parse_params(custom_scripts_directory, type="custom")) - def test_custom_scripts(self, config: Optional[CustomTestConfig] = None): - super().test_custom_scripts(config) - - @parameterized.expand(parse_params(custom_class_directory, type="custom")) - def test_custom_class(self, config: Optional[CustomTestConfig] = None): - super().test_custom_class(config) From 85e10cdb41e1eaa4dda5469da8dd79c0caddbcbc Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Thu, 21 Mar 2024 15:45:31 +0000 Subject: [PATCH 09/12] add testing targets --- tests/docs/testing_framework.md | 34 ++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/tests/docs/testing_framework.md b/tests/docs/testing_framework.md index 952e5c56c84..8a4c4d818f9 100644 --- a/tests/docs/testing_framework.md +++ b/tests/docs/testing_framework.md @@ -62,4 +62,36 @@ An example of an implementation of the `CustomIntegrationTest` can be found unde tests allowing us to run a subset of the tests when needed - Two new decorators are added in to check for package and compute requirements. If the requirements are not met, the test is skipped. Currently, `requires_torch` and -`requires_gpu` are added in and can be found under `testing_utils.py` \ No newline at end of file +`requires_gpu` are added in and can be found under `testing_utils.py` + +## Testing Targets + +### Unit Testing Targets: +- A unit test should be written for every utils, helper, or static function. + - Test cases should be written for all datatype combinations that the function takes as input + - Can have `smoke` tests but focus should be on `sanity` + +### Integration Testing Targets: +- An integration test should be written for every cli pathway that is exposed through `setup.py` + - All cli-arg combinations should be tested through a `smoke` check + (all may be overkill but ideally we're covering beyond the few important combinations) + - All **important** cli-arg combinations should be covered through either a `sanity` + check or a `regression` check + - A small model should be tested through a `sanity` check + - All other larger models should be tested through `regression` test types + +- An integration test should be written for every major/critical module + - All arg combinations should be tested through a `smoke` check + (all may be overkill but ideally we're covering beyond the few important combinations) + - All **important** arg combinations should be covered through either a `sanity` + check or a `regression` check + - A small model should be tested through a `sanity` check + - All other larger models should be tested through `regression` test types + +## End-to-end Testing Targets: +- Tests cascading repositories (sparseml --> vLLM) but will become more prominent as are +docker containers are furhter solidified + +## Cadence +- Ideally, large models and `regression` tests should be tested on a nightly cadence while +unit tests and `sanity` test should be tested on a per commit basis \ No newline at end of file From 62cd67620110e20f01835bb98eb80ff92c891be3 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Tue, 26 Mar 2024 19:38:11 +0000 Subject: [PATCH 10/12] add enums for test type and candence, additional config checks, remove extra assert True, and log message for custom test failures --- tests/custom_test.py | 6 +++++- tests/data.py | 19 ++++++++++++++----- tests/testing_utils.py | 16 ++++++++++++++-- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/tests/custom_test.py b/tests/custom_test.py index 3fe6f15653b..2fe4cdd9368 100644 --- a/tests/custom_test.py +++ b/tests/custom_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import runpy import unittest from typing import Optional @@ -19,6 +20,9 @@ from tests.data import CustomTestConfig +_LOGGER = logging.getLogger(__name__) + + class CustomTestCase(unittest.TestCase): """ CustomTestCase class. All custom test classes written should inherit from this @@ -78,5 +82,5 @@ def test_custom_class(self, config: Optional[CustomTestConfig] = None): raise Exception(output[-1]) for out in output.failures: + _LOGGER.error(out[-1]) assert False - assert True diff --git a/tests/data.py b/tests/data.py index da92ea558dc..f8e284d90c9 100644 --- a/tests/data.py +++ b/tests/data.py @@ -13,17 +13,26 @@ # limitations under the License. from dataclasses import dataclass -from typing import List, Union +from enum import Enum -# TODO: add enums -# test type as decorators? +# TODO: maybe test type as decorators? +class TestType(Enum): + SANITY = "sanity" + REGRESSION = "regression" + SMOKE = "smoke" + + +class Cadence(Enum): + COMMIT = "commit" + WEEKLY = "weekly" + NIGHTLY = "nightly" @dataclass class TestConfig: - test_type: str # sanity, regression, smoke - cadence: Union[str, List] # weekly, nightly, commit + test_type: TestType + cadence: Cadence @dataclass diff --git a/tests/testing_utils.py b/tests/testing_utils.py index a38215028e8..14373d25b22 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -13,6 +13,7 @@ # limitations under the License. import dataclasses +import enum import logging import os import unittest @@ -60,6 +61,12 @@ def _validate_test_config(config: dict): for f in dataclasses.fields(TestConfig): if f.name not in config: return False + config_value = config.get(f.name) + if issubclass(f.type, enum.Enum): + try: + f.type(config_value) + except ValueError: + raise False return True @@ -88,10 +95,15 @@ def parse_params( if type == "custom": config = CustomTestConfig(**config) else: - _validate_test_config(config) + if not _validate_test_config(config): + raise ValueError( + "The config provided does not comply with the expected " + "structure. See tests.data.TestConfig for the expected " + "fields." + ) config_dicts.append(config) else: logging.info( - f"Skipping testing model: {file} " f"for cadence: {config['cadence']}" + f"Skipping testing model: {file} for cadence: {config['cadence']}" ) return config_dicts From b956ba23dd28cd085e78264a0b9edc6ea3033b7e Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Wed, 27 Mar 2024 14:05:37 +0000 Subject: [PATCH 11/12] fix typo; update doc --- tests/docs/testing_framework.md | 3 ++- tests/testing_utils.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/docs/testing_framework.md b/tests/docs/testing_framework.md index 8a4c4d818f9..3af21691df1 100644 --- a/tests/docs/testing_framework.md +++ b/tests/docs/testing_framework.md @@ -90,7 +90,8 @@ the requirements are not met, the test is skipped. Currently, `requires_torch` a ## End-to-end Testing Targets: - Tests cascading repositories (sparseml --> vLLM) but will become more prominent as are -docker containers are furhter solidified +docker containers are furhter solidified. Goal would be to emulate common flows users +may follow ## Cadence - Ideally, large models and `regression` tests should be tested on a nightly cadence while diff --git a/tests/testing_utils.py b/tests/testing_utils.py index 14373d25b22..81853d0ca03 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -35,7 +35,7 @@ def is_torch_available(): return False -def is_gpu_avaialble(): +def is_gpu_available(): return False @@ -44,7 +44,7 @@ def requires_torch(test_case): def requires_gpu(test_case): - return unittest.skipUnless(is_gpu_avaialble(), "test requires GPU")(test_case) + return unittest.skipUnless(is_gpu_available(), "test requires GPU")(test_case) def _load_yaml(configs_directory, file): From 6010a29d77bf6743e2989f5e03f2ed5d98cf6d73 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Wed, 27 Mar 2024 11:04:35 -0400 Subject: [PATCH 12/12] Update tests/docs/testing_framework.md Co-authored-by: Rahul Tuli --- tests/docs/testing_framework.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docs/testing_framework.md b/tests/docs/testing_framework.md index 3af21691df1..8700bfd426f 100644 --- a/tests/docs/testing_framework.md +++ b/tests/docs/testing_framework.md @@ -89,7 +89,7 @@ the requirements are not met, the test is skipped. Currently, `requires_torch` a - All other larger models should be tested through `regression` test types ## End-to-end Testing Targets: -- Tests cascading repositories (sparseml --> vLLM) but will become more prominent as are +- Tests cascading repositories (sparseml --> vLLM) but will become more prominent as our docker containers are furhter solidified. Goal would be to emulate common flows users may follow