[Testing] Implementation of an updated testing framework

pyproject.toml

@@ -3,4 +3,10 @@ line-length = 88
 target-version = ['py36']
 
 [tool.pytest.ini_options]
-tmp_path_retention_policy = "none"
+tmp_path_retention_policy = "none"
+markers = [
+    "integration: integration tests",
+    "unit: unit tests",
+    "custom: custom integration tests",
+    "smoke: smoke tests"
+]

setup.py

@@ -120,6 +120,7 @@
     "tensorboard>=1.0,<2.9",
     "tensorboardX>=1.0",
     "evaluate>=0.4.1",
+    "parameterized",
 ]
 
 _docs_deps = [

tests/custom_test.py

@@ -0,0 +1,86 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import runpy
+import unittest
+from typing import Optional
+
+from tests.data import CustomTestConfig
+
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class CustomTestCase(unittest.TestCase):
+    """
+    CustomTestCase class. All custom test classes written should inherit from this
+    class. They will be subsequently tested in the test_custom_class function defined
+    within the CustomIntegrationTest.
+    """
+
+    ...
+
+
+# TODO: consider breaking this up into two classes, similar to non-custom
+# integration tests. Could then make use of parameterize_class instead
+class CustomIntegrationTest(unittest.TestCase):
+    """
+    Base Class for all custom integration tests.
+    """
+
+    custom_scripts_directory: str = None
+    custom_class_directory: str = None
+
+    def test_custom_scripts(self, config: Optional[CustomTestConfig] = None):
+        """
+        This test case will run all custom python scripts that reside in the directory
+        defined by custom_scripts_directory. For each custom python script, there
+        should be a corresponding yaml file which consists of the values defined by
+        the dataclass CustomTestConfig, including the field script_path which is
+        populated with the name of the python script. The test will fail if any
+        of the defined assertions in the script fail
+
+        :param config: config defined by the CustomTestConfig dataclass
+
+        """
+        if config is None:
+            self.skipTest("No custom scripts found. Testing test")
+        script_path = f"{self.custom_scripts_directory}/{config.script_path}"
+        runpy.run_path(script_path)
+
+    def test_custom_class(self, config: Optional[CustomTestConfig] = None):
+        """
+        This test case will run all custom test classes that reside in the directory
+        defined by custom_class_directory. For each custom test class, there
+        should be a corresponding yaml file which consists of the values defined by
+        the dataclass CustomTestConfig, including the field script_path which is
+        populated with the name of the python script. The test will fail if any
+        of the defined tests in the custom class fail.
+
+        :param config: config defined by the CustomTestConfig dataclass
+
+        """
+        if config is None:
+            self.skipTest("No custom class found. Testing test")
+        loader = unittest.TestLoader()
+        tests = loader.discover(self.custom_class_directory, pattern=config.script_path)
+        testRunner = unittest.runner.TextTestRunner()
+        output = testRunner.run(tests)
+        for out in output.errors:
+            raise Exception(output[-1])
+
+        for out in output.failures:
+            _LOGGER.error(out[-1])
+            assert False

tests/data.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+# TODO: maybe test type as decorators?
+class TestType(Enum):
+    SANITY = "sanity"
+    REGRESSION = "regression"
+    SMOKE = "smoke"
+
+
+class Cadence(Enum):
+    COMMIT = "commit"
+    WEEKLY = "weekly"
+    NIGHTLY = "nightly"
+
+
+@dataclass
+class TestConfig:
+    test_type: TestType
+    cadence: Cadence
+
+
+@dataclass
+class CustomTestConfig(TestConfig):
+    script_path: str

tests/docs/testing_framework.md

@@ -0,0 +1,98 @@
+# An Updated Testing Framework
+
+Below is a summary of the testing framework proposed for sparseml. 
+
+## Existing Tests
+
+### Integration Tests
+
+Existing integration tests are rewritten such that all values relevant to the particular
+test case are read from a config file, as opposed to hardcoded values in the test case
+itself or through overloaded pytest fixtures. Each config file should include one 
+combination of relevant parameters that are needed to be tested for that particular
+integration test. Each config file must at least have the values defined by the 
+`TestConfig` dataclass found under `tests/data`. These values include the `cadence`
+(weekly, commit, or nightly) and the `test_type` (sanity, smoke, or regression) for the
+particular test case. While the `test_type` is currently using a config value, we can
+expand it to use pytest markers instead. An example of this updated approach can be
+found in the export test case, `test_generation_export.py`
+
+### Unit Tests
+
+Unit tests are not changed significantly however, can be adapted to use the 
+`unittest.TestCase` base class. While this is not necessary to be used, it does
+seem like `unittest` provides overall greater readability compared to normal pytest
+tests. There is also the case where we can use both pytest and unittest for our test
+cases. This is not uncommon and also what transformers currently does. An example of 
+an updated test can be in the `test_export_data_new.py` test file. A note about using
+`unittest` is that it requires us to install the `parameterized` package for
+decorating test cases.
+
+## Custom Testing
+
+For the purpose of custom integration testing, two new workflows are now enabled 
+
+1. **Custom Script Testing**: Users can test their custom python script which is not 
+required to follow any specific structure. All asserts in the script will be validated
+2. **Custom Testing Class**: For slightly more structure, users can write their own
+testing class. The only requirement is that this testing class inherits from the base
+class `CustomTestCase` which can be found under `tests/custom_test`.
+
+To enable custom integration testing for any of the cases above, a test class must be
+written which inherits from `CustomIntegrationTest` under tests/custom_test. Within this
+class, two paths can be defined: `custom_scripts_directory` which points to the
+directory containing all the custom scripts which are to be tested and 
+`custom_class_directory` which points to the directory containing all the custom test
+classes.
+
+Similar to the non-custom integration testing, each custom integration test script or
+test class must include a .yaml file which includes 3 values 
+(defined by the `CustomTestConfig` dataclass found under `tests/data`): 
+`test_type` which indicates if the test is a sanity, smoke or regression test, 
+`cadence`: which dictates how often the test case runs (on commit, weekly, or nightly), 
+and the `script_path` which lists the name of the custom testing script or custom test
+class within the directory. 
+
+An example of an implementation of the `CustomIntegrationTest` can be found under
+`tests/examples`
+
+## Additional markers and decorators 
+
+- New markers are added in to mark tests as `unit`, `integration`, `smoke`, and `custom` 
+tests allowing us to run a subset of the tests when needed
+- Two new decorators are added in to check for package and compute requirements. If
+the requirements are not met, the test is skipped. Currently, `requires_torch` and 
+`requires_gpu` are added in and can be found under `testing_utils.py`
+
+## Testing Targets 
+
+### Unit Testing Targets:
+- A unit test should be written for every utils, helper, or static function. 
+    - Test cases should be written for all datatype combinations that the function takes as input 
+    - Can have `smoke` tests but focus should be on `sanity`
+
+### Integration Testing Targets:
+- An integration test should be written for every cli pathway that is exposed through `setup.py`
+    - All cli-arg combinations should be tested through a `smoke` check 
+    (all may be overkill but ideally we're covering beyond the few important combinations)
+    - All **important** cli-arg combinations should be covered through either a `sanity`
+    check or a `regression` check
+        - A small model should be tested through a `sanity` check  
+        - All other larger models should be tested through `regression` test types
+
+- An integration test should be written for every major/critical module 
+    - All arg combinations should be tested through a `smoke` check
+    (all may be overkill but ideally we're covering beyond the few important combinations)
+    - All **important** arg combinations should be covered through either a `sanity`
+    check or a `regression` check
+        - A small model should be tested through a `sanity` check  
+        - All other larger models should be tested through `regression` test types
+
+## End-to-end Testing Targets:
+- Tests cascading repositories (sparseml --> vLLM) but will become more prominent as our
+docker containers are furhter solidified. Goal would be to emulate common flows users
+may follow
+
+## Cadence
+- Ideally, large models and `regression` tests should be tested on a nightly cadence while
+unit tests and `sanity` test should be tested on a per commit basis

tests/examples/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/examples/generation_configs/custom_class/run.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tests.custom_test import CustomTestCase
+
+
+# Example custom class for testing
+class MyTests(CustomTestCase):
+    def test_something_else(self):
+        assert 1 == 1

tests/examples/generation_configs/custom_class/run.yaml

@@ -0,0 +1,3 @@
+cadence: "commit"
+test_type: "sanity"
+script_path: "run.py"

tests/examples/generation_configs/custom_script/test_python_script.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Example custom script for testing
+def do_something():
+    assert 1 == 1
+
+
+do_something()

tests/examples/generation_configs/custom_script/test_python_script.yaml

@@ -0,0 +1,3 @@
+cadence: "commit"
+test_type: "sanity"
+script_path: "test_python_script.py"

tests/examples/test_integration_custom.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+import pytest
+
+from parameterized import parameterized
+from tests.custom_test import CustomIntegrationTest
+from tests.data import CustomTestConfig
+from tests.testing_utils import parse_params
+
+
+@pytest.mark.custom
+class TestExampleIntegrationCustom(CustomIntegrationTest):
+    """
+    Integration test class which uses the base CustomIntegrationTest class.
+    """
+
+    custom_scripts_directory = "tests/examples/generation_configs/custom_script"
+    custom_class_directory = "tests/examples/generation_configs/custom_class"
+
+    @parameterized.expand(parse_params(custom_scripts_directory, type="custom"))
+    def test_custom_scripts(self, config: Optional[CustomTestConfig] = None):
+        super().test_custom_scripts(config)
+
+    @parameterized.expand(parse_params(custom_class_directory, type="custom"))
+    def test_custom_class(self, config: Optional[CustomTestConfig] = None):
+        super().test_custom_class(config)