fix tests for licesne_check filter

Signed-off-by: Shivdeep Singh <[email protected]>
IBM · Jun 11, 2024 · b539226 · b539226
1 parent 53a5df0
commit b539226
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 8 deletions.
diff --git a/transforms/code/license_check/python/Makefile b/transforms/code/license_check/python/Makefile
@@ -44,7 +44,7 @@ publish-dist:: .defaults.publish-dist
 test-image:: .transforms.python-test-image
 
 run-cli-sample:
-	$(MAKE) RUN_FILE=dpk_license_check_python/$(TRANSFORM_NAME)_python.py \
+	$(MAKE) RUN_FILE=$(TRANSFORM_NAME)_python.py \
                 RUN_ARGS="--data_local_config \"{ 'input_folder' : '../test-data/input', 'output_folder' : '../test-data/output'}\"  \
 		 --lc_license_column_name license							\
     	 --lc_licenses_file  ../test-data/sample_approved_licenses.json" 					\

diff --git a/transforms/code/license_check/python/src/dpk_license_check_python/transform_config.py b/transforms/code/license_check/python/src/dpk_license_check_python/transform_config.py
@@ -33,8 +33,8 @@ def _get_supported_licenses(license_file: str, data_access: DataAccess) -> list[
     logger.info(f"Getting supported licenses from file {license_file}")
     licenses_list = None
     try:
-        licenses_list_json = data_access.get_file(license_file)
-        licenses_list = json.loads(licenses_list_json)
+        licenses_list_json, _ = data_access.get_file(license_file)
+        licenses_list = json.loads(licenses_list_json.decode("utf-8"))
         logger.info(f"Read a list of {len(licenses_list)} licenses.")
     except Exception as e:
         logger.error(f"got and exception {e}")
@@ -57,15 +57,24 @@ def __init__(self, config: dict):
         deny = self.license_check["deny"]
         if not deny:
             self.transformer = AllowLicenseStatusTransformer(
-                license_column=license_column, allow_no_license=allow_no_license, licenses=licenses
+                license_column=license_column,
+                allow_no_license=allow_no_license,
+                licenses=licenses,
             )
         else:
             self.transformer = DenyLicenseStatusTransformer(
-                license_column=license_column, allow_no_license=allow_no_license, licenses=licenses
+                license_column=license_column,
+                allow_no_license=allow_no_license,
+                licenses=licenses,
             )
 
     def transform(self, table: pa.Table) -> tuple[list[pa.Table], dict]:
         """ """
+        if type(table) is tuple:
+            logger.info(f"We got a tuple. Extract table from tuple, if possible.")
+            extracted_table, _ = table
+            if type(extracted_table) is pa.Table:
+                table = extracted_table
 
         if not TransformUtils.validate_columns(table, [self.license_check["license_column_name"]]):
             return [], {}

diff --git a/transforms/code/license_check/python/test/test_license_check.py b/transforms/code/license_check/python/test/test_license_check.py
@@ -11,7 +11,7 @@
 ################################################################################
 
 import pyarrow as pa
-from data_processing.test_support.transform.transform_test import AbstractTransformTest
+from data_processing.test_support.transform import AbstractTableTransformTest
 from dpk_license_check_python.transform_config import (
     LICENSE_CHECK_PARAMS,
     LicenseCheckTransform,
@@ -34,7 +34,7 @@
 expected_metadata_list = [{"n_files": 1}, {}]
 
 
-class TestLicenseCheckTransform(AbstractTransformTest):
+class TestLicenseCheckTransform(AbstractTableTransformTest):
     """
     Extends the super-class to define the test data for the tests defined there.
     The name of this class MUST begin with the word Test so that pytest recognizes it as a test class.

diff --git a/transforms/code/license_check/python/test/test_license_check_python.py b/transforms/code/license_check/python/test/test_license_check_python.py
@@ -30,9 +30,11 @@ def get_test_transform_fixtures(self) -> list[tuple]:
         basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), basedir))
         fixtures = []
         launcher = PythonTransformLauncher(LicenseCheckTransformConfiguration())
+        license_file_path = os.path.join(basedir, "sample_approved_licenses.json")
+        print(license_file_path)
         config = {
             "lc_license_column_name": "license",
-            "lc_licenses_file": os.path.join(basedir, "sample_approved_licenses.json"),
+            "lc_licenses_file": license_file_path,
         }
         fixtures.append((launcher, config, basedir + "/input", basedir + "/expected"))
         return fixtures