Fix kfp workflows which were created with a bad template (#672)

* fix typo in kfp workflows Signed-off-by: David Wood <[email protected]> * fix another typo in kfp workflows Signed-off-by: David Wood <[email protected]> * innocuous change in .make.workflows to trigger kfp tests Signed-off-by: David Wood <[email protected]> * fix missing path in workflow templates Signed-off-by: David Wood <[email protected]> * fix repo_level_order_wf.py default float value of str type Signed-off-by: David Wood <[email protected]> * workflow generation fixes and kfp workflow to test for kfp_ray/Makefile Signed-off-by: David Wood <[email protected]> * disable kfp cicd for repo_level_ordering and text_encoder Signed-off-by: David Wood <[email protected]> * fix testing for makefile in kfp workflows Signed-off-by: David Wood <[email protected]> * remove kfp_support_list testing from transform's kfp git workflow Signed-off-by: David Wood <[email protected]> * disable kfp cicd on license_select Signed-off-by: David Wood <[email protected]> --------- Signed-off-by: David Wood <[email protected]>
IBM · Oct 7, 2024 · 87bff17 · 87bff17
1 parent d04454e
commit 87bff17
Show file tree

Hide file tree

Showing 25 changed files with 220 additions and 202 deletions.
diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile
@@ -10,39 +10,48 @@ LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_enco
 # A list that holds transforms that should not be tested with KFP
 
 transform-tests:
+	-rm -rf test-universal-*.yml
 	$(MAKE) TRANSFORM_SUBDIR=universal .transform-tests 
 	$(MAKE) TRANSFORM_SUBDIR=universal .transform-kfp-tests
+	-rm -rf test-language-*.yml
 	$(MAKE) TRANSFORM_SUBDIR=language .transform-tests 
 	$(MAKE) TRANSFORM_SUBDIR=language .transform-kfp-tests
+	-rm -rf test-code-*.yml
 	$(MAKE) TRANSFORM_SUBDIR=code .transform-tests 
 	$(MAKE) TRANSFORM_SUBDIR=code .transform-kfp-tests
 
 # Expects
 #   TRANSFORM_SUBDIR  transforms subdirectory (such as universal)
 .transform-tests:
 	@for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do	\
-	    dir=$$(basename $$i);							\
-	    yml=test-$(TRANSFORM_SUBDIR)-$$dir.yml;					\
-	    echo Generating $$yml;							\
-	    cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml;	\
+	    z=$$(ls $$i/Makefile*);							\
+	    if [ ! -z "$$z" ]; then							\
+	        dir=$$(basename $$i);							\
+	        yml=test-$(TRANSFORM_SUBDIR)-$$dir.yml;					\
+	        echo Generating $$yml;							\
+	        cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml;	\
+	    fi;										\
 	done
 
 .transform-kfp-tests:
 	@KFP_BLACK_LIST=$$(cd ../..; bash scripts/check-workflows.sh -show-kfp-black-list);		\
 	for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do	\
+	    z=$$(ls $$i/Makefile*);							\
+	    if [ ! -z "$$z" ]; then							\
 		dir=$$(basename $$i);				\
 		yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml;                                     \
 		if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ]; then		\
-			echo No kfp_ray directory for $$dir. Skipping generation of $$yml;                                                      \
+			echo No kfp_ray directory for $$dir. Skipping generation of $$yml;      \
 			continue;			\
 		fi;					\
 		z=$$(echo $${KFP_BLACK_LIST} | grep  $$dir);     \
 		if  [ ! -z "$$z" ]; then		\
-			echo $$dir is black listed. Skipping generation of $$yml;                                                      \
+			echo $$dir is black listed. Skipping generation of $$yml;               \
 			continue;			\
 		fi;					\
-		echo Generating $$yml;                                                      \
+		echo Generating $$yml;                  \
 		cat test-kfp-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml;    \
+	    fi;						\
 	done
 
 

diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml
@@ -13,7 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
-            - "transforms/.make.workflow"
+            - "transforms/.make.workflows"
             - "transforms/code/code2parquet/**"
             - "!kfp/**" # This is tested in separate workflow
             - "!data-processing-lib/**" # This is tested in separate workflow
@@ -27,7 +27,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
-            - "transforms/.make.workflow"
+            - "transforms/.make.workflows"
             - "transforms/code/code2parquet/**"
             - "!data-processing-lib/**" # This is tested in separate workflow
             - "!kfp/**" # This is tested in separate workflow
@@ -56,10 +56,10 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
-            - name: Test KFP libs (shared and v1) and run a workflow
+            - name: Test V1 KFP workflow for transforms/code/code2parquet
               timeout-minutes: 120
               run: |
-                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
+                  if [ -e "transforms/code/code2parquet/Makefile" -a -e "transforms/code/code2parquet/kfp_ray/Makefile" ]; then
                       export REPOROOT=$PWD
                       export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
                       source $K8S_SETUP_SCRIPTS/requirements.env
@@ -76,13 +76,13 @@ jobs:
                       chmod +x /tmp/mc
                       export DEPLOY_KUBEFLOW=1
                       make -C $K8S_SETUP_SCRIPTS setup
-                      make -C kfp/kfp_support_lib test
+                      #make -C kfp/kfp_support_lib test
                       make -C transforms/code/code2parquet workflow-build
                       source $K8S_SETUP_SCRIPTS/common.sh
                       make -C transforms/code/code2parquet workflow-test
                       echo "Run transforms/code/code2parquet completed"
                    else
-                      echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray"
+                      echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
                    fi
 
     test-kfp-v2:
@@ -99,10 +99,10 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
-            - name: Test KFP libs (shared and v2) and run a workflow
+            - name: Test V2 KFP workflow for transforms/code/code2parquet
               timeout-minutes: 120
               run: |
-                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
+                  if [ -e "transforms/code/code2parquet/Makefile" -a -e "transforms/code/code2parquet/kfp_ray/Makefile" ]; then
                       export REPOROOT=$PWD
                       export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
                       source $K8S_SETUP_SCRIPTS/requirements.env
@@ -120,11 +120,11 @@ jobs:
                       export DEPLOY_KUBEFLOW=1
                       export KFPv2=1
                       make -C $K8S_SETUP_SCRIPTS setup
-                      make -C kfp/kfp_support_lib test
+                      #make -C kfp/kfp_support_lib test
                       make -C transforms/code/code2parquet workflow-build
                       source $K8S_SETUP_SCRIPTS/common.sh
                       make -C transforms/code/code2parquet workflow-test
                       echo "Run transforms/code/code2parquet completed"
                   else
-                      echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray"
+                      echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
                   fi
diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml
@@ -13,7 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
-            - "transforms/.make.workflow"
+            - "transforms/.make.workflows"
             - "transforms/code/code_quality/**"
             - "!kfp/**" # This is tested in separate workflow
             - "!data-processing-lib/**" # This is tested in separate workflow
@@ -27,7 +27,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
-            - "transforms/.make.workflow"
+            - "transforms/.make.workflows"
             - "transforms/code/code_quality/**"
             - "!data-processing-lib/**" # This is tested in separate workflow
             - "!kfp/**" # This is tested in separate workflow
@@ -56,10 +56,10 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
-            - name: Test KFP libs (shared and v1) and run a workflow
+            - name: Test V1 KFP workflow for transforms/code/code_quality
               timeout-minutes: 120
               run: |
-                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
+                  if [ -e "transforms/code/code_quality/Makefile" -a -e "transforms/code/code_quality/kfp_ray/Makefile" ]; then
                       export REPOROOT=$PWD
                       export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
                       source $K8S_SETUP_SCRIPTS/requirements.env
@@ -76,13 +76,13 @@ jobs:
                       chmod +x /tmp/mc
                       export DEPLOY_KUBEFLOW=1
                       make -C $K8S_SETUP_SCRIPTS setup
-                      make -C kfp/kfp_support_lib test
+                      #make -C kfp/kfp_support_lib test
                       make -C transforms/code/code_quality workflow-build
                       source $K8S_SETUP_SCRIPTS/common.sh
                       make -C transforms/code/code_quality workflow-test
                       echo "Run transforms/code/code_quality completed"
                    else
-                      echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray"
+                      echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
                    fi
 
     test-kfp-v2:
@@ -99,10 +99,10 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
-            - name: Test KFP libs (shared and v2) and run a workflow
+            - name: Test V2 KFP workflow for transforms/code/code_quality
               timeout-minutes: 120
               run: |
-                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
+                  if [ -e "transforms/code/code_quality/Makefile" -a -e "transforms/code/code_quality/kfp_ray/Makefile" ]; then
                       export REPOROOT=$PWD
                       export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
                       source $K8S_SETUP_SCRIPTS/requirements.env
@@ -120,11 +120,11 @@ jobs:
                       export DEPLOY_KUBEFLOW=1
                       export KFPv2=1
                       make -C $K8S_SETUP_SCRIPTS setup
-                      make -C kfp/kfp_support_lib test
+                      #make -C kfp/kfp_support_lib test
                       make -C transforms/code/code_quality workflow-build
                       source $K8S_SETUP_SCRIPTS/common.sh
                       make -C transforms/code/code_quality workflow-test
                       echo "Run transforms/code/code_quality completed"
                   else
-                      echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray"
+                      echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
                   fi
diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml
@@ -13,7 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
-            - "transforms/.make.workflow"
+            - "transforms/.make.workflows"
             - "transforms/code/header_cleanser/**"
             - "!kfp/**" # This is tested in separate workflow
             - "!data-processing-lib/**" # This is tested in separate workflow
@@ -27,7 +27,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
-            - "transforms/.make.workflow"
+            - "transforms/.make.workflows"
             - "transforms/code/header_cleanser/**"
             - "!data-processing-lib/**" # This is tested in separate workflow
             - "!kfp/**" # This is tested in separate workflow
@@ -56,10 +56,10 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
-            - name: Test KFP libs (shared and v1) and run a workflow
+            - name: Test V1 KFP workflow for transforms/code/header_cleanser
               timeout-minutes: 120
               run: |
-                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
+                  if [ -e "transforms/code/header_cleanser/Makefile" -a -e "transforms/code/header_cleanser/kfp_ray/Makefile" ]; then
                       export REPOROOT=$PWD
                       export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
                       source $K8S_SETUP_SCRIPTS/requirements.env
@@ -76,13 +76,13 @@ jobs:
                       chmod +x /tmp/mc
                       export DEPLOY_KUBEFLOW=1
                       make -C $K8S_SETUP_SCRIPTS setup
-                      make -C kfp/kfp_support_lib test
+                      #make -C kfp/kfp_support_lib test
                       make -C transforms/code/header_cleanser workflow-build
                       source $K8S_SETUP_SCRIPTS/common.sh
                       make -C transforms/code/header_cleanser workflow-test
                       echo "Run transforms/code/header_cleanser completed"
                    else
-                      echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray"
+                      echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray/Makefile"
                    fi
 
     test-kfp-v2:
@@ -99,10 +99,10 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
-            - name: Test KFP libs (shared and v2) and run a workflow
+            - name: Test V2 KFP workflow for transforms/code/header_cleanser
               timeout-minutes: 120
               run: |
-                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
+                  if [ -e "transforms/code/header_cleanser/Makefile" -a -e "transforms/code/header_cleanser/kfp_ray/Makefile" ]; then
                       export REPOROOT=$PWD
                       export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
                       source $K8S_SETUP_SCRIPTS/requirements.env
@@ -120,11 +120,11 @@ jobs:
                       export DEPLOY_KUBEFLOW=1
                       export KFPv2=1
                       make -C $K8S_SETUP_SCRIPTS setup
-                      make -C kfp/kfp_support_lib test
+                      #make -C kfp/kfp_support_lib test
                       make -C transforms/code/header_cleanser workflow-build
                       source $K8S_SETUP_SCRIPTS/common.sh
                       make -C transforms/code/header_cleanser workflow-test
                       echo "Run transforms/code/header_cleanser completed"
                   else
-                      echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray"
+                      echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray/Makefile"
                   fi
diff --git a/.github/workflows/test-code-license_select-kfp.yml b/.github/workflows/test-code-license_select-kfp.yml
@@ -13,7 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
-            - "transforms/.make.workflow"
+            - "transforms/.make.workflows"
             - "transforms/code/license_select/**"
             - "!kfp/**" # This is tested in separate workflow
             - "!data-processing-lib/**" # This is tested in separate workflow
@@ -27,7 +27,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
-            - "transforms/.make.workflow"
+            - "transforms/.make.workflows"
             - "transforms/code/license_select/**"
             - "!data-processing-lib/**" # This is tested in separate workflow
             - "!kfp/**" # This is tested in separate workflow
@@ -56,10 +56,10 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
-            - name: Test KFP libs (shared and v1) and run a workflow
+            - name: Test V1 KFP workflow for transforms/code/license_select
               timeout-minutes: 120
               run: |
-                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
+                  if [ -e "transforms/code/license_select/Makefile" -a -e "transforms/code/license_select/kfp_ray/Makefile" ]; then
                       export REPOROOT=$PWD
                       export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
                       source $K8S_SETUP_SCRIPTS/requirements.env
@@ -76,13 +76,13 @@ jobs:
                       chmod +x /tmp/mc
                       export DEPLOY_KUBEFLOW=1
                       make -C $K8S_SETUP_SCRIPTS setup
-                      make -C kfp/kfp_support_lib test
+                      #make -C kfp/kfp_support_lib test
                       make -C transforms/code/license_select workflow-build
                       source $K8S_SETUP_SCRIPTS/common.sh
                       make -C transforms/code/license_select workflow-test
                       echo "Run transforms/code/license_select completed"
                    else
-                      echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray"
+                      echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray/Makefile"
                    fi
 
     test-kfp-v2:
@@ -99,10 +99,10 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
-            - name: Test KFP libs (shared and v2) and run a workflow
+            - name: Test V2 KFP workflow for transforms/code/license_select
               timeout-minutes: 120
               run: |
-                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
+                  if [ -e "transforms/code/license_select/Makefile" -a -e "transforms/code/license_select/kfp_ray/Makefile" ]; then
                       export REPOROOT=$PWD
                       export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
                       source $K8S_SETUP_SCRIPTS/requirements.env
@@ -120,11 +120,11 @@ jobs:
                       export DEPLOY_KUBEFLOW=1
                       export KFPv2=1
                       make -C $K8S_SETUP_SCRIPTS setup
-                      make -C kfp/kfp_support_lib test
+                      #make -C kfp/kfp_support_lib test
                       make -C transforms/code/license_select workflow-build
                       source $K8S_SETUP_SCRIPTS/common.sh
                       make -C transforms/code/license_select workflow-test
                       echo "Run transforms/code/license_select completed"
                   else
-                      echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray"
+                      echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray/Makefile"
                   fi