worklow readme, transform existence verification, disable build-libra…

…ry, and tools tests Signed-off-by: David Wood <[email protected]>
IBM · Sep 18, 2024 · e064e00 · e064e00
1 parent f9399c1
commit e064e00
Show file tree

Hide file tree

Showing 27 changed files with 244 additions and 38 deletions.
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
@@ -3,12 +3,25 @@
 Here we have the start of a system to automatically generated github workflows (currently only for transforms). 
 In general, the design is to use templates and `make` to generate/update the workflows.
 
+Goals
+1. Run only tests for a given transform when only the transform changes.
+Includes python, ray, spark and kfp_ray as available.  
+2. When the core dpk lib components files changes, test all transforms
+3. When the shared kfp components changes, test a randomly selected transform test
+   (We would like to avoid running all transform kfp tests in one PR)
+4. Extra credit: If .md or other non-code changes are made, run no tests. 
+
+Assumptions:
+1. All transforms will have test workflows.  A transform can disable its tests locally
+(temporarily?) by renaming its transforms/universal/noop/Makefile.disabled.
 ```
-make
+git clone ....
+...
+git checkout -b new-branch 
+make                # Creates new test*.yml workflows
 git commit -a -s -m "update workflows"
-git push
+git push --set-upstream origin new-branch
 ```
-
 should be sufficient.
 
 ## Transforms 
@@ -24,3 +37,6 @@ When a new transform is added to the repository,
 1. Run `make` in this directory to create the new test .yml for all transforms found in transforms/{universal,code,language} directories 
 1. commit and push the change to your branch with the new transform.
 
+## KFP
+
+## DPK libraries
diff --git a/.github/workflows/build-library.yml → .github/workflows/build-library.yml.old b/.github/workflows/build-library.yml → .github/workflows/build-library.yml.old
diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml
@@ -14,13 +14,21 @@ on:
         paths:
             - "transforms/code/code2parquet/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
     pull_request:
         branches:
             - "dev"
             - "releases/**"
         paths:
             - "transforms/code/code2parquet/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
 
 jobs:
     check_if_push_image:
@@ -84,7 +92,7 @@ jobs:
             - name: Test transform image in transforms/code/code2parquet
               run: |
                   if [ -e "transforms/code/code2parquet/Makefile" ]; then
-                      make -C data-processing-lib/spark DOCKER=docker image 
+                      make -C data-processing-lib/spark DOCKER=docker image
                       make -C transforms/code/code2parquet DOCKER=docker test-image
                   else
                       echo "transforms/code/code2parquet/Makefile not found - testing disabled for this transform."

diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml
@@ -14,13 +14,21 @@ on:
         paths:
             - "transforms/code/code_quality/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
     pull_request:
         branches:
             - "dev"
             - "releases/**"
         paths:
             - "transforms/code/code_quality/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
 
 jobs:
     check_if_push_image:
@@ -84,7 +92,7 @@ jobs:
             - name: Test transform image in transforms/code/code_quality
               run: |
                   if [ -e "transforms/code/code_quality/Makefile" ]; then
-                      make -C data-processing-lib/spark DOCKER=docker image 
+                      make -C data-processing-lib/spark DOCKER=docker image
                       make -C transforms/code/code_quality DOCKER=docker test-image
                   else
                       echo "transforms/code/code_quality/Makefile not found - testing disabled for this transform."

diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml
@@ -14,13 +14,21 @@ on:
         paths:
             - "transforms/code/header_cleanser/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
     pull_request:
         branches:
             - "dev"
             - "releases/**"
         paths:
             - "transforms/code/header_cleanser/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
 
 jobs:
     check_if_push_image:
@@ -84,7 +92,7 @@ jobs:
             - name: Test transform image in transforms/code/header_cleanser
               run: |
                   if [ -e "transforms/code/header_cleanser/Makefile" ]; then
-                      make -C data-processing-lib/spark DOCKER=docker image 
+                      make -C data-processing-lib/spark DOCKER=docker image
                       make -C transforms/code/header_cleanser DOCKER=docker test-image
                   else
                       echo "transforms/code/header_cleanser/Makefile not found - testing disabled for this transform."

diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml
@@ -14,13 +14,21 @@ on:
         paths:
             - "transforms/code/malware/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
     pull_request:
         branches:
             - "dev"
             - "releases/**"
         paths:
             - "transforms/code/malware/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
 
 jobs:
     check_if_push_image:
@@ -84,7 +92,7 @@ jobs:
             - name: Test transform image in transforms/code/malware
               run: |
                   if [ -e "transforms/code/malware/Makefile" ]; then
-                      make -C data-processing-lib/spark DOCKER=docker image 
+                      make -C data-processing-lib/spark DOCKER=docker image
                       make -C transforms/code/malware DOCKER=docker test-image
                   else
                       echo "transforms/code/malware/Makefile not found - testing disabled for this transform."

diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml
@@ -14,13 +14,21 @@ on:
         paths:
             - "transforms/code/proglang_select/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
     pull_request:
         branches:
             - "dev"
             - "releases/**"
         paths:
             - "transforms/code/proglang_select/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
 
 jobs:
     check_if_push_image:
@@ -84,7 +92,7 @@ jobs:
             - name: Test transform image in transforms/code/proglang_select
               run: |
                   if [ -e "transforms/code/proglang_select/Makefile" ]; then
-                      make -C data-processing-lib/spark DOCKER=docker image 
+                      make -C data-processing-lib/spark DOCKER=docker image
                       make -C transforms/code/proglang_select DOCKER=docker test-image
                   else
                       echo "transforms/code/proglang_select/Makefile not found - testing disabled for this transform."

diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml
@@ -14,13 +14,21 @@ on:
         paths:
             - "transforms/code/repo_level_ordering/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
     pull_request:
         branches:
             - "dev"
             - "releases/**"
         paths:
             - "transforms/code/repo_level_ordering/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
 
 jobs:
     check_if_push_image:
@@ -84,7 +92,7 @@ jobs:
             - name: Test transform image in transforms/code/repo_level_ordering
               run: |
                   if [ -e "transforms/code/repo_level_ordering/Makefile" ]; then
-                      make -C data-processing-lib/spark DOCKER=docker image 
+                      make -C data-processing-lib/spark DOCKER=docker image
                       make -C transforms/code/repo_level_ordering DOCKER=docker test-image
                   else
                       echo "transforms/code/repo_level_ordering/Makefile not found - testing disabled for this transform."

diff --git a/.github/workflows/test-kfp.yml b/.github/workflows/test-kfp.yml
@@ -12,6 +12,7 @@ on:
             - "kfp/**"
             - "!**.md"
             - "!**/doc/**"
+            - "!**/images/**"
             - "!**/.gitignore"
     pull_request:
         branches:
@@ -21,6 +22,7 @@ on:
             - "kfp/**"
             - "!**.md"
             - "!**/doc/**"
+            - "!**/images/**"
             - "!**/.gitignore"
 
 env:

diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml
@@ -14,13 +14,21 @@ on:
         paths:
             - "transforms/language/doc_chunk/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
     pull_request:
         branches:
             - "dev"
             - "releases/**"
         paths:
             - "transforms/language/doc_chunk/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
 
 jobs:
     check_if_push_image:
@@ -84,7 +92,7 @@ jobs:
             - name: Test transform image in transforms/language/doc_chunk
               run: |
                   if [ -e "transforms/language/doc_chunk/Makefile" ]; then
-                      make -C data-processing-lib/spark DOCKER=docker image 
+                      make -C data-processing-lib/spark DOCKER=docker image
                       make -C transforms/language/doc_chunk DOCKER=docker test-image
                   else
                       echo "transforms/language/doc_chunk/Makefile not found - testing disabled for this transform."

diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml
@@ -14,13 +14,21 @@ on:
         paths:
             - "transforms/language/doc_quality/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
     pull_request:
         branches:
             - "dev"
             - "releases/**"
         paths:
             - "transforms/language/doc_quality/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
 
 jobs:
     check_if_push_image:
@@ -84,7 +92,7 @@ jobs:
             - name: Test transform image in transforms/language/doc_quality
               run: |
                   if [ -e "transforms/language/doc_quality/Makefile" ]; then
-                      make -C data-processing-lib/spark DOCKER=docker image 
+                      make -C data-processing-lib/spark DOCKER=docker image
                       make -C transforms/language/doc_quality DOCKER=docker test-image
                   else
                       echo "transforms/language/doc_quality/Makefile not found - testing disabled for this transform."

diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml
@@ -14,13 +14,21 @@ on:
         paths:
             - "transforms/language/lang_id/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
     pull_request:
         branches:
             - "dev"
             - "releases/**"
         paths:
             - "transforms/language/lang_id/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
 
 jobs:
     check_if_push_image:
@@ -84,7 +92,7 @@ jobs:
             - name: Test transform image in transforms/language/lang_id
               run: |
                   if [ -e "transforms/language/lang_id/Makefile" ]; then
-                      make -C data-processing-lib/spark DOCKER=docker image 
+                      make -C data-processing-lib/spark DOCKER=docker image
                       make -C transforms/language/lang_id DOCKER=docker test-image
                   else
                       echo "transforms/language/lang_id/Makefile not found - testing disabled for this transform."

diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml
@@ -14,13 +14,21 @@ on:
         paths:
             - "transforms/language/pdf2parquet/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
     pull_request:
         branches:
             - "dev"
             - "releases/**"
         paths:
             - "transforms/language/pdf2parquet/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
 
 jobs:
     check_if_push_image:
@@ -84,7 +92,7 @@ jobs:
             - name: Test transform image in transforms/language/pdf2parquet
               run: |
                   if [ -e "transforms/language/pdf2parquet/Makefile" ]; then
-                      make -C data-processing-lib/spark DOCKER=docker image 
+                      make -C data-processing-lib/spark DOCKER=docker image
                       make -C transforms/language/pdf2parquet DOCKER=docker test-image
                   else
                       echo "transforms/language/pdf2parquet/Makefile not found - testing disabled for this transform."

diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml
@@ -14,13 +14,21 @@ on:
         paths:
             - "transforms/language/pii_redactor/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
     pull_request:
         branches:
             - "dev"
             - "releases/**"
         paths:
             - "transforms/language/pii_redactor/**"
             - "data-processing-lib/**"
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
 
 jobs:
     check_if_push_image:
@@ -84,7 +92,7 @@ jobs:
             - name: Test transform image in transforms/language/pii_redactor
               run: |
                   if [ -e "transforms/language/pii_redactor/Makefile" ]; then
-                      make -C data-processing-lib/spark DOCKER=docker image 
+                      make -C data-processing-lib/spark DOCKER=docker image
                       make -C transforms/language/pii_redactor DOCKER=docker test-image
                   else
                       echo "transforms/language/pii_redactor/Makefile not found - testing disabled for this transform."