From a887bf9f19f585593cf179f2f9297a1ae961ffd5 Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 18 Sep 2024 10:53:41 -0400 Subject: [PATCH] check for makefile in test transform workflow Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 15 ++++++++++++--- .github/workflows/test-code-code_quality.yml | 15 ++++++++++++--- .github/workflows/test-code-header_cleanser.yml | 15 ++++++++++++--- .github/workflows/test-code-malware.yml | 15 ++++++++++++--- .github/workflows/test-code-proglang_select.yml | 15 ++++++++++++--- .../workflows/test-code-repo_level_ordering.yml | 15 ++++++++++++--- .github/workflows/test-language-doc_chunk.yml | 15 ++++++++++++--- .github/workflows/test-language-doc_quality.yml | 15 ++++++++++++--- .github/workflows/test-language-lang_id.yml | 15 ++++++++++++--- .github/workflows/test-language-pdf2parquet.yml | 15 ++++++++++++--- .../workflows/test-language-pii_redactor.yml | 15 ++++++++++++--- .../workflows/test-language-text_encoder.yml | 15 ++++++++++++--- .github/workflows/test-transform.template | 17 +++++++++++++---- .github/workflows/test-universal-doc_id.yml | 15 ++++++++++++--- .github/workflows/test-universal-ededup.yml | 15 ++++++++++++--- .github/workflows/test-universal-fdedup.yml | 15 ++++++++++++--- .github/workflows/test-universal-filter.yml | 15 ++++++++++++--- .../workflows/test-universal-html2parquet.yml | 15 ++++++++++++--- .github/workflows/test-universal-noop.yml | 15 ++++++++++++--- .github/workflows/test-universal-profiler.yml | 15 ++++++++++++--- .github/workflows/test-universal-resize.yml | 15 ++++++++++++--- .../workflows/test-universal-tokenization.yml | 15 ++++++++++++--- 22 files changed, 265 insertions(+), 67 deletions(-) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 25f45e0a9..ee487ad87 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/code2parquet run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/code2parquet DOCKER=docker test-image + if [ -e "transforms/code/code2parquet/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/code2parquet DOCKER=docker test-image + else + echo "transforms/code/code2parquet/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/code2parquet publish + run: | + if [ -e "transforms/code/code2parquet/Makefile" ]; then + make -C transforms/code/code2parquet publish + else + echo "transforms/code/code2parquet/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 0dd33d6e0..f8a702360 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/code_quality run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/code_quality DOCKER=docker test-image + if [ -e "transforms/code/code_quality/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/code_quality DOCKER=docker test-image + else + echo "transforms/code/code_quality/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/code_quality publish + run: | + if [ -e "transforms/code/code_quality/Makefile" ]; then + make -C transforms/code/code_quality publish + else + echo "transforms/code/code_quality/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 38de3d1f5..9216ef5c4 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/header_cleanser run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/header_cleanser DOCKER=docker test-image + if [ -e "transforms/code/header_cleanser/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/header_cleanser DOCKER=docker test-image + else + echo "transforms/code/header_cleanser/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/header_cleanser publish + run: | + if [ -e "transforms/code/header_cleanser/Makefile" ]; then + make -C transforms/code/header_cleanser publish + else + echo "transforms/code/header_cleanser/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index 4c2c3caee..0dcb8c413 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/malware run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/malware DOCKER=docker test-image + if [ -e "transforms/code/malware/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/malware DOCKER=docker test-image + else + echo "transforms/code/malware/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/malware publish + run: | + if [ -e "transforms/code/malware/Makefile" ]; then + make -C transforms/code/malware publish + else + echo "transforms/code/malware/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index abc45af8c..e33974dc0 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/proglang_select run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/proglang_select DOCKER=docker test-image + if [ -e "transforms/code/proglang_select/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/proglang_select DOCKER=docker test-image + else + echo "transforms/code/proglang_select/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/proglang_select publish + run: | + if [ -e "transforms/code/proglang_select/Makefile" ]; then + make -C transforms/code/proglang_select publish + else + echo "transforms/code/proglang_select/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index af43684e7..43f741376 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/repo_level_ordering run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/repo_level_ordering DOCKER=docker test-image + if [ -e "transforms/code/repo_level_ordering/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/repo_level_ordering DOCKER=docker test-image + else + echo "transforms/code/repo_level_ordering/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/repo_level_ordering publish + run: | + if [ -e "transforms/code/repo_level_ordering/Makefile" ]; then + make -C transforms/code/repo_level_ordering publish + else + echo "transforms/code/repo_level_ordering/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index 481322aba..fb9c6aa56 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/doc_chunk run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/doc_chunk DOCKER=docker test-image + if [ -e "transforms/language/doc_chunk/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/doc_chunk DOCKER=docker test-image + else + echo "transforms/language/doc_chunk/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/doc_chunk publish + run: | + if [ -e "transforms/language/doc_chunk/Makefile" ]; then + make -C transforms/language/doc_chunk publish + else + echo "transforms/language/doc_chunk/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 6e7cbcfe8..920fb029b 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/doc_quality run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/doc_quality DOCKER=docker test-image + if [ -e "transforms/language/doc_quality/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/doc_quality DOCKER=docker test-image + else + echo "transforms/language/doc_quality/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/doc_quality publish + run: | + if [ -e "transforms/language/doc_quality/Makefile" ]; then + make -C transforms/language/doc_quality publish + else + echo "transforms/language/doc_quality/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index f297da4b7..0b4cde6cc 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/lang_id run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/lang_id DOCKER=docker test-image + if [ -e "transforms/language/lang_id/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/lang_id DOCKER=docker test-image + else + echo "transforms/language/lang_id/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/lang_id publish + run: | + if [ -e "transforms/language/lang_id/Makefile" ]; then + make -C transforms/language/lang_id publish + else + echo "transforms/language/lang_id/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 62f666b1a..cae8c4a77 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/pdf2parquet run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/pdf2parquet DOCKER=docker test-image + if [ -e "transforms/language/pdf2parquet/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/pdf2parquet DOCKER=docker test-image + else + echo "transforms/language/pdf2parquet/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/pdf2parquet publish + run: | + if [ -e "transforms/language/pdf2parquet/Makefile" ]; then + make -C transforms/language/pdf2parquet publish + else + echo "transforms/language/pdf2parquet/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 146c3620b..55a7ebe41 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/pii_redactor run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/pii_redactor DOCKER=docker test-image + if [ -e "transforms/language/pii_redactor/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/pii_redactor DOCKER=docker test-image + else + echo "transforms/language/pii_redactor/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/pii_redactor publish + run: | + if [ -e "transforms/language/pii_redactor/Makefile" ]; then + make -C transforms/language/pii_redactor publish + else + echo "transforms/language/pii_redactor/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 70549089b..379de4660 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/text_encoder run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/text_encoder DOCKER=docker test-image + if [ -e "transforms/language/text_encoder/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/text_encoder DOCKER=docker test-image + else + echo "transforms/language/text_encoder/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/text_encoder publish + run: | + if [ -e "transforms/language/text_encoder/Makefile" ]; then + make -C transforms/language/text_encoder publish + else + echo "transforms/language/text_encoder/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index d0206e705..af80abfc4 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -1,7 +1,7 @@ # # DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files # -name: Test - @TARGET_TRANSFORM_DIR@ +name: Test - @TARGET_TRANSFORM_DIR@ on: workflow_dispatch: @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in @TARGET_TRANSFORM_DIR@ run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-image + if [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-image + else + echo "@TARGET_TRANSFORM_DIR@/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C @TARGET_TRANSFORM_DIR@ publish + run: | + if [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ]; then + make -C @TARGET_TRANSFORM_DIR@ publish + else + echo "@TARGET_TRANSFORM_DIR@/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index b2c2c8224..85e14cb04 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/doc_id run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/doc_id DOCKER=docker test-image + if [ -e "transforms/universal/doc_id/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/doc_id DOCKER=docker test-image + else + echo "transforms/universal/doc_id/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/doc_id publish + run: | + if [ -e "transforms/universal/doc_id/Makefile" ]; then + make -C transforms/universal/doc_id publish + else + echo "transforms/universal/doc_id/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 9e6195034..92ed63829 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/ededup run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/ededup DOCKER=docker test-image + if [ -e "transforms/universal/ededup/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/ededup DOCKER=docker test-image + else + echo "transforms/universal/ededup/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/ededup publish + run: | + if [ -e "transforms/universal/ededup/Makefile" ]; then + make -C transforms/universal/ededup publish + else + echo "transforms/universal/ededup/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 0ddc4d43f..c6a83a2cf 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/fdedup run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/fdedup DOCKER=docker test-image + if [ -e "transforms/universal/fdedup/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/fdedup DOCKER=docker test-image + else + echo "transforms/universal/fdedup/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/fdedup publish + run: | + if [ -e "transforms/universal/fdedup/Makefile" ]; then + make -C transforms/universal/fdedup publish + else + echo "transforms/universal/fdedup/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 127bba7a5..ab285b88f 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/filter run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/filter DOCKER=docker test-image + if [ -e "transforms/universal/filter/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/filter DOCKER=docker test-image + else + echo "transforms/universal/filter/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/filter publish + run: | + if [ -e "transforms/universal/filter/Makefile" ]; then + make -C transforms/universal/filter publish + else + echo "transforms/universal/filter/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index c113e7800..d42f013ad 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/html2parquet run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/html2parquet DOCKER=docker test-image + if [ -e "transforms/universal/html2parquet/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/html2parquet DOCKER=docker test-image + else + echo "transforms/universal/html2parquet/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/html2parquet publish + run: | + if [ -e "transforms/universal/html2parquet/Makefile" ]; then + make -C transforms/universal/html2parquet publish + else + echo "transforms/universal/html2parquet/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 0f6f52b2d..04d7280f9 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/noop run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/noop DOCKER=docker test-image + if [ -e "transforms/universal/noop/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/noop DOCKER=docker test-image + else + echo "transforms/universal/noop/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/noop publish + run: | + if [ -e "transforms/universal/noop/Makefile" ]; then + make -C transforms/universal/noop publish + else + echo "transforms/universal/noop/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 0cdffae3b..4a8774a02 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/profiler run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/profiler DOCKER=docker test-image + if [ -e "transforms/universal/profiler/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/profiler DOCKER=docker test-image + else + echo "transforms/universal/profiler/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/profiler publish + run: | + if [ -e "transforms/universal/profiler/Makefile" ]; then + make -C transforms/universal/profiler publish + else + echo "transforms/universal/profiler/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index f5a4f37f8..ab4775ec0 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/resize run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/resize DOCKER=docker test-image + if [ -e "transforms/universal/resize/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/resize DOCKER=docker test-image + else + echo "transforms/universal/resize/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/resize publish + run: | + if [ -e "transforms/universal/resize/Makefile" ]; then + make -C transforms/universal/resize publish + else + echo "transforms/universal/resize/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 4411876f1..b6e5019fc 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/tokenization run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/tokenization DOCKER=docker test-image + if [ -e "transforms/universal/tokenization/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/tokenization DOCKER=docker test-image + else + echo "transforms/universal/tokenization/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/tokenization publish + run: | + if [ -e "transforms/universal/tokenization/Makefile" ]; then + make -C transforms/universal/tokenization publish + else + echo "transforms/universal/tokenization/Makefile not found - publishing disabled for this transform." + fi