Crawler transform #174
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files | |
| # | |
| name: Test KFP - transforms/universal/ededup | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - "dev" | |
| - "releases/**" | |
| tags: | |
| - "*" | |
| paths: | |
| - ".make.*" | |
| - "transforms/.make.workflows" | |
| - "transforms/universal/ededup/**" | |
| - "!kfp/**" # This is tested in separate workflow | |
| - "!data-processing-lib/**" # This is tested in separate workflow | |
| - "!**.md" | |
| - "!**/doc/**" | |
| - "!**/images/**" | |
| - "!**.gitignore" | |
| pull_request: | |
| branches: | |
| - "dev" | |
| - "releases/**" | |
| paths: | |
| - ".make.*" | |
| - "transforms/.make.workflows" | |
| - "transforms/universal/ededup/**" | |
| - "!data-processing-lib/**" # This is tested in separate workflow | |
| - "!kfp/**" # This is tested in separate workflow | |
| - "!**.md" | |
| - "!**/doc/**" | |
| - "!**/images/**" | |
| - "!**.gitignore" | |
| # taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test-kfp-v1: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Free up space in github runner | |
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| run: | | |
| df -h | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| df -h | |
| - name: Import environment variables | |
| run: | | |
| cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV | |
| echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV | |
| echo "REPOROOT=$PWD" >> $GITHUB_ENV | |
| echo "PATH=$PATH:/tmp" >> $GITHUB_ENV | |
| - name: Test V1 KFP workflow for transforms/universal/ededup | |
| timeout-minutes: 120 | |
| run: | | |
| KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list) | |
| if [ -e "transforms/universal/ededup/Makefile" -a -e "transforms/universal/ededup/kfp_ray/Makefile" ]; then | |
| transform=$(basename "transforms/universal/ededup") | |
| if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then | |
| $PWD/scripts/workflow_helper.sh install-tools | |
| $PWD/scripts/workflow_helper.sh test-workflow transforms/universal/ededup | |
| else | |
| $PWD/scripts/workflow_helper.sh build-workflow transforms/universal/ededup | |
| fi | |
| else | |
| echo "Skipping transforms/universal/ededup kfp test for lack of Makefile and/or kfp_ray/Makefile" | |
| fi | |
| test-kfp-v2: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Free up space in github runner | |
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| run: | | |
| df -h | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| df -h | |
| - name: Import environment variables | |
| run: | | |
| cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV | |
| echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV | |
| echo "REPOROOT=$PWD" >> $GITHUB_ENV | |
| echo "PATH=$PATH:/tmp" >> $GITHUB_ENV | |
| echo "KFPv2=1" >> $GITHUB_ENV | |
| - name: Test V2 KFP workflow for transforms/universal/ededup | |
| timeout-minutes: 120 | |
| run: | | |
| KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list) | |
| if [ -e "transforms/universal/ededup/Makefile" -a -e "transforms/universal/ededup/kfp_ray/Makefile" ]; then | |
| transform=$(basename "transforms/universal/ededup") | |
| if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then | |
| $PWD/scripts/workflow_helper.sh install-tools | |
| $PWD/scripts/workflow_helper.sh test-workflow transforms/universal/ededup | |
| else | |
| $PWD/scripts/workflow_helper.sh build-workflow transforms/universal/ededup | |
| fi | |
| else | |
| echo "Skipping transforms/universal/ededup kfp test for lack of Makefile and/or kfp_ray/Makefile" | |
| fi |