Packaging modules as single package with extra (data-prep-kit[ray,spark] and data-prep-kit-tranforms[all, ray, etc.]) #74
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files | |
| # | |
| name: Test KFP - transforms/universal/fdedup | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - "dev" | |
| - "releases/**" | |
| tags: | |
| - "*" | |
| paths: | |
| - ".make.*" | |
| - "transforms/.make.workflows" | |
| - "transforms/universal/fdedup/**" | |
| - "!kfp/**" # This is tested in separate workflow | |
| - "!data-processing-lib/**" # This is tested in separate workflow | |
| - "!**.md" | |
| - "!**/doc/**" | |
| - "!**/images/**" | |
| - "!**.gitignore" | |
| pull_request: | |
| branches: | |
| - "dev" | |
| - "releases/**" | |
| paths: | |
| - ".make.*" | |
| - "transforms/.make.workflows" | |
| - "transforms/universal/fdedup/**" | |
| - "!data-processing-lib/**" # This is tested in separate workflow | |
| - "!kfp/**" # This is tested in separate workflow | |
| - "!**.md" | |
| - "!**/doc/**" | |
| - "!**/images/**" | |
| - "!**.gitignore" | |
| # taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test-kfp-v1: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Free up space in github runner | |
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| run: | | |
| df -h | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| df -h | |
| - name: Test V1 KFP workflow for transforms/universal/fdedup | |
| timeout-minutes: 120 | |
| run: | | |
| if [ -e "transforms/universal/fdedup/Makefile" -a -e "transforms/universal/fdedup/kfp_ray/Makefile" ]; then | |
| export REPOROOT=$PWD | |
| export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup | |
| source $K8S_SETUP_SCRIPTS/requirements.env | |
| export PATH=$PATH:/tmp/ | |
| curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 | |
| chmod 777 /tmp/kind | |
| curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | |
| chmod 700 /tmp/get_helm.sh | |
| HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo | |
| chmod 777 /tmp/helm | |
| curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl | |
| chmod 777 /tmp/kubectl | |
| curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc | |
| chmod +x /tmp/mc | |
| export DEPLOY_KUBEFLOW=1 | |
| make -C $K8S_SETUP_SCRIPTS setup | |
| #make -C kfp/kfp_support_lib test | |
| make -C transforms/universal/fdedup workflow-build | |
| source $K8S_SETUP_SCRIPTS/common.sh | |
| make -C transforms/universal/fdedup workflow-test | |
| echo "Run transforms/universal/fdedup completed" | |
| else | |
| echo "Skipping transforms/universal/fdedup kfp test for lack of Makefile and/or kfp_ray/Makefile" | |
| fi | |
| test-kfp-v2: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Free up space in github runner | |
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| run: | | |
| df -h | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| df -h | |
| - name: Test V2 KFP workflow for transforms/universal/fdedup | |
| timeout-minutes: 120 | |
| run: | | |
| if [ -e "transforms/universal/fdedup/Makefile" -a -e "transforms/universal/fdedup/kfp_ray/Makefile" ]; then | |
| export REPOROOT=$PWD | |
| export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup | |
| source $K8S_SETUP_SCRIPTS/requirements.env | |
| export PATH=$PATH:/tmp/ | |
| curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 | |
| chmod 777 /tmp/kind | |
| curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | |
| chmod 700 /tmp/get_helm.sh | |
| HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo | |
| chmod 777 /tmp/helm | |
| curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl | |
| chmod 777 /tmp/kubectl | |
| curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc | |
| chmod +x /tmp/mc | |
| export DEPLOY_KUBEFLOW=1 | |
| export KFPv2=1 | |
| make -C $K8S_SETUP_SCRIPTS setup | |
| #make -C kfp/kfp_support_lib test | |
| make -C transforms/universal/fdedup workflow-build | |
| source $K8S_SETUP_SCRIPTS/common.sh | |
| make -C transforms/universal/fdedup workflow-test | |
| echo "Run transforms/universal/fdedup completed" | |
| else | |
| echo "Skipping transforms/universal/fdedup kfp test for lack of Makefile and/or kfp_ray/Makefile" | |
| fi |