test-universal-tokenization-kfp.yml

#
# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
#
name: Test KFP - transforms/universal/tokenization

on:
    workflow_dispatch:
    push:
        branches:
            - "dev"
            - "releases/**"
        tags:
            - "*"
        paths:
            - ".make.*"
            - "transforms/.make.workflows"
            - "transforms/universal/tokenization/**"
            - "!kfp/**" # This is tested in separate workflow
            - "!data-processing-lib/**" # This is tested in separate workflow
            - "!**.md"
            - "!**/doc/**"
            - "!**/images/**"
            - "!**.gitignore"
    pull_request:
        branches:
            - "dev"
            - "releases/**"
        paths:
            - ".make.*"
            - "transforms/.make.workflows"
            - "transforms/universal/tokenization/**"
            - "!data-processing-lib/**" # This is tested in separate workflow
            - "!kfp/**" # This is tested in separate workflow
            - "!**.md"
            - "!**/doc/**"
            - "!**/images/**"
            - "!**.gitignore"

# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre
concurrency:
    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
    cancel-in-progress: true

jobs:
    test-kfp-v1:
        runs-on: ubuntu-22.04
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Free up space in github runner
              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
              run: |
                  df -h
                  sudo rm -rf "/usr/local/share/boost"
                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                  df -h
            - name: Import environment variables
              run: |
                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
            - name: Test V1 KFP workflow for transforms/universal/tokenization
              timeout-minutes: 120
              run: |
                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                  if [ -e "transforms/universal/tokenization/Makefile" -a -e "transforms/universal/tokenization/kfp_ray/Makefile" ]; then
                     transform=$(basename "transforms/universal/tokenization")
                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
                        $PWD/scripts/workflow_helper.sh install-tools
                        $PWD/scripts/workflow_helper.sh test-workflow transforms/universal/tokenization   
                     else
                        $PWD/scripts/workflow_helper.sh build-workflow transforms/universal/tokenization
                     fi              
                  else
                     echo "Skipping transforms/universal/tokenization kfp test for lack of Makefile and/or kfp_ray/Makefile"
                   fi

    test-kfp-v2:
        runs-on: ubuntu-22.04
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Free up space in github runner
              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
              run: |
                  df -h
                  sudo rm -rf "/usr/local/share/boost"
                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                  df -h
            - name: Import environment variables
              run: |
                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
                  echo "KFPv2=1" >> $GITHUB_ENV
            - name: Test V2 KFP workflow for transforms/universal/tokenization
              timeout-minutes: 120
              run: |
                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                  if [ -e "transforms/universal/tokenization/Makefile" -a -e "transforms/universal/tokenization/kfp_ray/Makefile" ]; then
                     transform=$(basename "transforms/universal/tokenization")
                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
                        $PWD/scripts/workflow_helper.sh install-tools
                        $PWD/scripts/workflow_helper.sh test-workflow transforms/universal/tokenization   
                     else
                        $PWD/scripts/workflow_helper.sh build-workflow transforms/universal/tokenization
                     fi              
                  else
                     echo "Skipping transforms/universal/tokenization kfp test for lack of Makefile and/or kfp_ray/Makefile"
                   fi