Merge pull request #14 from shivdeep-singh-ibm/add_github_actions_wor…

…kflows

add ci workflows for github actions

.github/workflows/build-images.yml

@@ -0,0 +1,77 @@
+
+name: Build Transform Images
+
+on: 
+  workflow_dispatch:
+  push:
+    branches:
+      - "dev"
+  pull_request:
+    branches:
+      - "dev"
+jobs:
+  build-code:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build and Test Code Transforms
+      run: |
+         make -C transforms/code DOCKER=docker image test-image 
+  build-universal-doc_id:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build and Test Universal Transforms
+      run: |
+         #make -C transforms/universal/doc_id DOCKER=docker image test-image
+         echo "This is a placeholder."
+  build-universal-ededup:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build and Test Universal Transforms
+      run: |
+         make -C transforms/universal/ededup DOCKER=docker image test-image
+  build-universal-fdedup:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build and Test Universal Transforms
+      run: |
+         make -C transforms/universal/fdedup DOCKER=docker image test-image
+  build-universal-filter:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build and Test Universal Transforms
+      run: |
+         make -C transforms/universal/filter DOCKER=docker image test-image
+  build-universal-noop:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build and Test Universal Transforms
+      run: |
+         make -C transforms/universal/noop DOCKER=docker image test-image 
+  build-universal-tokenization:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build and Test Universal Transforms
+      run: |
+         make -C transforms/universal/tokenization DOCKER=docker image test-image
+  build-tools:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build and Test Universal Transforms
+      run: |
+         make -C tools/ingest2parquet DOCKER=docker image test-image

.github/workflows/build-library.yml

@@ -0,0 +1,24 @@
+
+name: Build Library
+
+on: 
+  workflow_dispatch:
+  push:
+    branches:
+      - "dev"
+  pull_request:
+    branches:
+      - "dev"
+jobs:
+  build-lib:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python:
+          - "3.11"
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build data-processing-lib
+      run: |
+         make -C data-processing-lib DOCKER=docker venv build

.github/workflows/deploy-docs.yml

@@ -0,0 +1,22 @@
+name: deploy docs 
+
+on:
+  workflow_dispatch:
+  push:
+    branches: 
+      - dev
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.x
+      - run: pip install mkdocs-material mkdocstrings[python] mkdocs-badges
+      - run: |
+           cp README.md ./data-processing-lib/doc/index.md
+           # remove badges 
+           cat README.md |sed  '/img\.shields\.io/d' > ./data-processing-lib/doc/index_.md 
+           cp doc/* ./data-processing-lib/doc/
+           cd data-processing-lib && mkdocs gh-deploy --force

.github/workflows/deploy-library.yml

@@ -0,0 +1,59 @@
+name: Build, Test, and Upload PyPI package
+
+on:
+    release:
+        types:
+            - published
+    workflow_dispatch:
+
+permissions:
+    contents: read
+    # see https://docs.pypi.org/trusted-publishers/
+    id-token: write
+
+jobs:
+    build-package:
+        name: Build and check packages
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout 
+              uses: actions/checkout@v4
+              with:
+                  # for setuptools-scm
+                  fetch-depth: 0
+            - name: Build Package for pypi 
+              run: |
+                make -C data-processing-lib build
+    publish-test-pypi:
+        name: Publish packages to test.pypi.org
+        # disabled 
+        if: false
+        runs-on: ubuntu-latest
+        needs: build-package
+
+        steps:
+            - name: Fetch build artifacts
+              uses: actions/download-artifact@v4
+              with:
+                  name: Packages
+                  path: dist
+
+            - name: Upload to Test PyPI
+              uses: pypa/gh-action-pypi-publish@release/v1
+              with:
+                  repository-url: https://test.pypi.org/legacy/
+
+    publish-pypi:
+        name: Publish release to pypi.org
+        runs-on: ubuntu-latest
+        needs: build-package
+        # disabled as of now
+        if: false
+        steps:
+            - name: Fetch build artifacts
+              uses: actions/download-artifact@v4
+              with:
+                  name: Packages
+                  path: dist
+            - name: Upload to PyPI
+              uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/deploy-transforms.yml

@@ -0,0 +1,32 @@
+name: Build, Test, and upload Transform images to quay.io
+
+on:
+    release:
+        types:
+            - published
+    workflow_dispatch:
+
+jobs:
+    build-images:
+        name: Build and check images
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout 
+              uses: actions/checkout@v4
+              with:
+                  # for setuptools-scm
+                  fetch-depth: 0
+            - name: Build Package for pypi 
+              run: |
+                make -C transforms image test-image
+    publish-images:
+        name: Publish packages to quay.io
+        # disabled 
+        if: false
+        runs-on: ubuntu-latest
+        needs: build-images
+
+        steps:
+            - name: Push images to quay.io registry
+              run: |
+                make -C transforms publish

.github/workflows/test.yml

@@ -0,0 +1,116 @@
+
+name: Test CI
+
+on: 
+  workflow_dispatch:
+  push:
+    branches:
+      - "dev"
+  pull_request:
+    branches:
+      - "dev"
+jobs:
+  test-lib:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Test data-processing-lib
+      run: |
+         make -C data-processing-lib DOCKER=docker venv test
+  test-code:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Test Code Transforms
+      run: |
+         make -C transforms/code DOCKER=docker venv test-src 
+  test-universal-doc_id:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Test Universal Transforms
+      # disabled since it requires more resources
+      if: false
+      run: |
+         make -C transforms/universal/doc_id DOCKER=docker venv test-src
+  test-universal-ededup:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Test Universal Transforms
+      run: |
+         make -C transforms/universal/ededup DOCKER=docker venv test-src
+  test-universal-fdedup:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Test Universal Transforms
+      run: |
+         make -C transforms/universal/fdedup DOCKER=docker venv test-src
+  test-universal-filter:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Test Universal Transforms
+      run: |
+         make -C transforms/universal/filter DOCKER=docker venv test-src
+  test-universal-noop:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Test Universal Transforms
+      run: |
+         make -C transforms/universal/noop DOCKER=docker venv test-src 
+  test-universal-tokenization:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Test Universal Transforms
+      run: |
+         make -C transforms/universal/tokenization DOCKER=docker venv test-src 
+  test-tools:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Test Universal Transforms
+      run: |
+         make -C tools DOCKER=docker venv test
+  test-kfp-lib:
+      runs-on: ubuntu-latest
+      steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Test Universal Transforms
+        run: |
+              source kind/requirements.env
+              export PATH=$PATH:/tmp/
+              curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
+              curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+              chmod 700 /tmp/get_helm.sh
+              HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
+              chmod 777 /tmp/helm
+              chmod 777 /tmp/kind
+              curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
+              chmod 777 /tmp/kubectl
+              curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
+              chmod +x /tmp/mc
+              export DEPLOY_KUBEFLOW=0
+              make -C kind setup 
+              make -C kfp/kfp_support_lib build test
+  test-kfp-compile:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Test Universal Transforms
+      run: |
+         make -C kfp/transform_workflows venv build