From c564e481c81b0ee312e34ac131d8dfe85d1f7dc4 Mon Sep 17 00:00:00 2001
From: Najib <32004868+nishaq503@users.noreply.github.com>
Date: Thu, 25 Jan 2024 11:38:40 -0500
Subject: [PATCH] CI: Added Docker workflow and switched to matrix strategy
 (#503)

* ci: using matrix strategy

* ci: added docker workflow

* ci: restricting release workflow to polusai repo only

* ci: fixing docker image check
---
 .github/workflows/docker.yml          | 59 +++++++++++++++++++++++++++
 .github/workflows/package-filter.yml  | 40 ++++++++++++++----
 .github/workflows/package-release.yml | 40 +++++++++---------
 .github/workflows/tests.yml           | 53 ++++++++++++------------
 4 files changed, 139 insertions(+), 53 deletions(-)
 create mode 100644 .github/workflows/docker.yml

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
new file mode 100644
index 000000000..503ca0501
--- /dev/null
+++ b/.github/workflows/docker.yml
@@ -0,0 +1,59 @@
+name: Docker Build and Push
+
+on:
+  workflow_call:
+    inputs:
+      matrix:
+        description: 'JSON-encoded matrix'
+        required: true
+        type: string
+      push:
+        description: 'Whether to push the image to Docker Hub'
+        required: true
+        type: boolean
+    secrets:
+      DOCKER_USERNAME:
+        description: 'Docker Hub username'
+        required: true
+      DOCKER_PASSWORD:
+        description: 'Docker Hub password'
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  docker:
+    name: Docker "${{ matrix.package_name }}"
+    strategy:
+      matrix: ${{fromJson(inputs.matrix)}}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Get | Docker Tag
+        id: docker_tag
+        run: |
+          package_dir="${{ matrix.package_dir }}"
+          version=$(cat ${package_dir}/VERSION)
+          tag=polusai/${{ matrix.package_name }}:${version}
+          echo "tag=${tag}" >> $GITHUB_OUTPUT
+      - name: Setup | Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login | DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Check | Image exists
+        run: |
+          tag=${{ steps.docker_tag.outputs.tag }}
+          docker pull ${tag} > /dev/null \
+              && $(echo "::error::${tag} already exists on DockerHub" && exit 1) \
+              || echo "success"
+      - name: Publish | Docker Image
+        uses: docker/build-push-action@v5
+        with:
+          context: "{{defaultContext}}:${{ matrix.package_dir }}"
+          push: ${{ inputs.push }}
+          tags: ${{ steps.docker_tag.outputs.tag }}
diff --git a/.github/workflows/package-filter.yml b/.github/workflows/package-filter.yml
index ac697e765..ee43ba18f 100644
--- a/.github/workflows/package-filter.yml
+++ b/.github/workflows/package-filter.yml
@@ -3,9 +3,9 @@ name: Package Filter
 on:
   workflow_call:
     outputs:
-      package_dirs:
+      matrix:
         description: "The directories containing the updated packages"
-        value: ${{ jobs.package-filter.outputs.package_dirs }}
+        value: ${{ jobs.package-filter.outputs.matrix }}
 
 permissions:
   contents: read
@@ -15,7 +15,7 @@ jobs:
     name: Filter for updated packages
     runs-on: ubuntu-latest
     outputs:
-      package_dirs: ${{ steps.package-filter.outputs.package_dirs }}
+      matrix: ${{ steps.package-filter.outputs.matrix }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -25,11 +25,18 @@ jobs:
       - name: Find Updated Packages
         id: package-filter
         run: |
-          CUR_DIR=$(pwd)
           PACKAGE_DIRS=""
 
           for changed_file in $(git diff --name-only origin/${{ github.base_ref }}...)
           do
+            pkg_dir=$(dirname ${changed_file})
+
+            # Exclude the root directory
+            if [[ "$pkg_dir" == "." ]]
+            then
+              continue
+            fi
+
             # Check if the changed file is a pyproject.toml file
             if [[ "$(basename ${changed_file})" == *"pyproject.toml"* ]]
             then
@@ -50,12 +57,31 @@ jobs:
             fi
           done
 
-          cd "$CUR_DIR"
-
           # Check if any packages were found
           if [[ -z "$PACKAGE_DIRS" ]]
           then
             echo "::error::No updated packages were found" && exit 1
           fi
 
-          echo "package_dirs=${PACKAGE_DIRS}" >> "$GITHUB_OUTPUT"
+          # Convert the package directories to JSON for the output matrix
+          JSON="{\"include\": ["
+          for package_dir in $PACKAGE_DIRS
+          do
+            package_name=$(basename $package_dir)
+            JSON_LINE="{\"package_dir\": \"${package_dir}\", \"package_name\": \"${package_name}\"},"
+            # Add the JSON line to the JSON string if it is not already included
+            if [[ ! "$JSON" == *"$JSON_LINE"* ]]
+            then
+              JSON="$JSON$JSON_LINE"
+            fi
+          done
+
+          # Remove trailing comma and add closing brackets
+          if [[ $JSON == *, ]]
+          then
+            JSON="${JSON%?}"
+          fi
+          JSON="$JSON]}"
+
+          # Set the output
+          echo "matrix=$( echo "$JSON" )" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/package-release.yml b/.github/workflows/package-release.yml
index 037501dec..c8e80ced7 100644
--- a/.github/workflows/package-release.yml
+++ b/.github/workflows/package-release.yml
@@ -13,13 +13,16 @@ permissions:
 jobs:
   package-filter:
     name: Filter for updated package
+    if: github.repository == 'polusai/polus-plugins'
     uses: ./.github/workflows/package-filter.yml
 
   package-release:
-    name: Bump release version of updated package
+    name: Release "${{ matrix.package_name }}"
     if: github.repository == 'polusai/polus-plugins'
-    runs-on: ubuntu-latest
     needs: package-filter
+    strategy:
+      matrix: ${{fromJson(needs.package-filter.outputs.matrix)}}
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -31,28 +34,27 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install bump2version
-      - name: Bump Version on all updated packages
+      - name: Bump Version
         run: |
-          CUR_DIR=$(pwd)
-
-          for package_dir in ${{ needs.package-filter.outputs.package_dirs }}
-          do
-            cd "$CUR_DIR/$package_dir"
-            bump2version release --no-commit
-          done
-
-          cd "$CUR_DIR"
+          cd "${{ matrix.package_dir }}"
+          bump2version release --no-commit
       - name: Commit and push all changed files
         env:
           CI_COMMIT_AUTHOR: Continuous Integration
         run: |
-          PKG_NAMES=""
-          for package_dir in ${{ needs.package-filter.outputs.package_dirs }}
-          do
-            PKG_NAMES="$PKG_NAMES $(basename $package_dir)"
-          done
-
           git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}"
           git config --global user.email "username@users.noreply.github.com"
-          git commit -a -m "build: Bumped release version for $PKG_NAMES"
+          git commit -a -m "build: Bumped release version for ${{ matrix.package_name }}"
           git push
+
+  docker:
+    name: Build Docker images
+    if: github.repository == 'polusai/polus-plugins'
+    needs: [package-filter, package-release]
+    uses: ./.github/workflows/docker.yml
+    with:
+      matrix: ${{ needs.package-filter.outputs.matrix }}
+      push: true
+    secrets:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 0a49f7d2f..1d505cf7c 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -6,11 +6,6 @@ on:
       - main
       - master
       - dev
-  workflow_call:
-    outputs:
-      package_dirs:
-        description: "The directories containing the updated packages"
-        value: ${{ jobs.package-filter.outputs.package_dirs }}
 
 permissions:
   contents: read
@@ -20,10 +15,12 @@ jobs:
     name: Filter for updated package
     uses: ./.github/workflows/package-filter.yml
 
-  package-tests:
-    name: Run the tests for updated packages.
-    runs-on: ubuntu-latest
+  tests:
+    name: Test "${{ matrix.package_name }}"
     needs: package-filter
+    strategy:
+      matrix: ${{fromJson(needs.package-filter.outputs.matrix)}}
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -38,26 +35,28 @@ jobs:
           pip install pre-commit
       - name: Run pre-commit hooks and check for changes
         run: |
-          CUR_DIR=$(pwd)
+          cd "${{ matrix.package_dir }}"
 
-          for package_dir in ${{ needs.package-filter.outputs.package_dirs }}
-          do
-            cd $package_dir
-            poetry run pre-commit run --files ./**/**
-            if [[ $(git status --porcelain) ]]
-            then
-              echo "::error::pre-commit hooks failed for $(basename ${package_dir})" && exit 1
-            fi
-            cd $CUR_DIR
-          done
+          poetry run pre-commit run --files ./**/**
+          if [[ $(git status --porcelain) ]]
+          then
+            echo "::error::pre-commit hooks failed for ${{ matrix.package_name }}" && exit 1
+          fi
       - name: Run tests
         run: |
-          CUR_DIR=$(pwd)
+          package_dir=${{ matrix.package_dir }}
+          cd $package_dir
+
+          poetry install
+          poetry run pytest -v
 
-          for package_dir in ${{ needs.package-filter.outputs.package_dirs }}
-          do
-            cd $package_dir
-            poetry update && poetry install
-            poetry run pytest -v
-            cd $CUR_DIR
-          done
+  docker:
+    name: Build Docker images
+    needs: [package-filter, tests]
+    uses: ./.github/workflows/docker.yml
+    with:
+      matrix: ${{ needs.package-filter.outputs.matrix }}
+      push: false
+    secrets:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_TOKEN }}