Enhance CI/CD infrastructure (#593)

Signed-off-by: chensuyue <[email protected]> Signed-off-by: Sun, Xuehao <[email protected]>
opea-project · Aug 15, 2024 · c26d0f6 · c26d0f6
1 parent e71aba0
commit c26d0f6
Show file tree

Hide file tree

Showing 61 changed files with 987 additions and 954 deletions.
diff --git a/.github/workflows/_example-workflow.yml b/.github/workflows/_example-workflow.yml
@@ -6,6 +6,9 @@ permissions: read-all
 on:
   workflow_call:
     inputs:
+      node:
+        required: true
+        type: string
       example:
         required: true
         type: string
@@ -37,48 +40,56 @@ on:
         default: "latest"
         required: false
         type: string
+      GenAIComps_branch:
+        default: "main"
+        required: false
+        type: string
 jobs:
 ####################################################################################################
 # Image Build
 ####################################################################################################
   build-images:
-    if: ${{ fromJSON(inputs.build) }}
-    strategy:
-      matrix:
-        node: ["docker-build-xeon", "docker-build-gaudi"]
-    runs-on: ${{ matrix.node }}
+    runs-on: "docker-build-${{ inputs.node }}"
     continue-on-error: true
     steps:
       - name: Clean Up Working Directory
-        run: |
-          sudo rm -rf ${{github.workspace}}/*
+        run: sudo rm -rf ${{github.workspace}}/*
 
       - name: Checkout out Repo
         uses: actions/checkout@v4
 
+      - name: Clone required Repo
+        run: |
+          cd ${{ github.workspace }}/${{ inputs.example }}/docker
+          build_compose_path=${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml
+          if [[ $(grep -c "tei-gaudi:" ${docker_compose_path}) != 0 ]]; then
+              git clone https://github.com/huggingface/tei-gaudi.git
+          fi
+          if [[ $(grep -c "vllm:" ${docker_compose_path}) != 0 ]]; then
+              git clone https://github.com/huggingface/tei-gaudi.git
+          fi
+          git clone https://github.com/opea-project/GenAIComps.git
+          cd GenAIComps && checkout ${{ inputs.GenAIComps_branch }} && cd ../
+
       - name: Build Image
+        if: ${{ fromJSON(inputs.build) }}
         uses: opea-project/validation/actions/image-build@main
         with:
-          work_dir: ${{ github.workspace }}/${{ inputs.example }}
-          docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.example }}-compose.yaml
+          work_dir: ${{ github.workspace }}/${{ inputs.example }}/docker
+          docker_compose_path: ${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml
           registry: ${OPEA_IMAGE_REPO}opea
           tag: ${{ inputs.tag }}
 
 ####################################################################################################
 # Trivy Scan
 ####################################################################################################
   image-list:
-    needs: [ build-images ]
-    if: ${{ fromJSON(inputs.scan) }}
+    needs: [build-images]
+    if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi' }}
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.scan-matrix.outputs.matrix }}
     steps:
-      - name: Harden Runner
-        uses: step-security/[email protected]
-        with:
-          egress-policy: audit
-
       - name: Checkout out Repo
         uses: actions/checkout@v4
 
@@ -90,34 +101,32 @@ jobs:
           echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT
 
   scan-images:
-    needs: [image-list]
-    if: ${{ fromJSON(inputs.scan) }}
-    runs-on: "docker-build-gaudi"
+    needs: [image-list, build-images]
+    if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi'}}
+    runs-on: "docker-build-${{ inputs.node }}"
     strategy:
       matrix:
         image: ${{ fromJSON(needs.image-list.outputs.matrix) }}
       fail-fast: false
     steps:
-      - name: Harden Runner
-        uses: step-security/[email protected]
-        with:
-          egress-policy: audit
-
       - name: Pull Image
-        run: docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+        run: |
+          docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+          echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV
 
       - name: Scan Container
         uses: opea-project/validation/actions/trivy-scan@main
         with:
-          image-ref: ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+          image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
           output: ${{ inputs.example }}-${{ matrix.image }}-scan.txt
 
       - name: Cleanup
         if: always()
         run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+
       - uses: actions/[email protected]
         with:
-          name: ${{ inputs.example }}-scan
+          name: ${{ inputs.example }}-${{ matrix.image }}-scan
           path: ${{ inputs.example }}-${{ matrix.image }}-scan.txt
           overwrite: true
 
@@ -127,15 +136,11 @@ jobs:
   test-example-compose:
     needs: [build-images]
     if: ${{ fromJSON(inputs.test_compose) }}
-    strategy:
-      matrix:
-        hardware: ["xeon", "gaudi"]
-      fail-fast: false
     uses: ./.github/workflows/_run-docker-compose.yml
     with:
       tag: ${{ inputs.tag }}
       example: ${{ inputs.example }}
-      hardware: ${{ matrix.hardware }}
+      hardware: ${{ inputs.node }}
     secrets: inherit
 
 
@@ -145,14 +150,10 @@ jobs:
   test-k8s-manifest:
     needs: [build-images]
     if: ${{ fromJSON(inputs.test_k8s) }}
-    strategy:
-      matrix:
-        hardware: ["xeon", "gaudi"]
-      fail-fast: false
     uses: ./.github/workflows/_manifest-e2e.yml
     with:
       example: ${{ inputs.example }}
-      hardware: ${{ matrix.hardware }}
+      hardware: ${{ inputs.node }}
       tag: ${{ inputs.tag }}
     secrets: inherit
 
@@ -162,7 +163,7 @@ jobs:
 ####################################################################################################
   publish:
     needs: [image-list, build-images, scan-images, test-example-compose]
-    if: ${{ fromJSON(inputs.publish) }}
+    if: ${{ fromJSON(inputs.publish) && inputs.node == 'gaudi' }}
     strategy:
       matrix:
         image: ${{ fromJSON(needs.image-list.outputs.matrix) }}

diff --git a/.github/workflows/_run-docker-compose.yml b/.github/workflows/_run-docker-compose.yml
@@ -72,6 +72,7 @@ jobs:
           sudo rm -rf ${{github.workspace}}/* || true
           docker system prune -f
           docker rmi $(docker images --filter reference="*/*/*:latest" -q) || true
+          docker rmi $(docker images --filter reference="*/*:ci" -q) || true
 
       - name: Checkout out Repo
         uses: actions/checkout@v4
@@ -93,7 +94,7 @@ jobs:
           test_case: ${{ matrix.test_case }}
         run: |
           cd ${{ github.workspace }}/$example/tests
-          export IMAGE_REPO=${OPEA_IMAGE_REPO}
+          if [[ "$IMAGE_REPO" == "" ]]; then export IMAGE_REPO="${OPEA_IMAGE_REPO}opea"; fi
           if [ -f ${test_case} ]; then timeout 30m bash ${test_case}; else echo "Test script {${test_case}} not found, skip test!"; fi
 
       - name: Clean up container

diff --git a/.github/workflows/manual-cd-workflow.yml b/.github/workflows/manual-cd-workflow.yml
@@ -5,9 +5,14 @@ name: Examples CD workflow on manual event
 on:
   workflow_dispatch:
     inputs:
+      nodes:
+        default: "gaudi,xeon"
+        description: "Hardware to run test"
+        required: true
+        type: string
       examples:
-        default: "AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation"
-        description: 'List of examples to test'
+        default: "ChatQnA"
+        description: 'List of examples to test [AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation]'
         required: true
         type: string
       tag:
@@ -41,39 +46,51 @@ on:
         required: false
         type: boolean
       publish_tags:
-        default: "latest,v0.9"
+        default: "latest,v1.0"
         description: 'Tag list apply to publish images'
         required: false
         type: string
+      GenAIComps_branch:
+        default: "main"
+        description: 'GenAIComps branch for image build'
+        required: false
+        type: string
 
 permissions: read-all
 jobs:
   get-test-matrix:
     runs-on: ubuntu-latest
     outputs:
-      matrix: ${{ steps.get-matrix.outputs.matrix }}
+      examples: ${{ steps.get-matrix.outputs.examples }}
+      nodes: ${{ steps.get-matrix.outputs.nodes }}
     steps:
     - name: Create Matrix
       id: get-matrix
       run: |
         examples=($(echo ${{ github.event.inputs.examples }} | tr ',' ' '))
         examples_json=$(printf '%s\n' "${examples[@]}" | sort -u | jq -R '.' | jq -sc '.')
-        echo "matrix=$examples_json" >> $GITHUB_OUTPUT
+        echo "examples=$examples_json" >> $GITHUB_OUTPUT
+        nodes=($(echo ${{ github.event.inputs.nodes }} | tr ',' ' '))
+        nodes_json=$(printf '%s\n' "${nodes[@]}" | sort -u | jq -R '.' | jq -sc '.')
+        echo "nodes=$nodes_json" >> $GITHUB_OUTPUT
 
   run-examples:
     needs: [get-test-matrix]
     strategy:
       matrix:
-        example: ${{ fromJson(needs.get-test-matrix.outputs.matrix) }}
+        example: ${{ fromJson(needs.get-test-matrix.outputs.examples) }}
+        node: ${{ fromJson(needs.get-test-matrix.outputs.nodes) }}
       fail-fast: false
     uses: ./.github/workflows/_example-workflow.yml
     with:
+      node: ${{ matrix.node }}
       example: ${{ matrix.example }}
       tag: ${{ inputs.tag }}
       build: ${{ fromJSON(inputs.build) }}
       scan: ${{ fromJSON(inputs.scan) }}
       test_compose: ${{ fromJSON(inputs.test_compose) }}
       test_k8s: ${{ fromJSON(inputs.test_k8s) }}
       publish: ${{ fromJSON(inputs.publish) }}
-      publish_tags: ${{ fromJSON(inputs.publish_tags) }}
+      publish_tags: ${{ inputs.publish_tags }}
+      GenAIComps_branch: ${{ inputs.GenAIComps_branch }}
     secrets: inherit
diff --git a/.github/workflows/pr-docker-compose-e2e.yml b/.github/workflows/pr-docker-compose-e2e.yml
@@ -4,7 +4,7 @@
 name: E2E test with docker compose
 
 on:
-  pull_request_target:
+  pull_request:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
@@ -21,29 +21,20 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  job1:
+  get-test-matrix:
     uses: ./.github/workflows/_get-test-matrix.yml
     with:
       diff_excluded_files: '.github|README.md|*.txt|deprecate|kubernetes|manifest|gmc|assets'
 
-  mega-image-build:
-    needs: job1
-    strategy:
-      matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
-    uses: ./.github/workflows/_image-build.yml
-    with:
-      image_tag: ${{ github.event.pull_request.head.sha }}
-      mega_service: "${{ matrix.example }}"
-      runner_label: "docker-build-${{ matrix.hardware }}"
-
   example-test:
-    needs: [job1, mega-image-build]
+    needs: [get-test-matrix]
     strategy:
-      matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
+      matrix: ${{ fromJSON(needs.get-test-matrix.outputs.run_matrix) }}
       fail-fast: false
     uses: ./.github/workflows/_run-docker-compose.yml
     with:
-      tag: ${{ needs.mega-image-build.outputs.image_tag }}
+      registry: "opea"
+      tag: "ci"
       example: ${{ matrix.example }}
       hardware: ${{ matrix.hardware }}
     secrets: inherit
diff --git a/AudioQnA/docker/docker_build_compose.yaml b/AudioQnA/docker/docker_build_compose.yaml
@@ -0,0 +1,54 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  audioqna:
+    build:
+      args:
+        http_proxy: ${http_proxy}
+        https_proxy: ${https_proxy}
+        no_proxy: ${no_proxy}
+      dockerfile: ./Dockerfile
+    image: ${REGISTRY:-opea}/audioqna:${TAG:-latest}
+  whisper-gaudi:
+    build:
+      context: GenAIComps
+      dockerfile: comps/asr/whisper/Dockerfile_hpu
+    extends: audioqna
+    image: ${REGISTRY:-opea}/whisper-gaudi:${TAG:-latest}
+  whisper:
+    build:
+      context: GenAIComps
+      dockerfile: comps/asr/whisper/Dockerfile
+    extends: audioqna
+    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
+  asr:
+    build:
+      context: GenAIComps
+      dockerfile: comps/asr/Dockerfile
+    extends: audioqna
+    image: ${REGISTRY:-opea}/asr:${TAG:-latest}
+  llm-tgi:
+    build:
+      context: GenAIComps
+      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+    extends: audioqna
+    image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
+  speecht5-gaudi:
+    build:
+      context: GenAIComps
+      dockerfile: comps/tts/speecht5/Dockerfile_hpu
+    extends: audioqna
+    image: ${REGISTRY:-opea}/speecht5-gaudi:${TAG:-latest}
+  speecht5:
+    build:
+      context: GenAIComps
+      dockerfile: comps/tts/speecht5/Dockerfile
+    extends: audioqna
+    image: ${REGISTRY:-opea}/speecht5:${TAG:-latest}
+  tts:
+    build:
+      context: GenAIComps
+      dockerfile: comps/tts/Dockerfile
+    extends: audioqna
+    image: ${REGISTRY:-opea}/tts:${TAG:-latest}