Improvements to Container CD

.github/workflows/comps-manual-ci.yaml

@@ -0,0 +1,54 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Build Specific Comps
+on:
+  workflow_dispatch:
+    inputs:
+      comps:
+        default: 'comps1, comps2'
+        description: 'List of comma-separated compss to test'
+        required: true
+        type: string
+      scan:
+        default: true
+        description: 'Scan all examples with Trivy'
+        required: false
+        type: boolean
+      test:
+        default: true
+        description: 'Test Examples'
+        required: false
+        type: boolean
+      publish:
+        default: false
+        description: 'Publish Images to Dockerhub'
+        required: false
+        type: boolean
+permissions: read-all
+jobs:
+  setup-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.get-matrix.outputs.matrix }}
+    steps:
+    - name: Create Matrix
+      id: get-matrix
+      run: |
+        COMPS=($(echo ${{ github.event.inputs.comps }} | tr ',' ' '))
+        COMPS_JSON=$(printf '%s\n' "${COMPS[@]}" | jq -R '.' | jq -sc '.')
+        echo "matrix=$COMPS_JSON" >> $GITHUB_OUTPUT
+  container-ci:
+    needs: [setup-matrix]
+    strategy:
+      matrix:
+        comp: ${{ fromJson(needs.setup-matrix.outputs.matrix) }}
+        experimental: [true]
+      fail-fast: false
+    uses: opea-project/genaiexamples/.github/workflows/container-ci.yaml@main
+    with:
+      example_dir: ${{ matrix.comp }}
+      scan: ${{ fromJSON(inputs.scan) }}
+      test: ${{ fromJSON(inputs.test) }}
+      publish: ${{ fromJSON(inputs.publish) }}
+    secrets: inherit

.github/workflows/comps-weekly-ci.yaml

@@ -0,0 +1,47 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Weekly Container Tests
+on:
+  schedule:
+  - cron: "0 0 * * 0"
+  workflow_dispatch: null
+  push:
+    tags: ['**'] # whenever a tag is created/modified
+permissions: read-all
+jobs:
+  get-comps:
+    runs-on: ubuntu-latest
+    outputs:
+      comps: ${{ steps.comp-list.outputs.FOLDERS }}
+    steps:
+    - name: Harden Runner
+      uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+      with:
+        egress-policy: audit
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - name: Output Comp Directories
+      id: comp-list
+      run: |
+        DOCKER_COMPOSE_PATHS=()
+        for path in $(find comps -name 'docker-compose.yaml'); do
+          DIR_PATH=$(dirname "$path")
+          DOCKER_COMPOSE_PATHS+=("${DIR_PATH}")
+        done
+        # Convert the array to a JSON array
+        DOCKER_COMPOSE_PATHS_JSON=$(printf '%s\n' "${DOCKER_COMPOSE_PATHS[@]}" | uniq | jq -R '.' | jq -sc '.')
+        echo "FOLDERS=$DOCKER_COMPOSE_PATHS_JSON" >> $GITHUB_OUTPUT
+  pipeline-ci:
+    needs: [get-comps]
+    strategy:
+      matrix:
+        comp: ${{ fromJson(needs.get-comps.outputs.comps) }}
+        experimental: [true]
+      fail-fast: false
+    uses: opea-project/genaiexamples/.github/workflows/container-ci.yaml@main
+    with:
+      example_dir: ${{ matrix.comp }}
+      scan: false
+      test: true
+      publish: false
+    secrets: inherit

comps/asr/README.md

@@ -60,26 +60,8 @@ Alternatively, you can also start the ASR microservice with Docker.
 
 ## 2.1 Build Images
 
-### 2.1.1 Whisper Server Image
-
-- Xeon CPU
-
-```bash
-cd ../..
-docker build -t opea/whisper:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/whisper/Dockerfile .
-```
-
-- Gaudi2 HPU
-
-```bash
-cd ../..
-docker build -t opea/whisper:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/whisper/Dockerfile_hpu .
-```
-
-### 2.1.2 ASR Service Image
-
 ```bash
-docker build -t opea/asr:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/Dockerfile .
+docker compose build
 ```
 
 ## 2.2 Start Whisper and ASR Service
@@ -95,7 +77,7 @@ docker run -p 7066:7066 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$htt
 - Gaudi2 HPU
 
 ```bash
-docker run -p 7066:7066 --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/whisper:latest
+docker run -p 7066:7066 --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/whisper-hpu:latest
 ```
 
 ### 2.2.2 Start ASR service

comps/asr/docker-compose.yaml

@@ -0,0 +1,23 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  whisper-xeon:
+    build:
+      args:
+        http_proxy: ${http_proxy}
+        https_proxy: ${https_proxy}
+        no_proxy: ${no_proxy}
+      context: ../../
+      dockerfile: comps/asr/whisper/Dockerfile
+    image: ${REGISTRY:-opea}/whisper:${GITHUB_RUN_NUMBER:-latest}
+  whipser-gaudi:
+    build:
+      dockerfile: comps/asr/whisper/Dockerfile_hpu
+    extends: whisper-xeon
+    image: ${REGISTRY-opea}/whisper-hpu:${GITHUB_RUN_NUMBER:-latest}
+  asr:
+    build:
+      dockerfile: comps/asr/Dockerfile
+    extends: whisper-xeon
+    image: ${REGISTRY:-opea}/asr:${GITHUB_RUN_NUMBER:-latest}