scope down pr to just audioqna and build+scan

Signed-off-by: tylertitsworth <[email protected]>
opea-project · Jul 10, 2024 · 108cb34 · 108cb34
1 parent 8bcac10
commit 108cb34
Show file tree

Hide file tree

Showing 45 changed files with 412 additions and 548 deletions.
diff --git a/.github/workflows/composite/docker-build/action.yml b/.github/workflows/composite/docker-build/action.yml
@@ -0,0 +1,81 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Source: https://github.com/intel/ai-containers/blob/main/.github/action.yml
+
+---
+name: Build Container Group
+description: Given the inputs found below, build all containers found in a docker-compose.yaml file for a given configuration
+author: [email protected]
+inputs:
+  group_dir:
+    description: Directory with docker-compose.yaml to build
+    required: true
+    type: string
+  env_overrides:
+    description: Bash Env Variable Overrides in `KEY=VAL && KEY2=VAL2` format
+    required: false
+    type: string
+  registry:
+    description: Container Registry URL
+    required: false
+    default: 'opea-project'
+    type: string
+outputs:
+  container-group:
+    description: "Container Group"
+    value: ${{ steps.container-output.outputs.group }}
+runs:
+  using: composite
+  steps:
+      # This step generates a random number to use as the project number
+      # which can help avoid collisions with parallel builds on the same system
+    - name: Generate Project Number
+      shell: bash
+      run: echo "project-number=$(shuf -i 0-10000 -n1)" >> $GITHUB_ENV
+    - name: Build Containers
+      shell: bash
+      run: |
+        REGISTRY=${{ inputs.registry }} \
+        COMPOSE_PROJECT_NAME=${{ env.project-number }} \
+        ${{ inputs.env_overrides }} docker compose -p ${{ env.project-number }} up --build --force-recreate --always-recreate-deps --no-start
+      working-directory: ${{ inputs.group_dir }}
+    - name: Print Containers
+      id: container-output
+      shell: bash
+      run: |
+        mkdir matrix
+        images=$(REGISTRY=${{ inputs.registry }} \
+        COMPOSE_PROJECT_NAME=${{ env.project-number }} \
+        ${{ inputs.env_overrides }} docker compose -p ${{ env.project-number }} images --format json)
+        for image in $(echo $images | jq -r --arg registry "$REGISTRY" '.[] | select(.Repository | contains($registry)) | .Tag'); do
+          echo "$image" > matrix/$image.txt
+        done
+        echo "group=${{ inputs.group_dir }}" | tr '/' '_' >> $GITHUB_OUTPUT
+      working-directory: ${{ inputs.group_dir }}
+    - uses: actions/upload-artifact@v4
+      with:
+        name: ${{ env.project-number }}-${{ steps.container-output.outputs.group }}
+        path: ${{ inputs.group_dir }}/matrix/*
+        retention-days: 1
+        overwrite: true
+    - name: Push Containers
+      shell: bash
+      if: ${{ !fromJson(inputs.no-push) }}
+      run: |
+        REGISTRY=${{ inputs.registry }} \
+        COMPOSE_PROJECT_NAME=${{ env.project-number }} \
+        ${{ inputs.env_overrides }} docker compose -p ${{ env.project-number }} push
+      working-directory: ${{ inputs.group_dir }}
+    - name: Un-Tag Containers
+      if: ${{ always() }}
+      shell: bash
+      run: |
+        REGISTRY=${{ inputs.registry }} \
+        COMPOSE_PROJECT_NAME=${{ env.project-number }} \
+        ${{ inputs.env_overrides }} docker compose -p ${{ env.project-number }} down --rmi all
+      working-directory: ${{ inputs.group_dir }}
+    - name: Remove Containers
+      if: ${{ always() }}
+      shell: bash
+      run: docker system prune --force
diff --git a/.github/workflows/composite/scan/action.yml b/.github/workflows/composite/scan/action.yml
@@ -0,0 +1,26 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Source: https://github.com/intel/ai-containers/blob/main/.github/scan/action.yml
+
+name: 'Aqua Security Trivy'
+description: 'Scans container images for vulnerabilities with Trivy without building the image. For use behind firewalls.'
+author: '[email protected]'
+inputs:
+  image-ref:
+    description: 'image reference(for backward compatibility)'
+    required: true
+  output:
+    description: 'writes results to a file with the specified file name'
+    required: true
+runs:
+  using: 'docker'
+  image: "docker://ghcr.io/aquasecurity/trivy"
+  entrypoint: trivy
+  args:
+    - '--timeout=30m'
+    - image
+    - '--format=sarif'
+    - '--no-progress'
+    - '--output=${{ inputs.output }}'
+    - ${{ inputs.image-ref }}
diff --git a/.github/workflows/container-ci.yml b/.github/workflows/container-ci.yml
@@ -1,20 +1,9 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
 
 name: Container Integration Tests
 on:
-  pull_request: null
+  pull_request
 permissions: read-all
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

diff --git a/.github/workflows/reuse-container-ci.yaml b/.github/workflows/reuse-container-ci.yaml
@@ -1,16 +1,5 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
 
 name: Container CI
 permissions: read-all
@@ -30,64 +19,24 @@ on:
         default: test-runner
         required: true
         type: string
-      no_build:
-        description: No build
-        default: false
-        required: true
-        type: boolean
-      no_push:
-        description: Do not push to Registry
-        required: true
-        default: false
-        type: boolean
-      no_start:
-        description: No Start
-        default: false
-        required: true
-        type: boolean
   workflow_call:
     inputs:
       group_dir:
         required: true
         type: string
-      no_push:
-        required: false
-        type: boolean
-      no_start:
-        required: false
-        type: boolean
 jobs:
-  ####################################################################################################
-  # Compose Build
-  ####################################################################################################
-  setup-build:
-    outputs:
-      matrix: ${{ steps.build-matrix.outputs.matrix }}
-    runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
-    steps:
-    - name: Harden Runner
-      uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
-      with:
-        egress-policy: audit
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-    - name: Set Matrix
-      id: build-matrix
-      run: echo "matrix=$(jq -c . < ${{ inputs.group_dir }}/.actions.json)" >> $GITHUB_OUTPUT
-    - name: Print Inputs
-      if: ${{ inputs.env_overrides }}
-      run: echo "Overrides - ${{ inputs.env_overrides }}" >> $GITHUB_STEP_SUMMARY
+####################################################################################################
+# Compose Build
+####################################################################################################
   build-containers:
-    needs: [setup-build]
-    env: ${{ matrix }}
-    runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
+    runs-on: ubuntu-latest
     strategy:
       matrix: ${{ fromJson(needs.setup-build.outputs.matrix) }}
       fail-fast: false
     outputs:
       group: ${{ steps.build-group.outputs.container-group }}
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-      if: ${{ !inputs.no_build }}
     - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
       with:
         registry: ${{ secrets.REGISTRY }}
@@ -97,21 +46,18 @@ jobs:
     - name: Build Container Group
       if: ${{ !inputs.no_build }}
       id: build-group
-      uses: intel/ai-containers/.github@main
+      uses: ./.github/workflows/composite/docker-build
       with:
         group_dir: ${{ inputs.group_dir }}
         env_overrides: ${{ inputs.env_overrides || env.env_overrides || '' }}
         registry: ${{ secrets.REGISTRY }}
-        repo: ${{ secrets.REPO }}
-        no-push: ${{ inputs.no_push }}
-        no-start: ${{ inputs.no_start }}
-        ####################################################################################################
-        # Trivy Scan
-        ####################################################################################################
+####################################################################################################
+# Trivy Scan
+####################################################################################################
   setup-scan:
     needs: [build-containers]
     if: ${{ github.event_name == 'pull_request' }}
-    runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
+    runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.scan-matrix.outputs.matrix }}
     steps:
@@ -128,7 +74,7 @@ jobs:
   scan-containers:
     needs: [setup-scan]
     if: ${{ !inputs.no_build && github.event_name == 'pull_request' }}
-    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
+    runs-on: ubuntu-latest
     env:
       GROUP_DIR: ${{ inputs.group_dir }}
     strategy:
@@ -153,7 +99,7 @@ jobs:
       env:
         GROUP_DIR: ${{ inputs.group_dir }}
     - name: Scan Container
-      uses: intel/ai-containers/.github/scan@main
+      uses: ./.github/workflows/composite/scan
       with:
         image-ref: ${{ secrets.REGISTRY }}/${{ env.group_dir }}:${{ matrix.container }}
         output: ${{ matrix.container }}-${{ env.group_dir }}-scan.sarif
@@ -165,48 +111,45 @@ jobs:
         sarif_file: '${{ matrix.container }}-${{ env.group_dir }}-scan.sarif'
         category: '${{ matrix.container }}'
       continue-on-error: true
-      ####################################################################################################
-      # Generic Test Runner
-      ####################################################################################################
-  setup-test:
-    needs: [build-containers]
-    runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
-    outputs:
-      matrix: ${{ steps.test-matrix.outputs.matrix }}
-    steps:
-    - name: Harden Runner
-      uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
-      with:
-        egress-policy: audit
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-    - name: Get Recipes
-      id: test-matrix
-      run: echo "matrix=$(find ${{ inputs.group_dir }} -type f -name 'tests.yaml' -exec dirname {} \; | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
-  test-containers:
-    needs: [setup-build, setup-test]
-    if: ${{ needs.setup-test.outputs.matrix != '[]' }}
-    runs-on: ${{ fromJson(needs.setup-build.outputs.matrix).runner_label }}
-    strategy:
-      matrix:
-        tests: ${{ fromJson(needs.setup-test.outputs.matrix) }}
-        experimental: [true]
-      fail-fast: false
-    steps:
-    - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
-      with:
-        egress-policy: audit
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-    - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
-      with:
-        registry: ${{ secrets.REGISTRY }}
-        username: ${{ secrets.REGISTRY_USER }}
-        password: ${{ secrets.REGISTRY_TOKEN }}
-    - name: Test Container Group
-      uses: intel/ai-containers/test-runner@main
-      with:
-        cache_registry: ${{ secrets.CACHE_REGISTRY }}
-        recipe_dir: ${{ inputs.group_dir }}
-        registry: ${{ secrets.REGISTRY }}
-        repo: ${{ secrets.REPO }}
-        test_dir: ${{ matrix.tests }}
-        token: ${{ secrets.GITHUB_TOKEN }}
+####################################################################################################
+# Generic Test Runner
+####################################################################################################
+  # setup-test:
+  #   needs: [build-containers]
+  #   runs-on: ubuntu-latest
+  #   outputs:
+  #     matrix: ${{ steps.test-matrix.outputs.matrix }}
+  #   steps:
+  #   - name: Harden Runner
+  #     uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+  #     with:
+  #       egress-policy: audit
+  #   - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+  #   - name: Get Recipes
+  #     id: test-matrix
+  #     run: echo "matrix=$(find ${{ inputs.group_dir }} -type f -name 'tests.yaml' -exec dirname {} \; | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+  # test-containers:
+  #   needs: [setup-build, setup-test]
+  #   if: ${{ needs.setup-test.outputs.matrix != '[]' }}
+  #   runs-on: ${{ fromJson(needs.setup-build.outputs.matrix).runner_label }}
+  #   strategy:
+  #     matrix:
+  #       tests: ${{ fromJson(needs.setup-test.outputs.matrix) }}
+  #       experimental: [true]
+  #     fail-fast: false
+  #   steps:
+  #   - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+  #     with:
+  #       egress-policy: audit
+  #   - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+  #   - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+  #     with:
+  #       registry: ${{ secrets.REGISTRY }}
+  #       username: ${{ secrets.REGISTRY_USER }}
+  #       password: ${{ secrets.REGISTRY_TOKEN }}
+  #   - name: Test Container Group
+  #     uses: ./.github/workflows/composite/test
+  #     with:
+  #       registry: ${{ secrets.REGISTRY }}
+  #       test_dir: ${{ matrix.tests }}
+  #       token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/AudioQnA/.actions.json b/AudioQnA/.actions.json
diff --git a/AudioQnA/docker/gaudi/docker_compose.yaml → AudioQnA/docker/gaudi/docker-compose.yaml b/AudioQnA/docker/gaudi/docker_compose.yaml → AudioQnA/docker/gaudi/docker-compose.yaml
@@ -90,7 +90,7 @@ services:
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
     restart: unless-stopped
   audioqna-gaudi-backend-server:
-    image: opea/audioqna:latest
+    image: ${REGISTRY:-opea}/audioqna:${GITHUB_RUN_NUMBER:-latest}
     container_name: audioqna-gaudi-backend-server
     depends_on:
       - asr

diff --git a/AudioQnA/docker/xeon/docker_compose.yaml → AudioQnA/docker/xeon/docker-compose.yaml b/AudioQnA/docker/xeon/docker_compose.yaml → AudioQnA/docker/xeon/docker-compose.yaml
@@ -2,8 +2,6 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-version: "3.8"
-
 services:
   whisper-service:
     image: opea/whisper:latest
@@ -73,7 +71,7 @@ services:
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
     restart: unless-stopped
   audioqna-xeon-backend-server:
-    image: opea/audioqna:latest
+    image: ${REGISTRY:-opea}/audioqna:${GITHUB_RUN_NUMBER:-latest}
     container_name: audioqna-xeon-backend-server
     depends_on:
       - asr

diff --git a/AudioQnA/tests/tests.yaml b/AudioQnA/tests/tests.yaml