From cfe54e21cebfdf4b7b4de735bedba3ce65c1044f Mon Sep 17 00:00:00 2001 From: Rafal Skolasinski Date: Wed, 28 Feb 2024 13:11:42 +0000 Subject: [PATCH] feat(build): use github actions to build docker images (#5380) * add workflow to push images using GHA * add option to choose environment * fix makefile target for tests * disable build of images via jenkins x * run workflow from master only on master * minor tweak --- .github/workflows/images.yml | 198 ++++++++++++++++++++++++++++- .lighthouse/jenkins-x/release.yaml | 78 ------------ testing/scripts/Makefile | 2 +- 3 files changed, 193 insertions(+), 85 deletions(-) delete mode 100644 .lighthouse/jenkins-x/release.yaml diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index e6de2cadf4..ff4116c700 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -1,6 +1,9 @@ -name: V2 Build docker images +name: Build & Push Docker Images on: + push: + branches: [ master ] + workflow_dispatch: inputs: docker-tag: @@ -8,11 +11,194 @@ on: default: 'latest' required: false +env: + GOLANG_VERSION: 1.20.9 + jobs: - placeholder: - name: placeholder + operator: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GOLANG_VERSION }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + registry: ${{ secrets.DOCKER_REGISTRY }} + + - name: Set default docker tag for builds from master + id: docker-tag + run: | + USER_INPUT="${{ github.event.inputs.docker-tag }}" + echo "value=${USER_INPUT:-latest}" >> $GITHUB_OUTPUT + + + - name: Build and push + working-directory: ./operator/ + env: + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + VERSION: ${{ steps.docker-tag.outputs.value }} + run: | + make docker-build docker-push + + executor: runs-on: ubuntu-latest - if: github.repository == 'seldonio/seldon-core' # Do not run this on forks. steps: - - name: Echo Message - run: echo "This is to make workflow available for V2 releases" + - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + registry: ${{ secrets.DOCKER_REGISTRY }} + + - name: Set default docker tag for builds from master + id: docker-tag + run: | + USER_INPUT="${{ github.event.inputs.docker-tag }}" + echo "value=${USER_INPUT:-latest}" >> $GITHUB_OUTPUT + + - name: Build and push + working-directory: ./executor/ + env: + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + VERSION: ${{ steps.docker-tag.outputs.value }} + run: | + make docker-build docker-push + + rclone-storage-initializer: + runs-on: ubuntu-latest + steps: + - name: Checkout Git Commit + uses: actions/checkout@v4 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + registry: ${{ secrets.DOCKER_REGISTRY }} + + - name: Set default docker tag for builds from master + id: docker-tag + run: | + USER_INPUT="${{ github.event.inputs.docker-tag }}" + echo "value=${USER_INPUT:-latest}" >> $GITHUB_OUTPUT + + - name: Build and push (Rclone Storage Initializer) + working-directory: ./components/rclone-storage-initializer + env: + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + VERSION: ${{ steps.docker-tag.outputs.value }} + run: | + make docker-build docker-push + + s2i-wrapper: + runs-on: ubuntu-latest + steps: + - name: Checkout Git Commit + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + registry: ${{ secrets.DOCKER_REGISTRY }} + + - name: Set default docker tag for builds from master + id: docker-tag + run: | + USER_INPUT="${{ github.event.inputs.docker-tag }}" + echo "value=${USER_INPUT:-latest}" >> $GITHUB_OUTPUT + + - name: Build and push (Conda Base Image) + working-directory: ./wrappers/s2i/python + env: + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + VERSION: ${{ steps.docker-tag.outputs.value }} + run: | + make docker-build-conda-base docker-push-conda-base + + - name: Build and push (Base Wrapper) + working-directory: ./wrappers/s2i/python + env: + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + VERSION: ${{ steps.docker-tag.outputs.value }} + run: | + make docker-build docker-push PYTHON_VERSION=3.8.10 + make docker-tag-base-python docker-push-base-python PYTHON_VERSION=3.8.10 + docker save -o /tmp/base-wrapper.tar ${DOCKER_REGISTRY}/seldon-core-s2i-python38:${VERSION} + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: base-wrapper + path: /tmp/base-wrapper.tar + + prepackaged-components: + runs-on: ubuntu-latest + needs: s2i-wrapper + strategy: + matrix: + server: + - servers/sklearnserver + - servers/xgboostserver + - servers/mlflowserver + - servers/tfserving_proxy + - components/alibi-explain-server + - components/alibi-detect-server + steps: + - name: Checkout Git Commit + uses: actions/checkout@v4 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + registry: ${{ secrets.DOCKER_REGISTRY }} + + - name: Set default docker tag for builds from master + id: docker-tag + run: | + USER_INPUT="${{ github.event.inputs.docker-tag }}" + echo "value=${USER_INPUT:-latest}" >> $GITHUB_OUTPUT + + - name: Install CLI tools from OpenShift Mirror + uses: redhat-actions/openshift-tools-installer@v1 + with: + github_pat: ${{ github.token }} + source: "github" + s2i: "latest" + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: base-wrapper + path: /tmp + + - name: Load image + run: | + docker load --input /tmp/base-wrapper.tar + + - name: Build and push (Prepackaged Server Image) + working-directory: ./${{ matrix.server }}/ + env: + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + VERSION: ${{ steps.docker-tag.outputs.value }} + run: | + make docker-build docker-push diff --git a/.lighthouse/jenkins-x/release.yaml b/.lighthouse/jenkins-x/release.yaml deleted file mode 100644 index a4a7091e3a..0000000000 --- a/.lighthouse/jenkins-x/release.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: PipelineRun -metadata: - creationTimestamp: null - name: release -spec: - pipelineSpec: - tasks: - - name: release-test-task - taskSpec: - stepTemplate: - name: "" - workingDir: /workspace/source - steps: - - image: uses:jenkins-x/jx3-pipeline-catalog/tasks/git-clone/git-clone-pr.yaml@versionStream - name: "" - - name: release-step - image: seldonio/core-builder:0.30 - env: - - name: DOCKER_BUILDKIT - value: "0" - - name: GOMAXPROCS - value: "1" - command: - - bash - - -c - - ./ci_build_and_push_images.sh - volumeMounts: - - mountPath: /lib/modules - name: modules - readOnly: true - - mountPath: /sys/fs/cgroup - name: cgroup - - name: dind-storage - mountPath: /var/lib/docker - - mountPath: /tekton/home/.docker - name: jenkins-docker-config-volume - resources: - requests: - cpu: 1 - memory: 6000Mi - ephemeral-storage: "100Gi" - limits: - cpu: 1 - memory: 6000Mi - ephemeral-storage: "100Gi" - securityContext: - privileged: true - imagePullPolicy: Always - volumes: - - name: modules - hostPath: - path: /lib/modules - type: Directory - - name: cgroup - hostPath: - path: /sys/fs/cgroup - type: Directory - - name: dind-storage - emptyDir: {} - - name: jenkins-docker-config-volume - secret: - items: - - key: config.json - path: config.json - secretName: jenkins-docker-cfg - - podTemplate: - nodeSelector: - cloud.google.com/gke-nodepool: general-pipelines-pool - tolerations: - - key: job-type - operator: Equal - value: general - effect: NoSchedule - serviceAccountName: tekton-bot - timeout: 6h0m0s -status: {} diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index b4063243ae..5128f0e2f2 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -54,7 +54,7 @@ kind_build_misc: kind_build_images: build_protos kind_build_operator kind_build_executor kind_build_prepackaged kind_build_alibi_explain kind_build_alibi_detect kind_build_rclone_storage_initializer .PHONY: kind_build_test_images -kind_build_test_images: make kind_build_test_models kind_build_misc +kind_build_test_images: kind_build_test_models kind_build_misc .PHONY: helm_setup helm_setup: