From 56f84ca548b876edb2df3db645b628eacd074fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:47:50 +0200 Subject: [PATCH 01/20] Allow to run docker build without logging to DockerHub --- .github/workflows/ci.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1d0b25c76..0da2c70245 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,8 +52,13 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Checkout code uses: actions/checkout@v2 + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV - name: DockerHub login uses: docker/login-action@v1 + if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -61,16 +66,21 @@ jobs: - name: Push dev run: | docker buildx create --use + + option=load + if [ -n "${{ env.DOCKERHUB_USERNAME }}" ] && [ -n "${{ env.DOCKERHUB_TOKEN }}" ]; then + option=push + fi + docker buildx build \ - --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-${{ github.sha }}-dev \ - --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-latest-dev \ - --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-${{ github.sha }}-dev \ - --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-latest-dev \ + --progress=plain \ + --cache-from type=gha, \ + --cache-to type=gha,mode=max \ -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-latest-dev \ -f ./docker/Dockerfile \ --target dev \ - --push \ + --$option \ ./ isort: From c87b039ed391ba10638b9c0cb24222176ff990a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:52:11 +0200 Subject: [PATCH 02/20] set DOCKERHUB_ORGANIZATION in env --- .github/workflows/ci.yml | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0da2c70245..57f03d70c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - run: yarn test --watchAll=false working-directory: frontend - build_and_push_dev: + build_dev: runs-on: ubuntu-latest steps: - name: Set up Docker Buildx @@ -56,6 +56,11 @@ jobs: run: | echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi - name: DockerHub login uses: docker/login-action@v1 if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN @@ -76,8 +81,8 @@ jobs: --progress=plain \ --cache-from type=gha, \ --cache-to type=gha,mode=max \ - -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ - -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-latest-dev \ + -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ + -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-latest-dev \ -f ./docker/Dockerfile \ --target dev \ --$option \ @@ -85,7 +90,7 @@ jobs: isort: runs-on: ubuntu-latest - needs: [build_and_push_dev] + needs: [build_dev] steps: - name: Checkout code uses: actions/checkout@v2 @@ -103,7 +108,7 @@ jobs: black: runs-on: ubuntu-latest - needs: [build_and_push_dev] + needs: [build_dev] steps: - name: Checkout code uses: actions/checkout@v2 @@ -121,7 +126,7 @@ jobs: flake8: runs-on: ubuntu-latest - needs: [build_and_push_dev] + needs: [build_dev] steps: - name: Checkout code uses: actions/checkout@v2 @@ -139,7 +144,7 @@ jobs: mypy: runs-on: ubuntu-latest - needs: [build_and_push_dev] + needs: [build_dev] steps: - name: Checkout code uses: actions/checkout@v2 @@ -155,8 +160,8 @@ jobs: ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ mypy . - build_and_push_dist: - needs: [build_and_push_dev] + build_dist: + needs: [build_dev] runs-on: ubuntu-latest steps: - name: Set up Docker Buildx @@ -198,7 +203,7 @@ jobs: unit_tests: runs-on: ubuntu-latest - needs: [build_and_push_dev] + needs: [build_dev] steps: - name: Checkout code uses: actions/checkout@v2 @@ -223,7 +228,7 @@ jobs: e2e_tests: runs-on: ubuntu-latest - needs: [build_and_push_dist] + needs: [build_dist] steps: - name: Checkout code uses: actions/checkout@v2 @@ -258,7 +263,7 @@ jobs: trivy: runs-on: ubuntu-latest - needs: [build_and_push_dist] + needs: [build_dist] steps: - name: Checkout code uses: actions/checkout@v2 From 43003b3faf7fbc8a22026f0be6a3b59f81bae639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:59:36 +0200 Subject: [PATCH 03/20] modify cache setup --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57f03d70c3..c1379ed6df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,8 +79,8 @@ jobs: docker buildx build \ --progress=plain \ - --cache-from type=gha, \ - --cache-to type=gha,mode=max \ + --cache-from type=gha \ + --cache-to type=gha \ -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-latest-dev \ -f ./docker/Dockerfile \ From 86f437cdb219a9a2d8750ee1a7a568ea8652585d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:10:31 +0200 Subject: [PATCH 04/20] create reusable workflow for dockerhub login --- .github/workflows/ci.yml | 26 ++++-------------- .github/workflows/dockerhub-login.yml | 38 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/dockerhub-login.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1379ed6df..1d822e125d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,28 +46,12 @@ jobs: working-directory: frontend build_dev: - runs-on: ubuntu-latest + uses: ./.github/workflows/dockerhub-login.yml + with: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKERHUB_ORGANIZATION: ${{ vars.DOCKERHUB_ORGANIZATION }} steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Checkout code - uses: actions/checkout@v2 - - name: Set up Docker environment - run: | - echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV - echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV - if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then - echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV - else - echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV - fi - - name: DockerHub login - uses: docker/login-action@v1 - if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Push dev run: | docker buildx create --use diff --git a/.github/workflows/dockerhub-login.yml b/.github/workflows/dockerhub-login.yml new file mode 100644 index 0000000000..4cb28db7e5 --- /dev/null +++ b/.github/workflows/dockerhub-login.yml @@ -0,0 +1,38 @@ +name: DockerHub Login + +on: + workflow_call: + inputs: + DOCKERHUB_USERNAME: + required: true + type: string + DOCKERHUB_TOKEN: + required: true + type: string + DOCKERHUB_ORGANIZATION: + required: false + type: string + default: unicef + +jobs: + setup: + runs-on: ubuntu-latest + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ inputs.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ inputs.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + echo "DOCKERHUB_ORGANIZATION=${{ inputs.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + + - name: DockerHub login + uses: docker/login-action@v1 + if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN + with: + username: ${{ inputs.DOCKERHUB_USERNAME }} + password: ${{ inputs.DOCKERHUB_TOKEN }} From d26144d99bd4965f2f94ee7cd0bbcb2efb893679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:14:58 +0200 Subject: [PATCH 05/20] make isort use reusable workflow, try to load image --- .github/workflows/ci.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d822e125d..ebb73b42bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,17 +73,13 @@ jobs: ./ isort: - runs-on: ubuntu-latest needs: [build_dev] + uses: ./.github/workflows/dockerhub-login.yml + with: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKERHUB_ORGANIZATION: ${{ vars.DOCKERHUB_ORGANIZATION }} steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: DockerHub login - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Check run: | docker run --rm -i \ From 6c586c63c01a47a450c5971ded8b571c383c9b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:17:31 +0200 Subject: [PATCH 06/20] check if image can be loaded --- .github/workflows/ci.yml | 46 ++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ebb73b42bb..b9f910bc77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,12 +46,28 @@ jobs: working-directory: frontend build_dev: - uses: ./.github/workflows/dockerhub-login.yml - with: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - DOCKERHUB_ORGANIZATION: ${{ vars.DOCKERHUB_ORGANIZATION }} + runs-on: ubuntu-latest steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi + - name: DockerHub login + uses: docker/login-action@v1 + if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Push dev run: | docker buildx create --use @@ -73,17 +89,25 @@ jobs: ./ isort: + runs-on: ubuntu-latest needs: [build_dev] - uses: ./.github/workflows/dockerhub-login.yml - with: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - DOCKERHUB_ORGANIZATION: ${{ vars.DOCKERHUB_ORGANIZATION }} steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi + - name: Check run: | docker run --rm -i \ - ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ + ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ isort . --check-only black: From 5294fb979c9c1989b0bffacd1c1610d4fb9fa70d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:27:43 +0200 Subject: [PATCH 07/20] try with artifact --- .github/workflows/ci.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9f910bc77..0d0fa79708 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,6 +88,18 @@ jobs: --$option \ ./ + - name: Save image to tar + if: !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + run: | + docker save ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev | gzip > dev-image.tar.gz + + - name: Upload Artifact + if: !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + uses: actions/upload-artifact@v4 + with: + name: dev-image + path: dev-image.tar.gz + isort: runs-on: ubuntu-latest needs: [build_dev] @@ -104,6 +116,18 @@ jobs: echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV fi + - name: Download Artifact + if: !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + uses: actions/download-artifact@v4 + with: + name: dev-image + path: /tmp/dev-image.tar.gz + + - name: Load image from tar + if: !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + run: | + gunzip -c /tmp/dev-image.tar.gz | docker load + - name: Check run: | docker run --rm -i \ From b7fd6f98f553453e1229947505c8b17368404528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:28:52 +0200 Subject: [PATCH 08/20] syntax fix --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d0fa79708..bc578d0c93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,12 +89,12 @@ jobs: ./ - name: Save image to tar - if: !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + if: ! ( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) run: | docker save ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev | gzip > dev-image.tar.gz - name: Upload Artifact - if: !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + if: ! ( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) uses: actions/upload-artifact@v4 with: name: dev-image @@ -117,14 +117,14 @@ jobs: fi - name: Download Artifact - if: !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + if: ! ( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) uses: actions/download-artifact@v4 with: name: dev-image path: /tmp/dev-image.tar.gz - name: Load image from tar - if: !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + if: ! ( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) run: | gunzip -c /tmp/dev-image.tar.gz | docker load From 91a8f6b90a16afde31ebc8c4b2ae5264e291aab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:32:34 +0200 Subject: [PATCH 09/20] syntax fix --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc578d0c93..28e6b77b80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: fi - name: DockerHub login uses: docker/login-action@v1 - if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN + if: ${{ env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -89,12 +89,12 @@ jobs: ./ - name: Save image to tar - if: ! ( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} run: | docker save ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev | gzip > dev-image.tar.gz - name: Upload Artifact - if: ! ( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} uses: actions/upload-artifact@v4 with: name: dev-image @@ -117,14 +117,14 @@ jobs: fi - name: Download Artifact - if: ! ( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} uses: actions/download-artifact@v4 with: name: dev-image path: /tmp/dev-image.tar.gz - name: Load image from tar - if: ! ( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} run: | gunzip -c /tmp/dev-image.tar.gz | docker load From 8ca49befe8a8d0d0cfe63c4542a7fbb0c7a4199f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:36:06 +0200 Subject: [PATCH 10/20] use docker-setup.yml --- .github/workflows/ci.yml | 30 ++++----------------- .github/workflows/docker-setup.yml | 22 ++++++++++++++++ .github/workflows/dockerhub-login.yml | 38 --------------------------- 3 files changed, 27 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/docker-setup.yml delete mode 100644 .github/workflows/dockerhub-login.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28e6b77b80..81af55b15d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,21 +52,8 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Checkout code uses: actions/checkout@v2 - - name: Set up Docker environment - run: | - echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV - echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV - if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then - echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV - else - echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV - fi - - name: DockerHub login - uses: docker/login-action@v1 - if: ${{ env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN }} - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Call Docker Setup Workflow + uses: ./.github/workflows/docker-setup.yml - name: Push dev run: | @@ -106,16 +93,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Docker environment - run: | - echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV - echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV - if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then - echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV - else - echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV - fi - + - name: Call Docker Setup Workflow + uses: ./.github/workflows/docker-setup.yml + - name: Download Artifact if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} uses: actions/download-artifact@v4 diff --git a/.github/workflows/docker-setup.yml b/.github/workflows/docker-setup.yml new file mode 100644 index 0000000000..67832bb582 --- /dev/null +++ b/.github/workflows/docker-setup.yml @@ -0,0 +1,22 @@ +name: Docker Setup + +on: workflow_call + +jobs: + setup-docker-env: + runs-on: ubuntu-latest + steps: + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + - name: DockerHub login + uses: docker/login-action@v1 + if: ${{ env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/dockerhub-login.yml b/.github/workflows/dockerhub-login.yml deleted file mode 100644 index 4cb28db7e5..0000000000 --- a/.github/workflows/dockerhub-login.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: DockerHub Login - -on: - workflow_call: - inputs: - DOCKERHUB_USERNAME: - required: true - type: string - DOCKERHUB_TOKEN: - required: true - type: string - DOCKERHUB_ORGANIZATION: - required: false - type: string - default: unicef - -jobs: - setup: - runs-on: ubuntu-latest - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Docker environment - run: | - echo "DOCKERHUB_USERNAME=${{ inputs.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV - echo "DOCKERHUB_TOKEN=${{ inputs.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV - echo "DOCKERHUB_ORGANIZATION=${{ inputs.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV - - - name: DockerHub login - uses: docker/login-action@v1 - if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN - with: - username: ${{ inputs.DOCKERHUB_USERNAME }} - password: ${{ inputs.DOCKERHUB_TOKEN }} From 3b8bc8b3ea738eb93961e694b311b0050ab8f8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:41:01 +0200 Subject: [PATCH 11/20] setup-docker-env action yml --- .github/workflows/ci.yml | 13 +++++++---- .github/workflows/docker-setup.yml | 22 ------------------- .github/workflows/setup-docker-env/action.yml | 22 +++++++++++++++++++ 3 files changed, 31 insertions(+), 26 deletions(-) delete mode 100644 .github/workflows/docker-setup.yml create mode 100644 .github/workflows/setup-docker-env/action.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81af55b15d..f5e7ae699d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,8 +52,11 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Checkout code uses: actions/checkout@v2 - - name: Call Docker Setup Workflow - uses: ./.github/workflows/docker-setup.yml + - name: Setup Docker Environment + uses: ./.github/actions/setup-docker-env + with: + dockerhub_organization: ${{ vars.DOCKERHUB_ORGANIZATION || 'unicef' }} + - name: Push dev run: | @@ -93,8 +96,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Call Docker Setup Workflow - uses: ./.github/workflows/docker-setup.yml + - name: Setup Docker Environment + uses: ./.github/actions/setup-docker-env + with: + dockerhub_organization: ${{ vars.DOCKERHUB_ORGANIZATION || 'unicef' }} - name: Download Artifact if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} diff --git a/.github/workflows/docker-setup.yml b/.github/workflows/docker-setup.yml deleted file mode 100644 index 67832bb582..0000000000 --- a/.github/workflows/docker-setup.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Docker Setup - -on: workflow_call - -jobs: - setup-docker-env: - runs-on: ubuntu-latest - steps: - - name: Set up Docker environment - run: | - echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV - echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV - if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then - echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV - else - echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV - - name: DockerHub login - uses: docker/login-action@v1 - if: ${{ env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN }} - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/setup-docker-env/action.yml b/.github/workflows/setup-docker-env/action.yml new file mode 100644 index 0000000000..ad7ceaa0e9 --- /dev/null +++ b/.github/workflows/setup-docker-env/action.yml @@ -0,0 +1,22 @@ +name: 'Setup Docker Environment' +description: 'Sets up Docker environment variables and logs into DockerHub' +inputs: + dockerhub_organization: + description: 'DockerHub organization' + required: false + default: 'unicef' +runs: + using: 'composite' + steps: + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + echo "DOCKERHUB_ORGANIZATION=${{ inputs.dockerhub_organization }}" >> $GITHUB_ENV + + - name: DockerHub login + uses: docker/login-action@v1 + if: ${{ env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} \ No newline at end of file From cc9f41deb7fe04879e973e82f5b9ec5c0cb2b4df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:42:37 +0200 Subject: [PATCH 12/20] move file --- .github/{workflows => actions}/setup-docker-env/action.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{workflows => actions}/setup-docker-env/action.yml (100%) diff --git a/.github/workflows/setup-docker-env/action.yml b/.github/actions/setup-docker-env/action.yml similarity index 100% rename from .github/workflows/setup-docker-env/action.yml rename to .github/actions/setup-docker-env/action.yml From 8a233dc92d2ea2f6436a8eb17f8261ce200c82fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:47:41 +0200 Subject: [PATCH 13/20] 'un'refactor - take a step back and make it work first, then make it pretty --- .github/actions/setup-docker-env/action.yml | 22 -------------- .github/workflows/ci.yml | 33 +++++++++++++++------ 2 files changed, 24 insertions(+), 31 deletions(-) delete mode 100644 .github/actions/setup-docker-env/action.yml diff --git a/.github/actions/setup-docker-env/action.yml b/.github/actions/setup-docker-env/action.yml deleted file mode 100644 index ad7ceaa0e9..0000000000 --- a/.github/actions/setup-docker-env/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: 'Setup Docker Environment' -description: 'Sets up Docker environment variables and logs into DockerHub' -inputs: - dockerhub_organization: - description: 'DockerHub organization' - required: false - default: 'unicef' -runs: - using: 'composite' - steps: - - name: Set up Docker environment - run: | - echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV - echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV - echo "DOCKERHUB_ORGANIZATION=${{ inputs.dockerhub_organization }}" >> $GITHUB_ENV - - - name: DockerHub login - uses: docker/login-action@v1 - if: ${{ env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN }} - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5e7ae699d..28e6b77b80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,11 +52,21 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Checkout code uses: actions/checkout@v2 - - name: Setup Docker Environment - uses: ./.github/actions/setup-docker-env + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi + - name: DockerHub login + uses: docker/login-action@v1 + if: ${{ env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN }} with: - dockerhub_organization: ${{ vars.DOCKERHUB_ORGANIZATION || 'unicef' }} - + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Push dev run: | @@ -96,11 +106,16 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Setup Docker Environment - uses: ./.github/actions/setup-docker-env - with: - dockerhub_organization: ${{ vars.DOCKERHUB_ORGANIZATION || 'unicef' }} - + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi + - name: Download Artifact if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} uses: actions/download-artifact@v4 From 00ece508c4a03aea31f6b3d8c6b212c38e486aa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:56:51 +0200 Subject: [PATCH 14/20] artifact --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28e6b77b80..a68ea81635 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,8 +97,8 @@ jobs: if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} uses: actions/upload-artifact@v4 with: - name: dev-image - path: dev-image.tar.gz + name: dev-artifact + path: /tmp/dev-image.tar.gz isort: runs-on: ubuntu-latest @@ -120,8 +120,8 @@ jobs: if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} uses: actions/download-artifact@v4 with: - name: dev-image - path: /tmp/dev-image.tar.gz + name: dev-artifact + path: /tmp - name: Load image from tar if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} From 716d102b0aa0cae61cffc73925105bbf1f594b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 12:12:01 +0200 Subject: [PATCH 15/20] artifact --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a68ea81635..1cf27de50e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: - name: Save image to tar if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} run: | - docker save ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev | gzip > dev-image.tar.gz + docker save ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev | gzip > /tmp/dev-image.tar.gz - name: Upload Artifact if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} From 0e7fbae11fed4e852f89607cd1ceef37885b0874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 12:28:19 +0200 Subject: [PATCH 16/20] apply docker env changes on static checks --- .github/workflows/ci.yml | 97 +++++++++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cf27de50e..1bd505c676 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,11 +140,27 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: DockerHub login - uses: docker/login-action@v1 + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi + + - name: Download Artifact + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + uses: actions/download-artifact@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + name: dev-artifact + path: /tmp + + - name: Load image from tar + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + run: | + gunzip -c /tmp/dev-image.tar.gz | docker load - name: Check run: | @@ -158,11 +174,27 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: DockerHub login - uses: docker/login-action@v1 + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi + + - name: Download Artifact + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + uses: actions/download-artifact@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + name: dev-artifact + path: /tmp + + - name: Load image from tar + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + run: | + gunzip -c /tmp/dev-image.tar.gz | docker load - name: Check run: | @@ -176,11 +208,27 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: DockerHub login - uses: docker/login-action@v1 + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi + + - name: Download Artifact + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + uses: actions/download-artifact@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + name: dev-artifact + path: /tmp + + - name: Load image from tar + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + run: | + gunzip -c /tmp/dev-image.tar.gz | docker load - name: Check run: | @@ -233,13 +281,30 @@ jobs: runs-on: ubuntu-latest needs: [build_dev] steps: + - name: Checkout code uses: actions/checkout@v2 - - name: DockerHub login - uses: docker/login-action@v1 + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi + + - name: Download Artifact + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + uses: actions/download-artifact@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + name: dev-artifact + path: /tmp + + - name: Load image from tar + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + run: | + gunzip -c /tmp/dev-image.tar.gz | docker load - name: Unit tests run: | From 40c81012c68c35b137bf50b7fd21d6316e6c8b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 12:37:05 +0200 Subject: [PATCH 17/20] change vars -> env --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bd505c676..7c37593b2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: docker buildx build \ --progress=plain \ --cache-from type=gha \ - --cache-to type=gha \ + --cache-to type=gha,mode=max \ -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-latest-dev \ -f ./docker/Dockerfile \ @@ -165,7 +165,7 @@ jobs: - name: Check run: | docker run --rm -i \ - ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ + ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ black . --check flake8: @@ -199,7 +199,7 @@ jobs: - name: Check run: | docker run --rm -i \ - ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ + ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ flake8 . mypy: @@ -233,7 +233,7 @@ jobs: - name: Check run: | docker run --rm -i \ - ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ + ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ mypy . build_dist: @@ -308,7 +308,7 @@ jobs: - name: Unit tests run: | - backend_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev docker compose \ + backend_image=${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev docker compose \ -f ./deployment/docker-compose.tst.yml \ run backend ./dev.sh test - name: Upload coverage to Codecov From 45291f482fed49ab8ddea2b757e31f5c40cce2cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 12:38:32 +0200 Subject: [PATCH 18/20] set cache scope --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c37593b2b..a46a60b908 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,8 +79,8 @@ jobs: docker buildx build \ --progress=plain \ - --cache-from type=gha \ - --cache-to type=gha,mode=max \ + --cache-from type=gha,scope=${{ github.ref_name }} \ + --cache-to type=gha,mode=max,scope=${{ github.ref_name }} \ -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev \ -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-latest-dev \ -f ./docker/Dockerfile \ From b6ce85a40cfbb8be642ed76914ce4bba804e919e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 12:55:39 +0200 Subject: [PATCH 19/20] dist image --- .github/workflows/ci.yml | 109 ++++++++++++++++++++++++++++++++------- 1 file changed, 89 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a46a60b908..58d64655a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,8 +244,18 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Checkout code uses: actions/checkout@v2 + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi - name: DockerHub login uses: docker/login-action@v1 + if: ${{ env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -254,34 +264,51 @@ jobs: run: | docker buildx create --use + option=load + if [ -n "${{ env.DOCKERHUB_USERNAME }}" ] && [ -n "${{ env.DOCKERHUB_TOKEN }}" ]; then + option=push + fi + # Base part of the command build_command="docker buildx build \ --progress=plain \ - --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-${{ github.sha }}-dev \ - --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-latest-dev \ - --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-${{ github.sha }}-dist \ - --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-latest-dist \ - --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-${{ github.sha }}-dist \ - --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-latest-dist \ - -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dist \ - -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }} \ + --cache-from ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-${{ github.sha }}-dev \ + --cache-from ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-latest-dev \ + --cache-from ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-${{ github.sha }}-dist \ + --cache-from ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-latest-dist \ + --cache-to ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-${{ github.sha }}-dist \ + --cache-to ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-latest-dist \ + -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dist \ + -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }} \ -f ./docker/Dockerfile \ --target dist \ - --push ./" + --$option ./" - if [ "${{ github.ref }}" = "refs/heads/master" ]; then + + if [ -n "${{ env.DOCKERHUB_USERNAME }}" ] && [ -n "${{ env.DOCKERHUB_TOKEN }}" ] && [ "${{ github.ref }}" = "refs/heads/master" ]; then version=$(python3 -c "import sys; version=None; [version:=line.split('=')[1].strip().strip('\"') for line in open('backend/pyproject.toml', 'r') if line.strip().startswith('version =')]; print(version if version else sys.exit(1))") - tagged_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope:core-$version + tagged_image=${{ env.DOCKERHUB_ORGANIZATION }}/hope:core-$version build_command="$build_command -t $tagged_image" fi eval $build_command + - name: Save image to tar + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + run: | + docker save ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dist | gzip > /tmp/dist-image.tar.gz + + - name: Upload Artifact + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + uses: actions/upload-artifact@v4 + with: + name: dist-artifact + path: /tmp/dist-image.tar.gz + unit_tests: runs-on: ubuntu-latest needs: [build_dev] steps: - - name: Checkout code uses: actions/checkout@v2 - name: Set up Docker environment @@ -325,11 +352,35 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: DockerHub login - uses: docker/login-action@v1 + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi + + - name: Download Dev Artifact + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + uses: actions/download-artifact@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + name: dev-artifact + path: /tmp + + - name: Download Dist Artifact + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + uses: actions/download-artifact@v4 + with: + name: dist-artifact + path: /tmp + + - name: Load images from tar + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + run: | + gunzip -c /tmp/dev-image.tar.gz | docker load + gunzip -c /tmp/dist-image.tar.gz | docker load - name: E2E tests run: | @@ -360,11 +411,28 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: DockerHub login - uses: docker/login-action@v1 + - name: Set up Docker environment + run: | + echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV + echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV + if [ -n "${{ vars.DOCKERHUB_ORGANIZATION }}" ]; then + echo "DOCKERHUB_ORGANIZATION=${{ vars.DOCKERHUB_ORGANIZATION }}" >> $GITHUB_ENV + else + echo "DOCKERHUB_ORGANIZATION=unicef" >> $GITHUB_ENV + fi + + - name: Download Dist Artifact + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + uses: actions/download-artifact@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + name: dist-artifact + path: /tmp + + - name: Load images from tar + if: ${{ !( env.DOCKERHUB_USERNAME && env.DOCKERHUB_TOKEN ) }} + run: | + gunzip -c /tmp/dist-image.tar.gz | docker load + - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master with: @@ -374,6 +442,7 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + deploy: runs-on: ubuntu-latest needs: [e2e_tests, unit_tests, isort, black, flake8, mypy, frontend_tests] From d9292d4cb00faaeebad50731663111098bd70bfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?= <17177420+wozniakpl@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:06:46 +0200 Subject: [PATCH 20/20] dist cache --- .github/workflows/ci.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58d64655a8..cf98b29161 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -272,12 +272,8 @@ jobs: # Base part of the command build_command="docker buildx build \ --progress=plain \ - --cache-from ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-${{ github.sha }}-dev \ - --cache-from ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-latest-dev \ - --cache-from ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-${{ github.sha }}-dist \ - --cache-from ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-latest-dist \ - --cache-to ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-${{ github.sha }}-dist \ - --cache-to ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-core-latest-dist \ + --cache-from type=gha,scope=${{ github.ref_name }} \ + --cache-to type=gha,mode=max,scope=${{ github.ref_name }} \ -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dist \ -t ${{ env.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }} \ -f ./docker/Dockerfile \