Skip to content

Commit

Permalink
Make CI smarter about when to rebuild the base images
Browse files Browse the repository at this point in the history
  • Loading branch information
gerrod3 committed Sep 26, 2024
1 parent c130256 commit 9fce634
Show file tree
Hide file tree
Showing 6 changed files with 91 additions and 82 deletions.
89 changes: 64 additions & 25 deletions .github/actions/base_images/action.yml
Original file line number Diff line number Diff line change
@@ -1,40 +1,83 @@
---
name: Build Base Images
description: Build the base images (pulp/base & pulp/pulp-ci-centos9)
description: Build the base images (pulp/base & pulp/pulp-ci-centos9) if needed
# Both ARM64 & x86-64 versions of each are built
# Save the images to a tar and upload to a cache using output 'pulp_ci_centos_id' as the key
# Use hashFiles(base_image_files, pulp-ci_image_files) + python_version as the key to the cache
inputs:
python_version:
required: true
description: "Python Version to use to build, e.g '3.9'"
outputs:
pulp_ci_centos_id:
value: ${{ steps.pulp_ci_centos_id.outputs.pulp_ci_centos_id }}
base_cache_key:
value: ${{ steps.hash_key.outputs.base_cache_key }}
description: "The cache key the built images were uploaded to."

runs:
using: "composite"
steps:
- uses: actions/checkout@v4

- name: Set the temporary image tag
- name: Calculate base images hash
id: hash_key
run: |
temp_base_tag="${GITHUB_REF_NAME%/*}"
python_version="${{ inputs.python_version }}"
echo "Building $temp_base_tag with python $python_version"
echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV
echo "PYTHON_VERSION=${python_version}" >> $GITHUB_ENV
hash=${{ hashFiles('images/Containerfile.core.base', 'images/pulp_ci_centos/Containerfile', 'images/assets/**', 'images/s6_assets/**') }}
echo "base image hash is ${hash}, python version is ${{ inputs.python_version }}"
echo "base_cache_key=${hash}-${{ inputs.python_version }}" >> $GITHUB_OUTPUT
shell: bash

- name: Restore previously cached images
id: cache
uses: actions/cache/restore@v3
with:
key: base-images=${{ steps.hash_key.outputs.base_cache_key }}
path: base-images.tar.gz

- name: Extract images if cached
if: steps.cache.outputs.cache-hit == 'true'
run: |
echo "Base Images were in cache"
podman load -i base-images.tar.gz
shell: bash

- name: Check for updates on cached images
if: steps.cache.outputs.cache-hit == 'true'
run: |
images=()
for IMAGE in base pulp-ci-centos9; do
for ARCH in arm64 amd64; do
podman run --pull=never pulp/${IMAGE}:ci-${ARCH} bash -c "dnf check-upgrade"
if [ $? -gt 0 ]; then
images+=('${IMAGE}-${ARCH}')
fi
done
done
echo "BUILD_IMAGES=[$(echo ${images[@]} | sed 's/ /, /g')]" >> $GITHUB_ENV
shell: bash

- name: Set images to build on cache miss
if: steps.cache.outputs.cache-hit != 'true'
run: echo "BUILD_IMAGES=['base-arm64', 'base-amd64', 'pulp-ci-centos9-arm64', 'pulp-ci-centos9-amd64']" >> $GITHUB_ENV
shell: bash

- name: Build images
if: env.BUILD_IMAGES
run: |
images=(${{ join(fromJSON(env.BUILD_IMAGES), ' ') }})
echo "Going to build images: ${images[@]}"
podman version
buildah version
sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes
for ARCH in arm64 amd64
do
podman build --platform linux/$ARCH --format docker --file images/Containerfile.core.base --tag pulp/base:${TEMP_BASE_TAG}-${ARCH} --build-arg PYTHON_VERSION=${PYTHON_VERSION} .
podman build --platform linux/$ARCH --format docker --file images/pulp_ci_centos/Containerfile --tag pulp/pulp-ci-centos9:${TEMP_BASE_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_BASE_TAG}-${ARCH} .
for image in "${images[@]}"; do
echo "Building image ${image}"
arch=${image:(-5):5}
case $image in
base-arm64 | base-amd64)
podman build --platform linux/$arch --format docker --file images/Containerfile.core.base --tag pulp/base:ci-$arch --build-arg PYTHON_VERSION=${{ inputs.python_version }} .
;;
pulp-ci-centos9-arm64 | pulp-ci-centos9-amd64)
podman build --platform linux/$arch --format docker --file images/pulp_ci_centos/Containerfile --tag pulp/pulp-ci-centos9:ci-$arch --build-arg FROM_TAG=ci-$arch .
;;
esac
done
shell: bash
# we use the docker format (default), even though it may not be the fastest,
Expand All @@ -43,20 +86,16 @@ runs:
# We should look into whether its possible to export just pulp-ci-centos,
# and tag the base image manually.
- name: Save podman images to tarball
id: pulp_ci_centos_id
if: env.BUILD_IMAGES
run: |
podman save -m -o base-images.tar pulp/base:${TEMP_BASE_TAG}-arm64 pulp/base:${TEMP_BASE_TAG}-amd64 pulp/pulp-ci-centos9:${TEMP_BASE_TAG}-arm64 pulp/pulp-ci-centos9:${TEMP_BASE_TAG}-amd64
# The id is unique to the image build (not the Containerfile) and will be used in the cache key
# If a workflow completes successfully, every workflow will generate a new cache.
# And if we re-run the entire workflow ("Re-run all jobs"), it will generate a new cache too.
# If we re-run a failed app-images job, it will use the existing cache from base-images
id=$(podman image inspect --format '{{ .Id }}' pulp/pulp-ci-centos9:${TEMP_BASE_TAG}-amd64)
echo "pulp_ci_centos_id=${id}" >> "$GITHUB_OUTPUT"
echo "pulp_ci_centos_id=${id}" >> "$GITHUB_ENV"
rm -f base-images.tar.gz
podman save -m -o base-images.tar pulp/base:ci-arm64 pulp/base:ci-amd64 pulp/pulp-ci-centos9:ci-arm64 pulp/pulp-ci-centos9:ci-amd64
gzip base-images.tar
shell: bash

- name: Cache podman images
if: env.BUILD_IMAGES
uses: actions/cache/save@v3
with:
key: base-images=${{ env.pulp_ci_centos_id }}
path: base-images.tar
key: base-images=${{ steps.hash_key.outputs.base_cache_key }}
path: base-images.tar.gz
42 changes: 6 additions & 36 deletions .github/actions/build_image/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,54 +19,24 @@ outputs:
app_branch:
value: ${{ steps.image_version_branch.outputs.app_branch }}
description: 'The pulpcore version branch that the built image matches'
app_arch_tag:
value: ${{ steps.image_tags.outputs.app_arch_tag }}
description: 'The temporary amd64 arch tag of the app image'

runs:
using: "composite"
steps:
- uses: actions/checkout@v4

- name: Set the temporary image tags
id: image_tags
run: |
if [ "${{ inputs.image_variant }}" == "nightly" ]; then
temp_app_tag="nightly"
else
temp_app_tag="${GITHUB_REF_NAME%/*}"
fi
temp_base_tag="${GITHUB_REF_NAME%/*}"
echo "Building $temp_app_tag from base $temp_base_tag"
echo "TEMP_APP_TAG=${temp_app_tag}" >> $GITHUB_ENV
echo "TEMP_BASE_TAG=${temp_base_tag}" >> $GITHUB_ENV
echo "app_arch_tag=${temp_app_tag}-amd64" >> $GITHUB_OUTPUT
shell: bash

- name: Set up Python
uses: actions/setup-python@v4

- name: Restore podman images from cache
uses: actions/cache/restore@v3
with:
key: base-images=${{ inputs.image_cache_key }}
path: base-images.tar
path: base-images.tar.gz

- name: Load podman images from tarball
run: |
podman load -i base-images.tar
shell: bash

- name: Install httpie and podman-compose
run: |
echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV
echo "Working around https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2024394"
curl -O http://archive.ubuntu.com/ubuntu/pool/universe/g/golang-github-containernetworking-plugins/containernetworking-plugins_1.1.1+ds1-3_amd64.deb
sudo dpkg -i containernetworking-plugins_1.1.1+ds1-3_amd64.deb
# Ubuntu 22.04 has old podman 3.4.4, we need podman-compose==1.0.3 to avoid an
# error with dependency contianers not being detected as running.
# "error generating dependency graph for container"
pip install httpie podman-compose==1.0.3
podman load -i base-images.tar.gz
shell: bash

- name: Build images
Expand All @@ -78,10 +48,10 @@ runs:
do
if [[ "${{ inputs.image_name }}" == "pulp-minimal" || "${{ inputs.image_name }}" == "galaxy-minimal" ]]; then
base_image=$(echo ${{ inputs.image_name }} | cut -d '-' -f1)
podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.core --tag pulp/${{ inputs.image_name }}:${TEMP_APP_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_BASE_TAG}-${ARCH} .
podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.webserver --tag pulp/${base_image}-web:${TEMP_APP_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_APP_TAG}-${ARCH} .
podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.core --tag pulp/${{ inputs.image_name }}:ci-${ARCH} --build-arg FROM_TAG=ci-${ARCH} .
podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.webserver --tag pulp/${base_image}-web:ci-${ARCH} --build-arg FROM_TAG=ci-${ARCH} .
else
podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile --tag pulp/${{ inputs.image_name }}:${TEMP_APP_TAG}-${ARCH} --build-arg FROM_TAG=${TEMP_BASE_TAG}-${ARCH} .
podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile --tag pulp/${{ inputs.image_name }}:ci-${ARCH} --build-arg FROM_TAG=ci-${ARCH} .
fi
done
podman images -a
Expand All @@ -96,7 +66,7 @@ runs:
else
pip_name="galaxy-ng"
fi
app_version=$(podman run --pull=never pulp/${{ inputs.image_name }}:${TEMP_APP_TAG}-amd64 bash -c "pip3 show ${pip_name} | sed -n -e 's/Version: //p'")
app_version=$(podman run --pull=never pulp/${{ inputs.image_name }}:ci-amd64 bash -c "pip3 show ${pip_name} | sed -n -e 's/Version: //p'")
app_branch=$(echo ${app_version} | grep -oP '\d+\.\d+')
echo "APP_VERSION: ${app_version}"
Expand Down
5 changes: 1 addition & 4 deletions .github/actions/publish_images/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,6 @@ inputs:
image_names:
description: 'Names of the images to be published'
required: true
build_tag:
description: 'Current tag of the local built container'
required: true
tags:
description: 'List of tags the images are to be published under'
required: true
Expand Down Expand Up @@ -56,7 +53,7 @@ runs:
for registry in ghcr.io docker.io quay.io; do
for image in ${{ inputs.image_names }}; do
for tag in ${{ inputs.tags }}; do
podman manifest create ${registry}/pulp/${image}:${tag} containers-storage:localhost/pulp/${image}:${{ inputs.build_tag }}-amd64 containers-storage:localhost/pulp/${image}:${{ inputs.build_tag }}-arm64
podman manifest create ${registry}/pulp/${image}:${tag} containers-storage:localhost/pulp/${image}:ci-amd64 containers-storage:localhost/pulp/${image}:ci-arm64
podman manifest push --all ${registry}/pulp/${image}:${tag} ${registry}/pulp/${image}:${tag}
done
done
Expand Down
23 changes: 16 additions & 7 deletions .github/actions/test_image/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,26 +12,35 @@ inputs:
app_branch:
description: 'The branch the app was built on'
required: true
app_arch_tag:
description: 'The temporary amd64 arch tag of the built app image'
required: true

runs:
using: "composite"
steps:
- name: Install httpie and podman-compose
run: |
echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV
echo "Working around https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2024394"
curl -O http://archive.ubuntu.com/ubuntu/pool/universe/g/golang-github-containernetworking-plugins/containernetworking-plugins_1.1.1+ds1-3_amd64.deb
sudo dpkg -i containernetworking-plugins_1.1.1+ds1-3_amd64.deb
# Ubuntu 22.04 has old podman 3.4.4, we need podman-compose==1.0.3 to avoid an
# error with dependency contianers not being detected as running.
# "error generating dependency graph for container"
pip install httpie podman-compose==1.0.3
shell: bash

- name: Test image with upgrade in s6 mode (pulp)
if: inputs.image_name == 'pulp'
run: |
# 3.20 has postgres 12 rather than 13
images/s6_assets/test.sh "pulp/${{ inputs.image_name }}:${{ inputs.app_arch_tag }}" http "quay.io/pulp/all-in-one-pulp:3.20"
images/s6_assets/test.sh "pulp/${{ inputs.image_name }}:ci-amd64" http "quay.io/pulp/all-in-one-pulp:3.20"
podman stop pulp
podman rm pulp
shell: bash

- name: Test the image in s6 mode (galaxy)
if: inputs.image_name == 'galaxy'
run: |
images/s6_assets/test.sh "pulp/${{ inputs.image_name }}:${{ inputs.app_arch_tag }}" https
images/s6_assets/test.sh "pulp/${{ inputs.image_name }}:ci-amd64" https
podman stop pulp
podman rm pulp
shell: bash
Expand All @@ -48,8 +57,8 @@ runs:
fi
else
FILE="compose.yml"
WEB_TAG="${{ inputs.app_arch_tag }}"
WEB_TAG="ci-amd64"
fi
base_image=$(echo ${{ inputs.image_name }} | cut -d '-' -f1)
images/compose/test.sh "${{ inputs.image_name }}:${{ inputs.app_arch_tag }}" "${base_image}-web:${WEB_TAG}" $FILE
images/compose/test.sh "${{ inputs.image_name }}:ci-amd64" "${base_image}-web:${WEB_TAG}" $FILE
shell: bash
6 changes: 2 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
image_variants: "${{ steps.image_variants.outputs.image_variants }}"
pulp_ci_centos_id: "${{ steps.build_base_images.outputs.pulp_ci_centos_id }}"
base_cache_key: "${{ steps.build_base_images.outputs.base_cache_key }}"
steps:
# We do not want to build nightly images unless it's a PR to the latest branch,
# or a branch/dispatch build on the latest branch.
Expand All @@ -76,7 +76,6 @@ jobs:
outputs:
app_version: ${{ steps.build_image.outputs.app_version }}
app_branch: ${{ steps.build_image.outputs.app_branch }}
app_arch_tag: ${{ steps.build_image.outputs.app_arch_tag }}
strategy:
fail-fast: false
matrix:
Expand All @@ -95,15 +94,14 @@ jobs:
with:
image_name: ${{ matrix.image_name }}
image_variant: ${{ matrix.image_variant }}
image_cache_key: ${{ needs.base-images.outputs.pulp_ci_centos_id }}
image_cache_key: ${{ needs.base-images.outputs.base_cache_key }}

- name: Test App Image
uses: "./.github/actions/test_image"
with:
image_name: ${{ matrix.image_name }}
image_variant: ${{ matrix.image_variant }}
app_branch: ${{ steps.build_image.outputs.app_branch }}
app_arch_tag: ${{ steps.build_image.outputs.app_arch_tag }}

- name: Logs
if: always()
Expand Down
8 changes: 2 additions & 6 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
base-images:
runs-on: ubuntu-latest
outputs:
pulp_ci_centos_id: ${{ steps.build_base_images.outputs.pulp_ci_centos_id }}
base_cache_key: ${{ steps.build_base_images.outputs.base_cache_key }}
image_variants: ${{ steps.image_variants.outputs.image_variants }}

steps:
Expand Down Expand Up @@ -138,7 +138,6 @@ jobs:
with:
image_names: "base pulp-ci-centos9"
tags: ${{ env.TAGS }}
build_tag: ${{ github.ref_name }}
github_token: ${{ secrets.GITHUB_TOKEN }}
docker_bot_username: ${{ secrets.DOCKER_BOT_USERNAME }}
docker_bot_password: ${{ secrets.DOCKER_BOT_PASSWORD }}
Expand All @@ -151,7 +150,6 @@ jobs:
outputs:
app_version: ${{ steps.build_image.outputs.app_version }}
app_branch: ${{ steps.build_image.outputs.app_branch }}
app_arch_tag: ${{ steps.build_image.outputs.app_arch_tag }}
strategy:
fail-fast: false
matrix:
Expand All @@ -170,15 +168,14 @@ jobs:
with:
image_name: ${{ matrix.image_name }}
image_variant: ${{ matrix.image_variant }}
image_cache_key: ${{ needs.base-images.outputs.pulp_ci_centos_id }}
image_cache_key: ${{ needs.base-images.outputs.base_cache_key }}

- name: Test App Image
uses: "./.github/actions/test_image"
with:
image_name: ${{ matrix.image_name }}
image_variant: ${{ matrix.image_variant }}
app_branch: ${{ steps.build_image.outputs.app_branch }}
app_arch_tag: ${{ steps.build_image.outputs.app_arch_tag }}

- name: Set tags
run: |
Expand Down Expand Up @@ -211,7 +208,6 @@ jobs:
with:
image_names: ${{ env.IMAGES }}
tags: ${{ env.TAGS }}
build_tag: ${{ matrix.image_variant == 'nightly' && 'nightly' || github.ref_name }}
github_token: ${{ secrets.GITHUB_TOKEN }}
docker_bot_username: ${{ secrets.DOCKER_BOT_USERNAME }}
docker_bot_password: ${{ secrets.DOCKER_BOT_PASSWORD }}
Expand Down

0 comments on commit 9fce634

Please sign in to comment.