ci.yaml

name: CFFI CI

on:
  push:
  pull_request:
    types: [ opened, synchronize, reopened ]
  schedule:
  - cron: '0 12 * * 1'
  workflow_dispatch:
    inputs:
      skip_artifact_upload:
        description: Skip (most) job artifact uploads?
        type: boolean
        default: true
      skip_ci_redundant_jobs:
        description: Skip CI redundant jobs?
        type: boolean
        default: true
      skip_slow_jobs:
        description: Skip slow/emulated jobs?
        type: boolean
        default: true

env:
  skip_ci_redundant_jobs: ${{ (github.event_name == 'workflow_dispatch' && inputs.skip_ci_redundant_jobs) || (github.event_name == 'pull_request' || github.event_name == 'push') }}
  skip_slow_jobs: ${{ (github.event_name == 'workflow_dispatch' && inputs.skip_slow_jobs) || (github.event_name == 'pull_request' || github.event_name == 'push') }}
  skip_artifact_upload: ${{ (github.event_name == 'workflow_dispatch' && inputs.skip_artifact_upload) || github.event_name != 'workflow_dispatch' }}

jobs:
  python_sdist:
    runs-on: ubuntu-22.04
    outputs:
      sdist_artifact_name: ${{ steps.build_sdist.outputs.sdist_artifact_name }}
      package_version: ${{ steps.build_sdist.outputs.package_version }}
    steps:
    - name: clone repo
      uses: actions/checkout@v4

    - name: install python
      uses: actions/setup-python@v5
      with:
        python-version: 3.x

    - name: build sdist
      id: build_sdist
      run: |
        rm -rf dist/
        python -m pip install build
        python -m build --sdist
        
        echo "sdist_artifact_name=$(ls ./dist)" >> "$GITHUB_OUTPUT"
        echo "package_version=$(ls ./dist | sed -En 's/cffi-(.+)\.tar\.gz/\1/p')" >> "$GITHUB_OUTPUT"

    - name: upload sdist artifact
      uses: actions/upload-artifact@v4
      with:
        name: ${{ steps.build_sdist.outputs.sdist_artifact_name }}
        path: dist/${{ steps.build_sdist.outputs.sdist_artifact_name }}
        if-no-files-found: error
      # always upload the sdist artifact- all the wheel build jobs require it

  make_linux_matrix:
    runs-on: ubuntu-22.04
    outputs:
      matrix_json: ${{ steps.make_matrix.outputs.matrix_json }}
    steps:
    - uses: actions/checkout@v4
    - name: make a matrix
      id: make_matrix
      uses: ./.github/actions/dynamatrix
      with:
        matrix_yaml: |
          include:
          - spec: cp38-manylinux_x86_64
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp39-manylinux_x86_64
            omit: ${{ env.skip_ci_redundant_jobs }}
            
          - spec: cp310-manylinux_x86_64
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp311-manylinux_x86_64
            omit: ${{ env.skip_ci_redundant_jobs }}
          
          - spec: cp312-manylinux_x86_64
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp313-manylinux_x86_64
  
          - spec: cp38-manylinux_i686
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp39-manylinux_i686
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp310-manylinux_i686
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp311-manylinux_i686
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp312-manylinux_i686
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp313-manylinux_i686
            omit: ${{ env.skip_ci_redundant_jobs }}
          
          - spec: cp39-musllinux_x86_64
            omit: ${{ env.skip_ci_redundant_jobs }}
          
          - spec: cp310-musllinux_x86_64
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp311-musllinux_x86_64
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp312-musllinux_x86_64
            omit: ${{ env.skip_ci_redundant_jobs }}
  
          - spec: cp313-musllinux_x86_64
  
          - spec: cp39-musllinux_i686
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp310-musllinux_i686
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp311-musllinux_i686
            omit: ${{ env.skip_ci_redundant_jobs }}

          #- spec: cp312-musllinux_i686  # busted as of 2024-05-17
          #  omit: ${{ env.skip_ci_redundant_jobs }}
  
          #- spec: cp313-musllinux_i686 # busted as of 2024-05-17
  
          - spec: cp39-musllinux_aarch64
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }}
          
          - spec: cp310-musllinux_aarch64
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }}

          - spec: cp311-musllinux_aarch64
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }}

          - spec: cp312-musllinux_aarch64
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }}
  
          - spec: cp313-musllinux_aarch64
            foreign_arch: true
            # test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs}}
  
          - spec: cp38-manylinux_aarch64
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs }}

          - spec: cp39-manylinux_aarch64
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}

          - spec: cp310-manylinux_aarch64
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}

          - spec: cp311-manylinux_aarch64
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}

          - spec: cp312-manylinux_aarch64
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}
  
          - spec: cp313-manylinux_aarch64
            foreign_arch: true
            # test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs }}
  
          - spec: cp38-manylinux_ppc64le
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs }}

          - spec: cp39-manylinux_ppc64le
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}

          - spec: cp310-manylinux_ppc64le
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}

          - spec: cp311-manylinux_ppc64le
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}

          - spec: cp312-manylinux_ppc64le
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}

          - spec: cp313-manylinux_ppc64le
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs }}
  
          - spec: cp38-manylinux_s390x
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs }}

          - spec: cp39-manylinux_s390x
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}

          - spec: cp310-manylinux_s390x
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}

          - spec: cp311-manylinux_s390x
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}

          - spec: cp312-manylinux_s390x
            foreign_arch: true
            test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs || env.skip_ci_redundant_jobs }}

          - spec: cp313-manylinux_s390x
            foreign_arch: true
            # test_args: '{package}/src/c'
            omit: ${{ env.skip_slow_jobs }}


  linux:
    needs: [python_sdist, make_linux_matrix]
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.make_linux_matrix.outputs.matrix_json) }}

    steps:
    - name: fetch sdist artifact
      id: fetch_sdist
      uses: actions/download-artifact@v4
      with:
        name: ${{ needs.python_sdist.outputs.sdist_artifact_name }}

    - name: configure docker foreign arch support
      uses: docker/setup-qemu-action@v3
      if: ${{ ! contains(matrix.spec, 'x86_64') }}

    - name: build/test wheels
      id: build
      env:
        CFLAGS: -Dffi_call=cffistatic_ffi_call  # override name for ffi_call to break hard if we linked against someone else's libffi
        CIBW_ARCHS_LINUX: all
        CIBW_BUILD: ${{ matrix.spec }}
        CIBW_BEFORE_BUILD: |
          set -eux && \
          curl -L -O https://github.com/libffi/libffi/archive/v3.4.2.tar.gz && \
          tar zxf v3.4.2.tar.gz && cd libffi-3.4.2 && \
          ((command -v apk && apk add libtool) || true) && \
          ./autogen.sh && \
          ./configure --without-gcc-arch --disable-docs --with-pic --enable-shared=no && \
          make install
        CIBW_ENVIRONMENT_PASS_LINUX: CFLAGS  # ensure that the build container can see our overridden build config
        CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_img || '' }}
        CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_img || '' }}
        CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux_img || '' }}
        CIBW_MANYLINUX_PPC64LE_IMAGE: ${{ matrix.manylinux_img || '' }}
        CIBW_MANYLINUX_S390X_IMAGE: ${{ matrix.manylinux_img || '' }}
        CIBW_MUSLLINUX_X86_64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_1' }}
        CIBW_MUSLLINUX_I686_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_1' }}
        CIBW_MUSLLINUX_AARCH64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_1' }}
        CIBW_PRERELEASE_PYTHONS: 'True'
        CIBW_TEST_REQUIRES: pytest setuptools  # 3.12+ no longer includes distutils, just always ensure setuptools is present
        CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest ${{ matrix.test_args || '{project}' }}  # default to test all
      run: |
        set -eux
        
        mkdir cffi
        
        tar zxf ${{ steps.fetch_sdist.outputs.download-path }}/cffi*.tar.gz --strip-components=1 -C cffi
        python -m pip install --upgrade "${{ matrix.cibw_version || 'cibuildwheel' }}"

        # actually build libffi + wheel (using env tweaks above)
        python -m cibuildwheel --output-dir dist ./cffi

        echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT"

    - name: upload artifacts
      uses: actions/upload-artifact@v4
      with:
        name: ${{ steps.build.outputs.artifact_name }}
        path: dist/*.whl
        if-no-files-found: error
      if: ${{ env.skip_artifact_upload != 'true' }}

  make_macos_matrix:
    runs-on: ubuntu-22.04
    outputs:
      matrix_json: ${{ steps.make_matrix.outputs.matrix_json }}
    steps:
    - uses: actions/checkout@v4
    - name: make a matrix
      id: make_matrix
      uses: ./.github/actions/dynamatrix
      with:
        matrix_yaml: |
          include:
          # build for x86_64 under the default hosted macOS 10.x x86_64 runner
          - spec: cp38-macosx_x86_64
            runs_on: [macos-13]
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp39-macosx_x86_64
            runs_on: [macos-13]
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp310-macosx_x86_64
            runs_on: [macos-13]
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp311-macosx_x86_64
            runs_on: [macos-13]
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp312-macosx_x86_64
            runs_on: [macos-13]
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp313-macosx_x86_64
            runs_on: [macos-13]
            # omit: ${{ env.skip_ci_redundant_jobs }}

           # FIXME: ? cp38-macosx_arm64 requires special handling and fails some test_zdist tests under cibw 2.1.2, skip it (so Apple's XCode python3 won't have a wheel)
          - spec: cp39-macosx_arm64
            deployment_target: '11.0'
            run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp310-macosx_arm64
            deployment_target: '11.0'
            run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
            omit: ${{ env.skip_ci_redundant_jobs}}

          - spec: cp311-macosx_arm64
            deployment_target: '11.0'
            run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp312-macosx_arm64
            deployment_target: '11.0'
            run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp313-macosx_arm64
            deployment_target: '11.0'
            run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}

  macos:
    needs: [python_sdist, make_macos_matrix]
    defaults:
      run:
        shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }}
    runs-on: ${{ matrix.runs_on || 'macos-14' }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.make_macos_matrix.outputs.matrix_json) }}

    steps:
    - name: fetch sdist artifact
      id: fetch_sdist
      uses: actions/download-artifact@v4
      with:
        name: ${{ needs.python_sdist.outputs.sdist_artifact_name }}

    - name: install python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'  # as of 2024-05, this has to be < 3.12 since the macos-13 runner image's
                                # built-in virtualenv/pip are pinned to busted versions that fail on newer Pythons

    - name: build wheel prereqs
      run: |
        set -eux
        python3 -m pip install --user --upgrade "${{ matrix.cibw_version || 'cibuildwheel' }}"
        brew uninstall --ignore-dependencies libffi 2>&1 || true

    - name: build/test wheels
      id: build
      env:
        CIBW_BUILD: ${{ matrix.spec }}
        CIBW_PRERELEASE_PYTHONS: 'True'
        CIBW_TEST_REQUIRES: pytest setuptools
        CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest
        MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
        SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
      run: |
        set -eux
        
        mkdir cffi
        
        tar zxf ${{ steps.fetch_sdist.outputs.download-path }}/cffi*.tar.gz --strip-components=1 -C cffi

        python3 -m cibuildwheel --output-dir dist cffi
        
        echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT"

    - name: upload artifacts
      uses: actions/upload-artifact@v4
      with:
        name: ${{ steps.build.outputs.artifact_name }}
        path: dist/*.whl
        if-no-files-found: error
      if: ${{ env.skip_artifact_upload != 'true' }}


  make_windows_matrix:
    runs-on: ubuntu-22.04
    outputs:
      matrix_json: ${{ steps.make_matrix.outputs.matrix_json }}
    steps:
    - uses: actions/checkout@v4
    - name: make a matrix
      id: make_matrix
      uses: ./.github/actions/dynamatrix
      with:
        matrix_yaml: |
          include:
          - spec: cp38-win_amd64
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp39-win_amd64
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp310-win_amd64
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp311-win_amd64
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp312-win_amd64
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp313-win_amd64
            # omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp38-win32
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp39-win32
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp310-win32
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp311-win32
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp312-win32
            omit: ${{ env.skip_ci_redundant_jobs }}

          - spec: cp313-win32
            omit: ${{ env.skip_ci_redundant_jobs }}

  windows:
    needs: [python_sdist, make_windows_matrix]
    runs-on: windows-2022
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.make_windows_matrix.outputs.matrix_json) }}

    steps:
    - name: fetch sdist artifact
      id: fetch_sdist
      uses: actions/download-artifact@v4
      with:
        name: ${{ needs.python_sdist.outputs.sdist_artifact_name }}

    - name: build/test wheels
      id: build
      env:
        CIBW_BUILD: ${{ matrix.spec }}
        CIBW_PRERELEASE_PYTHONS: 'True'
        CIBW_TEST_REQUIRES: pytest setuptools
        CIBW_TEST_COMMAND: 'python -m pytest {package}/src/c'
        # FIXME: /testing takes ~45min on Windows and has some failures...
        # CIBW_TEST_COMMAND='python -m pytest {package}/src/c {project}/testing'
      run: |
        set -eux
        
        mkdir cffi
        
        tar zxf cffi*.tar.gz --strip-components=1 -C cffi
        
        python -m pip install --upgrade pip
        pip install "${{ matrix.cibw_version || 'cibuildwheel'}}"
        python -m cibuildwheel --output-dir dist cffi
        
        echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT"

      shell: bash

    - name: upload artifacts
      uses: actions/upload-artifact@v4
      with:
        name: ${{ steps.build.outputs.artifact_name }}
        path: dist/*.whl
        if-no-files-found: error
      if: ${{ env.skip_artifact_upload != 'true' }}

  merge_artifacts:
    needs: [python_sdist, linux, macos, windows]
    runs-on: ubuntu-22.04
    steps:
    - name: merge all artifacts
      uses: actions/upload-artifact/merge@v4
      with:
        name: dist-cffi-${{ needs.python_sdist.outputs.package_version }}
        delete-merged: true
      if: ${{ env.skip_artifact_upload != 'true' }}


  check:
    if: always()
    needs: [python_sdist, linux, macos, windows, merge_artifacts]
    runs-on: ubuntu-latest
    steps:
    - name: Verify all previous jobs succeeded (provides a single check to sample for gating purposes)
      uses: re-actors/alls-green@release/v1
      with:
        jobs: ${{ toJSON(needs) }}