ci+cd.feat(wheel & pypi): improve and simplify the process and support more platforms

.github/workflows/ci.yml

@@ -8,6 +8,8 @@ on:
     pull_request:
     schedule:
         - cron:  '17 3 * * 0'
+    release:
+        types: [created]
 
 jobs:
     flake8:
@@ -127,40 +129,30 @@ jobs:
 
     wheels:
         name: Build and upload wheels
-        runs-on: ubuntu-latest
         strategy:
             matrix:
-                DOCKER_IMAGE:
-                - quay.io/pypa/manylinux2014_x86_64
-                # Disable i686 builds for now: no binary wheels for cryptography,
-                # source build fails, e.g. https://github.com/inducer/pyopencl/pull/421/checks?check_run_id=1781071632
-                # - quay.io/pypa/manylinux2014_i686
+                os: [ubuntu-latest]
+        runs-on: ${{ matrix.os }}
         steps:
         -   uses: actions/checkout@v2
-        -   name: "Main Script"
+        -   name: "build wheel"
             env:
-                TWINE_USERNAME: __token__
-                TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-                DOCKER_IMAGE: ${{ matrix.DOCKER_IMAGE }}
-
+                CIBW_PROJECT_REQUIRES_PYTHON: "~=3.6"
+                CIBW_SKIP: "pp36-*" # The docker image using doesn't support pp36
+                CIBW_BEFORE_BUILD: "./scripts/prepare-build-wheels.sh"
+                CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair {wheel} -w {dest_dir} --lib-sdir=/.libs && python ./scripts/fix-wheel.py {wheel} ~/deps/ocl-icd/COPYING"
+                CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014"
+                CIBW_MANYLINUX_PYPY_X86_64_IMAGE: "quay.io/pypa/manylinux2014_x86_64"
+                CIBW_ARCHS: "auto64" # 32bit unsupported for now
             run: |
-                pwd
-                ls -la
-
-                # Only perform upload for tag builds, otherwise unset TWINE_USERNAME to prevent
-                if ! [[ $GITHUB_REF == refs/tags* ]]; then
-                    echo "Not a tag build, GITHUB_REF is '$GITHUB_REF'. Unsetting TWINE_USERNAME"
-                    unset TWINE_USERNAME
-                fi
-
-                if [[ $DOCKER_IMAGE == *i686* ]]; then
-                    PRE_CMD=linux32
-                else
-                    PRE_CMD=""
-                fi
-
-                docker run --rm -v `pwd`:/io -e TWINE_USERNAME -e TWINE_PASSWORD $DOCKER_IMAGE $PRE_CMD /io/scripts/build-wheels.sh
-                ls wheelhouse/
+                pipx run --spec='cibuildwheel==1.11.1.post1' cibuildwheel --output-dir dist
+                ls -l ./dist/
+        -   name: "upload wheel"
+            uses: pypa/[email protected]
+            if: github.event_name == 'release' && github.event.action == 'created'
+            with:
+                user: __token__
+                password: ${{ secrets.TWINE_PASSWORD }}
 
     downstream_tests:
         strategy:

scripts/fix-wheel.py

@@ -5,20 +5,18 @@
 
 from delocate import wheeltools
 
-def add_library(paths):
-    wheel_fnames = glob('/io/wheelhouse/pyopencl*.whl')
-    for fname in wheel_fnames:
-        print('Processing', fname)
-        with wheeltools.InWheel(fname, fname):
-            libs_dir = os.path.join('pyopencl', '.libs')
-            os.makedirs(libs_dir, exist_ok=True)
-            for lib_path in paths:
-                shutil.copy2(lib_path, libs_dir)
+def add_library(fname, paths):
+    print('Processing', fname)
+    with wheeltools.InWheel(fname, fname):
+        libs_dir = os.path.join('pyopencl', '.libs')
+        os.makedirs(libs_dir, exist_ok=True)
+        for lib_path in paths:
+            shutil.copy2(lib_path, libs_dir)
 
 def main():
-    args = list(sys.argv)
-    args.pop(0)
-    add_library(args)
+    fname = sys.argv[1]
+    paths = sys.argv[2:]
+    add_library(fname, paths)
 
 if __name__ == '__main__':
     main()

scripts/prepare-build-wheels.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -e -x -o pipefail
+
+if [[ ! -d ~/deps ]]; then
+
+    # dependencies need to be setup
+
+    mkdir -p ~/deps
+    pushd ~/deps
+
+    curl https://tiker.net/tmp/.tmux.conf
+    yum install -y git yum openssl-devel ruby
+
+    git clone --branch v2.3.0 https://github.com/OCL-dev/ocl-icd
+    cd ocl-icd
+    curl -L -O https://raw.githubusercontent.com/conda-forge/ocl-icd-feedstock/e2c03e3ddb1ff86630ccf80dc7b87a81640025ea/recipe/install-headers.patch
+    git apply install-headers.patch
+    curl -L -O https://github.com/isuruf/ocl-icd/commit/3862386b51930f95d9ad1089f7157a98165d5a6b.patch
+    git apply 3862386b51930f95d9ad1089f7157a98165d5a6b.patch
+    autoreconf -i
+    chmod +x configure
+    ./configure --prefix=/usr
+    make -j$(nproc)
+    make install
+
+    popd
+fi
+
+# Compile wheels
+PYTHON_VERSION=$(python -c 'import platform; print(platform.python_version())')
+NUMPY_VERSION_SPEC='=='
+if [[ "${PYTHON_VERSION}" == '3.6'* ]]; then
+    NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.11.3"
+elif [[ "${PYTHON_VERSION}" == '3.7'* ]]; then
+    NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.14.5"
+elif [[ "${PYTHON_VERSION}" == '3.8'* ]]; then
+    NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.17.3"
+elif [[ "${PYTHON_VERSION}" == '3.9'* ]]; then
+    NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.19.5"
+else
+    # Unknown python version, let it unpinned instead
+    NUMPY_VERSION_SPEC=''
+fi
+# TODO: declear dependencies except numpy as build-time dependency as per PEP518
+# Build with the oldest numpy available to be compatible with newer ones
+pip install "numpy${NUMPY_VERSION_SPEC}" pybind11 mako
+
+# For bundling license files
+pip install delocate