diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21ba2fbae..8fd93c99d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,8 @@ on: pull_request: schedule: - cron: '17 3 * * 0' + release: + types: [created] jobs: flake8: @@ -127,40 +129,30 @@ jobs: wheels: name: Build and upload wheels - runs-on: ubuntu-latest strategy: matrix: - DOCKER_IMAGE: - - quay.io/pypa/manylinux2014_x86_64 - # Disable i686 builds for now: no binary wheels for cryptography, - # source build fails, e.g. https://github.com/inducer/pyopencl/pull/421/checks?check_run_id=1781071632 - # - quay.io/pypa/manylinux2014_i686 + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - name: "Main Script" + - name: "build wheel" env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - DOCKER_IMAGE: ${{ matrix.DOCKER_IMAGE }} - + CIBW_PROJECT_REQUIRES_PYTHON: "~=3.6" + CIBW_SKIP: "pp36-*" # The docker image using doesn't support pp36 + CIBW_BEFORE_BUILD: "./scripts/prepare-build-wheels.sh" + CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair {wheel} -w {dest_dir} --lib-sdir=/.libs && python ./scripts/fix-wheel.py {wheel} ~/deps/ocl-icd/COPYING" + CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014" + CIBW_MANYLINUX_PYPY_X86_64_IMAGE: "quay.io/pypa/manylinux2014_x86_64" + CIBW_ARCHS: "auto64" # 32bit unsupported for now run: | - pwd - ls -la - - # Only perform upload for tag builds, otherwise unset TWINE_USERNAME to prevent - if ! [[ $GITHUB_REF == refs/tags* ]]; then - echo "Not a tag build, GITHUB_REF is '$GITHUB_REF'. Unsetting TWINE_USERNAME" - unset TWINE_USERNAME - fi - - if [[ $DOCKER_IMAGE == *i686* ]]; then - PRE_CMD=linux32 - else - PRE_CMD="" - fi - - docker run --rm -v `pwd`:/io -e TWINE_USERNAME -e TWINE_PASSWORD $DOCKER_IMAGE $PRE_CMD /io/scripts/build-wheels.sh - ls wheelhouse/ + pipx run --spec='cibuildwheel==1.11.1.post1' cibuildwheel --output-dir dist + ls -l ./dist/ + - name: "upload wheel" + uses: pypa/gh-action-pypi-publish@v1.4.2 + if: github.event_name == 'release' && github.event.action == 'created' + with: + user: __token__ + password: ${{ secrets.TWINE_PASSWORD }} downstream_tests: strategy: diff --git a/scripts/build-wheels.sh b/scripts/build-wheels.sh deleted file mode 100755 index 4b08edc5b..000000000 --- a/scripts/build-wheels.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -set -e -x - -mkdir -p /deps -cd /deps - -function start_spinner { - if [ -n "$SPINNER_PID" ]; then - return - fi - - >&2 echo "Building libraries..." - # Start a process that runs as a keep-alive - # to avoid travis quitting if there is no output - (while true; do - sleep 60 - >&2 echo "Still building..." - done) & - SPINNER_PID=$! - disown -} - -function stop_spinner { - if [ ! -n "$SPINNER_PID" ]; then - return - fi - - kill $SPINNER_PID - unset SPINNER_PID - - >&2 echo "Building libraries finished." -} - -#start_spinner - -curl https://tiker.net/tmp/.tmux.conf -yum install -y git yum openssl-devel ruby - -git clone --branch v2.3.0 https://github.com/OCL-dev/ocl-icd -cd ocl-icd -curl -L -O https://raw.githubusercontent.com/conda-forge/ocl-icd-feedstock/e2c03e3ddb1ff86630ccf80dc7b87a81640025ea/recipe/install-headers.patch -git apply install-headers.patch -curl -L -O https://github.com/isuruf/ocl-icd/commit/3862386b51930f95d9ad1089f7157a98165d5a6b.patch -git apply 3862386b51930f95d9ad1089f7157a98165d5a6b.patch -autoreconf -i -chmod +x configure -./configure --prefix=/usr -make -j4 -make install -cd .. - -# Compile wheels -for PYBIN in /opt/python/*/bin; do - if [[ "${PYBIN}" == *cp36* ]]; then - NUMPY_VERSION="1.11.3" - elif [[ "${PYBIN}" == *cp37* ]]; then - NUMPY_VERSION="1.14.5" - elif [[ "${PYBIN}" == *cp38* ]]; then - NUMPY_VERSION="1.17.3" - elif [[ "${PYBIN}" == *cp39* ]]; then - NUMPY_VERSION="1.19.5" - else - continue - fi - # Build with the oldest numpy available to be compatible with newer ones - "${PYBIN}/pip" install "numpy==${NUMPY_VERSION}" pybind11 mako - "${PYBIN}/pip" wheel /io/ -w wheelhouse/ --no-deps -done - -# Bundle external shared libraries into the wheels -for whl in wheelhouse/pyopencl*.whl; do - auditwheel repair "$whl" -w /io/wheelhouse/ --lib-sdir=/.libs -done - -# Bundle license files - -/opt/python/cp39-cp39/bin/pip install delocate -/opt/python/cp39-cp39/bin/python /io/scripts/fix-wheel.py /deps/ocl-icd/COPYING - -if [[ "${TWINE_USERNAME}" == "" ]]; then - echo "TWINE_USERNAME not set. Skipping uploading wheels" - exit 0 -fi - -/opt/python/cp39-cp39/bin/pip install twine -for WHEEL in /io/wheelhouse/pyopencl*.whl; do - # dev - # /opt/python/cp39-cp39/bin/twine upload \ - # --skip-existing \ - # --repository-url https://test.pypi.org/legacy/ \ - # -u "${TWINE_USERNAME}" -p "${TWINE_PASSWORD}" \ - # "${WHEEL}" - # prod - /opt/python/cp39-cp39/bin/twine upload \ - --skip-existing \ - -u "${TWINE_USERNAME}" -p "${TWINE_PASSWORD}" \ - "${WHEEL}" -done - -#stop_spinner diff --git a/scripts/fix-wheel.py b/scripts/fix-wheel.py index df8f29e2c..f1a65b2f1 100644 --- a/scripts/fix-wheel.py +++ b/scripts/fix-wheel.py @@ -5,20 +5,18 @@ from delocate import wheeltools -def add_library(paths): - wheel_fnames = glob('/io/wheelhouse/pyopencl*.whl') - for fname in wheel_fnames: - print('Processing', fname) - with wheeltools.InWheel(fname, fname): - libs_dir = os.path.join('pyopencl', '.libs') - os.makedirs(libs_dir, exist_ok=True) - for lib_path in paths: - shutil.copy2(lib_path, libs_dir) +def add_library(fname, paths): + print('Processing', fname) + with wheeltools.InWheel(fname, fname): + libs_dir = os.path.join('pyopencl', '.libs') + os.makedirs(libs_dir, exist_ok=True) + for lib_path in paths: + shutil.copy2(lib_path, libs_dir) def main(): - args = list(sys.argv) - args.pop(0) - add_library(args) + fname = sys.argv[1] + paths = sys.argv[2:] + add_library(fname, paths) if __name__ == '__main__': main() diff --git a/scripts/prepare-build-wheels.sh b/scripts/prepare-build-wheels.sh new file mode 100755 index 000000000..0d0004965 --- /dev/null +++ b/scripts/prepare-build-wheels.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -e -x -o pipefail + +if [[ ! -d ~/deps ]]; then + + # dependencies need to be setup + + mkdir -p ~/deps + pushd ~/deps + + curl https://tiker.net/tmp/.tmux.conf + yum install -y git yum openssl-devel ruby + + git clone --branch v2.3.0 https://github.com/OCL-dev/ocl-icd + cd ocl-icd + curl -L -O https://raw.githubusercontent.com/conda-forge/ocl-icd-feedstock/e2c03e3ddb1ff86630ccf80dc7b87a81640025ea/recipe/install-headers.patch + git apply install-headers.patch + curl -L -O https://github.com/isuruf/ocl-icd/commit/3862386b51930f95d9ad1089f7157a98165d5a6b.patch + git apply 3862386b51930f95d9ad1089f7157a98165d5a6b.patch + autoreconf -i + chmod +x configure + ./configure --prefix=/usr + make -j$(nproc) + make install + + popd +fi + +# Compile wheels +PYTHON_VERSION=$(python -c 'import platform; print(platform.python_version())') +NUMPY_VERSION_SPEC='==' +if [[ "${PYTHON_VERSION}" == '3.6'* ]]; then + NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.11.3" +elif [[ "${PYTHON_VERSION}" == '3.7'* ]]; then + NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.14.5" +elif [[ "${PYTHON_VERSION}" == '3.8'* ]]; then + NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.17.3" +elif [[ "${PYTHON_VERSION}" == '3.9'* ]]; then + NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.19.5" +else + # Unknown python version, let it unpinned instead + NUMPY_VERSION_SPEC='' +fi +# TODO: declear dependencies except numpy as build-time dependency as per PEP518 +# Build with the oldest numpy available to be compatible with newer ones +pip install "numpy${NUMPY_VERSION_SPEC}" pybind11 mako + +# For bundling license files +pip install delocate