From fc423afab505db3edbeb102b81d0a10ef7995337 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Tue, 11 Jan 2022 22:21:23 +0000 Subject: [PATCH] python 3.10 --- .github/scripts/build-linux.sh | 13 +-- .github/scripts/build-macos.sh | 36 +++--- .github/scripts/build-windows.ps1 | 186 +++++++++++++++--------------- .github/scripts/check_sha256.sh | 21 ++++ .github/workflows/ci.yml | 72 +++++++----- README.rst | 37 +++++- dev-requirements.txt | 2 +- lensfunpy/_version.py | 2 +- test/{tests.py => test_basic.py} | 0 9 files changed, 211 insertions(+), 158 deletions(-) create mode 100755 .github/scripts/check_sha256.sh rename test/{tests.py => test_basic.py} (100%) diff --git a/.github/scripts/build-linux.sh b/.github/scripts/build-linux.sh index 513ea48..fdad643 100755 --- a/.github/scripts/build-linux.sh +++ b/.github/scripts/build-linux.sh @@ -1,8 +1,6 @@ #!/bin/bash set -e -x -bash --version - cd /io source .github/scripts/retry.sh @@ -10,9 +8,7 @@ source .github/scripts/retry.sh # List python versions ls /opt/python -if [ $PYTHON_VERSION == "3.5" ]; then - PYBIN="/opt/python/cp35-cp35m/bin" -elif [ $PYTHON_VERSION == "3.6" ]; then +if [ $PYTHON_VERSION == "3.6" ]; then PYBIN="/opt/python/cp36-cp36m/bin" elif [ $PYTHON_VERSION == "3.7" ]; then PYBIN="/opt/python/cp37-cp37m/bin" @@ -20,14 +16,13 @@ elif [ $PYTHON_VERSION == "3.8" ]; then PYBIN="/opt/python/cp38-cp38/bin" elif [ $PYTHON_VERSION == "3.9" ]; then PYBIN="/opt/python/cp39-cp39/bin" +elif [ $PYTHON_VERSION == "3.10" ]; then + PYBIN="/opt/python/cp310-cp310/bin" else echo "Unsupported Python version $PYTHON_VERSION" exit 1 fi -# Install build tools -retry yum install -y cmake - # Install liblensfun pushd external/lensfun cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=off -DINSTALL_HELPER_SCRIPTS=off . @@ -61,7 +56,7 @@ retry ${PYBIN}/pip install -r dev-requirements.txt retry ${PYBIN}/pip install -U numpy # scipy should trigger an update, but that doesn't happen pushd $HOME -${PYBIN}/nosetests --verbosity=3 --nocapture /io/test +${PYBIN}/pytest --verbosity=3 -s /io/test popd # Move wheel to dist/ folder for easier deployment diff --git a/.github/scripts/build-macos.sh b/.github/scripts/build-macos.sh index 574a7a6..090200f 100755 --- a/.github/scripts/build-macos.sh +++ b/.github/scripts/build-macos.sh @@ -3,6 +3,8 @@ set -e -x source .github/scripts/retry.sh +CHECK_SHA256=.github/scripts/check_sha256.sh + brew install pkg-config meson # General note: @@ -16,17 +18,6 @@ export MACOSX_DEPLOYMENT_TARGET=$MACOS_MIN_VERSION # The Python variant to install, see exception below. export PYTHON_INSTALLER_MACOS_VERSION=$MACOS_MIN_VERSION -# Work-around issue building on newer XCode versions. -# https://github.com/pandas-dev/pandas/issues/23424#issuecomment-446393981 -if [ $PYTHON_VERSION == "3.5" ]; then - # No 10.9 installer available, use 10.6. - # The resulting wheel platform tags still have 10.6 (=target of Python itself), - # even though technically the wheel should only be run on 10.9 upwards. - # This is fixed manually below by renaming the wheel. - # See https://github.com/pypa/wheel/issues/312. - export PYTHON_INSTALLER_MACOS_VERSION=10.6 -fi - # Install Python # Note: The GitHub Actions supplied Python versions are not used # as they are built without MACOSX_DEPLOYMENT_TARGET/-mmacosx-version-min @@ -57,19 +48,24 @@ pip freeze # See https://discourse.brew.sh/t/it-is-possible-to-build-packages-that-are-compatible-with-older-macos-versions/4421 LIB_INSTALL_PREFIX=$(pwd)/external/libs +export CMAKE_PREFIX_PATH=$LIB_INSTALL_PREFIX export PKG_CONFIG_PATH=$LIB_INSTALL_PREFIX/lib/pkgconfig export LIBRARY_PATH=$LIB_INSTALL_PREFIX/lib export PATH=$LIB_INSTALL_PREFIX/bin:$PATH # Install libffi (glib dependency) -curl -L --retry 3 https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz | tar xz +curl -L --retry 3 -o libffi.tar.gz https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz +$CHECK_SHA256 libffi.tar.gz d06ebb8e1d9a22d19e38d63fdb83954253f39bedc5d46232a05645685722ca37 +tar xzf libffi.tar.gz pushd libffi-3.2.1 ./configure --prefix=$LIB_INSTALL_PREFIX --disable-debug make install -j popd # Install gettext (glib dependency) -curl -L --retry 3 https://ftp.gnu.org/gnu/gettext/gettext-0.20.1.tar.xz | tar xz +curl -L --retry 3 -o gettext.tar.xz https://ftp.gnu.org/gnu/gettext/gettext-0.20.1.tar.xz +$CHECK_SHA256 gettext.tar.xz 53f02fbbec9e798b0faaf7c73272f83608e835c6288dd58be6c9bb54624a3800 +tar xzf gettext.tar.xz pushd gettext-0.20.1 ./configure --prefix=$LIB_INSTALL_PREFIX \ --disable-debug \ @@ -80,12 +76,13 @@ make install popd # Install glib (lensfun dependency) -curl -L --retry 3 https://download.gnome.org/sources/glib/2.69/glib-2.69.2.tar.xz | tar xz +curl -L --retry 3 -o glib.tar.xz https://download.gnome.org/sources/glib/2.69/glib-2.69.2.tar.xz +$CHECK_SHA256 glib.tar.xz a62249e35a8635175a697b3215f1df2b89e0fbb4adb520dcbe21a3ae1ebb8882 +tar xzf glib.tar.xz pushd glib-2.69.2 mkdir build cd build meson --prefix=$LIB_INSTALL_PREFIX \ - -Dinternal_pcre=true \ -Dselinux=disabled \ -Ddtrace=false \ -Dman=false \ @@ -105,15 +102,8 @@ export LDFLAGS=$CFLAGS export ARCHFLAGS=$CFLAGS # Build wheel -export CMAKE_PREFIX_PATH=$LIB_INSTALL_PREFIX python setup.py bdist_wheel -# Fix wheel platform tag, see above for details. -if [ $PYTHON_VERSION == "3.5" ]; then - filename=$(ls dist/*.whl) - mv -v "$filename" "${filename/macosx_10_6_intel/macosx_10_9_x86_64}" -fi - # List direct and indirect library dependencies mkdir tmp_wheel pushd tmp_wheel @@ -155,5 +145,5 @@ retry pip install -r dev-requirements.txt rm -rf $LIB_INSTALL_PREFIX mkdir tmp_for_test pushd tmp_for_test -nosetests --verbosity=3 --nocapture ../test +pytest --verbosity=3 -s ../test popd diff --git a/.github/scripts/build-windows.ps1 b/.github/scripts/build-windows.ps1 index 77e6028..a959e3c 100644 --- a/.github/scripts/build-windows.ps1 +++ b/.github/scripts/build-windows.ps1 @@ -1,9 +1,5 @@ $ErrorActionPreference = 'Stop' -function not-exist { -not (Test-Path $args) } -Set-Alias !exists not-exist -Option "Constant, AllScope" -Set-Alias exists Test-Path -Option "Constant, AllScope" - function exec { [CmdletBinding()] param([Parameter(Position=0,Mandatory=1)][scriptblock]$cmd) @@ -17,52 +13,84 @@ function exec { } } -function Init-VS { +function Initialize-Python { + if ($env:USE_CONDA -eq 1) { + $env:CONDA_ROOT = $pwd.Path + "\external\miniconda_$env:PYTHON_ARCH" + & .\.github\scripts\install-miniconda.ps1 + & $env:CONDA_ROOT\shell\condabin\conda-hook.ps1 + exec { conda update --yes -n base -c defaults conda } + } + # Check Python version/arch + exec { python -c "import platform; assert platform.python_version().startswith('$env:PYTHON_VERSION')" } + exec { python -c "import struct; assert struct.calcsize('P') * 8 == $env:PYTHON_ARCH" } +} + +function Create-VEnv { + [CmdletBinding()] + param([Parameter(Position=0,Mandatory=1)][string]$name) + if ($env:USE_CONDA -eq 1) { + exec { conda create --yes --name $name -c defaults --strict-channel-priority python=$env:PYTHON_VERSION --force } + } else { + exec { python -m venv env\$name } + } +} + +function Enter-VEnv { + [CmdletBinding()] + param([Parameter(Position=0,Mandatory=1)][string]$name) + if ($env:USE_CONDA -eq 1) { + conda activate $name + } else { + & .\env\$name\scripts\activate + } +} + +function Create-And-Enter-VEnv { + [CmdletBinding()] + param([Parameter(Position=0,Mandatory=1)][string]$name) + Create-VEnv $name + Enter-VEnv $name +} + +function Exit-VEnv { + if ($env:USE_CONDA -eq 1) { + conda deactivate + } else { + deactivate + } +} + +function Initialize-VS { # https://wiki.python.org/moin/WindowsCompilers # setuptools automatically selects the right compiler for building # the extension module. The following is mostly for building any - # dependencies like liblensfun. + # native dependencies, here via CMake. # https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line # https://docs.microsoft.com/en-us/cpp/porting/binary-compat-2015-2017 - $VS2017_ROOT = "C:\Program Files (x86)\Microsoft Visual Studio\2017" - $VS2019_ROOT = "C:\Program Files (x86)\Microsoft Visual Studio\2019" + $VS_ROOT = "C:\Program Files (x86)\Microsoft Visual Studio" + $VS_VERSIONS = @("2017", "2019") + $VS_EDITIONS = @("Enterprise", "Professional", "Community") + $VS_INIT_CMD_SUFFIX = "Common7\Tools\vsdevcmd.bat" $VS_ARCH = if ($env:PYTHON_ARCH -eq '32') { 'x86' } else { 'x64' } - - $PYTHON_VERSION_TUPLE = $env:PYTHON_VERSION.split('.') - $PYTHON_VERSION_MAJOR = $PYTHON_VERSION_TUPLE[0] - $PYTHON_VERSION_MINOR = $PYTHON_VERSION_TUPLE[1] - - if ($PYTHON_VERSION_MAJOR -eq '3') { - if ($PYTHON_VERSION_MINOR -le '4') { - throw ("Python <= 3.4 unsupported: $env:PYTHON_VERSION") - } - if (exists $VS2017_ROOT) { - $VS_VERSION = "2017" - if (exists "$VS2017_ROOT\Enterprise") { - $VS_ROOT = "$VS2017_ROOT\Enterprise" - } else { - $VS_ROOT = "$VS2017_ROOT\Community" + $VS_INIT_ARGS = "-arch=$VS_ARCH -no_logo" + + $found = $false + :outer foreach ($version in $VS_VERSIONS) { + foreach ($edition in $VS_EDITIONS) { + $VS_INIT_CMD = "$VS_ROOT\$version\$edition\$VS_INIT_CMD_SUFFIX" + if (Test-Path $VS_INIT_CMD) { + $found = $true + break outer } - $VS_INIT_CMD = "$VS_ROOT\Common7\Tools\vsdevcmd.bat" - $VS_INIT_ARGS = "-arch=$VS_ARCH -no_logo" - } elseif (exists $VS2019_ROOT) { - $VS_VERSION = "2019" - if (exists "$VS2019_ROOT\Enterprise") { - $VS_ROOT = "$VS2019_ROOT\Enterprise" - } else { - $VS_ROOT = "$VS2019_ROOT\Community" - } - $VS_INIT_CMD = "$VS_ROOT\Common7\Tools\vsdevcmd.bat" - $VS_INIT_ARGS = "-arch=$VS_ARCH -no_logo" - } else { - throw ("No suitable Visual Studio installation found") } - } else { - throw ("Unsupported Python version: $PYTHON_VERSION_MAJOR") } - Write-Host "Configuring VS$VS_VERSION for Python $env:PYTHON_VERSION on a $env:PYTHON_ARCH bit architecture" + + if (!$found) { + throw ("No suitable Visual Studio installation found") + } + Write-Host "Executing: $VS_INIT_CMD $VS_INIT_ARGS" # https://github.com/Microsoft/vswhere/wiki/Start-Developer-Command-Prompt @@ -85,66 +113,44 @@ if (!$env:NUMPY_VERSION) { throw "NUMPY_VERSION env var missing" } -Init-VS +Initialize-VS +Initialize-Python Get-ChildItem env: -$env:CONDA_ROOT = $pwd.Path + "\external\miniconda_$env:PYTHON_ARCH" -& .\.github\scripts\install-miniconda.ps1 - -& $env:CONDA_ROOT\shell\condabin\conda-hook.ps1 - -exec { conda update --yes -n base -c defaults conda } -exec { conda create --yes --name pyenv_build python=$env:PYTHON_VERSION numpy=$env:NUMPY_VERSION cython --force } -exec { conda activate pyenv_build } - -# Check that we have the expected version and architecture for Python -exec { python --version } -exec { python -c "import struct; assert struct.calcsize('P') * 8 == $env:PYTHON_ARCH" } -exec { python -c "import sys; print(sys.prefix)" } - -# output what's installed -exec { python -m pip freeze } - -# Build the compiled extension. -# -u disables output buffering which caused intermixing of lines -# when the external tools were started +# Build the wheel. +Create-And-Enter-VEnv build +exec { python -m pip install --upgrade pip wheel setuptools } +exec { python -m pip install --only-binary :all: numpy==$env:NUMPY_VERSION cython } exec { python -u setup.py bdist_wheel } +Exit-VEnv -# Necessary to avoid bug when switching to test env. -exec { conda deactivate } - -# Import test on a minimal environment -# (to catch DLL issues) -exec { conda create --yes --name pyenv_minimal python=$env:PYTHON_VERSION --force } -exec { conda activate pyenv_minimal } - -# Avoid using in-source package -New-Item -Force -ItemType directory tmp_for_test | out-null -cd tmp_for_test - +# Install and import in an empty environment. +# This is to catch DLL issues that may be hidden with dependencies. +Create-And-Enter-VEnv import-test python -m pip uninstall -y lensfunpy -ls ..\dist\*.whl | % { exec { python -m pip install $_ } } -exec { python -c "import lensfunpy" } +ls dist\*.whl | % { exec { python -m pip install $_ } } -# Necessary to avoid bug when switching to test env. -exec { conda deactivate } +# Avoid using in-source package during tests +mkdir -f tmp_for_test | out-null +pushd tmp_for_test +exec { python -c "import lensfunpy" } +popd -# Test -exec { conda create --yes --name pyenv_test python=$env:PYTHON_VERSION numpy scipy --force } -exec { conda activate pyenv_test } +Exit-VEnv -# Check that we have the expected version and architecture for Python -exec { python --version } -exec { python -c "import struct; assert struct.calcsize('P') * 8 == $env:PYTHON_ARCH" } -exec { python -c "import sys; print(sys.prefix)" } +# Run test suite with all required and optional dependencies +Create-And-Enter-VEnv testsuite +exec { python -m pip install --only-binary :all: numpy scipy } +python -m pip uninstall -y lensfunpy +ls dist\*.whl | % { exec { python -m pip install $_ } } +exec { python -m pip install -r dev-requirements.txt } -# output what's installed -exec { python -m pip freeze } +# Avoid using in-source package during tests +mkdir -f tmp_for_test | out-null +pushd tmp_for_test +exec { pytest --verbosity=3 -s ../test } +popd -python -m pip uninstall -y lensfunpy -ls ..\dist\*.whl | % { exec { python -m pip install $_ } } -exec { python -m pip install -r ..\dev-requirements.txt } -exec { nosetests --verbosity=3 --nocapture ../test } -cd .. +Exit-VEnv diff --git a/.github/scripts/check_sha256.sh b/.github/scripts/check_sha256.sh new file mode 100755 index 0000000..68b9196 --- /dev/null +++ b/.github/scripts/check_sha256.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +path=$1 +expected=$2 + +if [ "$(uname)" == "Darwin" ]; then + actual=$(shasum -a 256 "$path" | cut -d ' ' -f 1) +elif [ "$(uname)" == "Linux" ]; then + actual=$(sha256sum "$path" | cut -d ' ' -f 1) +else + echo "Unknown system: $(uname)" + exit 1 +fi + +if [ "$expected" != "$actual" ]; then + echo "CHECKSUM MISMATCH: $path" + echo "Expected: $expected" + echo "Actual: $actual" + exit 1 +fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 837a805..04b8d26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,17 +2,14 @@ name: CI on: [push, pull_request] +permissions: read-all + jobs: build: strategy: fail-fast: false matrix: config: - - os-image: ubuntu-latest - os-name: linux - docker-image: quay.io/pypa/manylinux2014_x86_64 - python-version: '3.6' - numpy-version: '1.11.*' - os-image: ubuntu-latest os-name: linux docker-image: quay.io/pypa/manylinux2014_x86_64 @@ -28,16 +25,12 @@ jobs: docker-image: quay.io/pypa/manylinux2014_x86_64 python-version: '3.9' numpy-version: '1.19.*' + - os-image: ubuntu-latest + os-name: linux + docker-image: quay.io/pypa/manylinux2014_x86_64 + python-version: '3.10' + numpy-version: '1.21.*' - # Ideally 10.6 (same as offical Python installer) but there - # are issues targeting that with newer XCode versions. - # As likely nobody is using 10.6 anymore, targeting 10.9 is acceptable. - # See https://github.com/pandas-dev/pandas/issues/23424#issuecomment-446393981. - - os-image: macos-latest - os-name: mac - macos-min-version: '10.9' - python-version: '3.6' - numpy-version: '1.11.*' - os-image: macos-latest os-name: mac macos-min-version: '10.9' @@ -53,27 +46,33 @@ jobs: macos-min-version: '10.9' python-version: '3.9' numpy-version: '1.19.*' + - os-image: macos-latest + os-name: mac + macos-min-version: '10.9' + python-version: '3.10' + numpy-version: '1.21.*' - - os-image: windows-latest - os-name: windows - python-version: '3.6' - python-arch: '64' - numpy-version: '1.11' - os-image: windows-latest os-name: windows python-version: '3.7' python-arch: '64' - numpy-version: '1.14' + numpy-version: '1.14.*' - os-image: windows-latest os-name: windows python-version: '3.8' python-arch: '64' - numpy-version: '1.17' + numpy-version: '1.17.*' - os-image: windows-latest os-name: windows python-version: '3.9' python-arch: '64' - numpy-version: '1.19' + numpy-version: '1.19.*' + - os-image: windows-latest + os-name: windows + python-version: '3.10' + python-arch: '64' + numpy-version: '1.21.*' + runs-on: ${{ matrix.config.os-image }} steps: @@ -96,6 +95,12 @@ jobs: PYTHON_VERSION: ${{ matrix.config.python-version }} NUMPY_VERSION: ${{ matrix.config.numpy-version }} + - name: Setup Python (Windows) + if: matrix.config.os-name == 'windows' + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.config.python-version }} + - name: Build wheels (Windows) if: matrix.config.os-name == 'windows' run: .github/scripts/build-windows.ps1 @@ -144,7 +149,7 @@ jobs: name: docs path: dist-docs - deploy: + publish-wheels: runs-on: ubuntu-latest needs: [build, docs] @@ -158,12 +163,6 @@ jobs: name: wheels path: dist - - name: Download docs HTML from artifact storage - uses: actions/download-artifact@v1 - with: - name: docs - path: dist-docs - - name: Setup Python uses: actions/setup-python@v1 @@ -172,6 +171,21 @@ jobs: pip install twine twine upload -u __token__ -p ${{ secrets.PYPI_TOKEN }} --skip-existing dist/* + publish-docs: + runs-on: ubuntu-latest + + needs: [publish-wheels] + + permissions: + contents: write + + steps: + - name: Download docs HTML from artifact storage + uses: actions/download-artifact@v1 + with: + name: docs + path: dist-docs + - name: Upload docs to GitHub Pages uses: peaceiris/actions-gh-pages@47a6d63ea8b47b19328e258563aa1fbe224c0a23 env: diff --git a/README.rst b/README.rst index d15f9f8..0d9fb3b 100644 --- a/README.rst +++ b/README.rst @@ -90,9 +90,12 @@ How to correct lens vignetting: Installation ------------ -Binary wheels for Linux, macOS, and Windows are provided for Python 3.5 - 3.8. -These can be installed with a simple ``pip install lensfunpy``. -Currently, Linux and macOS wheels are only available as 64 bit versions. +Install lensfunpy by running: +```sh +pip install lensfunpy +``` + +64-bit binary wheels are provided for Linux, macOS, and Windows. Installation from source on Linux/macOS --------------------------------------- @@ -139,6 +142,30 @@ for libraries by default in some Linux distributions. Note that on some systems the installation path may be slightly different, such as ``/usr/local/lib/x86_64-linux-gnu`` or ``/usr/local/lib64``. +Installation from source on Windows +----------------------------------- + +These instructions are experimental and support is not provided for them. +Typically, there should be no need to build manually since wheels are hosted on PyPI. + +You need to have Visual Studio installed to build lensfunpy. + +In a PowerShell window: + +.. code-block:: sh + + $env:USE_CONDA = '1' + $env:PYTHON_VERSION = '3.7' + $env:PYTHON_ARCH = '64' + $env:NUMPY_VERSION = '1.14.*' + git clone https://github.com/letmaik/lensfunpy + cd lensfunpy + .github/scripts/build-windows.ps1 + +The above will download all build dependencies (including a Python installation) +and is fully configured through the four environment variables. +Set ``USE_CONDA = '0'`` to build within an existing Python environment. + NumPy Dependency ---------------- @@ -147,10 +174,10 @@ lensfunpy depends on NumPy. The minimum supported NumPy version depends on your ========== ========= Python numpy ---------- --------- -3.5 >= 1.9 -3.6 >= 1.11 3.7 >= 1.14 3.8 >= 1.17 +3.9 >= 1.19 +3.10 >= 1.21 ========== ========= .. _lensfun: https://lensfun.github.io/ diff --git a/dev-requirements.txt b/dev-requirements.txt index 7a49684..73d41d5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,7 +10,7 @@ numpy scipy # test dependencies -nose +pytest # documentation dependencies sphinx_rtd_theme diff --git a/lensfunpy/_version.py b/lensfunpy/_version.py index d27d1f8..95fb1fc 100644 --- a/lensfunpy/_version.py +++ b/lensfunpy/_version.py @@ -1 +1 @@ -__version__ = "1.9.0" \ No newline at end of file +__version__ = "1.10.0" \ No newline at end of file diff --git a/test/tests.py b/test/test_basic.py similarity index 100% rename from test/tests.py rename to test/test_basic.py