From 0d421bf1839568b9ed133314cabbe0b1b93feb44 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 20 Mar 2021 22:31:15 +0800 Subject: [PATCH 1/9] Add support for cargo wrapper --- setuptools_rust/build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setuptools_rust/build.py b/setuptools_rust/build.py index 83e8ecb6..83260ec8 100644 --- a/setuptools_rust/build.py +++ b/setuptools_rust/build.py @@ -50,6 +50,7 @@ def initialize_options(self): self.build_temp = None self.plat_name = None self.target = os.getenv("CARGO_BUILD_TARGET") + self.cargo = os.getenv("CARGO", "cargo") def finalize_options(self): super().finalize_options() @@ -123,7 +124,7 @@ def build_extension(self, ext: RustExtension, target_triple=None): # Find where to put the temporary build files created by `cargo` metadata_command = [ - "cargo", + self.cargo, "metadata", "--manifest-path", ext.path, @@ -153,7 +154,7 @@ def build_extension(self, ext: RustExtension, target_triple=None): if executable: args = ( - ["cargo", "build", "--manifest-path", ext.path] + [self.cargo, "build", "--manifest-path", ext.path] + feature_args + target_args + list(ext.args or []) @@ -169,7 +170,7 @@ def build_extension(self, ext: RustExtension, target_triple=None): else: args = ( - ["cargo", "rustc", "--lib", "--manifest-path", ext.path] + [self.cargo, "rustc", "--lib", "--manifest-path", ext.path] + feature_args + target_args + list(ext.args or []) From 4d9112bce808395c0abc600cf20f879a3ff4e1b7 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 20 Mar 2021 22:42:24 +0800 Subject: [PATCH 2/9] Add cross test case --- .github/workflows/ci.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d6fa48b..099442fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,3 +155,32 @@ jobs: pip install rust_with_cffi/dist/rust_with_cffi*.whl python -c "from rust_with_cffi import rust; assert rust.rust_func() == 14" python -c "from rust_with_cffi.cffi import lib; assert lib.cffi_func() == 15" + + test-cross: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: 3.6 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - name: Install cross + run: cargo install cross + - name: Build package + run: pip install -e . + - name: Build an abi3 wheel using cross + shell: bash + env: + CARGO: cross + CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu + run: | + cd examples/rust_with_cffi/ + python --version + pip install wheel + python setup.py bdist_wheel --py-limited-api=cp35 + ls -la dist/ \ No newline at end of file From 706fd29667fe98f4ee79c18a05afd6911bcde281 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 20 Mar 2021 22:49:24 +0800 Subject: [PATCH 3/9] Hardcode cargo for cargo metadata command --- setuptools_rust/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setuptools_rust/build.py b/setuptools_rust/build.py index 83260ec8..9346832c 100644 --- a/setuptools_rust/build.py +++ b/setuptools_rust/build.py @@ -124,7 +124,7 @@ def build_extension(self, ext: RustExtension, target_triple=None): # Find where to put the temporary build files created by `cargo` metadata_command = [ - self.cargo, + "cargo", "metadata", "--manifest-path", ext.path, From 82fc90d6bd1583ff8293f380be4c94cbaa2605dc Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 20 Mar 2021 23:06:39 +0800 Subject: [PATCH 4/9] Use custom cross image --- .github/workflows/ci.yml | 10 +++++----- examples/namespace_package/Cross.toml | 2 ++ examples/namespace_package/Dockerfile | 8 ++++++++ 3 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 examples/namespace_package/Cross.toml create mode 100644 examples/namespace_package/Dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 099442fc..3be92786 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,7 +137,7 @@ jobs: cd examples/rust_with_cffi/ python --version pip install wheel - python setup.py bdist_wheel --py-limited-api=cp35 + python setup.py bdist_wheel --py-limited-api=cp36 ls -la dist/ # Now we switch to a differnet Python version and ensure we can install @@ -179,8 +179,8 @@ jobs: CARGO: cross CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu run: | - cd examples/rust_with_cffi/ - python --version - pip install wheel - python setup.py bdist_wheel --py-limited-api=cp35 + cd examples/namespace_package + docker build -t cross-pyo3:x86_64-unknown-linux-gnu . + python -m pip install wheel + python setup.py bdist_wheel --py-limited-api=cp36 ls -la dist/ \ No newline at end of file diff --git a/examples/namespace_package/Cross.toml b/examples/namespace_package/Cross.toml new file mode 100644 index 00000000..33207773 --- /dev/null +++ b/examples/namespace_package/Cross.toml @@ -0,0 +1,2 @@ +[target.x86_64-unknown-linux-gnu] +image = "cross-pyo3:x86_64-unknown-linux-gnu" diff --git a/examples/namespace_package/Dockerfile b/examples/namespace_package/Dockerfile new file mode 100644 index 00000000..ec3f35e8 --- /dev/null +++ b/examples/namespace_package/Dockerfile @@ -0,0 +1,8 @@ +FROM rustembedded/cross:x86_64-unknown-linux-gnu-0.2.1 + +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install -y python3.6 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 From cb4ba3b4320f0c58887f48b1bf9c883f8fb2969b Mon Sep 17 00:00:00 2001 From: messense Date: Fri, 26 Mar 2021 14:15:55 +0800 Subject: [PATCH 5/9] Test cross with aarch64-unknown-linux-gnu --- .github/workflows/ci.yml | 8 ++++---- examples/namespace_package/Cross.toml | 4 ++-- examples/namespace_package/Dockerfile | 9 +++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3be92786..1bcd8085 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,7 +163,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: 3.8 - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -177,10 +177,10 @@ jobs: shell: bash env: CARGO: cross - CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu + CARGO_BUILD_TARGET: aarch64-unknown-linux-gnu run: | cd examples/namespace_package - docker build -t cross-pyo3:x86_64-unknown-linux-gnu . + docker build -t cross-pyo3:aarch64-unknown-linux-gnu . python -m pip install wheel python setup.py bdist_wheel --py-limited-api=cp36 - ls -la dist/ \ No newline at end of file + ls -la dist/ diff --git a/examples/namespace_package/Cross.toml b/examples/namespace_package/Cross.toml index 33207773..f4517564 100644 --- a/examples/namespace_package/Cross.toml +++ b/examples/namespace_package/Cross.toml @@ -1,2 +1,2 @@ -[target.x86_64-unknown-linux-gnu] -image = "cross-pyo3:x86_64-unknown-linux-gnu" +[target.aarch64-unknown-linux-gnu] +image = "cross-pyo3:aarch64-unknown-linux-gnu" diff --git a/examples/namespace_package/Dockerfile b/examples/namespace_package/Dockerfile index ec3f35e8..536d5142 100644 --- a/examples/namespace_package/Dockerfile +++ b/examples/namespace_package/Dockerfile @@ -1,8 +1,9 @@ -FROM rustembedded/cross:x86_64-unknown-linux-gnu-0.2.1 +FROM rustembedded/cross:aarch64-unknown-linux-gnu -RUN apt-get update && \ +RUN dpkg --add-architecture arm64 && \ + apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository ppa:deadsnakes/ppa && \ apt-get update && \ - apt-get install -y python3.6 && \ - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 + apt-get install -y python3.8:arm64 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 From 8ad10587e99762841016801f9667523c4891418b Mon Sep 17 00:00:00 2001 From: messense Date: Fri, 26 Mar 2021 19:29:18 +0800 Subject: [PATCH 6/9] Copy aarch64 Python from manylinux2014_aarch64 image --- .github/workflows/ci.yml | 5 +++-- examples/namespace_package/Cross.toml | 7 +++++++ examples/namespace_package/Dockerfile | 10 +++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bcd8085..8c4b78db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,14 +173,15 @@ jobs: run: cargo install cross - name: Build package run: pip install -e . - - name: Build an abi3 wheel using cross + - name: Build wheel using cross shell: bash env: CARGO: cross CARGO_BUILD_TARGET: aarch64-unknown-linux-gnu + PYO3_CROSS_LIB_DIR: /opt/python/cp38-cp38/lib run: | cd examples/namespace_package docker build -t cross-pyo3:aarch64-unknown-linux-gnu . python -m pip install wheel - python setup.py bdist_wheel --py-limited-api=cp36 + python setup.py bdist_wheel --plat-name manylinux2014_aarch64 ls -la dist/ diff --git a/examples/namespace_package/Cross.toml b/examples/namespace_package/Cross.toml index f4517564..3b971ba6 100644 --- a/examples/namespace_package/Cross.toml +++ b/examples/namespace_package/Cross.toml @@ -1,2 +1,9 @@ [target.aarch64-unknown-linux-gnu] image = "cross-pyo3:aarch64-unknown-linux-gnu" + +[build.env] +passthrough = [ + "RUST_BACKTRACE", + "RUST_LOG", + "PYO3_CROSS_LIB_DIR", +] diff --git a/examples/namespace_package/Dockerfile b/examples/namespace_package/Dockerfile index 536d5142..0ecd9571 100644 --- a/examples/namespace_package/Dockerfile +++ b/examples/namespace_package/Dockerfile @@ -1,9 +1,13 @@ +FROM quay.io/pypa/manylinux2014_aarch64 AS manylinux + FROM rustembedded/cross:aarch64-unknown-linux-gnu -RUN dpkg --add-architecture arm64 && \ - apt-get update && \ +RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository ppa:deadsnakes/ppa && \ apt-get update && \ - apt-get install -y python3.8:arm64 && \ + apt-get install -y python3.8 && \ update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 + +COPY --from=manylinux /opt/_internal /opt/_internal +COPY --from=manylinux /opt/python /opt/python From aba5f159dc03d7c69026969821fedf40aaa16165 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 27 Mar 2021 21:53:05 +0800 Subject: [PATCH 7/9] Hack: Remove incorrect platform tag if we are cross compiling --- .github/workflows/ci.yml | 2 +- setuptools_rust/build.py | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c4b78db..abdce5da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,7 +75,7 @@ jobs: continue fi - tox -c $example_dir -e py + tox -c $example_dir -e py -vvvv done - name: Test macOS universal2 diff --git a/setuptools_rust/build.py b/setuptools_rust/build.py index 9346832c..311f9b69 100644 --- a/setuptools_rust/build.py +++ b/setuptools_rust/build.py @@ -4,6 +4,7 @@ import platform import shutil import sys +import sysconfig import subprocess from distutils.errors import ( CompileError, @@ -350,10 +351,42 @@ def get_dylib_ext_path(self, ext, target_fname): assert modpath not in build_ext.ext_map build_ext.ext_map[modpath] = ext try: - return build_ext.get_ext_fullpath(target_fname) + ext_path = build_ext.get_ext_fullpath(target_fname) finally: del build_ext.ext_map[modpath] + if '.abi3.' in ext_path: + return ext_path + # Examples: linux_x86_64, linux_i686, manylinux2014_aarch64, manylinux_2_24_armv7l + plat_name = self.plat_name.lower().replace('-', '_').replace('.', '_') + if not plat_name.startswith(('linux', 'manylinux')): + return ext_path + + arch_parts = [] + arch_found = False + for item in plat_name.split('_'): + if item.startswith(('linux', 'manylinux')): + continue + if item.isdigit() and not arch_found: + # manylinux_2_24_armv7l arch should be armv7l + continue + arch_found = True + arch_parts.append(item) + target_arch = '_'.join(arch_parts) + host_platform = sysconfig.get_platform() + host_arch = host_platform.rsplit('-', 1)[1] + # Remove incorrect platform tag if we are cross compiling + if target_arch and host_arch != target_arch: + ext_dir = os.path.dirname(ext_path) + ext_name = os.path.basename(ext_path) + # rust.cpython-38-x86_64-linux-gnu.so to (rust.cpython-38-x86_64-linux-gnu, .so) + ext_name, ext_ext = os.path.splitext(ext_name) + # rust.cpython-38-x86_64-linux-gnu to (rust, .cpython-38-x86_64-linux-gnu) + ext_name, _plat_tag = os.path.splitext(ext_name) + # rust.so, removed platform tag + ext_path = os.path.join(ext_dir, ext_name + ext_ext) + return ext_path + @staticmethod def create_universal2_binary(output_path, input_paths): # Try lipo first From 2f4e7d0145ae49c2513c9262eeb4451b46e86960 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 27 Mar 2021 22:01:44 +0800 Subject: [PATCH 8/9] Run cross built wheel tests with run-on-arch-action --- .github/workflows/ci.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abdce5da..aaa0e493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,3 +185,18 @@ jobs: python -m pip install wheel python setup.py bdist_wheel --plat-name manylinux2014_aarch64 ls -la dist/ + - uses: uraimo/run-on-arch-action@v2.0.5 + name: Install built wheel + with: + arch: aarch64 + distro: ubuntu20.04 + dockerRunArgs: | + --volume "${PWD}/examples/namespace_package:/io" + install: | + apt-get update + apt-get install -y --no-install-recommends python3 python3-pip + pip3 install -U pip + run: | + pip3 install namespace_package --no-index --find-links /io/dist/ --force-reinstall + python3 -c "from namespace_package import rust; assert rust.rust_func() == 14" + python3 -c "from namespace_package import python; assert python.python_func() == 15" From 59ef903b247eef75fc17e78b361ef067f0c47737 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 27 Mar 2021 23:22:11 +0800 Subject: [PATCH 9/9] Try to get plat_name from command line options --- setuptools_rust/setuptools_ext.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setuptools_rust/setuptools_ext.py b/setuptools_rust/setuptools_ext.py index a7936088..ab5ab8c6 100644 --- a/setuptools_rust/setuptools_ext.py +++ b/setuptools_rust/setuptools_ext.py @@ -106,9 +106,11 @@ def run(self): log.info("running build_rust") build_rust = self.get_finalized_command("build_rust") build_rust.inplace = self.inplace - build_rust.plat_name = self.plat_name build_rust.target = self.target build_rust.verbose = self.verbose + options = self.distribution.get_cmdline_options().get('bdist_wheel', {}) + plat_name = options.get('plat-name') or self.plat_name + build_rust.plat_name = plat_name build_rust.run() build_ext_base_class.run(self)