Merge pull request #20318 from wolfi-dev/python-dev

python dev

pipelines/py/collect-wheels.yaml

@@ -0,0 +1,28 @@
+name: Collect wheels built by a python pipeline
+
+needs:
+  packages:
+    - util-linux
+
+inputs:
+  dest:
+    description: the destination
+    default: ${{targets.subpkgdir}}/usr/share/wheels
+
+pipeline:
+  - name: "pip build ${{inputs.python}}"
+    runs: |
+      found=""
+      set --
+      for wheel in ./.wheels/*/*.whl; do
+          [ -f "$wheel" ] || continue
+          set -- "$@" "$wheel"
+      done
+      if [ $# -eq 0 ]; then
+          echo "No wheels found in $PWD/.wheels/*/*.whl"
+          exit 1
+      fi
+      mkdir -p ${{inputs.dest}}
+      echo "Found wheels:" "$@"
+      sha256sum "$@"
+      cp -v "$@" "${{inputs.dest}}"

pipelines/py/install-compile.yaml

@@ -0,0 +1,18 @@
+name: Setup, install and compile python
+
+inputs:
+  python:
+    description: which python to use
+    required: true
+  dest:
+    description: the destination
+    default: ${{targets.subpkgdir}}
+
+pipeline:
+  - name: "setup ${{inputs.python}}"
+    runs: |
+      ${{inputs.python}} setup.py install --no-compile --root="${{targets.subpkgdir}}" --prefix=/usr
+      rm -Rf build/
+  - name: "compileall ${{inputs.python}}"
+    runs: |
+      ${{inputs.python}} -m compileall --invalidation-mode=unchecked-hash -r100 ${{targets.subpkgdir}}/usr/lib/

pipelines/py/pip-build-install-bootstrap.yaml

@@ -0,0 +1,58 @@
+name: Use pip from a zipapp for bootstrap. Only for use with bootstrapping.
+
+needs:
+  packages:
+    - pip-zipapp
+
+inputs:
+  python:
+    description: which python to use
+    required: true
+  wheel:
+    description: build a wheel?
+    required: false
+    default: true
+  dest:
+    description: the destination
+    default: ${{targets.subpkgdir}}
+
+pipeline:
+  - name: "pip-bootstrap build"
+    runs: |
+      export SOURCE_DATE_EPOCH=315532800
+      py=${{inputs.python}}
+      pipzip=/usr/share/pip-zipapp/pip-zipapp.pyz
+      [ -e "$pipzip" ] || { echo "missing pip-zip - $pipzip does not exist"; exit 1; }
+      [ -f "$pipzip" ] || { echo "missing pip-zip - $pipzip is not a file"; exit 1; }
+      pyver=$("$py" -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')
+      sitepkgd=$("$py" -c 'import site; print(site.getsitepackages()[0])')
+      sitepkgd=${sitepkgd#/}
+      root=${{inputs.dest}}
+      wd="./.wheels/$pyver"
+      info() { echo "$@" 1>&2; }
+      vr() { info "execute:" "$@"; "$@"; }
+
+      tmpd=$(mktemp -d)
+      trap "rm -Rf '$tmpd'" EXIT
+
+      # --find-links to an empty dir and --no-index makes pip fully "offline"
+      distwheelsd="/usr/share/pip-zipapp/wheels"
+      mkdir -p "$distwheelsd"
+      info "$py is $pyver with site_packages dir '$sitepkgd'"
+
+      if [ "${{inputs.wheel}}" = "false" ]; then
+        info "skipping bdist-wheel"
+        vr "$py" "$pipzip" --python=/usr/bin/$py install --verbose --no-deps \
+            "--find-links=$distwheelsd" --no-index \
+            --force-reinstall --no-compile --prefix=/usr "--root=$root" .
+      else
+        vr "$py" "$pipzip" --python=/usr/bin/$py wheel --verbose "--wheel-dir=$wd" \
+            "--find-links=$distwheelsd" --no-index --no-deps .
+        set -- "$wd"/*.whl
+        [ -f "$1" ] || { info "did not produce a wheel in $wd"; exit 1; }
+        info "produced $# wheels: $*"
+        vr "$py" "$pipzip" --python=/usr/bin/$py install --verbose \
+            "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps \
+            --force-reinstall --no-compile --prefix=/usr "--root=$root" "$@"
+      fi
+      vr $py -m compileall --invalidation-mode=unchecked-hash -r100 "$root/$sitepkgd"

pipelines/py/pip-build-install.yaml

@@ -0,0 +1,35 @@
+name: Build and install python package with pip
+
+inputs:
+  python:
+    description: which python to use
+    required: true
+  dest:
+    description: the destination
+    default: ${{targets.subpkgdir}}
+
+pipeline:
+  - name: "pip build ${{inputs.python}}"
+    runs: |
+      export SOURCE_DATE_EPOCH=315532800
+      py=${{inputs.python}}
+      pyver=$("$py" -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')
+      sitepkgd=$("$py" -c 'import site; print(site.getsitepackages()[0])')
+      sitepkgd=${sitepkgd#/}
+      root=${{inputs.dest}}
+      wd="./.wheels/$pyver"
+      vr() { echo "execute:" "$@"; "$@"; }
+
+      tmpd=$(mktemp -d)
+      trap "rm -Rf '$tmpd'" EXIT
+
+      # --find-links to an empty dir and --no-index makes pip fully "offline"
+      distwheelsd="$tmpd/dist-wheels"
+      mkdir -p "$distwheelsd"
+      echo "$py is $pyver with site_packages dir '$sitepkgd'"
+      vr $py -m pip wheel --verbose "--wheel-dir=$wd" \
+          "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps .
+      vr $py -m pip install --verbose \
+          "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps \
+          --force-reinstall --no-compile --prefix=/usr "--root=$root" "$wd"/*.whl
+      vr $py -m compileall --invalidation-mode=unchecked-hash -r100 "$root/$sitepkgd"

py3-cython.yaml

@@ -0,0 +1,57 @@
+package:
+  name: py3-cython
+  version: 3.0.10
+  epoch: 1
+  description: Cython is an optimising static compiler for both the Python & the extended Cython programming languages.
+  copyright:
+    - license: Apache-2.0
+  dependencies:
+    provider-priority: 0
+
+vars:
+  pypi-package: cython
+
+data:
+  - name: py-versions
+    items:
+      3.10: "310"
+      3.11: "311"
+      3.12: "312"
+
+environment:
+  contents:
+    packages:
+      - build-base
+      - busybox
+      - py3-supported-pip
+      - py3-supported-python-dev
+      - py3-supported-wheel
+
+pipeline:
+  - uses: fetch
+    with:
+      expected-sha256: 00f97476cef9fcd9a89f9d2a49be3b518e1a74b91f377fe08c97fcb44bc0f7d7
+      uri: https://github.com/cython/cython/archive/${{package.version}}.tar.gz
+
+subpackages:
+  - range: py-versions
+    name: py${{range.key}}-${{vars.pypi-package}}
+    dependencies:
+      provider-priority: ${{range.value}}
+      provides:
+        - ${{package.name}}
+        - cython
+    pipeline:
+      - uses: py/pip-build-install
+        with:
+          python: python${{range.key}}
+      - uses: strip
+
+  - name: py3-wheels-${{vars.pypi-package}}
+    pipeline:
+      - uses: py/collect-wheels
+
+update:
+  enabled: true
+  github:
+    identifier: cython/cython

py3-flit-core.yaml

@@ -1,42 +1,71 @@
 package:
   name: py3-flit-core
   version: 3.9.0
-  epoch: 1
+  epoch: 2
   description: "simple packaging tool for simple packages (core)"
   copyright:
     - license: BSD-3-Clause
   dependencies:
-    runtime:
-      - python3
+    provider-priority: 0
+
+vars:
+  pypi-package: flit-core
+
+data:
+  - name: py-versions
+    items:
+      3.10: "310"
+      3.11: "311"
+      3.12: "312"
 
 environment:
   contents:
     packages:
       - build-base
       - busybox
-      - ca-certificates-bundle
-      - py3-installer
-      - python3
-      - wolfi-base
+      - py3-supported-python
 
 pipeline:
   - uses: fetch
     with:
       uri: https://files.pythonhosted.org/packages/source/f/flit/flit-${{package.version}}.tar.gz
       expected-sha256: d75edf5eb324da20d53570a6a6f87f51e606eee8384925cd66a90611140844c7
 
-  - runs: |
-      cd flit_core
-      python3 build_dists.py
-
-  - runs: |
-      cd flit_core
-      python3 -m installer -d "${{targets.destdir}}" dist/flit_core-${{package.version}}-py3-none-any.whl
+subpackages:
+  - range: py-versions
+    name: py${{range.key}}-${{vars.pypi-package}}
+    description: ${{vars.pypi-package}} installed for python${{range.key}}
+    dependencies:
+      provider-priority: ${{range.value}}
+      provides:
+        - ${{package.name}}
+    pipeline:
+      - runs: |
+          cd flit_core
+          rm -Rf dist/
+          python${{range.key}} -m flit_core.wheel
+          python${{range.key}} bootstrap_install.py \
+              --install-root=${{targets.subpkgdir}} \
+              dist/flit_core-${{package.version}}-py3-none-any.whl
+          mkdir -p /home/build/.wheels/${{range.key}}/
+          cp dist/flit_core-${{package.version}}-py3-none-any.whl \
+              /home/build/.wheels/${{range.key}}/
+          python${{range.key}} -m compileall --invalidation-mode=unchecked-hash -r100 \
+              ${{targets.subpkgdir}}
+          rm -rf "${{targets.subpkgdir}}"/usr/lib/python3*/site-packages/flit_core/tests
+      - uses: strip
 
-  - runs: |
-      rm -rf "${{targets.destdir}}"/usr/lib/python3*/site-packages/flit_core/tests
+  - name: py3-wheels-${{vars.pypi-package}}
+    pipeline:
+      - uses: py/collect-wheels
 
-  - uses: strip
+  - name: py3-supported-${{vars.pypi-package}}
+    description: meta package providing ${{vars.pypi-package}} for supported python versions.
+    dependencies:
+      runtime:
+        - py3.10-${{vars.pypi-package}}
+        - py3.11-${{vars.pypi-package}}
+        - py3.12-${{vars.pypi-package}}
 
 update:
   enabled: true