From 015c54ed640b0c381d024e4fc07beb85b12ef2c9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 15 Mar 2024 10:03:00 +0000 Subject: [PATCH 01/39] Make several core python packages build for mulitple python versions. At this point most things should use the py/pip-build-install pipeline and generally look like py3-pyyaml Also: * rename cython.yaml, numpy.yaml to py3- These are python packages and present in pypi, for consistency, name them like other python packages. Co-authored-by: Dimitri John Ledkov --- cython.yaml | 34 -------------- numpy.yaml | 61 ------------------------- pipelines/py/collect-wheels.yaml | 28 ++++++++++++ pipelines/py/install-compile.yaml | 25 ++++++++++ pipelines/py/pip-build-install.yaml | 44 ++++++++++++++++++ py3-cython.yaml | 63 +++++++++++++++++++++++++ py3-flit-core.yaml | 49 ++++++++++++++------ py3-gpep517.yaml | 48 ++++++++++++++----- py3-installer.yaml | 59 ++++++++++++++++++++++++ py3-numpy.yaml | 71 +++++++++++++++++++++++++++++ py3-pathspec.yaml | 35 ++++++++++---- py3-pip.yaml | 67 +++++++++++++++++++++++++++ py3-pyyaml.yaml | 39 +++++++++++----- py3-scipy.yaml | 3 +- py3-setuptools.yaml | 62 +++++++++++++++++++++++++ py3-supported-python.yaml | 29 ++++++++++++ py3-tomli.yaml | 31 +++++++++---- py3-wheel.yaml | 45 ++++++++++++++---- py3.10-installer.yaml | 38 --------------- py3.10-pip.yaml | 44 ------------------ py3.10-setuptools.yaml | 39 ---------------- py3.11-installer.yaml | 40 ---------------- py3.11-pip.yaml | 46 ------------------- py3.11-setuptools.yaml | 39 ---------------- py3.12-installer.yaml | 39 ---------------- py3.12-pip.yaml | 45 ------------------ py3.12-setuptools.yaml | 39 ---------------- pypi-pyyaml/pyyaml-cython3.patch | 36 +++++++++++++++ 28 files changed, 665 insertions(+), 533 deletions(-) delete mode 100644 cython.yaml delete mode 100644 numpy.yaml create mode 100644 pipelines/py/collect-wheels.yaml create mode 100644 pipelines/py/install-compile.yaml create mode 100644 pipelines/py/pip-build-install.yaml create mode 100644 py3-cython.yaml create mode 100644 py3-installer.yaml create mode 100644 py3-numpy.yaml create mode 100644 py3-pip.yaml create mode 100644 py3-setuptools.yaml create mode 100644 py3-supported-python.yaml delete mode 100644 py3.10-installer.yaml delete mode 100644 py3.10-pip.yaml delete mode 100644 py3.10-setuptools.yaml delete mode 100644 py3.11-installer.yaml delete mode 100644 py3.11-pip.yaml delete mode 100644 py3.11-setuptools.yaml delete mode 100644 py3.12-installer.yaml delete mode 100644 py3.12-pip.yaml delete mode 100644 py3.12-setuptools.yaml create mode 100644 pypi-pyyaml/pyyaml-cython3.patch diff --git a/cython.yaml b/cython.yaml deleted file mode 100644 index 5fc7a08532c..00000000000 --- a/cython.yaml +++ /dev/null @@ -1,34 +0,0 @@ -package: - name: cython - version: 3.0.9 - epoch: 0 - description: Cython is an optimising static compiler for both the Python & the extended Cython programming languages. - copyright: - - license: Apache-2.0 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3-setuptools - - python3 - - python3-dev - -pipeline: - - uses: fetch - with: - expected-sha256: ca7758a3a87340a93f5bde4747e4a5ff1708cf2df407dee533b63ef6cc845777 - uri: https://github.com/cython/cython/archive/${{package.version}}.tar.gz - - - runs: | - python setup.py build - python setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -update: - enabled: true - github: - identifier: cython/cython diff --git a/numpy.yaml b/numpy.yaml deleted file mode 100644 index b3fcf043398..00000000000 --- a/numpy.yaml +++ /dev/null @@ -1,61 +0,0 @@ -package: - name: numpy - version: 1.26.4 - epoch: 2 - description: "The fundamental package for scientific computing with Python." - copyright: - - license: BSD-3-Clause - -environment: - contents: - packages: - - bash - - build-base - - busybox - - cython - - gfortran - - git - - meson - - openblas-dev - - py3-gpep517 - - py3-packaging - - py3-pip - - py3-setuptools - - py3-wheel - - python3 - - python3-dev - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/numpy/numpy - tag: v${{package.version}} - expected-commit: 9815c16f449e12915ef35a8255329ba26dacd5c0 - recurse-submodules: true - - - runs: | - python3 setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -subpackages: - - name: f2py - description: "f2py for numpy (for python3)" - pipeline: - - runs: | - mkdir -p "${{targets.subpkgdir}}"/usr/bin - mv "${{targets.destdir}}"/usr/bin/f2py* "${{targets.subpkgdir}}"/usr/bin - dependencies: - runtime: - - python3-dev - - numpy - -update: - enabled: true - github: - identifier: numpy/numpy - # There are some v2 pre-releases - tag-filter: v1 - strip-prefix: v - strip-suffix: .dev0 diff --git a/pipelines/py/collect-wheels.yaml b/pipelines/py/collect-wheels.yaml new file mode 100644 index 00000000000..98ab7cbee2b --- /dev/null +++ b/pipelines/py/collect-wheels.yaml @@ -0,0 +1,28 @@ +name: Collect wheels built by a python pipeline + +needs: + packages: + - util-linux + +inputs: + dest: + description: the destination + default: ${{targets.subpkgdir}}/usr/share/wheels + +pipeline: + - name: "pip build ${{inputs.python}}" + runs: | + found="" + set -- + for wheel in ./.wheels/*/*.whl; do + [ -f "$wheel" ] || continue + set -- "$@" "$wheel" + done + if [ $# -eq 0 ]; then + echo "No wheels found in $PWD/.wheels/*/*.whl" + exit 1 + fi + mkdir -p ${{inputs.dest}} + echo "Found wheels:" "$@" + sha256sum "$@" + cp -v "$@" "${{inputs.dest}}" diff --git a/pipelines/py/install-compile.yaml b/pipelines/py/install-compile.yaml new file mode 100644 index 00000000000..ef6113f9df4 --- /dev/null +++ b/pipelines/py/install-compile.yaml @@ -0,0 +1,25 @@ +name: Setup, install and compile python + +inputs: + python: + description: which python to use + required: true + dest: + description: the destination + default: ${{targets.subpkgdir}} + +pipeline: + - name: "setup ${{inputs.python}}" + runs: | + ${{inputs.python}} setup.py install --no-compile --root="${{targets.subpkgdir}}" --prefix=/usr + rm -Rf build/ + - name: "compileall ${{inputs.python}}" + runs: | + ${{inputs.python}} -m compileall --invalidation-mode=unchecked-hash -r100 ${{targets.subpkgdir}}/usr/lib/ + pyver=$(${{inputs.python}} -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))') + for bin in ${{targets.subpkgdir}}/usr/bin/*; do + [ -f "$bin" ] && [ -x "$bin" ] || continue + d="${{targets.subpkgdir}}/usr/pybin/$pyver" + [ -d "$d" ] || mkdir -p "$d" + mv "$bin" "$d/" + done diff --git a/pipelines/py/pip-build-install.yaml b/pipelines/py/pip-build-install.yaml new file mode 100644 index 00000000000..283beb3f390 --- /dev/null +++ b/pipelines/py/pip-build-install.yaml @@ -0,0 +1,44 @@ +name: Build and install python package with pip + +inputs: + python: + description: which python to use + required: true + dest: + description: the destination + default: ${{targets.subpkgdir}} + +pipeline: + - name: "pip build ${{inputs.python}}" + runs: | + export SOURCE_DATE_EPOCH=315532800 + py=${{inputs.python}} + pyver=$("$py" -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))') + sitepkgd=$("$py" -c 'import site; print(site.getsitepackages()[0])') + sitepkgd=${sitepkgd#/} + root=${{inputs.dest}} + wd="./.wheels/$pyver" + vr() { echo "execute:" "$@"; "$@"; } + + tmpd=$(mktemp -d) + trap "rm -Rf '$tmpd'" EXIT + distwheelsd="$tmpd/dist-wheels" + mkdir -p "$distwheelsd" + echo "$py is $pyver with site_packages dir '$sitepkgd'" + vr $py -m pip wheel --verbose "--wheel-dir=$wd" \ + "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps . + vr $py -m pip install --verbose \ + "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps \ + --force-reinstall --no-compile --prefix=/usr "--root=$root" "$wd"/*.whl + vr $py -m compileall --invalidation-mode=unchecked-hash -r100 "$root/$sitepkgd" + + - name: "compileall ${{inputs.python}}" + runs: | + pyver=$(${{inputs.python}} -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))') + for bin in ${{inputs.dest}}/usr/bin/*; do + [ -f "$bin" ] && [ -x "$bin" ] || continue + d="${{inputs.dest}}/usr/pybin/$pyver" + [ -d "$d" ] || mkdir -p "$d" + mv "$bin" "$d/" + echo "moved $bin -> $d/" + done diff --git a/py3-cython.yaml b/py3-cython.yaml new file mode 100644 index 00000000000..5f061afb8cc --- /dev/null +++ b/py3-cython.yaml @@ -0,0 +1,63 @@ +package: + name: py3-cython + version: 3.0.9 + epoch: 1 + description: Cython is an optimising static compiler for both the Python & the extended Cython programming languages. + copyright: + - license: Apache-2.0 + dependencies: + runtime: + - py3.12-${{vars.pypi-package}} + provides: + - cython=${{package.version}} + +vars: + pypi-package: cython + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" + +environment: + contents: + packages: + - build-base + - busybox + - ca-certificates-bundle + - py3-supported-pip + - py3-supported-python-dev + +pipeline: + - uses: fetch + with: + expected-sha256: ca7758a3a87340a93f5bde4747e4a5ff1708cf2df407dee533b63ef6cc845777 + uri: https://github.com/cython/cython/archive/${{package.version}}.tar.gz + +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + pipeline: + - uses: py/pip-build-install + with: + python: python${{range.key}} + - uses: strip + + - name: py3-wheels-${{vars.pypi-package}} + pipeline: + - uses: py/collect-wheels + + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} + +update: + enabled: true + github: + identifier: cython/cython diff --git a/py3-flit-core.yaml b/py3-flit-core.yaml index 93f01fdaf6f..64d72cc262d 100644 --- a/py3-flit-core.yaml +++ b/py3-flit-core.yaml @@ -1,13 +1,23 @@ package: name: py3-flit-core version: 3.9.0 - epoch: 1 + epoch: 2 description: "simple packaging tool for simple packages (core)" copyright: - license: BSD-3-Clause dependencies: runtime: - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: flit-core + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: @@ -15,8 +25,8 @@ environment: - build-base - busybox - ca-certificates-bundle - - py3-installer - - python3 + - py3-supported-installer + - py3-supported-python - wolfi-base pipeline: @@ -25,18 +35,27 @@ pipeline: uri: https://files.pythonhosted.org/packages/source/f/flit/flit-${{package.version}}.tar.gz expected-sha256: d75edf5eb324da20d53570a6a6f87f51e606eee8384925cd66a90611140844c7 - - runs: | - cd flit_core - python3 build_dists.py - - - runs: | - cd flit_core - python3 -m installer -d "${{targets.destdir}}" dist/flit_core-${{package.version}}-py3-none-any.whl - - - runs: | - rm -rf "${{targets.destdir}}"/usr/lib/python3*/site-packages/flit_core/tests +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - runs: | + cd flit_core + rm -Rf dist/ + python${{range.key}} build_dists.py + python${{range.key}} -m installer -d "${{targets.subpkgdir}}" \ + dist/flit_core-${{package.version}}-py3-none-any.whl + rm -rf "${{targets.subpkgdir}}"/usr/lib/python3*/site-packages/flit_core/tests + - uses: strip - - uses: strip + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} update: enabled: true diff --git a/py3-gpep517.yaml b/py3-gpep517.yaml index e9bc710eb26..9d0b3db0154 100644 --- a/py3-gpep517.yaml +++ b/py3-gpep517.yaml @@ -1,15 +1,23 @@ package: name: py3-gpep517 version: "15" - epoch: 2 + epoch: 3 description: "PEP517 build system support for distros" copyright: - license: MIT dependencies: runtime: - - py3-installer - - py3-tomli - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: gpep517 + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: @@ -17,7 +25,7 @@ environment: - build-base - busybox - ca-certificates-bundle - - python3 + - py3-supported-python - wolfi-base pipeline: @@ -26,14 +34,30 @@ pipeline: uri: https://github.com/mgorny/gpep517/archive/v${{package.version}}/gpep517-v${{package.version}}.tar.gz expected-sha256: 368fa8dfb0775589e3f85d34df18395c22980de8520dc6447e778fc1d0102787 - - runs: | - _site_pkgs="$(python3 -c 'import site; print(site.getsitepackages()[0])')" - _py3ver=$(python3 -c 'import sys; print("{}.{}".format(sys.version_info.major, sys.version_info.minor))') - - mkdir -p "${{targets.destdir}}"/"$_site_pkgs" - cp -a gpep517 "${{targets.destdir}}"/$_site_pkgs/ +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: Easily download, build, install, upgrade, and uninstall Python packages + pipeline: + - runs: | + sitedir=usr/lib/python${{range.key}}/site-packages/ + mkdir -p "${{targets.subpkgdir}}/$sitedir" + cp -a gpep517 "${{targets.subpkgdir}}/$sitedir" + python${{range.key}} -m compileall --invalidation-mode=unchecked-hash -r100 \ + ${{targets.subpkgdir}}/usr/lib/ + - uses: strip + dependencies: + runtime: + - py${{range.key}}-tomli + - py${{range.key}}-installer - - uses: strip + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} update: enabled: true diff --git a/py3-installer.yaml b/py3-installer.yaml new file mode 100644 index 00000000000..ddb34cb4748 --- /dev/null +++ b/py3-installer.yaml @@ -0,0 +1,59 @@ +package: + name: py3-installer + version: 0.7.0 + epoch: 6 + description: A library for installing Python wheels. + copyright: + - license: "MIT" + dependencies: + runtime: + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: installer + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" + +environment: + contents: + packages: + - build-base + - busybox + - ca-certificates-bundle + - py3-supported-pip + - py3-supported-python + - wolfi-base + +pipeline: + - uses: git-checkout + with: + repository: https://github.com/pypa/installer.git + tag: ${{package.version}} + expected-commit: b1d39180f8548820d09ce992dfadff0a42242c48 + +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - runs: python${{range.key}} -m pip install . "--root=${{targets.subpkgdir}}" --prefix=/usr + - uses: strip + + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} + +update: + enabled: true + github: + identifier: pypa/installer + use-tag: true diff --git a/py3-numpy.yaml b/py3-numpy.yaml new file mode 100644 index 00000000000..10dc8822372 --- /dev/null +++ b/py3-numpy.yaml @@ -0,0 +1,71 @@ +package: + name: py3-numpy + version: 1.26.4 + epoch: 3 + description: "The fundamental package for scientific computing with Python." + copyright: + - license: BSD-3-Clause + dependencies: + runtime: + - py3.12-${{vars.pypi-package}} + provides: + - numpy + - f2py + +vars: + pypi-package: numpy + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" + +environment: + contents: + packages: + - bash + - build-base + - busybox + - gfortran + - git + - meson + - openblas-dev + - py3-supported-cython + - py3-supported-pip + - py3-supported-python-dev + - wolfi-base + +pipeline: + - uses: git-checkout + with: + repository: https://github.com/numpy/numpy + tag: v${{package.version}} + expected-commit: 9815c16f449e12915ef35a8255329ba26dacd5c0 + recurse-submodules: true + +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - runs: python${{range.key}} -m pip install . "--root=${{targets.subpkgdir}}" --prefix=/usr + - uses: strip + + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} + +update: + enabled: true + github: + identifier: numpy/numpy + # There are some v2 pre-releases + tag-filter: v1 + strip-prefix: v + strip-suffix: .dev0 diff --git a/py3-pathspec.yaml b/py3-pathspec.yaml index 71262c44ea6..c44e1a4ec82 100644 --- a/py3-pathspec.yaml +++ b/py3-pathspec.yaml @@ -1,13 +1,23 @@ package: name: py3-pathspec version: 0.12.1 - epoch: 0 + epoch: 1 description: "Utility library for gitignore style pattern matching of file paths" copyright: - license: MPL-2.0 dependencies: runtime: - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: pathspec + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: @@ -15,8 +25,8 @@ environment: - build-base - busybox - ca-certificates-bundle - - py3-setuptools - - python3 + - py3-supported-pip + - py3-supported-python - wolfi-base pipeline: @@ -25,13 +35,18 @@ pipeline: uri: https://files.pythonhosted.org/packages/source/p/pathspec/pathspec-${{package.version}}.tar.gz expected-sha512: 19d1a8ae5936e76eacd691b0e2fb33d0de376556751a9ae007f6b09d0aa36af7d171101df620cab3e93c9913be578ff3d6bba9d4460351ac248e5c9e015380ad - - runs: | - python3 setup.py build - - - runs: | - python3 setup.py install --prefix=/usr --root="${{targets.destdir}}" --skip-build +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: Easily download, build, install, upgrade, and uninstall Python packages + pipeline: + - uses: py/pip-build-install + with: + python: python${{range.key}} - - uses: strip + - name: py3-wheels-${{vars.pypi-package}} + pipeline: + - uses: py/collect-wheels update: enabled: true diff --git a/py3-pip.yaml b/py3-pip.yaml new file mode 100644 index 00000000000..a54c6001416 --- /dev/null +++ b/py3-pip.yaml @@ -0,0 +1,67 @@ +package: + name: py3-pip + version: "24.0" + epoch: 2 + description: The PyPA recommended tool for installing Python packages. + copyright: + - license: MIT + dependencies: + runtime: + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: pip + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" + +environment: + contents: + packages: + - build-base + - busybox + - ca-certificates-bundle + - py3-supported-python + - py3-supported-setuptools + - wolfi-base + +pipeline: + - uses: git-checkout + with: + repository: https://github.com/pypa/pip + expected-commit: ef78c129b1a966dbbbdb8ebfffc43723e89110d1 + tag: ${{package.version}} + +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: Easily download, build, install, upgrade, and uninstall Python packages + pipeline: + - uses: py/install-compile + with: + python: python${{range.key}} + - uses: strip + dependencies: + runtime: + - py${{range.key}}-flit-core + - py${{range.key}}-setuptools + - py${{range.key}}-wheel + + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} + +update: + enabled: true + shared: true + github: + identifier: pypa/pip + use-tag: true diff --git a/py3-pyyaml.yaml b/py3-pyyaml.yaml index 3d649a37327..c071cfd20fe 100644 --- a/py3-pyyaml.yaml +++ b/py3-pyyaml.yaml @@ -1,13 +1,23 @@ package: name: py3-pyyaml version: 6.0.1 - epoch: 4 + epoch: 5 description: "Python3 bindings for YAML" copyright: - license: MIT dependencies: runtime: - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: pyyaml + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: @@ -15,10 +25,9 @@ environment: - build-base - busybox - ca-certificates-bundle - - cython - - py3-setuptools - - python3 - - python3-dev + - py3-supported-cython + - py3-supported-pip + - py3-supported-python-dev - wolfi-base - yaml-dev @@ -34,13 +43,19 @@ pipeline: with: patches: pyyaml-cython3.patch - - runs: | - python3 setup.py build - - - runs: | - python3 setup.py install --prefix=/usr --root="${{targets.destdir}}" +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - uses: py/pip-build-install + with: + python: python${{range.key}} + - uses: strip - - uses: strip + - name: py3-wheels-${{vars.pypi-package}} + pipeline: + - uses: py/collect-wheels update: enabled: true diff --git a/py3-scipy.yaml b/py3-scipy.yaml index e4db2180f0f..2d37bc66fa7 100644 --- a/py3-scipy.yaml +++ b/py3-scipy.yaml @@ -1,7 +1,7 @@ package: name: py3-scipy version: 1.11.4 - epoch: 1 + epoch: 2 description: Fundamental algorithms for scientific computing in Python copyright: - license: BSD-3-Clause @@ -17,7 +17,6 @@ environment: - busybox - ca-certificates-bundle - cython~0 - - f2py - gfortran - numpy - openblas-dev diff --git a/py3-setuptools.yaml b/py3-setuptools.yaml new file mode 100644 index 00000000000..9d48a4114e1 --- /dev/null +++ b/py3-setuptools.yaml @@ -0,0 +1,62 @@ +package: + name: py3-setuptools + version: 69.2.0 + epoch: 1 + description: Easily download, build, install, upgrade, and uninstall Python packages + copyright: + - license: "MIT" + dependencies: + runtime: + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: setuptools + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" + +environment: + contents: + packages: + - build-base + - busybox + - ca-certificates-bundle + - py3-supported-python + - wolfi-base + environment: + # This is needed to work around the error "ValueError: ZIP does not support timestamps before 1980" + SOURCE_DATE_EPOCH: 315532800 + +pipeline: + - uses: fetch + with: + expected-sha256: 0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e + uri: https://files.pythonhosted.org/packages/source/s/setuptools/setuptools-${{package.version}}.tar.gz + +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: Easily download, build, install, upgrade, and uninstall Python packages + pipeline: + - uses: py/install-compile + with: + python: python${{range.key}} + - uses: strip + + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} + +update: + enabled: true + shared: true + release-monitor: + identifier: 4021 diff --git a/py3-supported-python.yaml b/py3-supported-python.yaml new file mode 100644 index 00000000000..230797b2786 --- /dev/null +++ b/py3-supported-python.yaml @@ -0,0 +1,29 @@ +package: + name: py3-supported-python + version: 1 + epoch: 1 + description: Easily download, build, install, upgrade, and uninstall Python packages + copyright: + - license: "MIT" + dependencies: + runtime: + - python-3.10-base + - python-3.11-base + - python-3.12-base + +environment: + contents: + packages: + - build-base + - busybox + +subpackages: + - name: ${{package.name}}-dev + dependencies: + runtime: + - python-3.10-base-dev + - python-3.11-base-dev + - python-3.12-base-dev + +update: + enabled: false diff --git a/py3-tomli.yaml b/py3-tomli.yaml index cfa1a19a541..825b6d78292 100644 --- a/py3-tomli.yaml +++ b/py3-tomli.yaml @@ -1,13 +1,23 @@ package: name: py3-tomli version: 2.0.1 - epoch: 4 + epoch: 5 description: "TOML parser" copyright: - license: MIT dependencies: runtime: - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: tomli + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: @@ -15,8 +25,8 @@ environment: - build-base - busybox - ca-certificates-bundle - - py3-installer - - python3 + - py3-supported-installer + - py3-supported-python - wolfi-base pipeline: @@ -26,10 +36,15 @@ pipeline: expected-sha256: 939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc extract: false - - runs: | - python3 -m installer -d "${{targets.destdir}}" tomli-${{package.version}}-py3-none-any.whl - - - uses: strip +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: Easily download, build, install, upgrade, and uninstall Python packages + pipeline: + - runs: | + python${{range.key}} -m installer -d "${{targets.subpkgdir}}" \ + tomli-${{package.version}}-py3-none-any.whl + - uses: strip update: enabled: true diff --git a/py3-wheel.yaml b/py3-wheel.yaml index b7dfeea24d2..1ace551f272 100644 --- a/py3-wheel.yaml +++ b/py3-wheel.yaml @@ -1,23 +1,33 @@ package: name: py3-wheel version: 0.43.0 - epoch: 0 + epoch: 1 description: "built-package format for Python" copyright: - license: MIT dependencies: runtime: - - py3-installer - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: wheel + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: packages: - busybox - ca-certificates-bundle - - py3-flit-core - - py3-gpep517 - - python3 + - py3-supported-flit-core + - py3-supported-gpep517 + - py3-supported-installer + - py3-supported-python - wolfi-base pipeline: @@ -26,11 +36,26 @@ pipeline: uri: https://github.com/pypa/wheel/archive/refs/tags/${{package.version}}.tar.gz expected-sha256: 23060d7cc8afafc2930554624b4bae7d58031830672048622c926675ab91e3b0 - - runs: | - python3 -m gpep517 build-wheel --wheel-dir dist --output-fd 1 - python3 -m installer -d "${{targets.destdir}}" dist/wheel-${{package.version}}-py3-none-any.whl +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - runs: | + distd=dist-${{range.key}} + rm -Rf $distd + python${{range.key}} -m gpep517 build-wheel --wheel-dir=$distd --output-fd 1 + python${{range.key}} -m installer -d "${{targets.subpkgdir}}" ${distd}/*.whl + rm ${{targets.subpkgdir}}/usr/bin/wheel + - uses: strip - - uses: strip + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} update: enabled: true diff --git a/py3.10-installer.yaml b/py3.10-installer.yaml deleted file mode 100644 index 58f59fcad57..00000000000 --- a/py3.10-installer.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Generated from https://pypi.org/project/installer/ -package: - name: py3.10-installer - version: 0.7.0 # When bumping this, also bump the provides below! - epoch: 4 - description: A library for installing Python wheels. - copyright: - - license: "MIT" - dependencies: - runtime: - - python-3.10 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.10-pip - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/installer.git - tag: ${{package.version}} - expected-commit: b1d39180f8548820d09ce992dfadff0a42242c48 - - - name: Python Install - runs: pip install . --prefix=/usr --root=${{targets.destdir}} - - - uses: strip - -update: - enabled: true - github: - identifier: pypa/installer - use-tag: true diff --git a/py3.10-pip.yaml b/py3.10-pip.yaml deleted file mode 100644 index a011bc5f78f..00000000000 --- a/py3.10-pip.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Generated from https://pypi.org/project/pip/ -package: - name: py3.10-pip - version: "24.0" - epoch: 0 - description: The PyPA recommended tool for installing Python packages. - copyright: - - license: MIT - dependencies: - runtime: - - python-3.10 - - py3.10-setuptools - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.10-setuptools - - python-3.10 - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/pip - expected-commit: ef78c129b1a966dbbbdb8ebfffc43723e89110d1 - tag: ${{package.version}} - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -update: - enabled: true - shared: true - github: - identifier: pypa/pip - use-tag: true diff --git a/py3.10-setuptools.yaml b/py3.10-setuptools.yaml deleted file mode 100644 index e99c012f3ba..00000000000 --- a/py3.10-setuptools.yaml +++ /dev/null @@ -1,39 +0,0 @@ -package: - name: py3.10-setuptools - version: 69.2.0 - epoch: 0 - description: Easily download, build, install, upgrade, and uninstall Python packages - copyright: - - license: "MIT" - dependencies: - runtime: - - python-3.10 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - python-3.10 - - wolfi-base - -pipeline: - - uses: fetch - with: - expected-sha256: 0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e - uri: https://files.pythonhosted.org/packages/source/s/setuptools/setuptools-${{package.version}}.tar.gz - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -update: - enabled: true - shared: true - release-monitor: - identifier: 4021 diff --git a/py3.11-installer.yaml b/py3.11-installer.yaml deleted file mode 100644 index 01e4ceba23a..00000000000 --- a/py3.11-installer.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Generated from https://pypi.org/project/installer/ -package: - name: py3.11-installer - version: 0.7.0 # When bumping this, also bump the provides below! - epoch: 3 - description: A library for installing Python wheels. - copyright: - - license: "MIT" - dependencies: - provides: - - py3-installer=${{package.full-version}} - runtime: - - python-3.11 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.11-pip - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/installer.git - tag: ${{package.version}} - expected-commit: b1d39180f8548820d09ce992dfadff0a42242c48 - - - name: Python Install - runs: pip install . --prefix=/usr --root=${{targets.destdir}} - - - uses: strip - -update: - enabled: true - github: - identifier: pypa/installer - use-tag: true diff --git a/py3.11-pip.yaml b/py3.11-pip.yaml deleted file mode 100644 index 7f5330da208..00000000000 --- a/py3.11-pip.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Generated from https://pypi.org/project/pip/ -package: - name: py3.11-pip - version: "24.0" - epoch: 0 - description: The PyPA recommended tool for installing Python packages. - copyright: - - license: MIT - dependencies: - provides: - - py3-pip=23.1.999 - runtime: - - python-3.11 - - py3.11-setuptools - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.11-setuptools - - python-3.11 - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/pip - expected-commit: ef78c129b1a966dbbbdb8ebfffc43723e89110d1 - tag: ${{package.version}} - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -update: - enabled: true - shared: true - github: - identifier: pypa/pip - use-tag: true diff --git a/py3.11-setuptools.yaml b/py3.11-setuptools.yaml deleted file mode 100644 index aaf2fad7cc4..00000000000 --- a/py3.11-setuptools.yaml +++ /dev/null @@ -1,39 +0,0 @@ -package: - name: py3.11-setuptools - version: 69.2.0 # When bumping this, also bump the provides below! - epoch: 0 - description: Easily download, build, install, upgrade, and uninstall Python packages - copyright: - - license: "MIT" - dependencies: - provides: - - py3-setuptools=69.0.0 # force the provides version lower than the py3.12-setuptools provides, so that the apk resolver favours py3.12-setuptools - runtime: - - python-3.11 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - python-3.11 - - wolfi-base - -pipeline: - - uses: fetch - with: - expected-sha256: 0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e - uri: https://files.pythonhosted.org/packages/source/s/setuptools/setuptools-${{package.version}}.tar.gz - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" - -update: - enabled: true - shared: true - release-monitor: - identifier: 4021 diff --git a/py3.12-installer.yaml b/py3.12-installer.yaml deleted file mode 100644 index 2bd39d4fc34..00000000000 --- a/py3.12-installer.yaml +++ /dev/null @@ -1,39 +0,0 @@ -package: - name: py3.12-installer - version: 0.7.0 - epoch: 5 - description: A library for installing Python wheels. - copyright: - - license: "MIT" - dependencies: - provides: - - py3-installer=${{package.full-version}} - runtime: - - python-3.12 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.12-pip - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/installer.git - tag: ${{package.version}} - expected-commit: b1d39180f8548820d09ce992dfadff0a42242c48 - - - name: Python Install - runs: pip install . --prefix=/usr --root=${{targets.destdir}} - - - uses: strip - -update: - enabled: true - github: - identifier: pypa/installer - use-tag: true diff --git a/py3.12-pip.yaml b/py3.12-pip.yaml deleted file mode 100644 index d394f8d3692..00000000000 --- a/py3.12-pip.yaml +++ /dev/null @@ -1,45 +0,0 @@ -package: - name: py3.12-pip - version: "24.0" - epoch: 1 - description: The PyPA recommended tool for installing Python packages. - copyright: - - license: MIT - dependencies: - provides: - - py3-pip=${{package.full-version}} - runtime: - - py3.12-setuptools - - python-3.12 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.12-setuptools - - python-3.12 - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/pip - expected-commit: ef78c129b1a966dbbbdb8ebfffc43723e89110d1 - tag: ${{package.version}} - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -update: - enabled: true - shared: true - github: - identifier: pypa/pip - use-tag: true diff --git a/py3.12-setuptools.yaml b/py3.12-setuptools.yaml deleted file mode 100644 index b1f776f3ff2..00000000000 --- a/py3.12-setuptools.yaml +++ /dev/null @@ -1,39 +0,0 @@ -package: - name: py3.12-setuptools - version: 69.2.0 - epoch: 0 - description: Easily download, build, install, upgrade, and uninstall Python packages - copyright: - - license: "MIT" - dependencies: - provides: - - py3-setuptools=${{package.full-version}} - runtime: - - python-3.12 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - python-3.12 - - wolfi-base - -pipeline: - - uses: fetch - with: - expected-sha256: 0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e - uri: https://files.pythonhosted.org/packages/source/s/setuptools/setuptools-${{package.version}}.tar.gz - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" --skip-build - -update: - enabled: true - shared: true - release-monitor: - identifier: 4021 diff --git a/pypi-pyyaml/pyyaml-cython3.patch b/pypi-pyyaml/pyyaml-cython3.patch new file mode 100644 index 00000000000..7ffbc243d10 --- /dev/null +++ b/pypi-pyyaml/pyyaml-cython3.patch @@ -0,0 +1,36 @@ +https://bugs.gentoo.org/898680 +https://github.com/yaml/pyyaml/pull/731 + +From 17dc5b6cd96dcfe64fd71789c771ca9b96d260e5 Mon Sep 17 00:00:00 2001 +From: "Andrew J. Hesford" +Date: Fri, 21 Jul 2023 09:50:00 -0400 +Subject: [PATCH] Fix builds with Cython 3 + +This is a *de minimis* fix for building with Cython 3. Recent Cython<3 +releases provided `Cython.Distutils.build_ext` as an alias to +`Cython.Distutils.old_build_ext.old_build_ext`; Cython 3 drops this +alias and instead uses a wholly new `Cython.Distutils.build_ext` that +does not provide the `cython_sources` function used in `setup.py`. + +Explicitly importing `old_build_ext` preserves the existing behavior for +recent Cython<3 and uses the correct behavior for Cython 3. Should the +import fail (*e.g.*, because the version of Cython available predates +the availability of `old_build_ext`), the import falls back to just +`Cython.Distutils.build_ext`. + +Signed-off-by: Andrew J. Hesford +--- a/setup.py ++++ b/setup.py +@@ -82,7 +82,11 @@ + with_cython = True + try: + from Cython.Distutils.extension import Extension as _Extension +- from Cython.Distutils import build_ext as _build_ext ++ try: ++ from Cython.Distutils.old_build_ext import old_build_ext as _build_ext ++ except ImportError: ++ from Cython.Distutils import build_ext as _build_ext ++ + with_cython = True + except ImportError: + if with_cython: \ No newline at end of file From d259615c5a20617262db34dae91c6f5fdb7845ca Mon Sep 17 00:00:00 2001 From: user Date: Fri, 29 Mar 2024 15:32:54 +0000 Subject: [PATCH 02/39] Resolve the dependency loop. flit-core has bootstrap_install.py to handle this, so we use it. pip doesn't stricktly depend on wheel, although buidling many projects assumes that it is there. --- pipelines/py/pip-build-install.yaml | 2 ++ py3-cython.yaml | 1 + py3-flit-core.yaml | 17 +++++++++++++---- py3-pip.yaml | 1 - 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pipelines/py/pip-build-install.yaml b/pipelines/py/pip-build-install.yaml index 283beb3f390..a675fb042f3 100644 --- a/pipelines/py/pip-build-install.yaml +++ b/pipelines/py/pip-build-install.yaml @@ -22,6 +22,8 @@ pipeline: tmpd=$(mktemp -d) trap "rm -Rf '$tmpd'" EXIT + + # --find-links to an empty dir and --no-index makes pip fully "offline" distwheelsd="$tmpd/dist-wheels" mkdir -p "$distwheelsd" echo "$py is $pyver with site_packages dir '$sitepkgd'" diff --git a/py3-cython.yaml b/py3-cython.yaml index 5f061afb8cc..79d17d6518e 100644 --- a/py3-cython.yaml +++ b/py3-cython.yaml @@ -29,6 +29,7 @@ environment: - ca-certificates-bundle - py3-supported-pip - py3-supported-python-dev + - py3-supported-wheel pipeline: - uses: fetch diff --git a/py3-flit-core.yaml b/py3-flit-core.yaml index 64d72cc262d..36b2b0fa66c 100644 --- a/py3-flit-core.yaml +++ b/py3-flit-core.yaml @@ -25,7 +25,6 @@ environment: - build-base - busybox - ca-certificates-bundle - - py3-supported-installer - py3-supported-python - wolfi-base @@ -43,12 +42,22 @@ subpackages: - runs: | cd flit_core rm -Rf dist/ - python${{range.key}} build_dists.py - python${{range.key}} -m installer -d "${{targets.subpkgdir}}" \ - dist/flit_core-${{package.version}}-py3-none-any.whl + python${{range.key}} -m flit_core.wheel + python${{range.key}} bootstrap_install.py \ + --install-root=${{targets.subpkgdir}} \ + dist/flit_core-${{package.version}}-py3-none-any.whl + mkdir -p /home/build/.wheels/${{range.key}}/ + cp dist/flit_core-${{package.version}}-py3-none-any.whl \ + /home/build/.wheels/${{range.key}}/ + python${{range.key}} -m compileall --invalidation-mode=unchecked-hash -r100 \ + ${{targets.subpkgdir}} rm -rf "${{targets.subpkgdir}}"/usr/lib/python3*/site-packages/flit_core/tests - uses: strip + - name: py3-wheels-${{vars.pypi-package}} + pipeline: + - uses: py/collect-wheels + - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. dependencies: diff --git a/py3-pip.yaml b/py3-pip.yaml index a54c6001416..2e4ee847cea 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -49,7 +49,6 @@ subpackages: runtime: - py${{range.key}}-flit-core - py${{range.key}}-setuptools - - py${{range.key}}-wheel - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. From 3a964132ebe26225b8510ec2875b506a8e617633 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Tue, 23 Apr 2024 22:20:56 +0100 Subject: [PATCH 03/39] Fun stuff Signed-off-by: Dimitri John Ledkov --- py3-pip.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/py3-pip.yaml b/py3-pip.yaml index 2e4ee847cea..6f6a8e1b196 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -49,6 +49,11 @@ subpackages: runtime: - py${{range.key}}-flit-core - py${{range.key}}-setuptools + scriptlets: + post-install: | + #!/usr/bin/python${{range.key}} + import glob, shutil + for b in glob.glob("/usr/pybin/${{range.key}}/*"): shutil.copy2(b, "/usr/bin/") - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. From 820e540033eeaa4648f7acd4568d31824b480544 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Fri, 26 Apr 2024 23:22:26 +0100 Subject: [PATCH 04/39] Revert "Fun stuff" hahhah just kidding This reverts commit 3a964132ebe26225b8510ec2875b506a8e617633. --- py3-pip.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/py3-pip.yaml b/py3-pip.yaml index 6f6a8e1b196..2e4ee847cea 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -49,11 +49,6 @@ subpackages: runtime: - py${{range.key}}-flit-core - py${{range.key}}-setuptools - scriptlets: - post-install: | - #!/usr/bin/python${{range.key}} - import glob, shutil - for b in glob.glob("/usr/pybin/${{range.key}}/*"): shutil.copy2(b, "/usr/bin/") - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. From 27b22d6f1c610d70f621c5b5ae2789f5d6cd9c21 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Fri, 26 Apr 2024 23:35:39 +0100 Subject: [PATCH 05/39] Drop pybins and allow to install them all Requires https://github.com/chainguard-dev/melange/pull/1166 Drop pybins, instead build all packages with commands in /usr/bin. Set replaces, replaces-priority to ensure they are apk installable. Also set provider-priority such that `apk add py3-pip` transparently installs py3.12-pip alone. This is all great, however not at all supported by apko. But imho should be supported. As one simply has to sort packages in replaces priority order prior to streaming. And allow overriding files between packages that have replaces declared. ```console ... (13/25) Installing python-3.10-base (3.10.14-r1) (14/25) Installing py3.10-flit-core (3.9.0-r2) (15/25) Installing py3.10-setuptools (69.5.1-r1) (16/25) Installing py3.10-pip (24.0-r2) (17/25) Installing python-3.11-base (3.11.9-r1) (18/25) Installing py3.11-flit-core (3.9.0-r2) (19/25) Installing py3.11-setuptools (69.5.1-r1) (20/25) Installing py3.11-pip (24.0-r2) (21/25) Installing python-3.12-base (3.12.3-r1) (22/25) Installing py3.12-flit-core (3.9.0-r2) (23/25) Installing py3.12-setuptools (69.5.1-r1) (24/25) Installing py3.12-pip (24.0-r2) (25/25) Installing py3-supported-pip (24.0-r2) OK: 197 MiB in 40 packages -rwxrwxr-x 1 root root 946 Apr 26 22:22 /usr/bin/pip3.12 -rwxrwxr-x 1 root root 946 Apr 26 22:22 /usr/bin/pip3.11 -rwxrwxr-x 1 root root 946 Apr 26 22:22 /usr/bin/pip3.10 -rwxrwxr-x 1 root root 940 Apr 26 22:22 /usr/bin/pip3 -rwxrwxr-x 1 root root 938 Apr 26 22:22 /usr/bin/pip ``` --- pipelines/py/install-compile.yaml | 7 ------- pipelines/py/pip-build-install.yaml | 11 ----------- py3-pip.yaml | 24 +++++++++++++++++++++--- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pipelines/py/install-compile.yaml b/pipelines/py/install-compile.yaml index ef6113f9df4..fd1dd68161f 100644 --- a/pipelines/py/install-compile.yaml +++ b/pipelines/py/install-compile.yaml @@ -16,10 +16,3 @@ pipeline: - name: "compileall ${{inputs.python}}" runs: | ${{inputs.python}} -m compileall --invalidation-mode=unchecked-hash -r100 ${{targets.subpkgdir}}/usr/lib/ - pyver=$(${{inputs.python}} -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))') - for bin in ${{targets.subpkgdir}}/usr/bin/*; do - [ -f "$bin" ] && [ -x "$bin" ] || continue - d="${{targets.subpkgdir}}/usr/pybin/$pyver" - [ -d "$d" ] || mkdir -p "$d" - mv "$bin" "$d/" - done diff --git a/pipelines/py/pip-build-install.yaml b/pipelines/py/pip-build-install.yaml index a675fb042f3..13d38b63b5f 100644 --- a/pipelines/py/pip-build-install.yaml +++ b/pipelines/py/pip-build-install.yaml @@ -33,14 +33,3 @@ pipeline: "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps \ --force-reinstall --no-compile --prefix=/usr "--root=$root" "$wd"/*.whl vr $py -m compileall --invalidation-mode=unchecked-hash -r100 "$root/$sitepkgd" - - - name: "compileall ${{inputs.python}}" - runs: | - pyver=$(${{inputs.python}} -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))') - for bin in ${{inputs.dest}}/usr/bin/*; do - [ -f "$bin" ] && [ -x "$bin" ] || continue - d="${{inputs.dest}}/usr/pybin/$pyver" - [ -d "$d" ] || mkdir -p "$d" - mv "$bin" "$d/" - echo "moved $bin -> $d/" - done diff --git a/py3-pip.yaml b/py3-pip.yaml index 2e4ee847cea..d5d84a14b48 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -5,7 +5,11 @@ package: description: The PyPA recommended tool for installing Python packages. copyright: - license: MIT + checks: + disabled: + - empty dependencies: + provider-priority: 0 runtime: - py3.12-${{vars.pypi-package}} @@ -15,9 +19,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "312" environment: contents: @@ -45,7 +49,21 @@ subpackages: with: python: python${{range.key}} - uses: strip + # ideally want If syntax here + # if: ${{range.key}} != 3.12 + # options: + # no-commands: true + options: + no-commands: true dependencies: + provides: + - py3-pip + replaces: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} + provider-priority: ${{range.value}} + replaces-priority: ${{range.value}} runtime: - py${{range.key}}-flit-core - py${{range.key}}-setuptools From 1aaf253dc62c5da1d44bfc02f88b7d834712e050 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 1 May 2024 00:12:06 +0100 Subject: [PATCH 06/39] wip --- py3-installer.yaml | 1 + py3-pip.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/py3-installer.yaml b/py3-installer.yaml index ddb34cb4748..ed8681aa030 100644 --- a/py3-installer.yaml +++ b/py3-installer.yaml @@ -41,6 +41,7 @@ subpackages: name: py${{range.key}}-${{vars.pypi-package}} description: ${{vars.pypi-package}} installed for python${{range.key}} pipeline: + - runs: head -n1 /usr/bin/pip* - runs: python${{range.key}} -m pip install . "--root=${{targets.subpkgdir}}" --prefix=/usr - uses: strip diff --git a/py3-pip.yaml b/py3-pip.yaml index d5d84a14b48..a74d2d07d7f 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -72,9 +72,9 @@ subpackages: description: meta package providing ${{vars.pypi-package}} for supported python versions. dependencies: runtime: - - py3.10-${{vars.pypi-package}} - - py3.11-${{vars.pypi-package}} - - py3.12-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}}=${{package.full-version}} + - py3.11-${{vars.pypi-package}}=${{package.full-version}} + - py3.10-${{vars.pypi-package}}=${{package.full-version}} update: enabled: true From dc75866a6a6f9b342e066a2e75161141cd92da98 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 15 Mar 2024 10:03:00 +0000 Subject: [PATCH 07/39] Make several core python packages build for mulitple python versions. At this point most things should use the py/pip-build-install pipeline and generally look like py3-pyyaml Also: * rename cython.yaml, numpy.yaml to py3- These are python packages and present in pypi, for consistency, name them like other python packages. Co-authored-by: Dimitri John Ledkov --- cython.yaml | 35 -------------- numpy.yaml | 61 ------------------------- pipelines/py/collect-wheels.yaml | 28 ++++++++++++ pipelines/py/install-compile.yaml | 25 ++++++++++ pipelines/py/pip-build-install.yaml | 44 ++++++++++++++++++ py3-cython.yaml | 63 +++++++++++++++++++++++++ py3-flit-core.yaml | 49 ++++++++++++++------ py3-gpep517.yaml | 48 ++++++++++++++----- py3-installer.yaml | 59 ++++++++++++++++++++++++ py3-numpy.yaml | 71 +++++++++++++++++++++++++++++ py3-pathspec.yaml | 35 ++++++++++---- py3-pip.yaml | 67 +++++++++++++++++++++++++++ py3-pyyaml.yaml | 39 +++++++++++----- py3-setuptools.yaml | 62 +++++++++++++++++++++++++ py3-supported-python.yaml | 29 ++++++++++++ py3-tomli.yaml | 31 +++++++++---- py3-wheel.yaml | 43 +++++++++++++---- py3.10-installer.yaml | 38 --------------- py3.10-pip.yaml | 46 ------------------- py3.10-setuptools.yaml | 39 ---------------- py3.11-installer.yaml | 40 ---------------- py3.11-pip.yaml | 48 ------------------- py3.11-setuptools.yaml | 39 ---------------- py3.12-installer.yaml | 39 ---------------- py3.12-pip.yaml | 47 ------------------- py3.12-setuptools.yaml | 39 ---------------- 26 files changed, 627 insertions(+), 537 deletions(-) delete mode 100644 cython.yaml delete mode 100644 numpy.yaml create mode 100644 pipelines/py/collect-wheels.yaml create mode 100644 pipelines/py/install-compile.yaml create mode 100644 pipelines/py/pip-build-install.yaml create mode 100644 py3-cython.yaml create mode 100644 py3-installer.yaml create mode 100644 py3-numpy.yaml create mode 100644 py3-pip.yaml create mode 100644 py3-setuptools.yaml create mode 100644 py3-supported-python.yaml delete mode 100644 py3.10-installer.yaml delete mode 100644 py3.10-pip.yaml delete mode 100644 py3.10-setuptools.yaml delete mode 100644 py3.11-installer.yaml delete mode 100644 py3.11-pip.yaml delete mode 100644 py3.11-setuptools.yaml delete mode 100644 py3.12-installer.yaml delete mode 100644 py3.12-pip.yaml delete mode 100644 py3.12-setuptools.yaml diff --git a/cython.yaml b/cython.yaml deleted file mode 100644 index de6de25ccf8..00000000000 --- a/cython.yaml +++ /dev/null @@ -1,35 +0,0 @@ -package: - name: cython - version: 3.0.10 - epoch: 2 - description: Cython is an optimising static compiler for both the Python & the extended Cython programming languages. - copyright: - - license: Apache-2.0 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3-setuptools - - python3 - - python3-dev - -pipeline: - - uses: git-checkout - with: - expected-commit: 7ae85318297ed306b84ecc72d0279e27090b25f3 - repository: https://github.com/cython/cython - tag: ${{package.version}} - - - runs: | - python setup.py build - python setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -update: - enabled: true - github: - identifier: cython/cython diff --git a/numpy.yaml b/numpy.yaml deleted file mode 100644 index b3fcf043398..00000000000 --- a/numpy.yaml +++ /dev/null @@ -1,61 +0,0 @@ -package: - name: numpy - version: 1.26.4 - epoch: 2 - description: "The fundamental package for scientific computing with Python." - copyright: - - license: BSD-3-Clause - -environment: - contents: - packages: - - bash - - build-base - - busybox - - cython - - gfortran - - git - - meson - - openblas-dev - - py3-gpep517 - - py3-packaging - - py3-pip - - py3-setuptools - - py3-wheel - - python3 - - python3-dev - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/numpy/numpy - tag: v${{package.version}} - expected-commit: 9815c16f449e12915ef35a8255329ba26dacd5c0 - recurse-submodules: true - - - runs: | - python3 setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -subpackages: - - name: f2py - description: "f2py for numpy (for python3)" - pipeline: - - runs: | - mkdir -p "${{targets.subpkgdir}}"/usr/bin - mv "${{targets.destdir}}"/usr/bin/f2py* "${{targets.subpkgdir}}"/usr/bin - dependencies: - runtime: - - python3-dev - - numpy - -update: - enabled: true - github: - identifier: numpy/numpy - # There are some v2 pre-releases - tag-filter: v1 - strip-prefix: v - strip-suffix: .dev0 diff --git a/pipelines/py/collect-wheels.yaml b/pipelines/py/collect-wheels.yaml new file mode 100644 index 00000000000..98ab7cbee2b --- /dev/null +++ b/pipelines/py/collect-wheels.yaml @@ -0,0 +1,28 @@ +name: Collect wheels built by a python pipeline + +needs: + packages: + - util-linux + +inputs: + dest: + description: the destination + default: ${{targets.subpkgdir}}/usr/share/wheels + +pipeline: + - name: "pip build ${{inputs.python}}" + runs: | + found="" + set -- + for wheel in ./.wheels/*/*.whl; do + [ -f "$wheel" ] || continue + set -- "$@" "$wheel" + done + if [ $# -eq 0 ]; then + echo "No wheels found in $PWD/.wheels/*/*.whl" + exit 1 + fi + mkdir -p ${{inputs.dest}} + echo "Found wheels:" "$@" + sha256sum "$@" + cp -v "$@" "${{inputs.dest}}" diff --git a/pipelines/py/install-compile.yaml b/pipelines/py/install-compile.yaml new file mode 100644 index 00000000000..ef6113f9df4 --- /dev/null +++ b/pipelines/py/install-compile.yaml @@ -0,0 +1,25 @@ +name: Setup, install and compile python + +inputs: + python: + description: which python to use + required: true + dest: + description: the destination + default: ${{targets.subpkgdir}} + +pipeline: + - name: "setup ${{inputs.python}}" + runs: | + ${{inputs.python}} setup.py install --no-compile --root="${{targets.subpkgdir}}" --prefix=/usr + rm -Rf build/ + - name: "compileall ${{inputs.python}}" + runs: | + ${{inputs.python}} -m compileall --invalidation-mode=unchecked-hash -r100 ${{targets.subpkgdir}}/usr/lib/ + pyver=$(${{inputs.python}} -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))') + for bin in ${{targets.subpkgdir}}/usr/bin/*; do + [ -f "$bin" ] && [ -x "$bin" ] || continue + d="${{targets.subpkgdir}}/usr/pybin/$pyver" + [ -d "$d" ] || mkdir -p "$d" + mv "$bin" "$d/" + done diff --git a/pipelines/py/pip-build-install.yaml b/pipelines/py/pip-build-install.yaml new file mode 100644 index 00000000000..283beb3f390 --- /dev/null +++ b/pipelines/py/pip-build-install.yaml @@ -0,0 +1,44 @@ +name: Build and install python package with pip + +inputs: + python: + description: which python to use + required: true + dest: + description: the destination + default: ${{targets.subpkgdir}} + +pipeline: + - name: "pip build ${{inputs.python}}" + runs: | + export SOURCE_DATE_EPOCH=315532800 + py=${{inputs.python}} + pyver=$("$py" -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))') + sitepkgd=$("$py" -c 'import site; print(site.getsitepackages()[0])') + sitepkgd=${sitepkgd#/} + root=${{inputs.dest}} + wd="./.wheels/$pyver" + vr() { echo "execute:" "$@"; "$@"; } + + tmpd=$(mktemp -d) + trap "rm -Rf '$tmpd'" EXIT + distwheelsd="$tmpd/dist-wheels" + mkdir -p "$distwheelsd" + echo "$py is $pyver with site_packages dir '$sitepkgd'" + vr $py -m pip wheel --verbose "--wheel-dir=$wd" \ + "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps . + vr $py -m pip install --verbose \ + "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps \ + --force-reinstall --no-compile --prefix=/usr "--root=$root" "$wd"/*.whl + vr $py -m compileall --invalidation-mode=unchecked-hash -r100 "$root/$sitepkgd" + + - name: "compileall ${{inputs.python}}" + runs: | + pyver=$(${{inputs.python}} -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))') + for bin in ${{inputs.dest}}/usr/bin/*; do + [ -f "$bin" ] && [ -x "$bin" ] || continue + d="${{inputs.dest}}/usr/pybin/$pyver" + [ -d "$d" ] || mkdir -p "$d" + mv "$bin" "$d/" + echo "moved $bin -> $d/" + done diff --git a/py3-cython.yaml b/py3-cython.yaml new file mode 100644 index 00000000000..5f061afb8cc --- /dev/null +++ b/py3-cython.yaml @@ -0,0 +1,63 @@ +package: + name: py3-cython + version: 3.0.9 + epoch: 1 + description: Cython is an optimising static compiler for both the Python & the extended Cython programming languages. + copyright: + - license: Apache-2.0 + dependencies: + runtime: + - py3.12-${{vars.pypi-package}} + provides: + - cython=${{package.version}} + +vars: + pypi-package: cython + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" + +environment: + contents: + packages: + - build-base + - busybox + - ca-certificates-bundle + - py3-supported-pip + - py3-supported-python-dev + +pipeline: + - uses: fetch + with: + expected-sha256: ca7758a3a87340a93f5bde4747e4a5ff1708cf2df407dee533b63ef6cc845777 + uri: https://github.com/cython/cython/archive/${{package.version}}.tar.gz + +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + pipeline: + - uses: py/pip-build-install + with: + python: python${{range.key}} + - uses: strip + + - name: py3-wheels-${{vars.pypi-package}} + pipeline: + - uses: py/collect-wheels + + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} + +update: + enabled: true + github: + identifier: cython/cython diff --git a/py3-flit-core.yaml b/py3-flit-core.yaml index 93f01fdaf6f..64d72cc262d 100644 --- a/py3-flit-core.yaml +++ b/py3-flit-core.yaml @@ -1,13 +1,23 @@ package: name: py3-flit-core version: 3.9.0 - epoch: 1 + epoch: 2 description: "simple packaging tool for simple packages (core)" copyright: - license: BSD-3-Clause dependencies: runtime: - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: flit-core + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: @@ -15,8 +25,8 @@ environment: - build-base - busybox - ca-certificates-bundle - - py3-installer - - python3 + - py3-supported-installer + - py3-supported-python - wolfi-base pipeline: @@ -25,18 +35,27 @@ pipeline: uri: https://files.pythonhosted.org/packages/source/f/flit/flit-${{package.version}}.tar.gz expected-sha256: d75edf5eb324da20d53570a6a6f87f51e606eee8384925cd66a90611140844c7 - - runs: | - cd flit_core - python3 build_dists.py - - - runs: | - cd flit_core - python3 -m installer -d "${{targets.destdir}}" dist/flit_core-${{package.version}}-py3-none-any.whl - - - runs: | - rm -rf "${{targets.destdir}}"/usr/lib/python3*/site-packages/flit_core/tests +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - runs: | + cd flit_core + rm -Rf dist/ + python${{range.key}} build_dists.py + python${{range.key}} -m installer -d "${{targets.subpkgdir}}" \ + dist/flit_core-${{package.version}}-py3-none-any.whl + rm -rf "${{targets.subpkgdir}}"/usr/lib/python3*/site-packages/flit_core/tests + - uses: strip - - uses: strip + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} update: enabled: true diff --git a/py3-gpep517.yaml b/py3-gpep517.yaml index e9bc710eb26..9d0b3db0154 100644 --- a/py3-gpep517.yaml +++ b/py3-gpep517.yaml @@ -1,15 +1,23 @@ package: name: py3-gpep517 version: "15" - epoch: 2 + epoch: 3 description: "PEP517 build system support for distros" copyright: - license: MIT dependencies: runtime: - - py3-installer - - py3-tomli - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: gpep517 + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: @@ -17,7 +25,7 @@ environment: - build-base - busybox - ca-certificates-bundle - - python3 + - py3-supported-python - wolfi-base pipeline: @@ -26,14 +34,30 @@ pipeline: uri: https://github.com/mgorny/gpep517/archive/v${{package.version}}/gpep517-v${{package.version}}.tar.gz expected-sha256: 368fa8dfb0775589e3f85d34df18395c22980de8520dc6447e778fc1d0102787 - - runs: | - _site_pkgs="$(python3 -c 'import site; print(site.getsitepackages()[0])')" - _py3ver=$(python3 -c 'import sys; print("{}.{}".format(sys.version_info.major, sys.version_info.minor))') - - mkdir -p "${{targets.destdir}}"/"$_site_pkgs" - cp -a gpep517 "${{targets.destdir}}"/$_site_pkgs/ +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: Easily download, build, install, upgrade, and uninstall Python packages + pipeline: + - runs: | + sitedir=usr/lib/python${{range.key}}/site-packages/ + mkdir -p "${{targets.subpkgdir}}/$sitedir" + cp -a gpep517 "${{targets.subpkgdir}}/$sitedir" + python${{range.key}} -m compileall --invalidation-mode=unchecked-hash -r100 \ + ${{targets.subpkgdir}}/usr/lib/ + - uses: strip + dependencies: + runtime: + - py${{range.key}}-tomli + - py${{range.key}}-installer - - uses: strip + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} update: enabled: true diff --git a/py3-installer.yaml b/py3-installer.yaml new file mode 100644 index 00000000000..ddb34cb4748 --- /dev/null +++ b/py3-installer.yaml @@ -0,0 +1,59 @@ +package: + name: py3-installer + version: 0.7.0 + epoch: 6 + description: A library for installing Python wheels. + copyright: + - license: "MIT" + dependencies: + runtime: + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: installer + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" + +environment: + contents: + packages: + - build-base + - busybox + - ca-certificates-bundle + - py3-supported-pip + - py3-supported-python + - wolfi-base + +pipeline: + - uses: git-checkout + with: + repository: https://github.com/pypa/installer.git + tag: ${{package.version}} + expected-commit: b1d39180f8548820d09ce992dfadff0a42242c48 + +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - runs: python${{range.key}} -m pip install . "--root=${{targets.subpkgdir}}" --prefix=/usr + - uses: strip + + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} + +update: + enabled: true + github: + identifier: pypa/installer + use-tag: true diff --git a/py3-numpy.yaml b/py3-numpy.yaml new file mode 100644 index 00000000000..10dc8822372 --- /dev/null +++ b/py3-numpy.yaml @@ -0,0 +1,71 @@ +package: + name: py3-numpy + version: 1.26.4 + epoch: 3 + description: "The fundamental package for scientific computing with Python." + copyright: + - license: BSD-3-Clause + dependencies: + runtime: + - py3.12-${{vars.pypi-package}} + provides: + - numpy + - f2py + +vars: + pypi-package: numpy + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" + +environment: + contents: + packages: + - bash + - build-base + - busybox + - gfortran + - git + - meson + - openblas-dev + - py3-supported-cython + - py3-supported-pip + - py3-supported-python-dev + - wolfi-base + +pipeline: + - uses: git-checkout + with: + repository: https://github.com/numpy/numpy + tag: v${{package.version}} + expected-commit: 9815c16f449e12915ef35a8255329ba26dacd5c0 + recurse-submodules: true + +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - runs: python${{range.key}} -m pip install . "--root=${{targets.subpkgdir}}" --prefix=/usr + - uses: strip + + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} + +update: + enabled: true + github: + identifier: numpy/numpy + # There are some v2 pre-releases + tag-filter: v1 + strip-prefix: v + strip-suffix: .dev0 diff --git a/py3-pathspec.yaml b/py3-pathspec.yaml index 71262c44ea6..c44e1a4ec82 100644 --- a/py3-pathspec.yaml +++ b/py3-pathspec.yaml @@ -1,13 +1,23 @@ package: name: py3-pathspec version: 0.12.1 - epoch: 0 + epoch: 1 description: "Utility library for gitignore style pattern matching of file paths" copyright: - license: MPL-2.0 dependencies: runtime: - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: pathspec + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: @@ -15,8 +25,8 @@ environment: - build-base - busybox - ca-certificates-bundle - - py3-setuptools - - python3 + - py3-supported-pip + - py3-supported-python - wolfi-base pipeline: @@ -25,13 +35,18 @@ pipeline: uri: https://files.pythonhosted.org/packages/source/p/pathspec/pathspec-${{package.version}}.tar.gz expected-sha512: 19d1a8ae5936e76eacd691b0e2fb33d0de376556751a9ae007f6b09d0aa36af7d171101df620cab3e93c9913be578ff3d6bba9d4460351ac248e5c9e015380ad - - runs: | - python3 setup.py build - - - runs: | - python3 setup.py install --prefix=/usr --root="${{targets.destdir}}" --skip-build +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: Easily download, build, install, upgrade, and uninstall Python packages + pipeline: + - uses: py/pip-build-install + with: + python: python${{range.key}} - - uses: strip + - name: py3-wheels-${{vars.pypi-package}} + pipeline: + - uses: py/collect-wheels update: enabled: true diff --git a/py3-pip.yaml b/py3-pip.yaml new file mode 100644 index 00000000000..a54c6001416 --- /dev/null +++ b/py3-pip.yaml @@ -0,0 +1,67 @@ +package: + name: py3-pip + version: "24.0" + epoch: 2 + description: The PyPA recommended tool for installing Python packages. + copyright: + - license: MIT + dependencies: + runtime: + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: pip + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" + +environment: + contents: + packages: + - build-base + - busybox + - ca-certificates-bundle + - py3-supported-python + - py3-supported-setuptools + - wolfi-base + +pipeline: + - uses: git-checkout + with: + repository: https://github.com/pypa/pip + expected-commit: ef78c129b1a966dbbbdb8ebfffc43723e89110d1 + tag: ${{package.version}} + +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: Easily download, build, install, upgrade, and uninstall Python packages + pipeline: + - uses: py/install-compile + with: + python: python${{range.key}} + - uses: strip + dependencies: + runtime: + - py${{range.key}}-flit-core + - py${{range.key}}-setuptools + - py${{range.key}}-wheel + + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} + +update: + enabled: true + shared: true + github: + identifier: pypa/pip + use-tag: true diff --git a/py3-pyyaml.yaml b/py3-pyyaml.yaml index 3d649a37327..c071cfd20fe 100644 --- a/py3-pyyaml.yaml +++ b/py3-pyyaml.yaml @@ -1,13 +1,23 @@ package: name: py3-pyyaml version: 6.0.1 - epoch: 4 + epoch: 5 description: "Python3 bindings for YAML" copyright: - license: MIT dependencies: runtime: - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: pyyaml + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: @@ -15,10 +25,9 @@ environment: - build-base - busybox - ca-certificates-bundle - - cython - - py3-setuptools - - python3 - - python3-dev + - py3-supported-cython + - py3-supported-pip + - py3-supported-python-dev - wolfi-base - yaml-dev @@ -34,13 +43,19 @@ pipeline: with: patches: pyyaml-cython3.patch - - runs: | - python3 setup.py build - - - runs: | - python3 setup.py install --prefix=/usr --root="${{targets.destdir}}" +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - uses: py/pip-build-install + with: + python: python${{range.key}} + - uses: strip - - uses: strip + - name: py3-wheels-${{vars.pypi-package}} + pipeline: + - uses: py/collect-wheels update: enabled: true diff --git a/py3-setuptools.yaml b/py3-setuptools.yaml new file mode 100644 index 00000000000..9d48a4114e1 --- /dev/null +++ b/py3-setuptools.yaml @@ -0,0 +1,62 @@ +package: + name: py3-setuptools + version: 69.2.0 + epoch: 1 + description: Easily download, build, install, upgrade, and uninstall Python packages + copyright: + - license: "MIT" + dependencies: + runtime: + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: setuptools + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" + +environment: + contents: + packages: + - build-base + - busybox + - ca-certificates-bundle + - py3-supported-python + - wolfi-base + environment: + # This is needed to work around the error "ValueError: ZIP does not support timestamps before 1980" + SOURCE_DATE_EPOCH: 315532800 + +pipeline: + - uses: fetch + with: + expected-sha256: 0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e + uri: https://files.pythonhosted.org/packages/source/s/setuptools/setuptools-${{package.version}}.tar.gz + +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: Easily download, build, install, upgrade, and uninstall Python packages + pipeline: + - uses: py/install-compile + with: + python: python${{range.key}} + - uses: strip + + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} + +update: + enabled: true + shared: true + release-monitor: + identifier: 4021 diff --git a/py3-supported-python.yaml b/py3-supported-python.yaml new file mode 100644 index 00000000000..230797b2786 --- /dev/null +++ b/py3-supported-python.yaml @@ -0,0 +1,29 @@ +package: + name: py3-supported-python + version: 1 + epoch: 1 + description: Easily download, build, install, upgrade, and uninstall Python packages + copyright: + - license: "MIT" + dependencies: + runtime: + - python-3.10-base + - python-3.11-base + - python-3.12-base + +environment: + contents: + packages: + - build-base + - busybox + +subpackages: + - name: ${{package.name}}-dev + dependencies: + runtime: + - python-3.10-base-dev + - python-3.11-base-dev + - python-3.12-base-dev + +update: + enabled: false diff --git a/py3-tomli.yaml b/py3-tomli.yaml index cfa1a19a541..825b6d78292 100644 --- a/py3-tomli.yaml +++ b/py3-tomli.yaml @@ -1,13 +1,23 @@ package: name: py3-tomli version: 2.0.1 - epoch: 4 + epoch: 5 description: "TOML parser" copyright: - license: MIT dependencies: runtime: - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: tomli + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: @@ -15,8 +25,8 @@ environment: - build-base - busybox - ca-certificates-bundle - - py3-installer - - python3 + - py3-supported-installer + - py3-supported-python - wolfi-base pipeline: @@ -26,10 +36,15 @@ pipeline: expected-sha256: 939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc extract: false - - runs: | - python3 -m installer -d "${{targets.destdir}}" tomli-${{package.version}}-py3-none-any.whl - - - uses: strip +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: Easily download, build, install, upgrade, and uninstall Python packages + pipeline: + - runs: | + python${{range.key}} -m installer -d "${{targets.subpkgdir}}" \ + tomli-${{package.version}}-py3-none-any.whl + - uses: strip update: enabled: true diff --git a/py3-wheel.yaml b/py3-wheel.yaml index 62205348a1a..9f7795ec9e9 100644 --- a/py3-wheel.yaml +++ b/py3-wheel.yaml @@ -7,17 +7,27 @@ package: - license: MIT dependencies: runtime: - - py3-installer - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: wheel + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: packages: - busybox - ca-certificates-bundle - - py3-flit-core - - py3-gpep517 - - python3 + - py3-supported-flit-core + - py3-supported-gpep517 + - py3-supported-installer + - py3-supported-python - wolfi-base pipeline: @@ -27,11 +37,26 @@ pipeline: tag: ${{package.version}} expected-commit: fa33dfd01fd665c1fd90097563b34bce4b5527ef - - runs: | - python3 -m gpep517 build-wheel --wheel-dir dist --output-fd 1 - python3 -m installer -d "${{targets.destdir}}" dist/wheel-${{package.version}}-py3-none-any.whl +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - runs: | + distd=dist-${{range.key}} + rm -Rf $distd + python${{range.key}} -m gpep517 build-wheel --wheel-dir=$distd --output-fd 1 + python${{range.key}} -m installer -d "${{targets.subpkgdir}}" ${distd}/*.whl + rm ${{targets.subpkgdir}}/usr/bin/wheel + - uses: strip - - uses: strip + - name: py3-supported-${{vars.pypi-package}} + description: meta package providing ${{vars.pypi-package}} for supported python versions. + dependencies: + runtime: + - py3.10-${{vars.pypi-package}} + - py3.11-${{vars.pypi-package}} + - py3.12-${{vars.pypi-package}} update: enabled: true diff --git a/py3.10-installer.yaml b/py3.10-installer.yaml deleted file mode 100644 index 58f59fcad57..00000000000 --- a/py3.10-installer.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Generated from https://pypi.org/project/installer/ -package: - name: py3.10-installer - version: 0.7.0 # When bumping this, also bump the provides below! - epoch: 4 - description: A library for installing Python wheels. - copyright: - - license: "MIT" - dependencies: - runtime: - - python-3.10 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.10-pip - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/installer.git - tag: ${{package.version}} - expected-commit: b1d39180f8548820d09ce992dfadff0a42242c48 - - - name: Python Install - runs: pip install . --prefix=/usr --root=${{targets.destdir}} - - - uses: strip - -update: - enabled: true - github: - identifier: pypa/installer - use-tag: true diff --git a/py3.10-pip.yaml b/py3.10-pip.yaml deleted file mode 100644 index 09a13cf9220..00000000000 --- a/py3.10-pip.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Generated from https://pypi.org/project/pip/ -package: - name: py3.10-pip - version: "24.0" - epoch: 1 - description: The PyPA recommended tool for installing Python packages. - copyright: - - license: MIT - dependencies: - runtime: - - python-3.10 - - py3.10-setuptools - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.10-setuptools - - python-3.10 - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/pip - expected-commit: ef78c129b1a966dbbbdb8ebfffc43723e89110d1 - tag: ${{package.version}} - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -update: - enabled: true - ignore-regex-patterns: - - 'b' - shared: true - github: - identifier: pypa/pip - use-tag: true diff --git a/py3.10-setuptools.yaml b/py3.10-setuptools.yaml deleted file mode 100644 index 2a2400fef3a..00000000000 --- a/py3.10-setuptools.yaml +++ /dev/null @@ -1,39 +0,0 @@ -package: - name: py3.10-setuptools - version: 69.5.1 - epoch: 1 - description: Easily download, build, install, upgrade, and uninstall Python packages - copyright: - - license: "MIT" - dependencies: - runtime: - - python-3.10 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - python-3.10 - - wolfi-base - -pipeline: - - uses: fetch - with: - expected-sha256: 6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 - uri: https://files.pythonhosted.org/packages/source/s/setuptools/setuptools-${{package.version}}.tar.gz - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -update: - enabled: true - shared: true - release-monitor: - identifier: 4021 diff --git a/py3.11-installer.yaml b/py3.11-installer.yaml deleted file mode 100644 index 01e4ceba23a..00000000000 --- a/py3.11-installer.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Generated from https://pypi.org/project/installer/ -package: - name: py3.11-installer - version: 0.7.0 # When bumping this, also bump the provides below! - epoch: 3 - description: A library for installing Python wheels. - copyright: - - license: "MIT" - dependencies: - provides: - - py3-installer=${{package.full-version}} - runtime: - - python-3.11 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.11-pip - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/installer.git - tag: ${{package.version}} - expected-commit: b1d39180f8548820d09ce992dfadff0a42242c48 - - - name: Python Install - runs: pip install . --prefix=/usr --root=${{targets.destdir}} - - - uses: strip - -update: - enabled: true - github: - identifier: pypa/installer - use-tag: true diff --git a/py3.11-pip.yaml b/py3.11-pip.yaml deleted file mode 100644 index 8eab4bb01fa..00000000000 --- a/py3.11-pip.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# Generated from https://pypi.org/project/pip/ -package: - name: py3.11-pip - version: "24.0" - epoch: 1 - description: The PyPA recommended tool for installing Python packages. - copyright: - - license: MIT - dependencies: - provides: - - py3-pip=23.1.999 - runtime: - - python-3.11 - - py3.11-setuptools - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.11-setuptools - - python-3.11 - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/pip - expected-commit: ef78c129b1a966dbbbdb8ebfffc43723e89110d1 - tag: ${{package.version}} - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -update: - enabled: true - ignore-regex-patterns: - - 'b' - shared: true - github: - identifier: pypa/pip - use-tag: true diff --git a/py3.11-setuptools.yaml b/py3.11-setuptools.yaml deleted file mode 100644 index 5bbce8b6497..00000000000 --- a/py3.11-setuptools.yaml +++ /dev/null @@ -1,39 +0,0 @@ -package: - name: py3.11-setuptools - version: 69.5.1 # When bumping this, also bump the provides below! - epoch: 1 - description: Easily download, build, install, upgrade, and uninstall Python packages - copyright: - - license: "MIT" - dependencies: - provides: - - py3-setuptools=69.0.0 # force the provides version lower than the py3.12-setuptools provides, so that the apk resolver favours py3.12-setuptools - runtime: - - python-3.11 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - python-3.11 - - wolfi-base - -pipeline: - - uses: fetch - with: - expected-sha256: 6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 - uri: https://files.pythonhosted.org/packages/source/s/setuptools/setuptools-${{package.version}}.tar.gz - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" - -update: - enabled: true - shared: true - release-monitor: - identifier: 4021 diff --git a/py3.12-installer.yaml b/py3.12-installer.yaml deleted file mode 100644 index 2bd39d4fc34..00000000000 --- a/py3.12-installer.yaml +++ /dev/null @@ -1,39 +0,0 @@ -package: - name: py3.12-installer - version: 0.7.0 - epoch: 5 - description: A library for installing Python wheels. - copyright: - - license: "MIT" - dependencies: - provides: - - py3-installer=${{package.full-version}} - runtime: - - python-3.12 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.12-pip - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/installer.git - tag: ${{package.version}} - expected-commit: b1d39180f8548820d09ce992dfadff0a42242c48 - - - name: Python Install - runs: pip install . --prefix=/usr --root=${{targets.destdir}} - - - uses: strip - -update: - enabled: true - github: - identifier: pypa/installer - use-tag: true diff --git a/py3.12-pip.yaml b/py3.12-pip.yaml deleted file mode 100644 index 8444d960e99..00000000000 --- a/py3.12-pip.yaml +++ /dev/null @@ -1,47 +0,0 @@ -package: - name: py3.12-pip - version: "24.0" - epoch: 2 - description: The PyPA recommended tool for installing Python packages. - copyright: - - license: MIT - dependencies: - provides: - - py3-pip=${{package.full-version}} - runtime: - - py3.12-setuptools - - python-3.12 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3.12-setuptools - - python-3.12 - - wolfi-base - -pipeline: - - uses: git-checkout - with: - repository: https://github.com/pypa/pip - expected-commit: ef78c129b1a966dbbbdb8ebfffc43723e89110d1 - tag: ${{package.version}} - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -update: - enabled: true - ignore-regex-patterns: - - 'b' - shared: true - github: - identifier: pypa/pip - use-tag: true diff --git a/py3.12-setuptools.yaml b/py3.12-setuptools.yaml deleted file mode 100644 index cd9839cecab..00000000000 --- a/py3.12-setuptools.yaml +++ /dev/null @@ -1,39 +0,0 @@ -package: - name: py3.12-setuptools - version: 69.5.1 - epoch: 1 - description: Easily download, build, install, upgrade, and uninstall Python packages - copyright: - - license: "MIT" - dependencies: - provides: - - py3-setuptools=${{package.full-version}} - runtime: - - python-3.12 - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - python-3.12 - - wolfi-base - -pipeline: - - uses: fetch - with: - expected-sha256: 6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 - uri: https://files.pythonhosted.org/packages/source/s/setuptools/setuptools-${{package.version}}.tar.gz - - - name: Python Build - runs: python setup.py build - - - name: Python Install - runs: python setup.py install --prefix=/usr --root="${{targets.destdir}}" --skip-build - -update: - enabled: true - shared: true - release-monitor: - identifier: 4021 From 0618e508deff930224caf58e6690e88a5541086c Mon Sep 17 00:00:00 2001 From: user Date: Fri, 29 Mar 2024 15:32:54 +0000 Subject: [PATCH 08/39] Resolve the dependency loop. flit-core has bootstrap_install.py to handle this, so we use it. pip doesn't stricktly depend on wheel, although buidling many projects assumes that it is there. --- pipelines/py/pip-build-install.yaml | 2 ++ py3-cython.yaml | 1 + py3-flit-core.yaml | 17 +++++++++++++---- py3-pip.yaml | 1 - 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pipelines/py/pip-build-install.yaml b/pipelines/py/pip-build-install.yaml index 283beb3f390..a675fb042f3 100644 --- a/pipelines/py/pip-build-install.yaml +++ b/pipelines/py/pip-build-install.yaml @@ -22,6 +22,8 @@ pipeline: tmpd=$(mktemp -d) trap "rm -Rf '$tmpd'" EXIT + + # --find-links to an empty dir and --no-index makes pip fully "offline" distwheelsd="$tmpd/dist-wheels" mkdir -p "$distwheelsd" echo "$py is $pyver with site_packages dir '$sitepkgd'" diff --git a/py3-cython.yaml b/py3-cython.yaml index 5f061afb8cc..79d17d6518e 100644 --- a/py3-cython.yaml +++ b/py3-cython.yaml @@ -29,6 +29,7 @@ environment: - ca-certificates-bundle - py3-supported-pip - py3-supported-python-dev + - py3-supported-wheel pipeline: - uses: fetch diff --git a/py3-flit-core.yaml b/py3-flit-core.yaml index 64d72cc262d..36b2b0fa66c 100644 --- a/py3-flit-core.yaml +++ b/py3-flit-core.yaml @@ -25,7 +25,6 @@ environment: - build-base - busybox - ca-certificates-bundle - - py3-supported-installer - py3-supported-python - wolfi-base @@ -43,12 +42,22 @@ subpackages: - runs: | cd flit_core rm -Rf dist/ - python${{range.key}} build_dists.py - python${{range.key}} -m installer -d "${{targets.subpkgdir}}" \ - dist/flit_core-${{package.version}}-py3-none-any.whl + python${{range.key}} -m flit_core.wheel + python${{range.key}} bootstrap_install.py \ + --install-root=${{targets.subpkgdir}} \ + dist/flit_core-${{package.version}}-py3-none-any.whl + mkdir -p /home/build/.wheels/${{range.key}}/ + cp dist/flit_core-${{package.version}}-py3-none-any.whl \ + /home/build/.wheels/${{range.key}}/ + python${{range.key}} -m compileall --invalidation-mode=unchecked-hash -r100 \ + ${{targets.subpkgdir}} rm -rf "${{targets.subpkgdir}}"/usr/lib/python3*/site-packages/flit_core/tests - uses: strip + - name: py3-wheels-${{vars.pypi-package}} + pipeline: + - uses: py/collect-wheels + - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. dependencies: diff --git a/py3-pip.yaml b/py3-pip.yaml index a54c6001416..2e4ee847cea 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -49,7 +49,6 @@ subpackages: runtime: - py${{range.key}}-flit-core - py${{range.key}}-setuptools - - py${{range.key}}-wheel - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. From 36975018065dce33bd1c5a640352fcadc60a44a5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 16 May 2024 15:51:35 +0000 Subject: [PATCH 09/39] Some cleanups of things. pip-build-install-bootstrap simplifies. You can get started with: make package/pip-zipapp \ package/py3-{supported-python,setuptools,installer,wheel,pip} Then most of core deps are built. --- pipelines/py/install-compile.yaml | 25 -------- pipelines/py/pip-build-install-bootstrap.yaml | 57 +++++++++++++++++++ py3-installer.yaml | 6 +- py3-pathspec.yaml | 2 +- py3-pip.yaml | 7 +-- py3-setuptools.yaml | 4 +- py3-six.yaml | 33 ++++++++--- py3-supported-python.yaml | 3 +- py3-tomli.yaml | 2 +- py3-wheel.yaml | 13 +---- 10 files changed, 96 insertions(+), 56 deletions(-) delete mode 100644 pipelines/py/install-compile.yaml create mode 100644 pipelines/py/pip-build-install-bootstrap.yaml diff --git a/pipelines/py/install-compile.yaml b/pipelines/py/install-compile.yaml deleted file mode 100644 index ef6113f9df4..00000000000 --- a/pipelines/py/install-compile.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Setup, install and compile python - -inputs: - python: - description: which python to use - required: true - dest: - description: the destination - default: ${{targets.subpkgdir}} - -pipeline: - - name: "setup ${{inputs.python}}" - runs: | - ${{inputs.python}} setup.py install --no-compile --root="${{targets.subpkgdir}}" --prefix=/usr - rm -Rf build/ - - name: "compileall ${{inputs.python}}" - runs: | - ${{inputs.python}} -m compileall --invalidation-mode=unchecked-hash -r100 ${{targets.subpkgdir}}/usr/lib/ - pyver=$(${{inputs.python}} -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))') - for bin in ${{targets.subpkgdir}}/usr/bin/*; do - [ -f "$bin" ] && [ -x "$bin" ] || continue - d="${{targets.subpkgdir}}/usr/pybin/$pyver" - [ -d "$d" ] || mkdir -p "$d" - mv "$bin" "$d/" - done diff --git a/pipelines/py/pip-build-install-bootstrap.yaml b/pipelines/py/pip-build-install-bootstrap.yaml new file mode 100644 index 00000000000..fa58bb4cf34 --- /dev/null +++ b/pipelines/py/pip-build-install-bootstrap.yaml @@ -0,0 +1,57 @@ +name: Use pip from a zipapp for bootstrap. Only for use with bootstrapping. + +needs: + packages: + - pip-zipapp + +inputs: + python: + description: which python to use + required: true + wheel: + description: build a wheel? + required: false + default: false + dest: + description: the destination + default: ${{targets.subpkgdir}} + +pipeline: + - name: "pip-bootstrap build" + runs: | + export SOURCE_DATE_EPOCH=315532800 + py=${{inputs.python}} + pipzip=/usr/share/pip-zipapp/pip-zipapp.pyz + [ -e "$pipzip" ] || { echo "missing pip-zip - $pipzip does not exist"; exit 1; } + [ -f "$pipzip" ] || { echo "missing pip-zip - $pipzip is not a file"; exit 1; } + pyver=$("$py" -c 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))') + sitepkgd=$("$py" -c 'import site; print(site.getsitepackages()[0])') + sitepkgd=${sitepkgd#/} + root=${{inputs.dest}} + wd="./.wheels/$pyver" + info() { echo "$@" 1>&2; } + vr() { info "execute:" "$@"; "$@"; } + + tmpd=$(mktemp -d) + trap "rm -Rf '$tmpd'" EXIT + + # --find-links to an empty dir and --no-index makes pip fully "offline" + distwheelsd="$tmpd/dist-wheels" + mkdir -p "$distwheelsd" + info "$py is $pyver with site_packages dir '$sitepkgd'" + + if [ "${{inputs.wheel}}" = "false" ]; then + info "skipping bdist-wheel" + vr "$py" "$pipzip" install --verbose --no-deps \ + --force-reinstall --no-compile --prefix=/usr "--root=$root" . + else + vr "$py" "$pipzip" wheel --verbose "--wheel-dir=$wd" \ + "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps . + set -- "$wd"/*.whl + [ -f "$1" ] || { info "did not produce a wheel in $wd"; exit 1; } + info "produced $# wheels: $*" + vr "$py" "$pipzip" install --verbose \ + "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps \ + --force-reinstall --no-compile --prefix=/usr "--root=$root" "$@" + fi + vr $py -m compileall --invalidation-mode=unchecked-hash -r100 "$root/$sitepkgd" diff --git a/py3-installer.yaml b/py3-installer.yaml index ddb34cb4748..c819d6884ae 100644 --- a/py3-installer.yaml +++ b/py3-installer.yaml @@ -25,7 +25,6 @@ environment: - build-base - busybox - ca-certificates-bundle - - py3-supported-pip - py3-supported-python - wolfi-base @@ -41,8 +40,9 @@ subpackages: name: py${{range.key}}-${{vars.pypi-package}} description: ${{vars.pypi-package}} installed for python${{range.key}} pipeline: - - runs: python${{range.key}} -m pip install . "--root=${{targets.subpkgdir}}" --prefix=/usr - - uses: strip + - uses: py/pip-build-install-bootstrap + with: + python: python${{range.key}} - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. diff --git a/py3-pathspec.yaml b/py3-pathspec.yaml index c44e1a4ec82..146fc215dfc 100644 --- a/py3-pathspec.yaml +++ b/py3-pathspec.yaml @@ -38,7 +38,7 @@ pipeline: subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} - description: Easily download, build, install, upgrade, and uninstall Python packages + description: ${{vars.pypi-package}} installed for python${{range.key}} pipeline: - uses: py/pip-build-install with: diff --git a/py3-pip.yaml b/py3-pip.yaml index 2e4ee847cea..acbc9e3d703 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -26,7 +26,6 @@ environment: - busybox - ca-certificates-bundle - py3-supported-python - - py3-supported-setuptools - wolfi-base pipeline: @@ -39,12 +38,12 @@ pipeline: subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} - description: Easily download, build, install, upgrade, and uninstall Python packages + description: ${{vars.pypi-package}} installed for python${{range.key}} pipeline: - - uses: py/install-compile + - uses: py/pip-build-install-bootstrap with: python: python${{range.key}} - - uses: strip + wheel: false dependencies: runtime: - py${{range.key}}-flit-core diff --git a/py3-setuptools.yaml b/py3-setuptools.yaml index 9d48a4114e1..f6f9c46a155 100644 --- a/py3-setuptools.yaml +++ b/py3-setuptools.yaml @@ -40,9 +40,9 @@ pipeline: subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} - description: Easily download, build, install, upgrade, and uninstall Python packages + description: ${{vars.pypi-package}} installed for python${{range.key}} pipeline: - - uses: py/install-compile + - uses: py/pip-build-install-bootstrap with: python: python${{range.key}} - uses: strip diff --git a/py3-six.yaml b/py3-six.yaml index 2eed069b61c..2d1a48a0511 100644 --- a/py3-six.yaml +++ b/py3-six.yaml @@ -7,7 +7,17 @@ package: - license: MIT dependencies: runtime: - - python3 + - py3.12-${{vars.pypi-package}} + +vars: + pypi-package: pyyaml + +data: + - name: py-versions + items: + 3.10: "" + 3.11: "" + 3.12: "" environment: contents: @@ -15,8 +25,7 @@ environment: - build-base - busybox - ca-certificates-bundle - - py3-setuptools - - python3 + - py3-supported-python - wolfi-base pipeline: @@ -25,13 +34,19 @@ pipeline: uri: https://files.pythonhosted.org/packages/source/s/six/six-${{package.version}}.tar.gz expected-sha256: 1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 - - runs: | - python3 setup.py build - - - runs: | - python3 setup.py install --prefix=/usr --root="${{targets.destdir}}" --skip-build +subpackages: + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - uses: py/pip-build-install + with: + python: python${{range.key}} + - uses: strip - - uses: strip + - name: py3-wheels-${{vars.pypi-package}} + pipeline: + - uses: py/collect-wheels update: enabled: true diff --git a/py3-supported-python.yaml b/py3-supported-python.yaml index 230797b2786..a75a9c5694d 100644 --- a/py3-supported-python.yaml +++ b/py3-supported-python.yaml @@ -2,7 +2,7 @@ package: name: py3-supported-python version: 1 epoch: 1 - description: Easily download, build, install, upgrade, and uninstall Python packages + description: metapackage to install all supported versions of python copyright: - license: "MIT" dependencies: @@ -19,6 +19,7 @@ environment: subpackages: - name: ${{package.name}}-dev + description: metapackage to install all supported versions of python-dev dependencies: runtime: - python-3.10-base-dev diff --git a/py3-tomli.yaml b/py3-tomli.yaml index 825b6d78292..519d134dcf9 100644 --- a/py3-tomli.yaml +++ b/py3-tomli.yaml @@ -39,7 +39,7 @@ pipeline: subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} - description: Easily download, build, install, upgrade, and uninstall Python packages + description: ${{vars.pypi-package}} installed for python${{range.key}} pipeline: - runs: | python${{range.key}} -m installer -d "${{targets.subpkgdir}}" \ diff --git a/py3-wheel.yaml b/py3-wheel.yaml index 9f7795ec9e9..a32e4f37afc 100644 --- a/py3-wheel.yaml +++ b/py3-wheel.yaml @@ -24,9 +24,6 @@ environment: packages: - busybox - ca-certificates-bundle - - py3-supported-flit-core - - py3-supported-gpep517 - - py3-supported-installer - py3-supported-python - wolfi-base @@ -42,13 +39,9 @@ subpackages: name: py${{range.key}}-${{vars.pypi-package}} description: ${{vars.pypi-package}} installed for python${{range.key}} pipeline: - - runs: | - distd=dist-${{range.key}} - rm -Rf $distd - python${{range.key}} -m gpep517 build-wheel --wheel-dir=$distd --output-fd 1 - python${{range.key}} -m installer -d "${{targets.subpkgdir}}" ${distd}/*.whl - rm ${{targets.subpkgdir}}/usr/bin/wheel - - uses: strip + - uses: py/pip-build-install-bootstrap + with: + python: python${{range.key}} - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. From 399b242b8f14bfc12940102b27f0298714e12f88 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Thu, 23 May 2024 12:39:33 +0100 Subject: [PATCH 10/39] Upgrade setuptools Signed-off-by: Dimitri John Ledkov --- py3-setuptools.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py3-setuptools.yaml b/py3-setuptools.yaml index 7fb96e47ed3..554c7fe9140 100644 --- a/py3-setuptools.yaml +++ b/py3-setuptools.yaml @@ -1,6 +1,6 @@ package: name: py3-setuptools - version: 69.5.1 + version: 70.0.0 epoch: 1 description: Easily download, build, install, upgrade, and uninstall Python packages copyright: @@ -34,7 +34,7 @@ environment: pipeline: - uses: fetch with: - expected-sha256: 6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 + expected-sha256: f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0 uri: https://files.pythonhosted.org/packages/source/s/setuptools/setuptools-${{package.version}}.tar.gz subpackages: From 91644307044ae4dca65d07a791cee95e811e791d Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Thu, 23 May 2024 12:57:30 +0100 Subject: [PATCH 11/39] Test setuptools Signed-off-by: Dimitri John Ledkov --- py3-setuptools.yaml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/py3-setuptools.yaml b/py3-setuptools.yaml index 554c7fe9140..36d31058934 100644 --- a/py3-setuptools.yaml +++ b/py3-setuptools.yaml @@ -6,6 +6,7 @@ package: copyright: - license: "MIT" dependencies: + provider-priority: 0 runtime: - py3.12-${{vars.pypi-package}} @@ -15,9 +16,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "312" environment: contents: @@ -41,6 +42,10 @@ subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} description: Easily download, build, install, upgrade, and uninstall Python packages + dependencies: + provider-priority: ${{range.value}} + provides: + - py3-setuptools pipeline: - uses: py/install-compile with: @@ -55,6 +60,11 @@ subpackages: - py3.11-${{vars.pypi-package}} - py3.12-${{vars.pypi-package}} + test: + pipeline: + - runs: | + ls /usr/lib/python3.*/site-packages | grep ${{vars.pypi-package}} | sort -u + update: enabled: true shared: true From 35b266533a6e728546d94f85e54b22b3fe16cd20 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Thu, 23 May 2024 13:16:04 +0100 Subject: [PATCH 12/39] fixes --- py3-flit-core.yaml | 13 ++++++++----- py3-gpep517.yaml | 16 +++++++++++----- py3-installer.yaml | 18 ++++++++++++------ py3-pip.yaml | 12 ++++++------ py3-setuptools.yaml | 7 ++----- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/py3-flit-core.yaml b/py3-flit-core.yaml index 36b2b0fa66c..9f61c9bc519 100644 --- a/py3-flit-core.yaml +++ b/py3-flit-core.yaml @@ -6,8 +6,7 @@ package: copyright: - license: BSD-3-Clause dependencies: - runtime: - - py3.12-${{vars.pypi-package}} + provider-priority: 0 vars: pypi-package: flit-core @@ -15,9 +14,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "312" environment: contents: @@ -38,6 +37,10 @@ subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} description: ${{vars.pypi-package}} installed for python${{range.key}} + dependencies: + provider-priority: ${{range.value}} + provides: + - ${{package.name}} pipeline: - runs: | cd flit_core diff --git a/py3-gpep517.yaml b/py3-gpep517.yaml index aefc593e44d..e5ff6eb8d1b 100644 --- a/py3-gpep517.yaml +++ b/py3-gpep517.yaml @@ -6,8 +6,7 @@ package: copyright: - license: MIT dependencies: - runtime: - - py3.12-${{vars.pypi-package}} + provider-priority: 0 vars: pypi-package: gpep517 @@ -15,9 +14,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "312" environment: contents: @@ -47,6 +46,9 @@ subpackages: ${{targets.subpkgdir}}/usr/lib/ - uses: strip dependencies: + provider-priority: ${{range.value}} + provides: + - ${{package.name}} runtime: - py${{range.key}}-tomli - py${{range.key}}-installer @@ -58,6 +60,10 @@ subpackages: - py3.10-${{vars.pypi-package}} - py3.11-${{vars.pypi-package}} - py3.12-${{vars.pypi-package}} + test: + pipeline: + - runs: | + find /usr/lib/python3.*/site-packages -name ${{vars.pypi-package}} update: enabled: true diff --git a/py3-installer.yaml b/py3-installer.yaml index ed8681aa030..415b2ac81d6 100644 --- a/py3-installer.yaml +++ b/py3-installer.yaml @@ -6,8 +6,7 @@ package: copyright: - license: "MIT" dependencies: - runtime: - - py3.12-${{vars.pypi-package}} + provider-priority: 0 vars: pypi-package: installer @@ -15,9 +14,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "312" environment: contents: @@ -40,8 +39,11 @@ subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} description: ${{vars.pypi-package}} installed for python${{range.key}} + dependencies: + provider-priority: ${{range.value}} + provides: + - ${{package.name}} pipeline: - - runs: head -n1 /usr/bin/pip* - runs: python${{range.key}} -m pip install . "--root=${{targets.subpkgdir}}" --prefix=/usr - uses: strip @@ -52,6 +54,10 @@ subpackages: - py3.10-${{vars.pypi-package}} - py3.11-${{vars.pypi-package}} - py3.12-${{vars.pypi-package}} + test: + pipeline: + - runs: | + find /usr/lib/python3.*/site-packages -name ${{vars.pypi-package}} update: enabled: true diff --git a/py3-pip.yaml b/py3-pip.yaml index a74d2d07d7f..cd06c28a186 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -49,12 +49,8 @@ subpackages: with: python: python${{range.key}} - uses: strip - # ideally want If syntax here - # if: ${{range.key}} != 3.12 - # options: - # no-commands: true options: - no-commands: true + no-versioned-commands: true dependencies: provides: - py3-pip @@ -63,7 +59,6 @@ subpackages: - py3.11-${{vars.pypi-package}} - py3.12-${{vars.pypi-package}} provider-priority: ${{range.value}} - replaces-priority: ${{range.value}} runtime: - py${{range.key}}-flit-core - py${{range.key}}-setuptools @@ -76,6 +71,11 @@ subpackages: - py3.11-${{vars.pypi-package}}=${{package.full-version}} - py3.10-${{vars.pypi-package}}=${{package.full-version}} + test: + pipeline: + - runs: | + ls /usr/lib/python3.*/site-packages | grep ${{vars.pypi-package}} | sort -u + update: enabled: true shared: true diff --git a/py3-setuptools.yaml b/py3-setuptools.yaml index 36d31058934..6922ac1b46d 100644 --- a/py3-setuptools.yaml +++ b/py3-setuptools.yaml @@ -7,8 +7,6 @@ package: - license: "MIT" dependencies: provider-priority: 0 - runtime: - - py3.12-${{vars.pypi-package}} vars: pypi-package: setuptools @@ -45,7 +43,7 @@ subpackages: dependencies: provider-priority: ${{range.value}} provides: - - py3-setuptools + - ${{package.name}} pipeline: - uses: py/install-compile with: @@ -59,11 +57,10 @@ subpackages: - py3.10-${{vars.pypi-package}} - py3.11-${{vars.pypi-package}} - py3.12-${{vars.pypi-package}} - test: pipeline: - runs: | - ls /usr/lib/python3.*/site-packages | grep ${{vars.pypi-package}} | sort -u + find /usr/lib/python3.*/site-packages -name ${{vars.pypi-package}} update: enabled: true From 19004cf3c880fb0dd1401f85b0bf43e93cdbc5ea Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Thu, 23 May 2024 15:25:03 +0100 Subject: [PATCH 13/39] tomli Signed-off-by: Dimitri John Ledkov --- py3-tomli.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/py3-tomli.yaml b/py3-tomli.yaml index 825b6d78292..788033ab6c1 100644 --- a/py3-tomli.yaml +++ b/py3-tomli.yaml @@ -6,8 +6,7 @@ package: copyright: - license: MIT dependencies: - runtime: - - py3.12-${{vars.pypi-package}} + provider-priority: 0 vars: pypi-package: tomli @@ -15,9 +14,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "0" environment: contents: @@ -40,6 +39,10 @@ subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} description: Easily download, build, install, upgrade, and uninstall Python packages + dependencies: + provider-priority: ${{range.value}} + provides: + - ${{package.name}} pipeline: - runs: | python${{range.key}} -m installer -d "${{targets.subpkgdir}}" \ From 54e9d982cf8b0c008f0b98e2b6f2888cb3ee9f31 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Thu, 23 May 2024 15:38:15 +0100 Subject: [PATCH 14/39] foo --- py3-pip.yaml | 2 -- py3-wheel.yaml | 16 +++++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/py3-pip.yaml b/py3-pip.yaml index cd06c28a186..8e8e313250b 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -10,8 +10,6 @@ package: - empty dependencies: provider-priority: 0 - runtime: - - py3.12-${{vars.pypi-package}} vars: pypi-package: pip diff --git a/py3-wheel.yaml b/py3-wheel.yaml index 9f7795ec9e9..a766e6cc859 100644 --- a/py3-wheel.yaml +++ b/py3-wheel.yaml @@ -6,8 +6,7 @@ package: copyright: - license: MIT dependencies: - runtime: - - py3.12-${{vars.pypi-package}} + provider-priority: 0 vars: pypi-package: wheel @@ -15,9 +14,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "312" environment: contents: @@ -41,6 +40,13 @@ subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} description: ${{vars.pypi-package}} installed for python${{range.key}} + dependencies: + provides: + - py3-${{vars.pypi-package}} + provider-priority: ${{range.value}} + runtime: + - py${{range.key}}-flit-core + - py${{range.key}}-setuptools pipeline: - runs: | distd=dist-${{range.key}} From 45b0e269ca44ff94fe3c37d846cee3e7e1512132 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 28 May 2024 15:39:46 +0000 Subject: [PATCH 15/39] Add pip-zipapp.yaml file. --- pip-zipapp.yaml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 pip-zipapp.yaml diff --git a/pip-zipapp.yaml b/pip-zipapp.yaml new file mode 100644 index 00000000000..bea82330faa --- /dev/null +++ b/pip-zipapp.yaml @@ -0,0 +1,32 @@ +package: + name: pip-zipapp + version: 24.0 + epoch: 0 + description: "Pip as a python zipapp" + copyright: + - license: MIT + +environment: + contents: + packages: + - busybox + +pipeline: + - uses: fetch + with: + uri: https://bootstrap.pypa.io/pip/zipapp/pip-${{package.version}}.pyz + expected-sha256: 1af141650f2b1e71bf9d856db74324042b4bfe3a36c37f8c2b3e2d8d6b3da411 + extract: false + + - runs: | + destd=${{targets.destdir}} + sdir=usr/share/${{package.name}} + fname=${{package.name}}.pyz + mkdir -p "$destd/$sdir" + cp "pip-${{package.version}}.pyz" "$destd/$sdir/$fname" + chmod 0644 "$destd/$sdir/$fname" + +update: + enabled: true + github: + identifier: pypa/pip From 634fa9312a3156f6868be5a9d7a5a86ba528d6d5 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Tue, 28 May 2024 17:24:18 +0100 Subject: [PATCH 16/39] Remove no longer needed cython-0 Signed-off-by: Dimitri John Ledkov --- cython-0.yaml | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 cython-0.yaml diff --git a/cython-0.yaml b/cython-0.yaml deleted file mode 100644 index 90a0091d207..00000000000 --- a/cython-0.yaml +++ /dev/null @@ -1,40 +0,0 @@ -package: - name: cython-0 - version: 0.29.37.1 - epoch: 0 - description: Cython is an optimising static compiler for both the Python & the extended Cython programming languages. - copyright: - - license: Apache-2.0 - dependencies: - provides: - - cython=${{package.version}}-r${{package.epoch}} - -environment: - contents: - packages: - - build-base - - busybox - - ca-certificates-bundle - - py3-setuptools - - python3 - - python3-dev - -pipeline: - - uses: git-checkout - with: - expected-commit: 598be7f9fd253c9e44f051f2235abedc66c022c1 - repository: https://github.com/cython/cython - tag: ${{package.version}} - - - runs: | - python setup.py build - python setup.py install --prefix=/usr --root="${{targets.destdir}}" - - - uses: strip - -update: - enabled: true - github: - identifier: cython/cython - use-tag: true - tag-filter: 0. From 68b9a2f2b94cb0a76ec3f82b5533880b725768f9 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Tue, 28 May 2024 17:50:20 +0100 Subject: [PATCH 17/39] Split wheel binary into -bin subpackage Signed-off-by: Dimitri John Ledkov --- py3-wheel.yaml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/py3-wheel.yaml b/py3-wheel.yaml index 610a8d37576..9d40d90ffcd 100644 --- a/py3-wheel.yaml +++ b/py3-wheel.yaml @@ -37,14 +37,24 @@ subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} description: ${{vars.pypi-package}} installed for python${{range.key}} + pipeline: + - uses: py/pip-build-install-bootstrap + with: + python: python${{range.key}} + + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}}-bin + description: Executable binaries for ${{vars.pypi-package}} installed for python${{range.key}} dependencies: + runtime: + - py${{range.key}}-${{vars.pypi-package}} provides: - py3-${{vars.pypi-package}} provider-priority: ${{range.value}} pipeline: - - uses: py/pip-build-install-bootstrap - with: - python: python${{range.key}} + - runs: | + mkdir -p ${{targets.contextdir}}/usr + mv ${{targets.contextdir}}/../py${{range.key}}-${{vars.pypi-package}}/usr/bin ${{targets.contextdir}}/usr - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. From a2c8c0edb98a8a6a2affadfae9b5ff589fcdbbc4 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Tue, 28 May 2024 18:11:49 +0100 Subject: [PATCH 18/39] Make py3-pip coinstallable Signed-off-by: Dimitri John Ledkov --- py3-pip.yaml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/py3-pip.yaml b/py3-pip.yaml index a72b66af5a4..192fafd43cf 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -36,7 +36,10 @@ pipeline: subpackages: - range: py-versions - name: py${{range.key}}-${{vars.pypi-package}} + # Historically py3.X-pip installed /usr/bin/pip + # Hence co-installable modules only package is called -base + # And /usr/bin/pip conflicting package is called py3.X-pip + name: py${{range.key}}-${{vars.pypi-package}}-base description: ${{vars.pypi-package}} installed for python${{range.key}} pipeline: - uses: py/pip-build-install-bootstrap @@ -44,20 +47,32 @@ subpackages: python: python${{range.key}} wheel: false dependencies: - provider-priority: ${{range.value}} - provides: - - py3-pip runtime: - py${{range.key}}-flit-core - py${{range.key}}-setuptools + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}} + description: Executable binaries for ${{vars.pypi-package}} installed for python${{range.key}} + dependencies: + runtime: + - py${{range.key}}-${{vars.pypi-package}}-base + provides: + - py3-${{vars.pypi-package}} + provider-priority: ${{range.value}} + pipeline: + - runs: | + mkdir -p ${{targets.contextdir}}/usr/bin + mv ${{targets.contextdir}}/../py${{range.key}}-${{vars.pypi-package}}-base/usr/bin/pip ${{targets.contextdir}}/usr/bin/ + mv ${{targets.contextdir}}/../py${{range.key}}-${{vars.pypi-package}}-base/usr/bin/pip3 ${{targets.contextdir}}/usr/bin/ + - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. dependencies: runtime: - - py3.12-${{vars.pypi-package}}=${{package.full-version}} - - py3.11-${{vars.pypi-package}}=${{package.full-version}} - - py3.10-${{vars.pypi-package}}=${{package.full-version}} + - py3.12-${{vars.pypi-package}}-base + - py3.11-${{vars.pypi-package}}-base + - py3.10-${{vars.pypi-package}}-base update: enabled: true From 667f6a86d4d93f9e441e4649cda67bc7542e75d5 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Tue, 28 May 2024 18:19:44 +0100 Subject: [PATCH 19/39] py3-supported-cython is not coinstallable, and not needed Signed-off-by: Dimitri John Ledkov --- py3-cython.yaml | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/py3-cython.yaml b/py3-cython.yaml index 93873a6979b..14e64b212b5 100644 --- a/py3-cython.yaml +++ b/py3-cython.yaml @@ -6,10 +6,7 @@ package: copyright: - license: Apache-2.0 dependencies: - runtime: - - py3.12-${{vars.pypi-package}} - provides: - - cython=${{package.version}} + provider-priority: 0 vars: pypi-package: cython @@ -17,9 +14,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "312" environment: contents: @@ -40,6 +37,11 @@ pipeline: subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} + dependencies: + provider-priority: ${{range.value}} + provides: + - ${{package.name}} + - cyphon pipeline: - uses: py/pip-build-install with: @@ -50,14 +52,6 @@ subpackages: pipeline: - uses: py/collect-wheels - - name: py3-supported-${{vars.pypi-package}} - description: meta package providing ${{vars.pypi-package}} for supported python versions. - dependencies: - runtime: - - py3.10-${{vars.pypi-package}} - - py3.11-${{vars.pypi-package}} - - py3.12-${{vars.pypi-package}} - update: enabled: true github: From 3c0e1c78b33b18082328af2bac012bddb05d416b Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Tue, 28 May 2024 18:29:23 +0100 Subject: [PATCH 20/39] Fix py3-tomli priorities Signed-off-by: Dimitri John Ledkov --- py3-tomli.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py3-tomli.yaml b/py3-tomli.yaml index c546d89a54a..601791b09bb 100644 --- a/py3-tomli.yaml +++ b/py3-tomli.yaml @@ -16,7 +16,7 @@ data: items: 3.10: "310" 3.11: "311" - 3.12: "0" + 3.12: "312" environment: contents: From 83eae213d45dd06bc2c3326361ca6dea05b2b0bc Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Tue, 28 May 2024 18:49:22 +0100 Subject: [PATCH 21/39] py3-six fix up environment --- py3-six.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/py3-six.yaml b/py3-six.yaml index 2d1a48a0511..23a71541b92 100644 --- a/py3-six.yaml +++ b/py3-six.yaml @@ -6,8 +6,7 @@ package: copyright: - license: MIT dependencies: - runtime: - - py3.12-${{vars.pypi-package}} + provider-priority: 0 vars: pypi-package: pyyaml @@ -15,9 +14,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "312" environment: contents: @@ -25,7 +24,9 @@ environment: - build-base - busybox - ca-certificates-bundle + - py3-supported-pip - py3-supported-python + - py3-supported-wheel - wolfi-base pipeline: @@ -38,6 +39,10 @@ subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} description: ${{vars.pypi-package}} installed for python${{range.key}} + dependencies: + provides: + - py3-${{vars.pypi-package}} + provider-priority: ${{range.value}} pipeline: - uses: py/pip-build-install with: From 038bf1f2c2e8d3200636b4aedf65d11be384fb78 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Tue, 28 May 2024 18:55:30 +0100 Subject: [PATCH 22/39] Fix up py3-numpy --- py3-numpy.yaml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/py3-numpy.yaml b/py3-numpy.yaml index 10dc8822372..c26cb84b78e 100644 --- a/py3-numpy.yaml +++ b/py3-numpy.yaml @@ -6,11 +6,7 @@ package: copyright: - license: BSD-3-Clause dependencies: - runtime: - - py3.12-${{vars.pypi-package}} - provides: - - numpy - - f2py + provider-priority: 0 vars: pypi-package: numpy @@ -18,9 +14,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "312" environment: contents: @@ -32,7 +28,7 @@ environment: - git - meson - openblas-dev - - py3-supported-cython + - py3-cython - py3-supported-pip - py3-supported-python-dev - wolfi-base @@ -52,14 +48,14 @@ subpackages: pipeline: - runs: python${{range.key}} -m pip install . "--root=${{targets.subpkgdir}}" --prefix=/usr - uses: strip - - - name: py3-supported-${{vars.pypi-package}} - description: meta package providing ${{vars.pypi-package}} for supported python versions. dependencies: runtime: - - py3.10-${{vars.pypi-package}} - - py3.11-${{vars.pypi-package}} - - py3.12-${{vars.pypi-package}} + - py${{range.key}}-${{vars.pypi-package}} + provides: + - py3-${{vars.pypi-package}} + - numpy + - f2py + provider-priority: ${{range.value}} update: enabled: true From 4ba63ea7e4c2f7568b5f6ae148d14ea5e2bc96da Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 29 May 2024 12:17:34 +0100 Subject: [PATCH 23/39] Fixup py3-pathspec Signed-off-by: Dimitri John Ledkov --- py3-pathspec.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/py3-pathspec.yaml b/py3-pathspec.yaml index 146fc215dfc..07434e8a5aa 100644 --- a/py3-pathspec.yaml +++ b/py3-pathspec.yaml @@ -6,8 +6,7 @@ package: copyright: - license: MPL-2.0 dependencies: - runtime: - - py3.12-${{vars.pypi-package}} + provider-priority: 0 vars: pypi-package: pathspec @@ -15,9 +14,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "312" environment: contents: @@ -43,6 +42,10 @@ subpackages: - uses: py/pip-build-install with: python: python${{range.key}} + dependencies: + provides: + - py3-${{vars.pypi-package}} + provider-priority: ${{range.value}} - name: py3-wheels-${{vars.pypi-package}} pipeline: From 264959166fd913abe5455900e43623dbed6c5540 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 29 May 2024 12:23:29 +0100 Subject: [PATCH 24/39] Fixes Signed-off-by: Dimitri John Ledkov --- py3-pyyaml.yaml | 13 ++++++++----- py3-setuptools.yaml | 2 +- py3-six.yaml | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/py3-pyyaml.yaml b/py3-pyyaml.yaml index c071cfd20fe..0ccab9d6cd7 100644 --- a/py3-pyyaml.yaml +++ b/py3-pyyaml.yaml @@ -6,8 +6,7 @@ package: copyright: - license: MIT dependencies: - runtime: - - py3.12-${{vars.pypi-package}} + provider-priority: 0 vars: pypi-package: pyyaml @@ -15,9 +14,9 @@ vars: data: - name: py-versions items: - 3.10: "" - 3.11: "" - 3.12: "" + 3.10: "310" + 3.11: "311" + 3.12: "312" environment: contents: @@ -47,6 +46,10 @@ subpackages: - range: py-versions name: py${{range.key}}-${{vars.pypi-package}} description: ${{vars.pypi-package}} installed for python${{range.key}} + dependencies: + provider-priority: ${{range.value}} + provides: + - py3-${{vars.pypi-package}} pipeline: - uses: py/pip-build-install with: diff --git a/py3-setuptools.yaml b/py3-setuptools.yaml index d42c71b336a..5a5a06db3c1 100644 --- a/py3-setuptools.yaml +++ b/py3-setuptools.yaml @@ -43,7 +43,7 @@ subpackages: dependencies: provider-priority: ${{range.value}} provides: - - ${{package.name}} + - py3-${{vars.pypi-package}} pipeline: - uses: py/pip-build-install-bootstrap with: diff --git a/py3-six.yaml b/py3-six.yaml index 23a71541b92..88ab2e47fc5 100644 --- a/py3-six.yaml +++ b/py3-six.yaml @@ -9,7 +9,7 @@ package: provider-priority: 0 vars: - pypi-package: pyyaml + pypi-package: six data: - name: py-versions From 958c89deaa2573f68af216a301b7592c95dd6d24 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 29 May 2024 12:26:02 +0100 Subject: [PATCH 25/39] Remove duplicate patch Signed-off-by: Dimitri John Ledkov --- pypi-pyyaml/pyyaml-cython3.patch | 36 -------------------------------- 1 file changed, 36 deletions(-) delete mode 100644 pypi-pyyaml/pyyaml-cython3.patch diff --git a/pypi-pyyaml/pyyaml-cython3.patch b/pypi-pyyaml/pyyaml-cython3.patch deleted file mode 100644 index 7ffbc243d10..00000000000 --- a/pypi-pyyaml/pyyaml-cython3.patch +++ /dev/null @@ -1,36 +0,0 @@ -https://bugs.gentoo.org/898680 -https://github.com/yaml/pyyaml/pull/731 - -From 17dc5b6cd96dcfe64fd71789c771ca9b96d260e5 Mon Sep 17 00:00:00 2001 -From: "Andrew J. Hesford" -Date: Fri, 21 Jul 2023 09:50:00 -0400 -Subject: [PATCH] Fix builds with Cython 3 - -This is a *de minimis* fix for building with Cython 3. Recent Cython<3 -releases provided `Cython.Distutils.build_ext` as an alias to -`Cython.Distutils.old_build_ext.old_build_ext`; Cython 3 drops this -alias and instead uses a wholly new `Cython.Distutils.build_ext` that -does not provide the `cython_sources` function used in `setup.py`. - -Explicitly importing `old_build_ext` preserves the existing behavior for -recent Cython<3 and uses the correct behavior for Cython 3. Should the -import fail (*e.g.*, because the version of Cython available predates -the availability of `old_build_ext`), the import falls back to just -`Cython.Distutils.build_ext`. - -Signed-off-by: Andrew J. Hesford ---- a/setup.py -+++ b/setup.py -@@ -82,7 +82,11 @@ - with_cython = True - try: - from Cython.Distutils.extension import Extension as _Extension -- from Cython.Distutils import build_ext as _build_ext -+ try: -+ from Cython.Distutils.old_build_ext import old_build_ext as _build_ext -+ except ImportError: -+ from Cython.Distutils import build_ext as _build_ext -+ - with_cython = True - except ImportError: - if with_cython: \ No newline at end of file From 68014830b12e5336f4c2246fa22b641136a3df5a Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 29 May 2024 12:29:45 +0100 Subject: [PATCH 26/39] Split numpy bins, in case something uses them Signed-off-by: Dimitri John Ledkov --- py3-numpy.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/py3-numpy.yaml b/py3-numpy.yaml index c26cb84b78e..30dffc06977 100644 --- a/py3-numpy.yaml +++ b/py3-numpy.yaml @@ -57,6 +57,20 @@ subpackages: - f2py provider-priority: ${{range.value}} + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}}-bin + description: Executable binaries for ${{vars.pypi-package}} installed for python${{range.key}} + dependencies: + runtime: + - py${{range.key}}-${{vars.pypi-package}} + provides: + - py3-${{vars.pypi-package}} + provider-priority: ${{range.value}} + pipeline: + - runs: | + mkdir -p ${{targets.contextdir}}/usr + mv ${{targets.contextdir}}/../py${{range.key}}-${{vars.pypi-package}}/usr/bin ${{targets.contextdir}}/usr + update: enabled: true github: From 96125aa60d89e13a72d4bb371603554000c76625 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 29 May 2024 12:38:22 +0100 Subject: [PATCH 27/39] Fixup gpep517 dist-info and binaries Signed-off-by: Dimitri John Ledkov --- py3-gpep517.yaml | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/py3-gpep517.yaml b/py3-gpep517.yaml index 5ed83df658d..7b92ae4d136 100644 --- a/py3-gpep517.yaml +++ b/py3-gpep517.yaml @@ -24,7 +24,9 @@ environment: - build-base - busybox - ca-certificates-bundle + - py3-supported-pip - py3-supported-python + - py3-supported-wheel - wolfi-base pipeline: @@ -38,13 +40,9 @@ subpackages: name: py${{range.key}}-${{vars.pypi-package}} description: Easily download, build, install, upgrade, and uninstall Python packages pipeline: - - runs: | - sitedir=usr/lib/python${{range.key}}/site-packages/ - mkdir -p "${{targets.subpkgdir}}/$sitedir" - cp -a gpep517 "${{targets.subpkgdir}}/$sitedir" - python${{range.key}} -m compileall --invalidation-mode=unchecked-hash -r100 \ - ${{targets.subpkgdir}}/usr/lib/ - - uses: strip + - uses: py/pip-build-install + with: + python: python${{range.key}} dependencies: provider-priority: ${{range.value}} provides: @@ -53,6 +51,20 @@ subpackages: - py${{range.key}}-tomli - py${{range.key}}-installer + - range: py-versions + name: py${{range.key}}-${{vars.pypi-package}}-bin + description: Executable binaries for ${{vars.pypi-package}} installed for python${{range.key}} + dependencies: + runtime: + - py${{range.key}}-${{vars.pypi-package}} + provides: + - py3-${{vars.pypi-package}} + provider-priority: ${{range.value}} + pipeline: + - runs: | + mkdir -p ${{targets.contextdir}}/usr + mv ${{targets.contextdir}}/../py${{range.key}}-${{vars.pypi-package}}/usr/bin ${{targets.contextdir}}/usr + - name: py3-supported-${{vars.pypi-package}} description: meta package providing ${{vars.pypi-package}} for supported python versions. dependencies: From bc08d0f5412b6e74a47e35a5008085e04e99aff9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 30 May 2024 13:36:13 +0000 Subject: [PATCH 28/39] Include some bootstrap wheels from pypi in pip-zipapp Now this will work, and gets dist-info installed for setuptools. make package/pip-zipapp \ package/py3-{supported-python,setuptools,installer,wheel,pip} Other changes here: * switch to bootstrap workflow building wheels * point it at the included wheels dir to allow it to install dependencies * tell pip-zipapp which python to use (it doesn't just use the one it was invoked with) * drop '--no-build-isolation' from 'wheel', which allows pip to install the wheels it has, but use '--no-index' which tells it not to use the internet. --- pip-zipapp.yaml | 28 +++++++++++++++++++ pipelines/py/pip-build-install-bootstrap.yaml | 13 +++++---- py3-pip.yaml | 1 - 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/pip-zipapp.yaml b/pip-zipapp.yaml index bea82330faa..e307ad1d27c 100644 --- a/pip-zipapp.yaml +++ b/pip-zipapp.yaml @@ -17,6 +17,31 @@ pipeline: uri: https://bootstrap.pypa.io/pip/zipapp/pip-${{package.version}}.pyz expected-sha256: 1af141650f2b1e71bf9d856db74324042b4bfe3a36c37f8c2b3e2d8d6b3da411 extract: false + - uses: fetch + with: + uri: "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl" + expected-sha256: "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53" + extract: false + - uses: fetch + with: + uri: "https://files.pythonhosted.org/packages/de/88/70c5767a0e43eb4451c2200f07d042a4bcd7639276003a9c54a68cfcc1f8/setuptools-70.0.0-py3-none-any.whl" + expected-sha256: "54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4" + extract: false + - uses: fetch + with: + uri: "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl" + expected-sha256: "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc" + extract: false + - uses: fetch + with: + uri: "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl" + expected-sha256: "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81" + extract: false + - uses: fetch + with: + uri: "https://files.pythonhosted.org/packages/38/45/618e84e49a6c51e5dd15565ec2fcd82ab273434f236b8f108f065ded517a/flit_core-3.9.0-py3-none-any.whl" + expected-sha256: "7aada352fb0c7f5538c4fafeddf314d3a6a92ee8e2b1de70482329e42de70301" + extract: false - runs: | destd=${{targets.destdir}} @@ -26,6 +51,9 @@ pipeline: cp "pip-${{package.version}}.pyz" "$destd/$sdir/$fname" chmod 0644 "$destd/$sdir/$fname" + mkdir -p "$destd/$sdir/wheels" + cp *.whl "$destd/$sdir/wheels/" + update: enabled: true github: diff --git a/pipelines/py/pip-build-install-bootstrap.yaml b/pipelines/py/pip-build-install-bootstrap.yaml index fa58bb4cf34..865c749a7e0 100644 --- a/pipelines/py/pip-build-install-bootstrap.yaml +++ b/pipelines/py/pip-build-install-bootstrap.yaml @@ -11,7 +11,7 @@ inputs: wheel: description: build a wheel? required: false - default: false + default: true dest: description: the destination default: ${{targets.subpkgdir}} @@ -36,21 +36,22 @@ pipeline: trap "rm -Rf '$tmpd'" EXIT # --find-links to an empty dir and --no-index makes pip fully "offline" - distwheelsd="$tmpd/dist-wheels" + distwheelsd="/usr/share/pip-zipapp/wheels" mkdir -p "$distwheelsd" info "$py is $pyver with site_packages dir '$sitepkgd'" if [ "${{inputs.wheel}}" = "false" ]; then info "skipping bdist-wheel" - vr "$py" "$pipzip" install --verbose --no-deps \ + vr "$py" "$pipzip" --python=/usr/bin/$py install --verbose --no-deps \ + "--find-links=$distwheelsd" --no-index \ --force-reinstall --no-compile --prefix=/usr "--root=$root" . else - vr "$py" "$pipzip" wheel --verbose "--wheel-dir=$wd" \ - "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps . + vr "$py" "$pipzip" --python=/usr/bin/$py wheel --verbose "--wheel-dir=$wd" \ + "--find-links=$distwheelsd" --no-index --no-deps . set -- "$wd"/*.whl [ -f "$1" ] || { info "did not produce a wheel in $wd"; exit 1; } info "produced $# wheels: $*" - vr "$py" "$pipzip" install --verbose \ + vr "$py" "$pipzip" --python=/usr/bin/$py install --verbose \ "--find-links=$distwheelsd" --no-index --no-build-isolation --no-deps \ --force-reinstall --no-compile --prefix=/usr "--root=$root" "$@" fi diff --git a/py3-pip.yaml b/py3-pip.yaml index 192fafd43cf..84e82630fe4 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -45,7 +45,6 @@ subpackages: - uses: py/pip-build-install-bootstrap with: python: python${{range.key}} - wheel: false dependencies: runtime: - py${{range.key}}-flit-core From ccee10b16823dc9163536794b3ef291437410727 Mon Sep 17 00:00:00 2001 From: Pris Nasrat Date: Mon, 17 Jun 2024 11:23:54 +0000 Subject: [PATCH 29/39] py3-pip make update check ignore beta b tags Signed-off-by: Pris Nasrat --- py3-pip.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/py3-pip.yaml b/py3-pip.yaml index 84e82630fe4..48f04c4cf48 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -74,6 +74,8 @@ subpackages: - py3.10-${{vars.pypi-package}}-base update: + ignore-regex-patterns: + - b enabled: true shared: true github: From 6cab0aa426d940fb084a86e4ac723990bf8079d3 Mon Sep 17 00:00:00 2001 From: Pris Nasrat Date: Mon, 17 Jun 2024 14:19:39 +0000 Subject: [PATCH 30/39] py3-tomli epoch bump for merge Signed-off-by: Pris Nasrat --- py3-tomli.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py3-tomli.yaml b/py3-tomli.yaml index 601791b09bb..5017941f7cd 100644 --- a/py3-tomli.yaml +++ b/py3-tomli.yaml @@ -1,7 +1,7 @@ package: name: py3-tomli version: 2.0.1 - epoch: 5 + epoch: 8 description: "TOML parser" copyright: - license: MIT From 3710cfd4471d45048f8b75d8c37abd2f3db1523d Mon Sep 17 00:00:00 2001 From: Pris Nasrat Date: Mon, 17 Jun 2024 14:42:45 +0000 Subject: [PATCH 31/39] pip-zipapp lint clean Signed-off-by: Pris Nasrat --- pip-zipapp.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pip-zipapp.yaml b/pip-zipapp.yaml index e307ad1d27c..202366a826d 100644 --- a/pip-zipapp.yaml +++ b/pip-zipapp.yaml @@ -17,26 +17,31 @@ pipeline: uri: https://bootstrap.pypa.io/pip/zipapp/pip-${{package.version}}.pyz expected-sha256: 1af141650f2b1e71bf9d856db74324042b4bfe3a36c37f8c2b3e2d8d6b3da411 extract: false + - uses: fetch with: uri: "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl" expected-sha256: "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53" extract: false + - uses: fetch with: uri: "https://files.pythonhosted.org/packages/de/88/70c5767a0e43eb4451c2200f07d042a4bcd7639276003a9c54a68cfcc1f8/setuptools-70.0.0-py3-none-any.whl" expected-sha256: "54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4" extract: false + - uses: fetch with: uri: "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl" expected-sha256: "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc" extract: false + - uses: fetch with: uri: "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl" expected-sha256: "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81" extract: false + - uses: fetch with: uri: "https://files.pythonhosted.org/packages/38/45/618e84e49a6c51e5dd15565ec2fcd82ab273434f236b8f108f065ded517a/flit_core-3.9.0-py3-none-any.whl" From 1f219d8b2775f4a1166e598d712d236845d26c57 Mon Sep 17 00:00:00 2001 From: Pris Nasrat Date: Mon, 17 Jun 2024 15:44:40 +0000 Subject: [PATCH 32/39] py3-cython: fix typo in provides cython Signed-off-by: Pris Nasrat --- py3-cython.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py3-cython.yaml b/py3-cython.yaml index 14e64b212b5..4345ac895ae 100644 --- a/py3-cython.yaml +++ b/py3-cython.yaml @@ -1,7 +1,7 @@ package: name: py3-cython version: 3.0.10 - epoch: 1 + epoch: 2 description: Cython is an optimising static compiler for both the Python & the extended Cython programming languages. copyright: - license: Apache-2.0 @@ -41,7 +41,7 @@ subpackages: provider-priority: ${{range.value}} provides: - ${{package.name}} - - cyphon + - cython pipeline: - uses: py/pip-build-install with: From 0f4779b002fd3005a3eb2696fb9e9dc84822431d Mon Sep 17 00:00:00 2001 From: Pris Nasrat Date: Mon, 17 Jun 2024 15:45:43 +0000 Subject: [PATCH 33/39] py3-cython fix spurious epoch bump Signed-off-by: Pris Nasrat --- py3-cython.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py3-cython.yaml b/py3-cython.yaml index 4345ac895ae..4338fc752d4 100644 --- a/py3-cython.yaml +++ b/py3-cython.yaml @@ -1,7 +1,7 @@ package: name: py3-cython version: 3.0.10 - epoch: 2 + epoch: 1 description: Cython is an optimising static compiler for both the Python & the extended Cython programming languages. copyright: - license: Apache-2.0 From bf6d717368bbbd9d43bbfcc562eefdc678c9cb6b Mon Sep 17 00:00:00 2001 From: Pris Nasrat Date: Mon, 17 Jun 2024 15:50:10 +0000 Subject: [PATCH 34/39] Revert "Remove no longer needed cython-0" This reverts commit 634fa9312a3156f6868be5a9d7a5a86ba528d6d5. Various packages still depend on `cython~0` Signed-off-by: Pris Nasrat --- cython-0.yaml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 cython-0.yaml diff --git a/cython-0.yaml b/cython-0.yaml new file mode 100644 index 00000000000..90a0091d207 --- /dev/null +++ b/cython-0.yaml @@ -0,0 +1,40 @@ +package: + name: cython-0 + version: 0.29.37.1 + epoch: 0 + description: Cython is an optimising static compiler for both the Python & the extended Cython programming languages. + copyright: + - license: Apache-2.0 + dependencies: + provides: + - cython=${{package.version}}-r${{package.epoch}} + +environment: + contents: + packages: + - build-base + - busybox + - ca-certificates-bundle + - py3-setuptools + - python3 + - python3-dev + +pipeline: + - uses: git-checkout + with: + expected-commit: 598be7f9fd253c9e44f051f2235abedc66c022c1 + repository: https://github.com/cython/cython + tag: ${{package.version}} + + - runs: | + python setup.py build + python setup.py install --prefix=/usr --root="${{targets.destdir}}" + + - uses: strip + +update: + enabled: true + github: + identifier: cython/cython + use-tag: true + tag-filter: 0. From f4cfefb3095c52af9e6b759f41d0302583e46156 Mon Sep 17 00:00:00 2001 From: Pris Nasrat Date: Mon, 17 Jun 2024 16:10:39 +0000 Subject: [PATCH 35/39] py3-pyyaml: require py3-cython clean before build to ensure versioned cpython libs created for all py versions Signed-off-by: Pris Nasrat --- py3-pyyaml.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/py3-pyyaml.yaml b/py3-pyyaml.yaml index 0ccab9d6cd7..6cb3598df38 100644 --- a/py3-pyyaml.yaml +++ b/py3-pyyaml.yaml @@ -24,9 +24,10 @@ environment: - build-base - busybox - ca-certificates-bundle - - py3-supported-cython + - py3-cython - py3-supported-pip - py3-supported-python-dev + - py3-supported-wheel - wolfi-base - yaml-dev @@ -51,6 +52,7 @@ subpackages: provides: - py3-${{vars.pypi-package}} pipeline: + - runs: python${{range.key}} setup.py clean --all - uses: py/pip-build-install with: python: python${{range.key}} From 0fcf53918cc4c154c443628a9f90849587ecfcdc Mon Sep 17 00:00:00 2001 From: Pris Nasrat Date: Mon, 17 Jun 2024 16:42:50 +0000 Subject: [PATCH 36/39] python packages: simplify build env deps build-base pulls in wolfi-baselayout and transitively ca-certificates-bundle Signed-off-by: Pris Nasrat --- py3-cython.yaml | 1 - py3-flit-core.yaml | 2 -- py3-gpep517.yaml | 2 -- py3-installer.yaml | 2 -- py3-pathspec.yaml | 2 -- py3-pip.yaml | 2 -- py3-pyyaml.yaml | 2 -- py3-setuptools.yaml | 2 -- py3-six.yaml | 2 -- py3-tomli.yaml | 2 -- py3-wheel.yaml | 2 -- 11 files changed, 21 deletions(-) diff --git a/py3-cython.yaml b/py3-cython.yaml index 4338fc752d4..242c20d952e 100644 --- a/py3-cython.yaml +++ b/py3-cython.yaml @@ -23,7 +23,6 @@ environment: packages: - build-base - busybox - - ca-certificates-bundle - py3-supported-pip - py3-supported-python-dev - py3-supported-wheel diff --git a/py3-flit-core.yaml b/py3-flit-core.yaml index 9f61c9bc519..fc9e9ce60b1 100644 --- a/py3-flit-core.yaml +++ b/py3-flit-core.yaml @@ -23,9 +23,7 @@ environment: packages: - build-base - busybox - - ca-certificates-bundle - py3-supported-python - - wolfi-base pipeline: - uses: fetch diff --git a/py3-gpep517.yaml b/py3-gpep517.yaml index 7b92ae4d136..6916a90512e 100644 --- a/py3-gpep517.yaml +++ b/py3-gpep517.yaml @@ -23,11 +23,9 @@ environment: packages: - build-base - busybox - - ca-certificates-bundle - py3-supported-pip - py3-supported-python - py3-supported-wheel - - wolfi-base pipeline: - uses: fetch diff --git a/py3-installer.yaml b/py3-installer.yaml index 6a828b62654..0759ff78720 100644 --- a/py3-installer.yaml +++ b/py3-installer.yaml @@ -23,9 +23,7 @@ environment: packages: - build-base - busybox - - ca-certificates-bundle - py3-supported-python - - wolfi-base pipeline: - uses: git-checkout diff --git a/py3-pathspec.yaml b/py3-pathspec.yaml index 07434e8a5aa..573ae2cab61 100644 --- a/py3-pathspec.yaml +++ b/py3-pathspec.yaml @@ -23,10 +23,8 @@ environment: packages: - build-base - busybox - - ca-certificates-bundle - py3-supported-pip - py3-supported-python - - wolfi-base pipeline: - uses: fetch diff --git a/py3-pip.yaml b/py3-pip.yaml index 48f04c4cf48..532c4171d9f 100644 --- a/py3-pip.yaml +++ b/py3-pip.yaml @@ -23,9 +23,7 @@ environment: packages: - build-base - busybox - - ca-certificates-bundle - py3-supported-python - - wolfi-base pipeline: - uses: git-checkout diff --git a/py3-pyyaml.yaml b/py3-pyyaml.yaml index 6cb3598df38..9e953b58984 100644 --- a/py3-pyyaml.yaml +++ b/py3-pyyaml.yaml @@ -23,12 +23,10 @@ environment: packages: - build-base - busybox - - ca-certificates-bundle - py3-cython - py3-supported-pip - py3-supported-python-dev - py3-supported-wheel - - wolfi-base - yaml-dev pipeline: diff --git a/py3-setuptools.yaml b/py3-setuptools.yaml index 5a5a06db3c1..86881e80366 100644 --- a/py3-setuptools.yaml +++ b/py3-setuptools.yaml @@ -23,9 +23,7 @@ environment: packages: - build-base - busybox - - ca-certificates-bundle - py3-supported-python - - wolfi-base environment: # This is needed to work around the error "ValueError: ZIP does not support timestamps before 1980" SOURCE_DATE_EPOCH: 315532800 diff --git a/py3-six.yaml b/py3-six.yaml index 88ab2e47fc5..8222f585bfc 100644 --- a/py3-six.yaml +++ b/py3-six.yaml @@ -23,11 +23,9 @@ environment: packages: - build-base - busybox - - ca-certificates-bundle - py3-supported-pip - py3-supported-python - py3-supported-wheel - - wolfi-base pipeline: - uses: fetch diff --git a/py3-tomli.yaml b/py3-tomli.yaml index 3f43de1b499..5d5f010d590 100644 --- a/py3-tomli.yaml +++ b/py3-tomli.yaml @@ -23,10 +23,8 @@ environment: packages: - build-base - busybox - - ca-certificates-bundle - py3-supported-installer - py3-supported-python - - wolfi-base pipeline: - uses: fetch diff --git a/py3-wheel.yaml b/py3-wheel.yaml index 9d40d90ffcd..d746f4405d2 100644 --- a/py3-wheel.yaml +++ b/py3-wheel.yaml @@ -22,9 +22,7 @@ environment: contents: packages: - busybox - - ca-certificates-bundle - py3-supported-python - - wolfi-base pipeline: - uses: git-checkout From ef27c60e9354969d51a422f35c3bd0745d47d943 Mon Sep 17 00:00:00 2001 From: Pris Nasrat Date: Mon, 17 Jun 2024 18:51:02 +0000 Subject: [PATCH 37/39] py3-gpep517 explicit setuptools dep Signed-off-by: Pris Nasrat --- py3-gpep517.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/py3-gpep517.yaml b/py3-gpep517.yaml index 6916a90512e..9f81c81cd8a 100644 --- a/py3-gpep517.yaml +++ b/py3-gpep517.yaml @@ -25,6 +25,7 @@ environment: - busybox - py3-supported-pip - py3-supported-python + - py3-supported-setuptools - py3-supported-wheel pipeline: From 8a0720d838031328210a87785f5770b98f888ba2 Mon Sep 17 00:00:00 2001 From: Pris Nasrat Date: Mon, 17 Jun 2024 19:23:19 +0000 Subject: [PATCH 38/39] py3-tomli don't use python/import Need to use explicit py version in import test See https://github.com/chainguard-dev/melange/issues/1299 Signed-off-by: Pris Nasrat --- py3-tomli.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py3-tomli.yaml b/py3-tomli.yaml index 5d5f010d590..f0767bc01bd 100644 --- a/py3-tomli.yaml +++ b/py3-tomli.yaml @@ -54,6 +54,6 @@ update: test: pipeline: - - uses: python/import - with: - import: tomli + - name: Verify tomli installation + runs: | + python3.12 -c "import tomli" || exit3 1 From dd74c20b220e6228c38461e000c93bc38ba165e4 Mon Sep 17 00:00:00 2001 From: Pris Nasrat Date: Mon, 17 Jun 2024 19:24:06 +0000 Subject: [PATCH 39/39] py3-tomli fix typo Signed-off-by: Pris Nasrat --- py3-tomli.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py3-tomli.yaml b/py3-tomli.yaml index f0767bc01bd..be4c5436a1b 100644 --- a/py3-tomli.yaml +++ b/py3-tomli.yaml @@ -56,4 +56,4 @@ test: pipeline: - name: Verify tomli installation runs: | - python3.12 -c "import tomli" || exit3 1 + python3.12 -c "import tomli" || exit 1