From 10e326a50df0bf5678b7860ddc7c934eeb908444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Vask=C3=B3?= <1771332+vlaci@users.noreply.github.com> Date: Thu, 14 Dec 2023 17:41:42 +0100 Subject: [PATCH] nix: publish package as an overlay --- flake.lock | 34 ----- flake.nix | 395 ++++++++++---------------------------------------- pyperscan.nix | 136 +++++++++++++++++ tests.nix | 111 ++++++++++++++ 4 files changed, 323 insertions(+), 353 deletions(-) create mode 100644 pyperscan.nix create mode 100644 tests.nix diff --git a/flake.lock b/flake.lock index 092f4f5..e81294f 100644 --- a/flake.lock +++ b/flake.lock @@ -57,24 +57,6 @@ "type": "github" } }, - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1701680307, - "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1702272962, @@ -96,24 +78,8 @@ "advisory-db": "advisory-db", "crane": "crane", "fenix": "fenix", - "flake-utils": "flake-utils", "nixpkgs": "nixpkgs" } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 2dce401..3a83aa8 100644 --- a/flake.nix +++ b/flake.nix @@ -13,339 +13,96 @@ inputs.rust-analyzer-src.follows = ""; }; - flake-utils.url = "github:numtide/flake-utils"; - advisory-db = { url = "github:rustsec/advisory-db"; flake = false; }; }; - outputs = { self, nixpkgs, crane, fenix, flake-utils, advisory-db, ... }: - flake-utils.lib.eachDefaultSystem (system: - let - pkgs = import nixpkgs { - inherit system; - }; - inherit (pkgs) lib makeRustPlatform python3Packages; - - craneLib = crane.lib.${system}; - cppFilter = path: _type: builtins.match ".*/hyperscan-sys/(wrapper.h|hyperscan|vectorscan).*$" path != null; - - sourceFilter = path: type: - (cppFilter path type) || (craneLib.filterCargoSources path type); - - src = lib.cleanSourceWith { - src = craneLib.path ./.; - filter = sourceFilter; - }; - - buildInputs = with pkgs; [ - python3 - ] ++ lib.optional stdenv.isDarwin libiconv - ++ lib.optional (system == "x86_64-linux") hyperscan - ++ lib.optional (system != "x86_64-linux") (vectorscan.override { boost = boost183; }); - - rust-toolchain = fenix.packages.${system}.complete.toolchain; - rust-toolchain-llvm-tools = fenix.packages.${system}.complete.withComponents [ - "llvm-tools" - "cargo" - "rustc" - ]; - craneLibLLvmTools = craneLib.overrideToolchain rust-toolchain-llvm-tools; - - commonArgs = { - inherit src buildInputs; - - # python package build will recompile PyO3 when built with maturin - # as there are different build features are used for the extension module - # and the standalone dylib which is used for tests and benchmarks - doNotLinkInheritedArtifacts = true; - }; - - # Build *just* the cargo dependencies, so we can reuse - # all of that work (e.g. via cachix) when running in CI - cargoArtifacts = craneLib.buildDepsOnly commonArgs; - - # Build the actual crate itself, reusing the dependency - # artifacts from above. - libpyperscan = craneLib.buildPackage (commonArgs // { - inherit cargoArtifacts; + outputs = { self, nixpkgs, crane, fenix, advisory-db, ... }@inputs: + let + supportedSystems = [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + nixpkgsFor = forAllSystems (system: import nixpkgs { inherit system; overlays = [ self.overlays.default ]; }); + + in + { + overlays.default = final: prev: let + stdenv = if prev.stdenv.isDarwin then prev.clang11Stdenv else prev.stdenv; + rustPlatform = final.makeRustPlatform { inherit stdenv; inherit (final) rustc cargo; }; + in { + pyperscan = final.callPackage (import ./pyperscan.nix inputs) { inherit stdenv rustPlatform; }; + }; + checks = forAllSystems (system: + let + inherit (nixpkgsFor.${system}) pyperscan rustPlatform; nativeBuildInputs = with rustPlatform; [ bindgenHook ]; - }); - - rustPlatform = makeRustPlatform { - cargo = rust-toolchain; - rustc = rust-toolchain; - }; - - rustPlatform-cov = makeRustPlatform { - cargo = rust-toolchain-llvm-tools; - rustc = rust-toolchain-llvm-tools; - }; - - pyFilter = path: _type: builtins.match ".*pyi?$|.*/py.typed$|.*/pyproject.toml|.*/README.md$|.*/LICENSE" path != null; - testFilter = p: t: builtins.match ".*/(tests|tests/.*\.py)$" p != null; - - mkNativeBuildInputs = { rustPlatform, extra ? [ ] }: with rustPlatform; [ - bindgenHook - cargoSetupHook - maturinBuildHook - ] ++ extra; - - pyperscan = - let - drv = pkgs.callPackage - ({ lib - , stdenv - , python3Packages - , rustPlatform - , hyperscan - , vectorscan - , boost - , cmake - , pkg-config - , ragel - , util-linux - , vendorHyperscan ? false - , vendorVectorscan ? false - }: - - assert vendorHyperscan -> !vendorVectorscan; - assert vendorVectorscan -> !vendorHyperscan; - - let - inherit (lib) optional optionals; - vendor = vendorHyperscan || vendorVectorscan; - cargo_toml = builtins.fromTOML (builtins.readFile ./Cargo.toml); - in - python3Packages.buildPythonPackage - (commonArgs // { - inherit (cargo_toml.workspace.package) version; - - pname = "pyperscan"; - format = "pyproject"; - - src = lib.cleanSourceWith { - src = craneLib.path ./.; - filter = p: t: (pyFilter p t) || (sourceFilter p t); - }; - - strictDeps = true; - doCheck = false; - - cargoDeps = rustPlatform.importCargoLock { - lockFile = ./Cargo.lock; - }; - - maturinBuildFlags = (optionals vendorHyperscan [ "-F hyperscan" ]) ++ (optionals (vendorVectorscan) [ "-F vectorscan" ]); - - buildInputs = with pkgs; [ - python3 - ] ++ lib.optional stdenv.isDarwin libiconv - ++ lib.optional vendor boost - ++ lib.optional (!vendor && system == "x86_64-linux") hyperscan - ++ lib.optional (!vendor && system != "x86_64-linux") (vectorscan.override { boost = boost183; }); - - nativeBuildInputs = mkNativeBuildInputs { - inherit rustPlatform; - extra = ( - optionals vendor [ cmake ragel ] - ++ optional (vendor && stdenv.isLinux) util-linux - ); - }; - - dontUseCmakeConfigure = true; - - passthru = { - shared = drv; - hyperscan = drv.override { vendorHyperscan = true; }; - vectorscan = drv.override { vendorVectorscan = true; }; - tests.checks = with python3Packages; buildPythonPackage - { - inherit (pyperscan) version; - pname = "${pyperscan.pname}-tests-checks"; - format = "other"; - - src = lib.cleanSourceWith { - src = ./.; - filter = p: t: (pyFilter p t) || (testFilter p t); - }; - - dontBuild = true; - dontInstall = true; - - nativeCheckInputs = [ - black - mypy - pkgs.ruff - pytest - pyperscan - ]; - - checkPhase = '' - black . - mypy . - ruff . - ''; - }; - tests.coverage = - let - pyperscan-cov = pyperscan.overridePythonAttrs (with pkgs; super: { - pname = "${super.pname}-coverage"; - nativeBuildInputs = mkNativeBuildInputs { - rustPlatform = rustPlatform-cov; - extra = [ - cargo-llvm-cov - ]; - }; - preConfigure = (super.preConfigure or "") + '' - source <(cargo llvm-cov show-env --export-prefix) - export RUSTFLAGS="-C target-feature=-crt-static -C instrument-coverage -C llvm-args=--instrprof-atomic-counter-update-all --cfg=coverage --cfg=coverage_nightly --cfg=trybuild_no_target" - ''; - }); - in - with python3Packages; buildPythonPackage - { - inherit (pyperscan) version cargoDeps; - pname = "${pyperscan.pname}-tests-coverage"; - format = "other"; - - src = lib.cleanSourceWith { - src = ./.; - filter = p: t: (sourceFilter p t) || (testFilter p t); - }; - - dontBuild = true; - dontInstall = true; - - preCheck = '' - source <(cargo llvm-cov show-env --export-prefix) - LLVM_COV_FLAGS=$(echo -n $(find ${pyperscan-cov} -name "*.so")) - export LLVM_COV_FLAGS - ''; - postCheck = '' - rm -r $out - cargo llvm-cov report -vv --ignore-filename-regex cargo-vendor-dir --codecov --output-path $out - ''; - - nativeBuildInputs = - (with rustPlatform-cov; [ - cargoSetupHook - ]); + inherit (pyperscan) cargoArtifacts craneLib commonArgs libpyperscan src; + in + pyperscan.passthru.tests // { + # Build the crate as part of `nix flake check` for convenience + inherit libpyperscan; + + # Run clippy (and deny all warnings) on the crate source, + # again, resuing the dependency artifacts from above. + # + # Note that this is done as a separate derivation so that + # we can block the CI if there are issues here, but not + # prevent downstream consumers from building our crate by itself. + libpyperscan-clippy = craneLib.cargoClippy (commonArgs // { + inherit nativeBuildInputs cargoArtifacts; + cargoClippyExtraArgs = "--all-targets -- --deny warnings"; + }); + + libpyperscan-doc = craneLib.cargoDoc (commonArgs // { + inherit nativeBuildInputs cargoArtifacts; + }); + + # Check formatting + libpyperscan-fmt = craneLib.cargoFmt { + inherit src; + }; - nativeCheckInputs = with pkgs; [ - rust-toolchain-llvm-tools - cargo-llvm-cov - pyperscan-cov - pytestCheckHook - ]; - }; - tests.pytest = with python3Packages; buildPythonPackage - { - inherit (pyperscan) version; - pname = "${pyperscan.pname}-tests-pytest"; - format = "other"; + # Audit dependencies + libpyperscan-audit = craneLib.cargoAudit { + inherit src advisory-db; + }; + } + ); - src = lib.cleanSourceWith { - src = ./.; - filter = p: t: (testFilter p t); - }; + formatter = forAllSystems (system: nixpkgsFor.${system}.nixpkgs-fmt); - dontBuild = true; - dontInstall = true; + packages = forAllSystems (system: + let - nativeCheckInputs = [ - pyperscan - pytestCheckHook - ]; - }; - }; + inherit (nixpkgsFor.${system}) pyperscan; + in + { + inherit pyperscan; + default = pyperscan; + }); - })) - { - inherit rustPlatform; - vectorscan = pkgs.vectorscan.override { boost = pkgs.boost183; }; - boost = pkgs.boost183; - }; - in - drv; - in - { - checks = - let - nativeBuildInputs = with rustPlatform; [ - bindgenHook + devShells = forAllSystems (system: + let + pkgs = nixpkgsFor.${system}; + in + { + default = with pkgs; mkShell { + inputsFrom = [ pyperscan.libpyperscan ]; + buildInputs = [ + just + maturin + pdm + podman + pre-commit + boost + cmake + ragel + fenix.packages.${system}.complete.rust-analyzer ]; - in - pyperscan.passthru.tests // { - # Build the crate as part of `nix flake check` for convenience - inherit libpyperscan; - - # Run clippy (and deny all warnings) on the crate source, - # again, resuing the dependency artifacts from above. - # - # Note that this is done as a separate derivation so that - # we can block the CI if there are issues here, but not - # prevent downstream consumers from building our crate by itself. - libpyperscan-clippy = craneLib.cargoClippy (commonArgs // { - inherit nativeBuildInputs cargoArtifacts; - cargoClippyExtraArgs = "--all-targets -- --deny warnings"; - }); - - libpyperscan-doc = craneLib.cargoDoc (commonArgs // { - inherit nativeBuildInputs cargoArtifacts; - }); - - # Check formatting - libpyperscan-fmt = craneLib.cargoFmt { - inherit src; - }; - - # Audit dependencies - libpyperscan-audit = craneLib.cargoAudit { - inherit src advisory-db; - }; - } // lib.optionalAttrs - (system == "x86_64-linux") - { - # Check code coverage (note: this will not upload coverage anywhere) - libpyperscan-coverage = craneLibLLvmTools.cargoLlvmCov { - inherit src buildInputs nativeBuildInputs cargoArtifacts; - cargoLlvmCovExtraArgs = "--ignore-filename-regex /nix/store --codecov --output-path $out"; - }; - }; - - formatter = pkgs.nixpkgs-fmt; - - packages = with pkgs; - { - default = pyperscan; - }; - - devShells = - let - inherit (pkgs.lib) filter hasSuffix; - noHooks = filter (drv: !(hasSuffix "hook.sh" drv.name)); - in - rec { - default = with pkgs; mkShell { - inputsFrom = with self.checks.${system}; [ libpyperscan ]; - buildInputs = [ - just - maturin - pdm - podman - pre-commit - rust-toolchain - boost - cmake - ragel - fenix.packages.${system}.complete.rust-analyzer - ]; - }; }; - }); + }); + }; } diff --git a/pyperscan.nix b/pyperscan.nix new file mode 100644 index 0000000..2041daa --- /dev/null +++ b/pyperscan.nix @@ -0,0 +1,136 @@ +inputs: + +{ lib +, stdenv +, makeRustPlatform +, rustPlatform +, cargo +, rustc +, maturin +, cargo-llvm-cov +, system +, libiconv +, boost +, hyperscan +, vectorscan +, python3 +, ruff +, vendorHyperscan ? false +, vendorVectorscan ? false +, coverage ? false +, pyperscan +, zstd +}: + +assert vendorHyperscan -> !vendorVectorscan; +assert vendorVectorscan -> !vendorHyperscan; + +let + inherit (lib) optional optionals optionalString; + craneLib = inputs.crane.lib.${system}; + + cppFilter = path: _type: builtins.match ".*/hyperscan-sys/(wrapper.h|hyperscan|vectorscan).*$" path != null; + + pyFilter = path: _type: builtins.match ".*pyi?$|.*/py.typed$|.*/pyproject.toml|.*/README.md$|.*/LICENSE" path != null; + testFilter = p: t: builtins.match ".*/(tests|tests/.*\.py|examples|examples/.*\.py)$" p != null; + sourceFilter = path: type: + (cppFilter path type) || (craneLib.filterCargoSources path type); + + vendor = vendorHyperscan || vendorVectorscan; + buildInputs = [ + python3 + ] ++ optional stdenv.isDarwin libiconv + ++ optional vendor boost + ++ optional (system == "x86_64-linux") hyperscan + ++ optional (system != "x86_64-linux") vectorscan; + + src = lib.cleanSourceWith { + src = craneLib.path ./.; + filter = p: t: (pyFilter p t) || (sourceFilter p t); + }; + commonArgs = { + inherit src buildInputs; + + # python package build will recompile PyO3 when built with maturin + # as there are different build features are used for the extension module + # and the standalone dylib which is used for tests and benchmarks + doNotLinkInheritedArtifacts = true; + }; + + # Build *just* the cargo dependencies, so we can reuse + # all of that work (e.g. via cachix) when running in CI + cargoArtifacts = craneLib.buildDepsOnly commonArgs; + + # Build the actual crate itself, reusing the dependency + # artifacts from above. + libpyperscan = craneLib.buildPackage (commonArgs // { + inherit cargoArtifacts; + nativeBuildInputs = with rustPlatform; [ + bindgenHook + ]; + }); + + cargo_toml = builtins.fromTOML (builtins.readFile ./Cargo.toml); + + drv = python3.pkgs.buildPythonPackage { + pname = "pyperscan" + + optionalString vendorHyperscan "-hyperscan" + + optionalString vendorVectorscan "-vectorscan" + + optionalString coverage "-coverage"; + format = "pyproject"; + + + inherit src buildInputs; + inherit (cargo_toml.workspace.package) version; + + # python package build will recompile PyO3 when built with maturin + # as there are different build features are used for the extension module + # and the standalone dylib which is used for tests and benchmarks + doNotLinkInheritedArtifacts = true; + dontUseCmakeConfigure = true; + + strictDeps = true; + doCheck = false; + + cargoDeps = rustPlatform.importCargoLock { + lockFile = ./Cargo.lock; + }; + + maturinBuildFlags = + (optionals vendorHyperscan [ "-F hyperscan" ]) + ++ (optionals (vendorVectorscan) [ "-F vectorscan" ]); + + nativeBuildInputs = with rustPlatform; [ + bindgenHook + cargoSetupHook + (maturinBuildHook.override { pkgsHostTarget = { inherit maturin cargo rustc; }; }) + zstd + ] ++ optional (vendor && stdenv.isLinux) util-linux + ++ optional coverage cargo-llvm-cov; + + preConfigure = optionalString coverage '' + source <(cargo llvm-cov show-env --export-prefix) + ''; + preBuild = '' + tar -xf ${cargoArtifacts}/* + find target/release + pwd + ''; + postBuild = '' + find target/release + ''; + + passthru = { + inherit cargoArtifacts craneLib commonArgs libpyperscan; + shared = drv; + hyperscan = drv.override { vendorHyperscan = true; }; + vectorscan = drv.override { vendorVectorscan = true; }; + + tests = import ./tests.nix { + inherit lib stdenv system makeRustPlatform rustPlatform pyFilter testFilter cargo-llvm-cov python3 ruff pyperscan; + inherit (inputs) fenix; + }; + }; + }; +in +drv diff --git a/tests.nix b/tests.nix new file mode 100644 index 0000000..9a398d4 --- /dev/null +++ b/tests.nix @@ -0,0 +1,111 @@ +{ lib +, stdenv +, system +, fenix +, makeRustPlatform +, rustPlatform +, pyFilter +, testFilter +, cargo-llvm-cov +, python3 +, pyperscan +, ruff +}: + +{ + checks = with python3.pkgs; buildPythonPackage { + inherit (pyperscan) version; + pname = "${pyperscan.pname}-tests-checks"; + format = "other"; + + src = lib.cleanSourceWith { + src = ./.; + filter = p: t: (pyFilter p t) || (testFilter p t); + }; + + dontBuild = true; + dontInstall = true; + + nativeCheckInputs = [ + mypy + ruff + pyperscan + ]; + + checkPhase = '' + #mypy . + ruff . + ''; + }; + + coverage = + let + rust-toolchain-llvm-tools = fenix.packages.${system}.complete.withComponents [ + "llvm-tools-preview" + "cargo" + "rustc" + ]; + rustPlatform-cov = makeRustPlatform { + inherit stdenv; + cargo = rust-toolchain-llvm-tools; + rustc = rust-toolchain-llvm-tools; + }; + pyperscan-cov = pyperscan.override { coverage = true; rustPlatform = rustPlatform-cov; rustc = rust-toolchain-llvm-tools; cargo = rust-toolchain-llvm-tools;}; + in + with python3.pkgs; buildPythonPackage { + + inherit (pyperscan-cov) version cargoDeps; + pname = "${pyperscan-cov.pname}-tests-pytest"; + format = "other"; + + src = lib.cleanSourceWith { + src = ./.; + filter = p: t: (pyFilter p t) || (testFilter p t) || builtins.match "Cargo.(toml|lock)" != null; + + }; + + dontBuild = true; + dontInstall = true; + + nativeCheckInputs = with python3.pkgs; [ + cargo-llvm-cov + pytestCheckHook + pyperscan-cov + rust-toolchain-llvm-tools + ]; + + nativeBuildInputs = with rustPlatform-cov; [ + cargoSetupHook + ]; + + preConfigure = '' + source <(cargo llvm-cov show-env --export-prefix) + LLVM_COV_FLAGS=$(python -c 'import pyperscan; print(pyperscan._pyperscan.__file__, end="")') + export LLVM_COV_FLAGS + ''; + + postCheck = '' + rm -r $out + cargo llvm-cov report -vv --ignore-filename-regex cargo-vendor-dir --codecov --output-path $out + ''; + }; + pytest = python3.pkgs.buildPythonPackage + { + inherit (pyperscan) version; + pname = "${pyperscan.pname}-tests-pytest"; + format = "other"; + + src = lib.cleanSourceWith { + src = ./.; + filter = p: t: (testFilter p t); + }; + + dontBuild = true; + dontInstall = true; + + nativeCheckInputs = with python3.pkgs; [ + pyperscan + pytestCheckHook + ]; + }; +}