Allow running tests with cffi installed

Quality of live improvement, esp, with IDEs.

.github/workflows/test.yml

@@ -92,9 +92,6 @@ jobs:
           python_version=$(echo ${{ matrix.python-version }} | sed -e s/-dev//)
           echo "PYTHON_VERSION=$python_version" >> "${GITHUB_ENV}"
       - uses: astral-sh/setup-uv@v3
-      - name: Install cffi
-        if: ${{ !contains(matrix.python-version, 'pypy') }}
-        run: uv pip install --system cffi
       - name: Install python packages
         run: |
           uv tool install virtualenv
@@ -282,7 +279,7 @@ jobs:
         run: |
           set -ex
           apk add cargo python3-dev libffi-dev py3-pip curl \
-            bash tar zstd git patchelf py3-cffi py3-virtualenv
+            bash tar zstd git patchelf py3-virtualenv
       - uses: actions/checkout@v4
       - name: Fix git permissions
         run: git config --global --add safe.directory $GITHUB_WORKSPACE

Cargo.toml

@@ -135,9 +135,10 @@ pretty_assertions = { version = "1.3.0", optional = true }
 
 [dev-dependencies]
 expect-test = "1.4.1"
-rstest = "0.22.0"
+fs2 = "0.4.3"
 indoc = "2.0.3"
 pretty_assertions = "1.3.0"
+rstest = "0.22.0"
 rustversion = "1.0.9"
 time = { version = "0.3.34", features = ["macros"] }
 trycmd = "0.15.0"

guide/src/contributing.md

@@ -50,8 +50,7 @@ Ready to contribute? Here's how to setup maturin for local development.
    ```bash
    $ cargo test
    ```
-   Note that in order to run tests you need to install `virtualenv` and
-   `cffi` (`pip3 install cffi virtualenv`).
+   Note that in order to run tests you need to install `virtualenv` (`pip install virtualenv`).
 6. make sure your changes are well formatted and pass the linting checks by
    installing [pre-commit](https://pre-commit.com/) and running
    ```bash

tests/common/integration.rs

@@ -1,13 +1,17 @@
-use crate::common::{check_installed, create_virtualenv, maybe_mock_cargo, test_python_path};
+use crate::common::{
+    check_installed, create_virtualenv, create_virtualenv_name, maybe_mock_cargo, test_python_path,
+};
 use anyhow::{bail, Context, Result};
 #[cfg(feature = "zig")]
 use cargo_zigbuild::Zig;
 use clap::Parser;
+use fs2::FileExt;
 use fs_err::File;
-use maturin::{BuildOptions, PlatformTag, PythonInterpreter};
+use maturin::{BuildOptions, PlatformTag, PythonInterpreter, Target};
+use normpath::PathExt;
 use std::collections::HashSet;
 use std::env;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::str;
 
@@ -69,9 +73,55 @@ pub fn test_integration(
         false
     };
 
+    // One scope up to extend the lifetime
+    let venvs_dir = Path::new("test-crates")
+        .normalize()?
+        .into_path_buf()
+        .join("venvs");
+    let cffi_provider = "cffi-provider";
+    let cffi_venv = venvs_dir.join(cffi_provider);
+    let target_triple = Target::from_target_triple(None)?;
+    let python = target_triple.get_venv_python(&cffi_venv);
+
     if let Some(interp) = python_interp.as_ref() {
         cli.push("--interpreter");
         cli.push(interp);
+    } else {
+        // Install cffi in a separate environment
+
+        // All tests try to use this venv at the same time, so we need to make sure only one
+        // modifies it at a time and that during that time, no other test reads it.
+        let file = File::create(venvs_dir.join("cffi-provider.lock"))?;
+        file.file().lock_exclusive()?;
+        if !dbg!(venvs_dir.join(cffi_provider)).is_dir() {
+            println!("CRAETAITROAPTGIIDSOGRKADS");
+            dbg!(create_virtualenv_name(
+                cffi_provider,
+                python_interp.clone().map(PathBuf::from)
+            )?);
+            let pip_install_cffi = [
+                "-m",
+                "pip",
+                "--disable-pip-version-check",
+                "install",
+                "cffi",
+            ];
+            let output = Command::new(&python)
+                .args(pip_install_cffi)
+                .status()
+                //.output()
+                .context(format!("pip install cffi failed with {python:?}"))?;
+            if !output.success() {
+                bail!("Installing cffi into {} failed", cffi_venv.display());
+            }
+        }
+        file.file().unlock()?;
+        cli.push("--interpreter");
+        cli.push(
+            python
+                .to_str()
+                .context("non-utf8 python interpreter path")?,
+        );
     }
 
     let options: BuildOptions = BuildOptions::try_parse_from(cli)?;

tests/common/mod.rs

@@ -125,12 +125,20 @@ pub fn create_virtualenv(name: &str, python_interp: Option<PathBuf>) -> Result<(
         Ok(python_impl) => format!("{name}-{python_impl}"),
         Err(_) => name.to_string(),
     };
+
+    let venv_dir = create_virtualenv_name(&venv_name, interp)?;
+
+    let target = Target::from_target_triple(None)?;
+    let python = target.get_venv_python(&venv_dir);
+    Ok((venv_dir, python))
+}
+
+pub fn create_virtualenv_name(venv_name: &str, interp: Option<PathBuf>) -> Result<PathBuf> {
     let venv_dir = PathBuf::from("test-crates")
         .normalize()?
         .into_path_buf()
         .join("venvs")
         .join(venv_name);
-    let target = Target::from_target_triple(None)?;
 
     if venv_dir.is_dir() {
         fs::remove_dir_all(&venv_dir)?;
@@ -153,9 +161,7 @@ pub fn create_virtualenv(name: &str, python_interp: Option<PathBuf>) -> Result<(
             str::from_utf8(&output.stderr)?
         );
     }
-
-    let python = target.get_venv_python(&venv_dir);
-    Ok((venv_dir, python))
+    Ok(venv_dir)
 }
 
 /// Creates conda environments