feat: run each benchmark in its own process

src/pytest_codspeed/isolation.py

@@ -0,0 +1,90 @@
+"""Process isolation: every benchmark runs in a fresh pytest process, so the
+allocator state, caches and threads that other benchmarks or setup code leave
+behind can't change its measurement."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from dataclasses import dataclass
+from enum import Enum
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pathlib import Path
+    from typing import Sequence
+
+# Isolation is on unless this is `0` or `false`.
+PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"
+# Set on each child process: the URI of the only benchmark it runs.
+ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"
+
+ENABLED_VALUES = ("1", "true")
+DISABLED_VALUES = ("0", "false")
+
+# Shells report a child killed by signal N as exit code 128 + N.
+SIGNAL_EXIT_CODE_OFFSET = 128
+
+# The parent already printed the session header and summary, so children only
+# print what is needed to debug a failure.
+CHILD_PYTEST_ARGS = ("-q",)
+
+
+class IsolationKind(Enum):
+    # All benchmarks share this process and its allocator state.
+    SHARED_PROCESS = "shared-process"
+    # Every benchmark runs in its own child process.
+    PROCESS_PER_BENCHMARK = "process-per-benchmark"
+    # A child spawned by PROCESS_PER_BENCHMARK, restricted to one URI.
+    ISOLATED_BENCHMARK = "isolated-benchmark"
+
+
+@dataclass(frozen=True)
+class Isolation:
+    """How the plugin runs each benchmark."""
+
+    kind: IsolationKind
+    # Only set for ISOLATED_BENCHMARK.
+    uri: str | None = None
+
+    @staticmethod
+    def current() -> Isolation:
+        """The isolation of this process, read from the environment. The plugin
+        calls it once per pytest session; no process-wide cache, so in-process
+        runs (`pytester`) can use different settings."""
+        uri = os.environ.get(ISOLATED_BENCHMARK_ENV)
+        if uri is not None:
+            return Isolation(IsolationKind.ISOLATED_BENCHMARK, uri)
+
+        value = os.environ.get(PROCESS_ISOLATION_ENV)
+        if value is None or value in ENABLED_VALUES:
+            return Isolation(IsolationKind.PROCESS_PER_BENCHMARK)
+        if value in DISABLED_VALUES:
+            return Isolation(IsolationKind.SHARED_PROCESS)
+        raise ValueError(
+            f"{PROCESS_ISOLATION_ENV} must be `true` or `false`, got `{value}`"
+        )
+
+    def only_benchmark(self) -> str | None:
+        """The only benchmark this process may run. It overrides CLI filters."""
+        if self.kind is IsolationKind.ISOLATED_BENCHMARK:
+            return self.uri
+        return None
+
+
+def run_in_child_process(uri: str, args: Sequence[str], cwd: Path) -> int:
+    """Runs pytest again with the same arguments, limited to `uri`, and returns
+    the exit code of the child in the shell convention."""
+    # Keep the parent's output before the child's.
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    # `sys.orig_argv` only exists on Python 3.10+, and pytest can run in-process
+    # (e.g. `pytest.main`), so rebuild the command from pytest's own arguments.
+    command = [sys.executable, "-m", "pytest", *args, *CHILD_PYTEST_ARGS]
+    env = {**os.environ, ISOLATED_BENCHMARK_ENV: uri}
+    returncode = subprocess.run(command, cwd=cwd, env=env).returncode
+    if returncode < 0:
+        return SIGNAL_EXIT_CODE_OFFSET - returncode
+    return returncode

src/pytest_codspeed/plugin.py

@@ -21,6 +21,12 @@
     PedanticOptions,
 )
 from pytest_codspeed.instruments import MeasurementMode, get_instrument_from_mode
+from pytest_codspeed.instruments.analysis import AnalysisInstrument
+from pytest_codspeed.isolation import (
+    Isolation,
+    IsolationKind,
+    run_in_child_process,
+)
 from pytest_codspeed.utils import (
     BEFORE_PYTEST_8_1_1,
     IS_PYTEST_BENCHMARK_INSTALLED,
@@ -117,9 +123,24 @@ class CodSpeedPlugin:
     config: CodSpeedConfig
     disabled_plugins: tuple[str, ...]
     profile_folder: Path | None
+    isolation: Isolation
     benchmark_count: int = field(default=0, hash=False, compare=False)
 
 
+def get_isolation(is_codspeed_enabled: bool, instrument: Instrument) -> Isolation:
+    """Only the instrumented modes are isolated, and only when the hooks measure.
+    Walltime and unmeasured runs keep sharing one process."""
+    shared = Isolation(IsolationKind.SHARED_PROCESS)
+    if not is_codspeed_enabled or not isinstance(instrument, AnalysisInstrument):
+        return shared
+    if not instrument.should_measure:
+        return shared
+    try:
+        return Isolation.current()
+    except ValueError as error:
+        raise pytest.UsageError(str(error)) from error
+
+
 PLUGIN_NAME = "codspeed_plugin"
 
 
@@ -170,13 +191,15 @@ def pytest_configure(config: pytest.Config):
 
     codspeed_config = CodSpeedConfig.from_pytest_config(config)
 
+    instrument_obj = instrument(codspeed_config, mode)
     plugin = CodSpeedPlugin(
         disabled_plugins=tuple(disabled_plugins),
         is_codspeed_enabled=is_codspeed_enabled,
         mode=mode,
-        instrument=instrument(codspeed_config, mode),
+        instrument=instrument_obj,
         config=codspeed_config,
         profile_folder=Path(profile_folder) if profile_folder else None,
+        isolation=get_isolation(is_codspeed_enabled, instrument_obj),
     )
     config.pluginmanager.register(plugin, PLUGIN_NAME)
 
@@ -263,6 +286,22 @@ def pytest_collection_modifyitems(
         config.hook.pytest_deselected(items=deselected)
         items[:] = selected
 
+    only = plugin.isolation.only_benchmark()
+    if only is None:
+        return
+    # The parent already applied the user's filters, so an exact URI match is
+    # all a child needs.
+    other: list[pytest.Item] = []
+    kept: list[pytest.Item] = []
+    for item in items:
+        uri, _ = get_git_relative_uri_and_name(item.nodeid, config.rootpath)
+        if uri == only:
+            kept.append(item)
+        else:
+            other.append(item)
+    config.hook.pytest_deselected(items=other)
+    items[:] = kept
+
 
 def _measure(
     plugin: CodSpeedPlugin,
@@ -295,6 +334,31 @@ def _measure(
             gc.enable()
 
 
+@pytest.hookimpl(tryfirst=True)
+def pytest_pyfunc_call(pyfuncitem: pytest.Function):
+    """Runs the benchmark in a child process instead of calling the test function
+    here, so the parent never executes the test body."""
+    plugin = get_plugin(pyfuncitem.config)
+    if plugin.isolation.kind is not IsolationKind.PROCESS_PER_BENCHMARK:
+        return None
+    if not should_benchmark_item(pyfuncitem):
+        return None
+
+    assert isinstance(plugin.instrument, AnalysisInstrument)
+    config = pyfuncitem.config
+    uri, _ = get_git_relative_uri_and_name(pyfuncitem.nodeid, config.rootpath)
+    exit_code = run_in_child_process(
+        uri, config.invocation_params.args, config.invocation_params.dir
+    )
+    if exit_code != 0:
+        pytest.fail(
+            f"benchmark {uri} failed in its isolated process (exit code {exit_code})",
+            pytrace=False,
+        )
+    plugin.instrument.benchmark_count += 1
+    return True
+
+
 def wrap_runtest(
     plugin: CodSpeedPlugin,
     node: pytest.Item,
@@ -315,6 +379,10 @@ def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None):
         # Defer to the default test protocol since no benchmarking is needed
         return None
 
+    if plugin.isolation.kind is IsolationKind.PROCESS_PER_BENCHMARK:
+        # The child measures it, `pytest_pyfunc_call` only spawns the child.
+        return None
+
     if has_benchmark_fixture(item):
         # Instrumentation is handled by the fixture
         return None

tests/conftest.py

@@ -58,6 +58,13 @@
 )
 
 
+@pytest.fixture(autouse=True)
+def disable_process_isolation(monkeypatch):
+    # Most tests run the benchmark body in the test process (`pytester.runpytest`),
+    # which process isolation would move into a child. Isolation tests opt in.
+    monkeypatch.setenv("CODSPEED_PROCESS_ISOLATION", "false")
+
+
 @pytest.fixture(scope="function")
 def codspeed_env(monkeypatch):
     @contextmanager

tests/test_pytest_plugin.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 from conftest import (
     IS_PERF_TRAMPOLINE_SUPPORTED,
@@ -438,3 +440,114 @@ def foo():
     result.stdout.fnmatch_lines(
         ["*RuntimeError: The benchmark fixture can only be used once per test*"]
     )
+
+
+ISOLATION_BENCH_COUNT = 3
+
+ISOLATION_PID_FIXTURE = """
+import os
+
+import pytest
+
+
+@pytest.mark.parametrize("index", range(3))
+def test_bench(benchmark, index):
+    benchmark(lambda: open("pids.txt", "a").write(f"{os.getpid()}\\n"))
+"""
+
+
+def run_isolation_subprocess(
+    pytester: pytest.Pytester,
+    monkeypatch: pytest.MonkeyPatch,
+    isolation: str | None = None,
+) -> pytest.RunResult:
+    # Children are real processes, so the parent must be one too.
+    monkeypatch.setenv("CODSPEED_ENV", "1")
+    monkeypatch.setenv("CODSPEED_RUNNER_MODE", "simulation")
+    if isolation is None:
+        monkeypatch.delenv("CODSPEED_PROCESS_ISOLATION", raising=False)
+    else:
+        monkeypatch.setenv("CODSPEED_PROCESS_ISOLATION", isolation)
+    return pytester.runpytest_subprocess("--codspeed", "-p", "no:cacheprovider")
+
+
+def read_pids(pytester: pytest.Pytester):
+    return (pytester.path / "pids.txt").read_text().split()
+
+
+def test_isolation_runs_each_benchmark_in_its_own_process(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    pytester.makepyfile(ISOLATION_PID_FIXTURE)
+    result = run_isolation_subprocess(pytester, monkeypatch)
+    assert result.ret == 0
+    assert f"{ISOLATION_BENCH_COUNT} benchmarked" in result.stdout.str()
+    pids = read_pids(pytester)
+    # The body runs more than once per benchmark (warmup), so compare PIDs only.
+    assert len(set(pids)) == ISOLATION_BENCH_COUNT
+
+
+@pytest.mark.parametrize("value", ["0", "false"])
+def test_isolation_disabled_shares_one_process(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, value: str
+) -> None:
+    pytester.makepyfile(ISOLATION_PID_FIXTURE)
+    result = run_isolation_subprocess(pytester, monkeypatch, value)
+    assert result.ret == 0
+    pids = read_pids(pytester)
+    assert len(set(pids)) == 1
+
+
+def test_isolation_parent_does_not_run_the_body(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    pytester.makepyfile(
+        """
+        def test_result(benchmark):
+            result = benchmark(lambda: 42)
+            assert result == 42
+        """
+    )
+    result = run_isolation_subprocess(pytester, monkeypatch)
+    assert result.ret == 0
+    result.assert_outcomes(passed=1)
+
+
+def test_isolation_failing_benchmark_fails_only_that_test(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    pytester.makepyfile(
+        """
+        def mark(name):
+            with open("ran.txt", "a") as f:
+                f.write(name + "\\n")
+
+        def test_before(benchmark):
+            benchmark(mark, "before")
+
+        def test_raises(benchmark):
+            def fn():
+                raise ValueError("boom")
+            benchmark(fn)
+
+        def test_after(benchmark):
+            benchmark(mark, "after")
+        """
+    )
+    result = run_isolation_subprocess(pytester, monkeypatch)
+    assert result.ret == pytest.ExitCode.TESTS_FAILED
+    result.assert_outcomes(passed=2, failed=1)
+    result.stdout.fnmatch_lines(["*failed in its isolated process (exit code 1)*"])
+    ran = set((pytester.path / "ran.txt").read_text().split())
+    assert ran == {"before", "after"}
+
+
+def test_isolation_invalid_env_value_errors(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    pytester.makepyfile(ISOLATION_PID_FIXTURE)
+    result = run_isolation_subprocess(pytester, monkeypatch, "garbage")
+    assert result.ret != 0
+    result.stderr.fnmatch_lines(
+        ["*CODSPEED_PROCESS_ISOLATION must be `true` or `false`, got `garbage`*"]
+    )