From 3f80fbd7d7aa0de733b5826b207dc6bd424c33e8 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:48 +0200 Subject: [PATCH 1/3] feat(isolation): add per-benchmark process isolation All benchmarks of a pytest session share one process, so the allocator state, caches and threads that one benchmark leaves behind change the measurement of every benchmark after it. Adding or changing one benchmark can then shift unrelated results. Add `Isolation`, read from the environment: - `CODSPEED_PROCESS_ISOLATION` is on by default (unset, `1`, `true`) and off with `0` or `false`. Any other value is an error. - `CODSPEED_ISOLATED_BENCHMARK` is set by the parent on each child and holds the URI of the only benchmark that child runs. `run_in_child_process` re-runs pytest as `python -m pytest -q` in the original directory. `sys.orig_argv` is not used because it needs Python 3.10 and the package supports 3.9; pytest's own invocation args also work for in-process `pytest.main` runs. Stdout and stderr are flushed first so the parent's output stays before the child's. A child killed by signal N reports exit code 128 + N, as shells do. Re-exec instead of fork(): fork only copies the calling thread. --- src/pytest_codspeed/isolation.py | 90 ++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 src/pytest_codspeed/isolation.py diff --git a/src/pytest_codspeed/isolation.py b/src/pytest_codspeed/isolation.py new file mode 100644 index 0000000..d52ff5d --- /dev/null +++ b/src/pytest_codspeed/isolation.py @@ -0,0 +1,90 @@ +"""Process isolation: every benchmark runs in a fresh pytest process, so the +allocator state, caches and threads that other benchmarks or setup code leave +behind can't change its measurement.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + from typing import Sequence + +# Isolation is on unless this is `0` or `false`. +PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION" +# Set on each child process: the URI of the only benchmark it runs. +ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK" + +ENABLED_VALUES = ("1", "true") +DISABLED_VALUES = ("0", "false") + +# Shells report a child killed by signal N as exit code 128 + N. +SIGNAL_EXIT_CODE_OFFSET = 128 + +# The parent already printed the session header and summary, so children only +# print what is needed to debug a failure. +CHILD_PYTEST_ARGS = ("-q",) + + +class IsolationKind(Enum): + # All benchmarks share this process and its allocator state. + SHARED_PROCESS = "shared-process" + # Every benchmark runs in its own child process. + PROCESS_PER_BENCHMARK = "process-per-benchmark" + # A child spawned by PROCESS_PER_BENCHMARK, restricted to one URI. + ISOLATED_BENCHMARK = "isolated-benchmark" + + +@dataclass(frozen=True) +class Isolation: + """How the plugin runs each benchmark.""" + + kind: IsolationKind + # Only set for ISOLATED_BENCHMARK. + uri: str | None = None + + @staticmethod + def current() -> Isolation: + """The isolation of this process, read from the environment. The plugin + calls it once per pytest session; no process-wide cache, so in-process + runs (`pytester`) can use different settings.""" + uri = os.environ.get(ISOLATED_BENCHMARK_ENV) + if uri is not None: + return Isolation(IsolationKind.ISOLATED_BENCHMARK, uri) + + value = os.environ.get(PROCESS_ISOLATION_ENV) + if value is None or value in ENABLED_VALUES: + return Isolation(IsolationKind.PROCESS_PER_BENCHMARK) + if value in DISABLED_VALUES: + return Isolation(IsolationKind.SHARED_PROCESS) + raise ValueError( + f"{PROCESS_ISOLATION_ENV} must be `true` or `false`, got `{value}`" + ) + + def only_benchmark(self) -> str | None: + """The only benchmark this process may run. It overrides CLI filters.""" + if self.kind is IsolationKind.ISOLATED_BENCHMARK: + return self.uri + return None + + +def run_in_child_process(uri: str, args: Sequence[str], cwd: Path) -> int: + """Runs pytest again with the same arguments, limited to `uri`, and returns + the exit code of the child in the shell convention.""" + # Keep the parent's output before the child's. + sys.stdout.flush() + sys.stderr.flush() + + # `sys.orig_argv` only exists on Python 3.10+, and pytest can run in-process + # (e.g. `pytest.main`), so rebuild the command from pytest's own arguments. + command = [sys.executable, "-m", "pytest", *args, *CHILD_PYTEST_ARGS] + env = {**os.environ, ISOLATED_BENCHMARK_ENV: uri} + returncode = subprocess.run(command, cwd=cwd, env=env).returncode + if returncode < 0: + return SIGNAL_EXIT_CODE_OFFSET - returncode + return returncode From 57780613422474c6364e452d488848433ecf42ba Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:51 +0200 Subject: [PATCH 2/3] feat(plugin): run each benchmark in its own process Wire process isolation into the plugin for the simulation and memory modes, and only when the instrument hooks are active. Walltime and runs outside CodSpeed keep sharing one process. - Parent: a `pytest_pyfunc_call` hook spawns a child for each benchmark item and returns, so the parent never runs the test body. Fixture setup and teardown still run in the parent. `pytest_runtest_protocol` does not wrap marker-only benchmarks in the parent, so no measurement markers surround the child spawn. - Child: `pytest_collection_modifyitems` deselects every item whose URI is not `CODSPEED_ISOLATED_BENCHMARK`, which overrides `-k` and node id selection. The child measures and reports the benchmark as before. - A failing child fails only its test item with its exit code in the message, so the run continues and ends with TESTS_FAILED, like a raising benchmark in a shared process. - An invalid `CODSPEED_PROCESS_ISOLATION` value is a `pytest.UsageError`. --- src/pytest_codspeed/plugin.py | 70 ++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/src/pytest_codspeed/plugin.py b/src/pytest_codspeed/plugin.py index 2b50a8b..ebf4299 100644 --- a/src/pytest_codspeed/plugin.py +++ b/src/pytest_codspeed/plugin.py @@ -21,6 +21,12 @@ PedanticOptions, ) from pytest_codspeed.instruments import MeasurementMode, get_instrument_from_mode +from pytest_codspeed.instruments.analysis import AnalysisInstrument +from pytest_codspeed.isolation import ( + Isolation, + IsolationKind, + run_in_child_process, +) from pytest_codspeed.utils import ( BEFORE_PYTEST_8_1_1, IS_PYTEST_BENCHMARK_INSTALLED, @@ -117,9 +123,24 @@ class CodSpeedPlugin: config: CodSpeedConfig disabled_plugins: tuple[str, ...] profile_folder: Path | None + isolation: Isolation benchmark_count: int = field(default=0, hash=False, compare=False) +def get_isolation(is_codspeed_enabled: bool, instrument: Instrument) -> Isolation: + """Only the instrumented modes are isolated, and only when the hooks measure. + Walltime and unmeasured runs keep sharing one process.""" + shared = Isolation(IsolationKind.SHARED_PROCESS) + if not is_codspeed_enabled or not isinstance(instrument, AnalysisInstrument): + return shared + if not instrument.should_measure: + return shared + try: + return Isolation.current() + except ValueError as error: + raise pytest.UsageError(str(error)) from error + + PLUGIN_NAME = "codspeed_plugin" @@ -170,13 +191,15 @@ def pytest_configure(config: pytest.Config): codspeed_config = CodSpeedConfig.from_pytest_config(config) + instrument_obj = instrument(codspeed_config, mode) plugin = CodSpeedPlugin( disabled_plugins=tuple(disabled_plugins), is_codspeed_enabled=is_codspeed_enabled, mode=mode, - instrument=instrument(codspeed_config, mode), + instrument=instrument_obj, config=codspeed_config, profile_folder=Path(profile_folder) if profile_folder else None, + isolation=get_isolation(is_codspeed_enabled, instrument_obj), ) config.pluginmanager.register(plugin, PLUGIN_NAME) @@ -263,6 +286,22 @@ def pytest_collection_modifyitems( config.hook.pytest_deselected(items=deselected) items[:] = selected + only = plugin.isolation.only_benchmark() + if only is None: + return + # The parent already applied the user's filters, so an exact URI match is + # all a child needs. + other: list[pytest.Item] = [] + kept: list[pytest.Item] = [] + for item in items: + uri, _ = get_git_relative_uri_and_name(item.nodeid, config.rootpath) + if uri == only: + kept.append(item) + else: + other.append(item) + config.hook.pytest_deselected(items=other) + items[:] = kept + def _measure( plugin: CodSpeedPlugin, @@ -295,6 +334,31 @@ def _measure( gc.enable() +@pytest.hookimpl(tryfirst=True) +def pytest_pyfunc_call(pyfuncitem: pytest.Function): + """Runs the benchmark in a child process instead of calling the test function + here, so the parent never executes the test body.""" + plugin = get_plugin(pyfuncitem.config) + if plugin.isolation.kind is not IsolationKind.PROCESS_PER_BENCHMARK: + return None + if not should_benchmark_item(pyfuncitem): + return None + + assert isinstance(plugin.instrument, AnalysisInstrument) + config = pyfuncitem.config + uri, _ = get_git_relative_uri_and_name(pyfuncitem.nodeid, config.rootpath) + exit_code = run_in_child_process( + uri, config.invocation_params.args, config.invocation_params.dir + ) + if exit_code != 0: + pytest.fail( + f"benchmark {uri} failed in its isolated process (exit code {exit_code})", + pytrace=False, + ) + plugin.instrument.benchmark_count += 1 + return True + + def wrap_runtest( plugin: CodSpeedPlugin, node: pytest.Item, @@ -315,6 +379,10 @@ def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None): # Defer to the default test protocol since no benchmarking is needed return None + if plugin.isolation.kind is IsolationKind.PROCESS_PER_BENCHMARK: + # The child measures it, `pytest_pyfunc_call` only spawns the child. + return None + if has_benchmark_fixture(item): # Instrumentation is handled by the fixture return None From c5f9e54038b7a411ac723567ed3d7cc24bc9bcfd Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:52 +0200 Subject: [PATCH 3/3] test: cover process isolation end to end Run real pytest subprocesses in simulation mode and check that: - each benchmark runs in its own process by default, and all share one process with `CODSPEED_PROCESS_ISOLATION` set to `0` or `false`; - `benchmark(...)` returns the target's value with isolation on; - a raising benchmark fails only its test, the other benchmarks still run, and the run exits with TESTS_FAILED; - an invalid `CODSPEED_PROCESS_ISOLATION` value fails with a clear error. An autouse fixture sets `CODSPEED_PROCESS_ISOLATION=false` for the other tests. They run pytest in-process with `pytester` and check results that only exist where the benchmark body runs, which isolation moves into a child. The isolation tests set the variable themselves. --- tests/conftest.py | 7 +++ tests/test_pytest_plugin.py | 113 ++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index e8a5625..6222dad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -58,6 +58,13 @@ ) +@pytest.fixture(autouse=True) +def disable_process_isolation(monkeypatch): + # Most tests run the benchmark body in the test process (`pytester.runpytest`), + # which process isolation would move into a child. Isolation tests opt in. + monkeypatch.setenv("CODSPEED_PROCESS_ISOLATION", "false") + + @pytest.fixture(scope="function") def codspeed_env(monkeypatch): @contextmanager diff --git a/tests/test_pytest_plugin.py b/tests/test_pytest_plugin.py index 452e9da..e03518c 100644 --- a/tests/test_pytest_plugin.py +++ b/tests/test_pytest_plugin.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from conftest import ( IS_PERF_TRAMPOLINE_SUPPORTED, @@ -438,3 +440,114 @@ def foo(): result.stdout.fnmatch_lines( ["*RuntimeError: The benchmark fixture can only be used once per test*"] ) + + +ISOLATION_BENCH_COUNT = 3 + +ISOLATION_PID_FIXTURE = """ +import os + +import pytest + + +@pytest.mark.parametrize("index", range(3)) +def test_bench(benchmark, index): + benchmark(lambda: open("pids.txt", "a").write(f"{os.getpid()}\\n")) +""" + + +def run_isolation_subprocess( + pytester: pytest.Pytester, + monkeypatch: pytest.MonkeyPatch, + isolation: str | None = None, +) -> pytest.RunResult: + # Children are real processes, so the parent must be one too. + monkeypatch.setenv("CODSPEED_ENV", "1") + monkeypatch.setenv("CODSPEED_RUNNER_MODE", "simulation") + if isolation is None: + monkeypatch.delenv("CODSPEED_PROCESS_ISOLATION", raising=False) + else: + monkeypatch.setenv("CODSPEED_PROCESS_ISOLATION", isolation) + return pytester.runpytest_subprocess("--codspeed", "-p", "no:cacheprovider") + + +def read_pids(pytester: pytest.Pytester): + return (pytester.path / "pids.txt").read_text().split() + + +def test_isolation_runs_each_benchmark_in_its_own_process( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + pytester.makepyfile(ISOLATION_PID_FIXTURE) + result = run_isolation_subprocess(pytester, monkeypatch) + assert result.ret == 0 + assert f"{ISOLATION_BENCH_COUNT} benchmarked" in result.stdout.str() + pids = read_pids(pytester) + # The body runs more than once per benchmark (warmup), so compare PIDs only. + assert len(set(pids)) == ISOLATION_BENCH_COUNT + + +@pytest.mark.parametrize("value", ["0", "false"]) +def test_isolation_disabled_shares_one_process( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, value: str +) -> None: + pytester.makepyfile(ISOLATION_PID_FIXTURE) + result = run_isolation_subprocess(pytester, monkeypatch, value) + assert result.ret == 0 + pids = read_pids(pytester) + assert len(set(pids)) == 1 + + +def test_isolation_parent_does_not_run_the_body( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + pytester.makepyfile( + """ + def test_result(benchmark): + result = benchmark(lambda: 42) + assert result == 42 + """ + ) + result = run_isolation_subprocess(pytester, monkeypatch) + assert result.ret == 0 + result.assert_outcomes(passed=1) + + +def test_isolation_failing_benchmark_fails_only_that_test( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + pytester.makepyfile( + """ + def mark(name): + with open("ran.txt", "a") as f: + f.write(name + "\\n") + + def test_before(benchmark): + benchmark(mark, "before") + + def test_raises(benchmark): + def fn(): + raise ValueError("boom") + benchmark(fn) + + def test_after(benchmark): + benchmark(mark, "after") + """ + ) + result = run_isolation_subprocess(pytester, monkeypatch) + assert result.ret == pytest.ExitCode.TESTS_FAILED + result.assert_outcomes(passed=2, failed=1) + result.stdout.fnmatch_lines(["*failed in its isolated process (exit code 1)*"]) + ran = set((pytester.path / "ran.txt").read_text().split()) + assert ran == {"before", "after"} + + +def test_isolation_invalid_env_value_errors( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + pytester.makepyfile(ISOLATION_PID_FIXTURE) + result = run_isolation_subprocess(pytester, monkeypatch, "garbage") + assert result.ret != 0 + result.stderr.fnmatch_lines( + ["*CODSPEED_PROCESS_ISOLATION must be `true` or `false`, got `garbage`*"] + )