diff --git a/src/pytest_codspeed/isolation.py b/src/pytest_codspeed/isolation.py new file mode 100644 index 0000000..d52ff5d --- /dev/null +++ b/src/pytest_codspeed/isolation.py @@ -0,0 +1,90 @@ +"""Process isolation: every benchmark runs in a fresh pytest process, so the +allocator state, caches and threads that other benchmarks or setup code leave +behind can't change its measurement.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + from typing import Sequence + +# Isolation is on unless this is `0` or `false`. +PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION" +# Set on each child process: the URI of the only benchmark it runs. +ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK" + +ENABLED_VALUES = ("1", "true") +DISABLED_VALUES = ("0", "false") + +# Shells report a child killed by signal N as exit code 128 + N. +SIGNAL_EXIT_CODE_OFFSET = 128 + +# The parent already printed the session header and summary, so children only +# print what is needed to debug a failure. +CHILD_PYTEST_ARGS = ("-q",) + + +class IsolationKind(Enum): + # All benchmarks share this process and its allocator state. + SHARED_PROCESS = "shared-process" + # Every benchmark runs in its own child process. + PROCESS_PER_BENCHMARK = "process-per-benchmark" + # A child spawned by PROCESS_PER_BENCHMARK, restricted to one URI. + ISOLATED_BENCHMARK = "isolated-benchmark" + + +@dataclass(frozen=True) +class Isolation: + """How the plugin runs each benchmark.""" + + kind: IsolationKind + # Only set for ISOLATED_BENCHMARK. + uri: str | None = None + + @staticmethod + def current() -> Isolation: + """The isolation of this process, read from the environment. The plugin + calls it once per pytest session; no process-wide cache, so in-process + runs (`pytester`) can use different settings.""" + uri = os.environ.get(ISOLATED_BENCHMARK_ENV) + if uri is not None: + return Isolation(IsolationKind.ISOLATED_BENCHMARK, uri) + + value = os.environ.get(PROCESS_ISOLATION_ENV) + if value is None or value in ENABLED_VALUES: + return Isolation(IsolationKind.PROCESS_PER_BENCHMARK) + if value in DISABLED_VALUES: + return Isolation(IsolationKind.SHARED_PROCESS) + raise ValueError( + f"{PROCESS_ISOLATION_ENV} must be `true` or `false`, got `{value}`" + ) + + def only_benchmark(self) -> str | None: + """The only benchmark this process may run. It overrides CLI filters.""" + if self.kind is IsolationKind.ISOLATED_BENCHMARK: + return self.uri + return None + + +def run_in_child_process(uri: str, args: Sequence[str], cwd: Path) -> int: + """Runs pytest again with the same arguments, limited to `uri`, and returns + the exit code of the child in the shell convention.""" + # Keep the parent's output before the child's. + sys.stdout.flush() + sys.stderr.flush() + + # `sys.orig_argv` only exists on Python 3.10+, and pytest can run in-process + # (e.g. `pytest.main`), so rebuild the command from pytest's own arguments. + command = [sys.executable, "-m", "pytest", *args, *CHILD_PYTEST_ARGS] + env = {**os.environ, ISOLATED_BENCHMARK_ENV: uri} + returncode = subprocess.run(command, cwd=cwd, env=env).returncode + if returncode < 0: + return SIGNAL_EXIT_CODE_OFFSET - returncode + return returncode diff --git a/src/pytest_codspeed/plugin.py b/src/pytest_codspeed/plugin.py index 2b50a8b..ebf4299 100644 --- a/src/pytest_codspeed/plugin.py +++ b/src/pytest_codspeed/plugin.py @@ -21,6 +21,12 @@ PedanticOptions, ) from pytest_codspeed.instruments import MeasurementMode, get_instrument_from_mode +from pytest_codspeed.instruments.analysis import AnalysisInstrument +from pytest_codspeed.isolation import ( + Isolation, + IsolationKind, + run_in_child_process, +) from pytest_codspeed.utils import ( BEFORE_PYTEST_8_1_1, IS_PYTEST_BENCHMARK_INSTALLED, @@ -117,9 +123,24 @@ class CodSpeedPlugin: config: CodSpeedConfig disabled_plugins: tuple[str, ...] profile_folder: Path | None + isolation: Isolation benchmark_count: int = field(default=0, hash=False, compare=False) +def get_isolation(is_codspeed_enabled: bool, instrument: Instrument) -> Isolation: + """Only the instrumented modes are isolated, and only when the hooks measure. + Walltime and unmeasured runs keep sharing one process.""" + shared = Isolation(IsolationKind.SHARED_PROCESS) + if not is_codspeed_enabled or not isinstance(instrument, AnalysisInstrument): + return shared + if not instrument.should_measure: + return shared + try: + return Isolation.current() + except ValueError as error: + raise pytest.UsageError(str(error)) from error + + PLUGIN_NAME = "codspeed_plugin" @@ -170,13 +191,15 @@ def pytest_configure(config: pytest.Config): codspeed_config = CodSpeedConfig.from_pytest_config(config) + instrument_obj = instrument(codspeed_config, mode) plugin = CodSpeedPlugin( disabled_plugins=tuple(disabled_plugins), is_codspeed_enabled=is_codspeed_enabled, mode=mode, - instrument=instrument(codspeed_config, mode), + instrument=instrument_obj, config=codspeed_config, profile_folder=Path(profile_folder) if profile_folder else None, + isolation=get_isolation(is_codspeed_enabled, instrument_obj), ) config.pluginmanager.register(plugin, PLUGIN_NAME) @@ -263,6 +286,22 @@ def pytest_collection_modifyitems( config.hook.pytest_deselected(items=deselected) items[:] = selected + only = plugin.isolation.only_benchmark() + if only is None: + return + # The parent already applied the user's filters, so an exact URI match is + # all a child needs. + other: list[pytest.Item] = [] + kept: list[pytest.Item] = [] + for item in items: + uri, _ = get_git_relative_uri_and_name(item.nodeid, config.rootpath) + if uri == only: + kept.append(item) + else: + other.append(item) + config.hook.pytest_deselected(items=other) + items[:] = kept + def _measure( plugin: CodSpeedPlugin, @@ -295,6 +334,31 @@ def _measure( gc.enable() +@pytest.hookimpl(tryfirst=True) +def pytest_pyfunc_call(pyfuncitem: pytest.Function): + """Runs the benchmark in a child process instead of calling the test function + here, so the parent never executes the test body.""" + plugin = get_plugin(pyfuncitem.config) + if plugin.isolation.kind is not IsolationKind.PROCESS_PER_BENCHMARK: + return None + if not should_benchmark_item(pyfuncitem): + return None + + assert isinstance(plugin.instrument, AnalysisInstrument) + config = pyfuncitem.config + uri, _ = get_git_relative_uri_and_name(pyfuncitem.nodeid, config.rootpath) + exit_code = run_in_child_process( + uri, config.invocation_params.args, config.invocation_params.dir + ) + if exit_code != 0: + pytest.fail( + f"benchmark {uri} failed in its isolated process (exit code {exit_code})", + pytrace=False, + ) + plugin.instrument.benchmark_count += 1 + return True + + def wrap_runtest( plugin: CodSpeedPlugin, node: pytest.Item, @@ -315,6 +379,10 @@ def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None): # Defer to the default test protocol since no benchmarking is needed return None + if plugin.isolation.kind is IsolationKind.PROCESS_PER_BENCHMARK: + # The child measures it, `pytest_pyfunc_call` only spawns the child. + return None + if has_benchmark_fixture(item): # Instrumentation is handled by the fixture return None diff --git a/tests/conftest.py b/tests/conftest.py index e8a5625..6222dad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -58,6 +58,13 @@ ) +@pytest.fixture(autouse=True) +def disable_process_isolation(monkeypatch): + # Most tests run the benchmark body in the test process (`pytester.runpytest`), + # which process isolation would move into a child. Isolation tests opt in. + monkeypatch.setenv("CODSPEED_PROCESS_ISOLATION", "false") + + @pytest.fixture(scope="function") def codspeed_env(monkeypatch): @contextmanager diff --git a/tests/test_pytest_plugin.py b/tests/test_pytest_plugin.py index 452e9da..e03518c 100644 --- a/tests/test_pytest_plugin.py +++ b/tests/test_pytest_plugin.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from conftest import ( IS_PERF_TRAMPOLINE_SUPPORTED, @@ -438,3 +440,114 @@ def foo(): result.stdout.fnmatch_lines( ["*RuntimeError: The benchmark fixture can only be used once per test*"] ) + + +ISOLATION_BENCH_COUNT = 3 + +ISOLATION_PID_FIXTURE = """ +import os + +import pytest + + +@pytest.mark.parametrize("index", range(3)) +def test_bench(benchmark, index): + benchmark(lambda: open("pids.txt", "a").write(f"{os.getpid()}\\n")) +""" + + +def run_isolation_subprocess( + pytester: pytest.Pytester, + monkeypatch: pytest.MonkeyPatch, + isolation: str | None = None, +) -> pytest.RunResult: + # Children are real processes, so the parent must be one too. + monkeypatch.setenv("CODSPEED_ENV", "1") + monkeypatch.setenv("CODSPEED_RUNNER_MODE", "simulation") + if isolation is None: + monkeypatch.delenv("CODSPEED_PROCESS_ISOLATION", raising=False) + else: + monkeypatch.setenv("CODSPEED_PROCESS_ISOLATION", isolation) + return pytester.runpytest_subprocess("--codspeed", "-p", "no:cacheprovider") + + +def read_pids(pytester: pytest.Pytester): + return (pytester.path / "pids.txt").read_text().split() + + +def test_isolation_runs_each_benchmark_in_its_own_process( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + pytester.makepyfile(ISOLATION_PID_FIXTURE) + result = run_isolation_subprocess(pytester, monkeypatch) + assert result.ret == 0 + assert f"{ISOLATION_BENCH_COUNT} benchmarked" in result.stdout.str() + pids = read_pids(pytester) + # The body runs more than once per benchmark (warmup), so compare PIDs only. + assert len(set(pids)) == ISOLATION_BENCH_COUNT + + +@pytest.mark.parametrize("value", ["0", "false"]) +def test_isolation_disabled_shares_one_process( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, value: str +) -> None: + pytester.makepyfile(ISOLATION_PID_FIXTURE) + result = run_isolation_subprocess(pytester, monkeypatch, value) + assert result.ret == 0 + pids = read_pids(pytester) + assert len(set(pids)) == 1 + + +def test_isolation_parent_does_not_run_the_body( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + pytester.makepyfile( + """ + def test_result(benchmark): + result = benchmark(lambda: 42) + assert result == 42 + """ + ) + result = run_isolation_subprocess(pytester, monkeypatch) + assert result.ret == 0 + result.assert_outcomes(passed=1) + + +def test_isolation_failing_benchmark_fails_only_that_test( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + pytester.makepyfile( + """ + def mark(name): + with open("ran.txt", "a") as f: + f.write(name + "\\n") + + def test_before(benchmark): + benchmark(mark, "before") + + def test_raises(benchmark): + def fn(): + raise ValueError("boom") + benchmark(fn) + + def test_after(benchmark): + benchmark(mark, "after") + """ + ) + result = run_isolation_subprocess(pytester, monkeypatch) + assert result.ret == pytest.ExitCode.TESTS_FAILED + result.assert_outcomes(passed=2, failed=1) + result.stdout.fnmatch_lines(["*failed in its isolated process (exit code 1)*"]) + ran = set((pytester.path / "ran.txt").read_text().split()) + assert ran == {"before", "after"} + + +def test_isolation_invalid_env_value_errors( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + pytester.makepyfile(ISOLATION_PID_FIXTURE) + result = run_isolation_subprocess(pytester, monkeypatch, "garbage") + assert result.ret != 0 + result.stderr.fnmatch_lines( + ["*CODSPEED_PROCESS_ISOLATION must be `true` or `false`, got `garbage`*"] + )