From c0aeffb4070658f39560a06652c11e126dda5fc2 Mon Sep 17 00:00:00 2001 From: Mark Falco Date: Wed, 21 Oct 2026 09:00:14 -0400 Subject: [PATCH 1/2] Support benchmark selection by Python implementation --- doc/custom_benchmarks.rst | 25 +- doc/usage.rst | 9 +- pyperformance/_benchmark.py | 5 + pyperformance/_benchmark_metadata.py | 14 + pyperformance/cli.py | 48 ++- pyperformance/commands.py | 5 +- .../tests/test_benchmark_compatibility.py | 273 ++++++++++++++++++ 7 files changed, 356 insertions(+), 23 deletions(-) create mode 100644 pyperformance/tests/test_benchmark_compatibility.py diff --git a/doc/custom_benchmarks.rst b/doc/custom_benchmarks.rst index 01028f70..71dc3a6d 100644 --- a/doc/custom_benchmarks.rst +++ b/doc/custom_benchmarks.rst @@ -324,21 +324,26 @@ All other PEP 621 fields are optional (e.g. ``requires-python = ">=3.8"``, The ``[tool.pyperformance]`` Section ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -=============== ===== === === === -field type R B F -=============== ===== === === === -tool.name str X X -tool.tags [str] X -tool.extra_opts [str] X -tool.inherits file -tool.runscript file X -tool.datadir file X -=============== ===== === === === +=========================== ===== === === === +field type R B F +=========================== ===== === === === +tool.name str X X +tool.tags [str] X +tool.python_implementations [str] X +tool.extra_opts [str] X +tool.inherits file +tool.runscript file X +tool.datadir file X +=========================== ===== === === === "R": required "B": inferred from the inherited metadata "F": inferred from filename * tags: optional list of names to group benchmarks +* python_implementations: optional nonempty list of supported names from + ``sys.implementation.name``, such as ``["cpython"]``. Names are normalized + to lowercase. Omitting this field allows every implementation unless the + inherited metadata restricts it. A supplied list replaces the inherited list. * extra_opts: optional list of args to pass to ``tool.runscript`` * runscript: the benchmark script to use instead of run_benchmark.py. diff --git a/doc/usage.rst b/doc/usage.rst index 95e9c25d..8c7ccd25 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -559,9 +559,16 @@ significant. Omitting the ``-b`` option will result in the default group of benchmarks being run Omitting ``-b`` is the same as specifying `-b default`. -To run every benchmark pyperformance knows about, use ``-b all``. To see a full +To select every benchmark eligible for the target interpreter, use ``-b all``. To see a full list of all available benchmarks, use `--help`. +The ``run`` and ``list`` commands check compatibility against the interpreter +selected by ``--python``. A benchmark must support its Python language version +and, when restricted by metadata, its implementation. Unsupported implementations +are reported on stderr and omitted from groups and explicit selections. If no +benchmarks remain, ``run`` reports that no benchmark was run. +``list_groups`` continues to show declared group membership without filtering. + Negative benchmarks specifications are also supported: `-b -2to3` will run every benchmark in the default group except for 2to3 (this is the same as `-b default,-2to3`). `-b all,-django` will run all benchmarks except the Django diff --git a/pyperformance/_benchmark.py b/pyperformance/_benchmark.py index 17c34a06..0a134225 100644 --- a/pyperformance/_benchmark.py +++ b/pyperformance/_benchmark.py @@ -167,6 +167,11 @@ def extra_opts(self): def python(self): return SpecifierSet(self._get_metadata_value("python", "")) + @property + def python_implementations(self): + """Return supported implementation names, or an empty sequence for all.""" + return self._get_metadata_value("python_implementations", ()) + # Other metadata keys: # * base # * dependencies diff --git a/pyperformance/_benchmark_metadata.py b/pyperformance/_benchmark_metadata.py index 8a2fd4f9..43eec5c9 100644 --- a/pyperformance/_benchmark_metadata.py +++ b/pyperformance/_benchmark_metadata.py @@ -26,6 +26,7 @@ "metafile": None, "name": None, "tags": None, + "python_implementations": None, "datadir": None, "runscript": None, "extra_opts": None, @@ -38,6 +39,7 @@ # metafile # tags # python +# python_implementations # dependencies # (from requirements.in) # requirements # (from lock file or requirements.txt) # datadir @@ -219,6 +221,18 @@ def _resolve_value(field, value, rootdir): raise ValueError("Invalid tag 'all'") elif tag == "": raise ValueError("Invalid empty tag") + elif field == "python_implementations": + if not isinstance(value, list): + raise TypeError("python_implementations should be a list of names") + if not value: + raise ValueError("python_implementations should not be empty") + implementations = [] + for name in value: + if not isinstance(name, str): + raise TypeError("python_implementations should be a list of names") + _utils.check_name(name) + implementations.append(name.lower()) + value = implementations elif field == "datadir": if not os.path.isabs(value): value = os.path.join(rootdir, value) diff --git a/pyperformance/cli.py b/pyperformance/cli.py index cac15c33..2b1cfb53 100644 --- a/pyperformance/cli.py +++ b/pyperformance/cli.py @@ -313,14 +313,31 @@ def _manifest_from_options(options): return _manifest.load_manifest(options.manifest) -def _benchmarks_from_options(options): +def _benchmarks_from_options(options, python): + """Resolve selected benchmarks for the interpreter the command will use. + + options: Parsed options containing the manifest and benchmark selection. + python: Target executable as a string, or information returned by _pythoninfo.get_info(). + Return: Eligible benchmarks, or None when no benchmarks were requested. + """ if not getattr(options, "benchmarks", None): return None + + from pyperformance import _pythoninfo + + info = _pythoninfo.get_info(python) if isinstance(python, str) else python manifest = _manifest_from_options(options) - return _select_benchmarks(options.benchmarks, manifest) + return _select_benchmarks(options.benchmarks, manifest, info) + +def _select_benchmarks(raw, manifest, info): + """Resolve a selection and omit benchmarks unsupported by the target. -def _select_benchmarks(raw, manifest): + raw: Benchmark names and groups in command-line selection syntax. + manifest: Catalog used to expand the selection. + info: Target interpreter information returned by _pythoninfo.get_info(). + Return: Eligible benchmarks after explicit exclusions are applied. + """ from pyperformance import _benchmark_selections # Get the raw list of benchmarks. @@ -342,14 +359,23 @@ def parse_entry(o, s): # Get the selections. selected = [] - this_python_version = ".".join(map(str, sys.version_info[:3])) + python_version = ".".join(map(str, info.sys.version_info[:3])) + implementation = info.sys.implementation.name.lower() for bench in _benchmark_selections.iter_selections(manifest, parsed_infos): if isinstance(bench, str): logging.warning(f"no benchmark named {bench!r}") continue - # Filter out any benchmarks that can't be run on the Python version we're running - if this_python_version in bench.python: - selected.append(bench) + implementations = bench.python_implementations + if implementations and implementation not in implementations: + supported = ", ".join(implementations) + print( + f"Skipping {bench.name}: target implementation {implementation} is not supported (supported: {supported})", + file=sys.stderr, + ) + continue + if python_version not in bench.python: + continue + selected.append(bench) return selected @@ -376,10 +402,10 @@ def _main(): action = options.venv_action if action == "create": - benchmarks = _benchmarks_from_options(options) + benchmarks = _benchmarks_from_options(options, info or sys.executable) cmd_venv_create(options, root, info, benchmarks) elif action == "recreate": - benchmarks = _benchmarks_from_options(options) + benchmarks = _benchmarks_from_options(options, info or sys.executable) cmd_venv_recreate(options, root, info, benchmarks) elif action == "remove": cmd_venv_remove(options, root) @@ -402,12 +428,12 @@ def _main(): cmd_show(options) sys.exit() elif options.action == "run": - benchmarks = _benchmarks_from_options(options) + benchmarks = _benchmarks_from_options(options, options.python) cmd_run(options, benchmarks) elif options.action == "compare": cmd_compare(options) elif options.action == "list": - benchmarks = _benchmarks_from_options(options) + benchmarks = _benchmarks_from_options(options, options.python) cmd_list(options, benchmarks) elif options.action == "list_groups": manifest = _manifest_from_options(options) diff --git a/pyperformance/commands.py b/pyperformance/commands.py index 7fc6603b..6c543669 100644 --- a/pyperformance/commands.py +++ b/pyperformance/commands.py @@ -178,7 +178,10 @@ def cmd_run(options, benchmarks): print('ERROR: "%s" is not an absolute path' % executable) sys.exit(1) - suite, errors = run_benchmarks(benchmarks, executable, options) + if benchmarks: + suite, errors = run_benchmarks(benchmarks, executable, options) + else: + suite, errors = None, [] if not suite: print("ERROR: No benchmark was run") diff --git a/pyperformance/tests/test_benchmark_compatibility.py b/pyperformance/tests/test_benchmark_compatibility.py new file mode 100644 index 00000000..b3f1190c --- /dev/null +++ b/pyperformance/tests/test_benchmark_compatibility.py @@ -0,0 +1,273 @@ +import contextlib +import io +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from pyperformance import ( + _benchmark, + _benchmark_metadata, + _manifest, + _pythoninfo, + _venv, + cli, + commands, +) + + +def target_info(name, version, implementation_version=None): + """Return minimal interpreter information for selection tests. + + name: Value reported by sys.implementation.name. + version: Python language version. + implementation_version: Implementation version, defaulting to the language version. + Return: Interpreter information containing the fields used by selection. + """ + implementation = SimpleNamespace( + name=name, version=implementation_version or version + ) + return SimpleNamespace( + sys=SimpleNamespace(version_info=version, implementation=implementation) + ) + + +class BenchmarkCompatibilityTests(unittest.TestCase): + """Exercise compatibility metadata and selection for the target interpreter.""" + + def setUp(self): + """Provide an isolated directory for benchmark metadata.""" + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.cpython = target_info("cpython", (3, 12, 0)) + self.pypy = target_info("pypy", (3, 11, 0), (7, 3, 23)) + + def write_benchmark(self, name, tool_text="", requires_python=">=3.10"): + """Write one benchmark's metadata in the isolated directory. + + name: Benchmark name used for its directory and metadata. + tool_text: Additional TOML lines in the pyperformance tool section. + requires_python: Python language-version requirement. + Return: Path to the new metadata file. + """ + directory = self.root / ("bm_" + name) + directory.mkdir() + path = directory / "pyproject.toml" + path.write_text( + f'[project]\nname = "pyperformance_bm_{name}"\nversion = "1.0"\n' + f'requires-python = "{requires_python}"\n\n' + f'[tool.pyperformance]\nname = "{name}"\n{tool_text}\n' + ) + return path + + def make_manifest(self): + """Create a catalog with portable, restricted, and version-gated cases. + + Return: The loaded manifest and its file path. + """ + paths = { + "portable": self.write_benchmark("portable"), + "cpython_only": self.write_benchmark( + "cpython_only", 'python_implementations = ["cpython"]' + ), + "new_python": self.write_benchmark("new_python", requires_python=">=3.12"), + } + path = self.root / "MANIFEST" + text = "[benchmarks]\n\nname\tmetafile\n" + text += "".join(f"{name}\t{metafile}\n" for name, metafile in paths.items()) + text += "\n[group default]\nportable\ncpython_only\nnew_python\n" + text += "\n[group gc]\nportable\ncpython_only\n" + path.write_text(text) + return _manifest.load_manifest(str(path)), path + + def select(self, selection, manifest, info): + """Resolve one selection while retaining its user-facing notices. + + selection: Command-line benchmark selection. + manifest: Catalog containing the test benchmarks. + info: Target interpreter information. + Return: Sorted selected names and captured stderr. + """ + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + selected = cli._select_benchmarks(selection, manifest, info) + return sorted(benchmark.name for benchmark in selected), stderr.getvalue() + + def test_omitted_metadata_allows_every_implementation(self): + """An unrestricted benchmark should expose no implementation filter.""" + path = self.write_benchmark("portable") + metadata, _ = _benchmark_metadata.load_metadata(str(path)) + self.assertNotIn("python_implementations", metadata) + benchmark = _benchmark.Benchmark( + _benchmark.BenchmarkSpec("portable", "1.0"), str(path) + ) + self.assertFalse(benchmark.python_implementations) + + def test_metadata_normalizes_names(self): + """Names should be case-insensitive without requiring a central registry.""" + path = self.write_benchmark( + "restricted", 'python_implementations = ["CPython", "PyPy", "New_Python"]' + ) + metadata, _ = _benchmark_metadata.load_metadata(str(path)) + self.assertEqual( + metadata["python_implementations"], ["cpython", "pypy", "new_python"] + ) + + def test_metadata_inherits_and_replaces_restrictions(self): + """Variants should inherit a restriction unless they supply another list.""" + base = self.write_benchmark("base", 'python_implementations = ["cpython"]') + inherited = base.parent / "bm_inherited.toml" + inherited.write_text('[tool.pyperformance]\nname = "inherited"\n') + overridden = base.parent / "bm_overridden.toml" + overridden.write_text( + '[tool.pyperformance]\nname = "overridden"\npython_implementations = ["PyPy"]\n' + ) + + metadata, _ = _benchmark_metadata.load_metadata(str(inherited)) + self.assertEqual(metadata["python_implementations"], ["cpython"]) + metadata, _ = _benchmark_metadata.load_metadata(str(overridden)) + self.assertEqual(metadata["python_implementations"], ["pypy"]) + + def test_invalid_implementation_metadata(self): + """Malformed restrictions should fail while metadata is loaded.""" + cases = ( + ('"cpython"', TypeError), + ("[]", ValueError), + ("[1]", TypeError), + ('[""]', ValueError), + ('["bad name"]', ValueError), + ) + for index, (value, error) in enumerate(cases): + with self.subTest(value=value): + path = self.write_benchmark( + f"invalid_{index}", "python_implementations = " + value + ) + with self.assertRaises(error): + _benchmark_metadata.load_metadata(str(path)) + + def test_selection_uses_target_compatibility(self): + """Names and groups should use the target language and implementation.""" + manifest, _ = self.make_manifest() + cases = ( + ("default", ["cpython_only", "new_python", "portable"], ["portable"], True), + ("all", ["cpython_only", "new_python", "portable"], ["portable"], True), + ("gc", ["cpython_only", "portable"], ["portable"], True), + ("portable", ["portable"], ["portable"], False), + ("cpython_only", ["cpython_only"], [], True), + ("portable,cpython_only", ["cpython_only", "portable"], ["portable"], True), + ("all,-cpython_only", ["new_python", "portable"], ["portable"], False), + ) + notice = "Skipping cpython_only: target implementation pypy is not supported (supported: cpython)\n" + for selection, cpython_names, pypy_names, skipped in cases: + with self.subTest(selection=selection, implementation="cpython"): + self.assertEqual( + self.select(selection, manifest, self.cpython), (cpython_names, "") + ) + with self.subTest(selection=selection, implementation="pypy"): + self.assertEqual( + self.select(selection, manifest, self.pypy), + (pypy_names, notice if skipped else ""), + ) + + def test_implementation_notice_when_version_is_also_unsupported(self): + """An implementation restriction should remain visible when both checks fail.""" + metadata = self.write_benchmark( + "cpython_new", + 'python_implementations = ["cpython"]', + requires_python=">=3.12", + ) + path = self.root / "MANIFEST" + path.write_text(f"[benchmarks]\n\nname\tmetafile\ncpython_new\t{metadata}\n") + manifest = _manifest.load_manifest(str(path)) + names, stderr = self.select("cpython_new", manifest, self.pypy) + self.assertEqual(names, []) + self.assertEqual( + stderr, + "Skipping cpython_new: target implementation pypy is not supported (supported: cpython)\n", + ) + + def test_empty_run_stops_before_benchmark_preparation(self): + """An empty selection should fail before preparing benchmark environments.""" + options = SimpleNamespace(output=None, python=sys.executable) + stdout = io.StringIO() + with ( + mock.patch("pyperformance.run.run_benchmarks") as run_benchmarks, + contextlib.redirect_stdout(stdout), + ): + with self.assertRaises(SystemExit) as error: + commands.cmd_run(options, []) + self.assertEqual(error.exception.code, 1) + self.assertIn("ERROR: No benchmark was run", stdout.getvalue()) + run_benchmarks.assert_not_called() + + def test_dispatch_selects_for_the_effective_interpreter(self): + """Commands should select for the interpreter they pass to execution.""" + _, manifest_path = self.make_manifest() + + def get_info(executable=sys.executable): + """Return the synthetic interpreter named by the command.""" + if executable == "/target-pypy": + return self.pypy + if executable == sys.executable: + return self.cpython + raise AssertionError(f"unexpected interpreter: {executable}") + + cases = ( + ("run", None, False, ["portable"]), + ("list", None, False, ["portable"]), + ("venv", "create", False, ["portable"]), + ("venv", "recreate", False, ["portable"]), + ("venv", "create", True, ["cpython_only", "new_python", "portable"]), + ("venv", "recreate", True, ["cpython_only", "new_python", "portable"]), + ) + for action, venv_action, explicit_venv, expected in cases: + with self.subTest( + action=action, venv_action=venv_action, explicit_venv=explicit_venv + ): + options = SimpleNamespace( + action=action, + python="/target-pypy", + benchmarks="all", + manifest=str(manifest_path), + venv="/explicit-venv" if explicit_venv else None, + venv_action=venv_action, + ) + command_name = ( + "cmd_venv_" + venv_action if action == "venv" else "cmd_" + action + ) + with contextlib.ExitStack() as stack: + stack.enter_context( + mock.patch.object(cli, "is_installed", return_value=True) + ) + stack.enter_context( + mock.patch.object( + cli, "parse_args", return_value=(mock.Mock(), options) + ) + ) + stack.enter_context( + mock.patch.object(_pythoninfo, "get_info", side_effect=get_info) + ) + stack.enter_context( + mock.patch.object( + _venv, "get_venv_root", return_value="/generated-venv" + ) + ) + command = stack.enter_context(mock.patch.object(cli, command_name)) + stack.enter_context(contextlib.redirect_stderr(io.StringIO())) + cli._main() + command.assert_called_once() + selected = command.call_args.args[-1] + self.assertEqual( + sorted(benchmark.name for benchmark in selected), expected + ) + if action == "venv": + self.assertIs( + command.call_args.args[-2], None if explicit_venv else self.pypy + ) + + +if __name__ == "__main__": + unittest.main() From 6e9d6b6164613a711a1833f2e073fde186dd9121 Mon Sep 17 00:00:00 2001 From: Mark Falco Date: Wed, 21 Oct 2026 09:00:32 -0400 Subject: [PATCH 2/2] Add GC benchmarks for large heaps and collection phases --- doc/benchmarks.rst | 57 ++++ pyperformance/data-files/benchmarks/MANIFEST | 7 + .../bm_btree/bm_btree_gc_only_frozen.toml | 4 + .../bm_btree/run_frozen_benchmark.py | 100 ++++++ .../bm_gc_collect/bm_gc_collect_large.toml | 3 + .../bm_gc_collect_self_lists.toml | 3 + .../bm_gc_collect/run_large_benchmark.py | 74 +++++ .../bm_gc_collect/run_self_lists_benchmark.py | 94 ++++++ .../bm_gc_resurrection_full.toml | 3 + .../bm_gc_resurrection_partial.toml | 3 + .../bm_gc_weakref_recheck.toml | 3 + .../bm_gc_resurrection/pyproject.toml | 10 + .../bm_gc_resurrection/run_benchmark.py | 312 ++++++++++++++++++ .../bm_gc_tuple_survivors/pyproject.toml | 9 + .../bm_gc_tuple_survivors/run_benchmark.py | 91 +++++ .../tests/test_benchmark_compatibility.py | 24 ++ 16 files changed, 797 insertions(+) create mode 100644 pyperformance/data-files/benchmarks/bm_btree/bm_btree_gc_only_frozen.toml create mode 100644 pyperformance/data-files/benchmarks/bm_btree/run_frozen_benchmark.py create mode 100644 pyperformance/data-files/benchmarks/bm_gc_collect/bm_gc_collect_large.toml create mode 100644 pyperformance/data-files/benchmarks/bm_gc_collect/bm_gc_collect_self_lists.toml create mode 100644 pyperformance/data-files/benchmarks/bm_gc_collect/run_large_benchmark.py create mode 100644 pyperformance/data-files/benchmarks/bm_gc_collect/run_self_lists_benchmark.py create mode 100644 pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_resurrection_full.toml create mode 100644 pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_resurrection_partial.toml create mode 100644 pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_weakref_recheck.toml create mode 100644 pyperformance/data-files/benchmarks/bm_gc_resurrection/pyproject.toml create mode 100644 pyperformance/data-files/benchmarks/bm_gc_resurrection/run_benchmark.py create mode 100644 pyperformance/data-files/benchmarks/bm_gc_tuple_survivors/pyproject.toml create mode 100644 pyperformance/data-files/benchmarks/bm_gc_tuple_survivors/run_benchmark.py diff --git a/doc/benchmarks.rst b/doc/benchmarks.rst index 7701d512..ccc42b23 100644 --- a/doc/benchmarks.rst +++ b/doc/benchmarks.rst @@ -117,6 +117,13 @@ skips the operations after creation. Note that this benchmark does not create any reference cycles that the garbage collector will need to break to free memory. +The ``btree_gc_only_frozen`` variant freezes the startup heap before creating +the tree. An ordinary holder references a frozen anchor so the measured full +collection runs with references into a nonempty permanent generation. Tree +construction, freezing, and unfreezing are outside timing. The permanent +generation must be empty when this benchmark starts. This variant runs on +CPython, which provides the required GC freeze API. + chameleon --------- @@ -250,6 +257,56 @@ Create 100,000 point objects which compute ``math.cos()``, ``math.sin()`` and than testing performance of class attributes. +gc_collect_large, gc_collect_self_lists +--------------------------------------- + +``gc_collect_large`` uses the same cyclic graph as ``gc_collect`` with 10,000 +independent bidirectional rings of 21 nodes each, for 210,000 garbage nodes. +``gc_collect_self_lists`` uses 1,000,000 independent one-element lists that +each contain themselves. Its small per-object traversal emphasizes the cost +of processing a large garbage list. + +Both benchmarks keep the graph rooted until ``gc.disable()`` is called, then +time only the explicit full collection. PyPy may still run minor collections +while automatic major collections and finalizers are disabled. Construction +and checks are outside timing. ``--cycles`` controls the number of rings or +self-referential lists. + + +gc_weakref_recheck, gc_resurrection_partial, gc_resurrection_full +----------------------------------------------------------------- + +Collect 10,000 independent rings of 21 nodes each with one callback event: + +* ``gc_weakref_recheck`` runs one weakref callback that observes a cleared + reference and retains no fixture objects. +* ``gc_resurrection_partial`` runs one finalizer that retains half the rings. +* ``gc_resurrection_full`` runs one finalizer that retains every ring. + +The resurrection variants add one finalizing node and a list of roots. Timing +includes the collections needed to reclaim unrescued garbage. If an older +collector defers all garbage after resurrection, the partial variant includes +the second collection needed to finish that work. Setup, validation, and the +final collection of rescued objects are outside timing. + +These variants run on CPython and use its integer reclamation counts to verify +the measured work. ``--cycles`` controls the number of rings and must be even. + + +gc_tuple_survivors +------------------ + +Retain 200,000 fresh tuples containing 16 ``None`` items each and time +``gc.collect(0)``. Construction and cleanup are outside timing. ``--entries`` +and ``--width`` control the population and tuple size. + +On GIL-enabled CPython, the retained root is aged before the tuples are created, +and the young-generation collection can untrack the tuples. Free-threaded +CPython scans the full GC heap for this call. PyPy also performs a full +collection and may reclaim construction temporaries, so those results have a +broader scope than the GIL-enabled CPython result. + + genshi ------ diff --git a/pyperformance/data-files/benchmarks/MANIFEST b/pyperformance/data-files/benchmarks/MANIFEST index 35777453..bc64648f 100644 --- a/pyperformance/data-files/benchmarks/MANIFEST +++ b/pyperformance/data-files/benchmarks/MANIFEST @@ -28,11 +28,18 @@ base64 bpe_tokeniser btree btree_gc_only +btree_gc_only_frozen concurrent_imap coroutines coverage gc_traversal gc_collect +gc_collect_large +gc_collect_self_lists +gc_weakref_recheck +gc_resurrection_partial +gc_resurrection_full +gc_tuple_survivors generators chameleon chaos diff --git a/pyperformance/data-files/benchmarks/bm_btree/bm_btree_gc_only_frozen.toml b/pyperformance/data-files/benchmarks/bm_btree/bm_btree_gc_only_frozen.toml new file mode 100644 index 00000000..f1c94efa --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_btree/bm_btree_gc_only_frozen.toml @@ -0,0 +1,4 @@ +[tool.pyperformance] +name = "btree_gc_only_frozen" +python_implementations = ["cpython"] +runscript = "run_frozen_benchmark.py" diff --git a/pyperformance/data-files/benchmarks/bm_btree/run_frozen_benchmark.py b/pyperformance/data-files/benchmarks/bm_btree/run_frozen_benchmark.py new file mode 100644 index 00000000..44ed8101 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_btree/run_frozen_benchmark.py @@ -0,0 +1,100 @@ +"""Measure B-tree collection after freezing the startup heap.""" + +import gc +import random + +import pyperf + +from run_benchmark import NUM_NODES, RANDOM_SEED, add_metadata, make_records, make_tree + + +def benchmark_collection(loops, nodes): + """Time full collections of a B-tree created after freezing the heap. + + loops: Number of trees and measured collections. + nodes: Number of records in each tree. + Return: Total collection wall time in seconds, excluding setup and checks. + Requires an empty permanent generation; unfreezes the heap and restores + the caller's automatic-GC state on return. + """ + if not all(hasattr(gc, name) for name in ("freeze", "unfreeze", "get_freeze_count")): + raise RuntimeError("benchmark requires the GC freeze API") + if gc.get_freeze_count(): + raise RuntimeError("benchmark requires an empty permanent generation") + get_debug = getattr(gc, "get_debug", None) + if get_debug is not None and get_debug(): + raise RuntimeError("benchmark requires GC debug flags to be disabled") + + was_enabled = gc.isenabled() + must_unfreeze = False + total_time = 0.0 + try: + gc.enable() + gc.collect() + if gc.garbage: + raise RuntimeError("benchmark starts with uncollectable garbage") + anchor = [] + if not gc.is_tracked(anchor): + raise RuntimeError("frozen anchor is not tracked") + # An interrupted freeze may already have moved objects. + must_unfreeze = True + gc.freeze() + if gc.get_freeze_count() == 0: + raise RuntimeError("freeze did not populate the permanent generation") + + # The new holder gives the ordinary heap a reference to a frozen object. + holder = [anchor] + records = list(make_records(nodes)) + for _ in range(loops): + gc.enable() + random.seed(RANDOM_SEED) + tree = make_tree(nodes, records) + gc.disable() + started = pyperf.perf_counter() + collected = gc.collect() + total_time += pyperf.perf_counter() - started + + if collected is not None and collected != 0: + raise RuntimeError(f"live-tree collection reclaimed {collected} objects") + if len(tree) != nodes: + raise RuntimeError("collection changed the tree size") + for index in (0, nodes // 2, nodes - 1): + if tree[index] is not records[index]: + raise RuntimeError(f"collection changed record {index}") + if holder[0] is not anchor or not gc.is_tracked(holder) or not gc.is_tracked(anchor): + raise RuntimeError("collection changed the frozen anchor or its holder") + if gc.get_freeze_count() == 0 or gc.garbage: + raise RuntimeError("collection lost the permanent generation or produced garbage") + tree = None + finally: + try: + if must_unfreeze: + gc.unfreeze() + finally: + if was_enabled: + gc.enable() + else: + gc.disable() + return total_time + + +def add_cmdline_args(command, args): + """Pass the selected tree size to each worker. + + command: Worker command to extend. + args: Parsed benchmark arguments. + Return: None. + """ + command.extend(("--nodes", str(args.nodes))) + + +if __name__ == "__main__": + runner = pyperf.Runner(add_cmdline_args=add_cmdline_args, processes=1, loops=1, values=3, warmups=1, min_time=0.4) + runner.argparser.add_argument("--nodes", type=int, default=NUM_NODES, help="Number of records in each B-tree.") + args = runner.parse_args() + if args.nodes < 1: + runner.argparser.error("--nodes must be positive") + add_metadata(runner) + runner.metadata["description"] = "Full collection of a B-tree created after freezing the startup heap" + runner.metadata["btree_num_nodes"] = args.nodes + runner.bench_time_func("btree_gc_only_frozen", benchmark_collection, args.nodes) diff --git a/pyperformance/data-files/benchmarks/bm_gc_collect/bm_gc_collect_large.toml b/pyperformance/data-files/benchmarks/bm_gc_collect/bm_gc_collect_large.toml new file mode 100644 index 00000000..aab2eecd --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_gc_collect/bm_gc_collect_large.toml @@ -0,0 +1,3 @@ +[tool.pyperformance] +name = "gc_collect_large" +runscript = "run_large_benchmark.py" diff --git a/pyperformance/data-files/benchmarks/bm_gc_collect/bm_gc_collect_self_lists.toml b/pyperformance/data-files/benchmarks/bm_gc_collect/bm_gc_collect_self_lists.toml new file mode 100644 index 00000000..2a470b01 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_gc_collect/bm_gc_collect_self_lists.toml @@ -0,0 +1,3 @@ +[tool.pyperformance] +name = "gc_collect_self_lists" +runscript = "run_self_lists_benchmark.py" diff --git a/pyperformance/data-files/benchmarks/bm_gc_collect/run_large_benchmark.py b/pyperformance/data-files/benchmarks/bm_gc_collect/run_large_benchmark.py new file mode 100644 index 00000000..63624429 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_gc_collect/run_large_benchmark.py @@ -0,0 +1,74 @@ +"""Measure full collection of a larger population of cyclic garbage.""" + +import gc + +import pyperf + +from run_benchmark import LINKS, create_gc_cycles + + +CYCLES = 10_000 + + +def benchmark_collection(loops, cycles, links): + """Time full collections of independent reference cycles. + + loops: Number of measured collections. + cycles: Number of independent rings in each collection. + links: Number of nodes added after each ring's first node. + Return: Total collection wall time in seconds, excluding setup and checks. + The caller's automatic-GC state is restored on return. + """ + get_debug = getattr(gc, "get_debug", None) + if get_debug is not None and get_debug(): + raise RuntimeError("benchmark requires GC debug flags to be disabled") + was_enabled = gc.isenabled() + total_time = 0.0 + try: + for _ in range(loops): + gc.enable() + gc.collect() + if gc.garbage: + raise RuntimeError("benchmark starts with uncollectable garbage") + roots = create_gc_cycles(cycles, links) + + # Keep the graph rooted until the explicit collection is ready. + gc.disable() + roots = None + started = pyperf.perf_counter() + collected = gc.collect() + total_time += pyperf.perf_counter() - started + + if collected is not None and collected < cycles * (links + 1): + raise RuntimeError("collection reclaimed fewer nodes than the fixture created") + if gc.garbage: + raise RuntimeError("collection produced uncollectable garbage") + finally: + if was_enabled: + gc.enable() + else: + gc.disable() + return total_time + + +def add_cmdline_args(command, args): + """Pass the selected graph size to each worker. + + command: Worker command to extend. + args: Parsed benchmark arguments. + Return: None. + """ + command.extend(("--cycles", str(args.cycles))) + + +if __name__ == "__main__": + runner = pyperf.Runner(add_cmdline_args=add_cmdline_args) + runner.argparser.add_argument("--cycles", type=int, default=CYCLES, help="Number of independent reference cycles.") + args = runner.parse_args() + if args.cycles < 1: + runner.argparser.error("--cycles must be positive") + runner.metadata["description"] = "Full collection of many independent reference cycles" + runner.metadata["gc_cycles"] = args.cycles + runner.metadata["gc_links"] = LINKS + runner.metadata["gc_nodes"] = args.cycles * (LINKS + 1) + runner.bench_time_func("gc_collect_large", benchmark_collection, args.cycles, LINKS) diff --git a/pyperformance/data-files/benchmarks/bm_gc_collect/run_self_lists_benchmark.py b/pyperformance/data-files/benchmarks/bm_gc_collect/run_self_lists_benchmark.py new file mode 100644 index 00000000..3162f4be --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_gc_collect/run_self_lists_benchmark.py @@ -0,0 +1,94 @@ +"""Measure full collection of synthetic, minimal cyclic containers.""" + +import gc + +import pyperf + + +CYCLES = 1_000_000 + + +def create_self_lists(cycles): + """Return rooted one-element lists that each contain themselves. + + cycles: Number of independent self-referential lists. + Return: An outer list owning every cycle; no child is retained elsewhere. + """ + roots = [] + for _ in range(cycles): + node = [] + node.append(node) + roots.append(node) + return roots + + +def benchmark_collection(loops, cycles): + """Time full collections of independent self-referential lists. + + loops: Number of measured collections. + cycles: Number of lists in each collection. + Return: Total collection wall time in seconds, excluding setup and checks. + The caller's automatic-GC state is restored on return or failure. + """ + get_debug = getattr(gc, "get_debug", None) + if get_debug is not None and get_debug(): + raise RuntimeError("benchmark requires GC debug flags to be disabled") + was_enabled = gc.isenabled() + total_time = 0.0 + completed = False + roots = None + try: + for _ in range(loops): + gc.enable() + gc.collect() + if gc.garbage: + raise RuntimeError("benchmark starts with uncollectable garbage") + roots = create_self_lists(cycles) + + # Keep every cycle rooted until automatic collection is disabled. + gc.disable() + roots = None + started = pyperf.perf_counter() + collected = gc.collect() + total_time += pyperf.perf_counter() - started + + if collected is not None and collected < cycles: + raise RuntimeError("collection reclaimed fewer lists than the fixture created") + if gc.garbage: + raise RuntimeError("collection produced uncollectable garbage") + completed = True + finally: + roots = None + try: + if not completed: + # A failed worker should not leave its released cycles behind. + gc.collect() + finally: + if was_enabled: + gc.enable() + else: + gc.disable() + return total_time + + +def add_cmdline_args(command, args): + """Pass the selected population size to each worker. + + command: Worker command to extend. + args: Parsed benchmark arguments. + Return: None. + """ + command.extend(("--cycles", str(args.cycles))) + + +if __name__ == "__main__": + runner = pyperf.Runner(add_cmdline_args=add_cmdline_args) + runner.argparser.add_argument("--cycles", type=int, default=CYCLES, help="Number of independent self-referential lists.") + args = runner.parse_args() + if args.cycles < 1: + runner.argparser.error("--cycles must be positive") + runner.metadata["description"] = "Full collection of synthetic one-element self-referential lists" + runner.metadata["gc_cycles"] = args.cycles + runner.metadata["gc_nodes"] = args.cycles + runner.metadata["gc_references"] = args.cycles + runner.bench_time_func("gc_collect_self_lists", benchmark_collection, args.cycles) diff --git a/pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_resurrection_full.toml b/pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_resurrection_full.toml new file mode 100644 index 00000000..f25eacd0 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_resurrection_full.toml @@ -0,0 +1,3 @@ +[tool.pyperformance] +name = "gc_resurrection_full" +extra_opts = ["--mode", "full"] diff --git a/pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_resurrection_partial.toml b/pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_resurrection_partial.toml new file mode 100644 index 00000000..cb72bfee --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_resurrection_partial.toml @@ -0,0 +1,3 @@ +[tool.pyperformance] +name = "gc_resurrection_partial" +extra_opts = ["--mode", "partial"] diff --git a/pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_weakref_recheck.toml b/pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_weakref_recheck.toml new file mode 100644 index 00000000..13b721e0 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_gc_resurrection/bm_gc_weakref_recheck.toml @@ -0,0 +1,3 @@ +[tool.pyperformance] +name = "gc_weakref_recheck" +extra_opts = ["--mode", "weakref"] diff --git a/pyperformance/data-files/benchmarks/bm_gc_resurrection/pyproject.toml b/pyperformance/data-files/benchmarks/bm_gc_resurrection/pyproject.toml new file mode 100644 index 00000000..91a2dba9 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_gc_resurrection/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "pyperformance_bm_gc_resurrection" +requires-python = ">=3.8" +dependencies = ["pyperf"] +urls = {repository = "https://github.com/python/pyperformance"} +dynamic = ["version"] + +[tool.pyperformance] +name = "gc_resurrection" +python_implementations = ["cpython"] diff --git a/pyperformance/data-files/benchmarks/bm_gc_resurrection/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_gc_resurrection/run_benchmark.py new file mode 100644 index 00000000..6945efaa --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_gc_resurrection/run_benchmark.py @@ -0,0 +1,312 @@ +"""Measure collection when callbacks require garbage to be checked again.""" + +import gc +import weakref + +import pyperf + + +CYCLES = 10_000 +RING_SIZE = 21 + + +class Node: + """A ring member with a version-independent tracked-object count. + + next: Next ring member. + prev: Previous ring member. + __weakref__: Slot used by the weakref callback variant. + """ + + __slots__ = ("next", "prev", "__weakref__") + + def __init__(self): + """Create an unlinked ring member. + + Return: None. + """ + self.next = None + self.prev = None + + +class Rescuer(Node): + """A ring member whose finalizer retains a selected part of the graph. + + payload: List of ring roots retained with this object. + calls: Number of finalizer calls for the current population. + holder: Preallocated external root slot receiving this object. + """ + + __slots__ = ("payload",) + calls = 0 + holder = [None] + + def __init__(self, payload): + """Create an unlinked member carrying the selected roots. + + payload: List of ring roots to retain during finalization. + Return: None. + """ + super().__init__() + self.payload = payload + + def __del__(self): + """Record finalization and retain this object in the external slot. + + Return: None. + """ + Rescuer.calls += 1 + Rescuer.holder[0] = self + + +weakref_calls = 0 +weakref_was_cleared = False + + +def record_weakref(reference): + """Record one callback without retaining a fixture object. + + reference: Weakref supplied by the collector. + Return: None. + """ + global weakref_calls, weakref_was_cleared + weakref_calls += 1 + weakref_was_cleared = reference() is None + + +def create_cycles(cycles): + """Create independent bidirectional rings. + + cycles: Number of rings, each containing RING_SIZE nodes. + Return: List holding one root for every ring. + """ + roots = [] + for _ in range(cycles): + first = Node() + previous = first + for _ in range(RING_SIZE - 1): + node = Node() + previous.next = node + node.prev = previous + previous = node + previous.next = first + first.prev = previous + roots.append(first) + return roots + + +def prepare_collection(cycles, mode): + """Create the selected graph while retaining all its rings. + + cycles: Even number of rings, at least two. + mode: weakref, partial, or full. + Return: Construction roots and the retained callback weakref, if any. + """ + global weakref_calls, weakref_was_cleared + assert cycles >= 2 and cycles % 2 == 0 + if mode not in ("weakref", "partial", "full"): + raise ValueError("unknown resurrection mode: " + mode) + if Rescuer.holder[0] is not None: + raise RuntimeError("a prior rescued graph is still retained") + Rescuer.calls = 0 + weakref_calls = 0 + weakref_was_cleared = False + roots = create_cycles(cycles) + if mode == "weakref": + return roots, weakref.ref(roots[0], record_weakref) + + if mode == "partial": + selected = cycles // 2 + else: + assert mode == "full" + selected = cycles + # Copy even the full set so both modes drop the same construction root. + rescuer = Rescuer(roots[:selected]) + first = roots[0] + successor = first.next + first.next = rescuer + rescuer.prev = first + rescuer.next = successor + successor.prev = rescuer + return roots, None + + +def check_events(mode, reference): + """Check the callback events for one population. + + mode: weakref, partial, or full. + reference: Retained weakref in weakref mode, otherwise None. + Return: None after checking event counts and weakref state. + """ + expected_callbacks = 1 if mode == "weakref" else 0 + expected_finalizers = 0 if mode == "weakref" else 1 + if weakref_calls != expected_callbacks: + raise RuntimeError(f"observed {weakref_calls} weakref callbacks, expected {expected_callbacks}") + if Rescuer.calls != expected_finalizers: + raise RuntimeError(f"observed {Rescuer.calls} finalizers, expected {expected_finalizers}") + if mode == "weakref": + if reference is None or reference() is not None or not weakref_was_cleared: + raise RuntimeError("weakref callback did not observe a cleared reference") + if Rescuer.holder[0] is not None: + raise RuntimeError("weakref callback unexpectedly retained a graph") + else: + assert mode in ("partial", "full") + if reference is not None or weakref_was_cleared: + raise RuntimeError("resurrection unexpectedly recorded a weakref callback") + + +def check_rescued_graph(cycles, mode, complete): + """Check ownership and selected links in the rescued graph. + + cycles: Number of rings created for this population. + mode: partial or full. + complete: Whether to inspect every retained ring instead of local links. + Return: Expected number of original ring nodes retained. + """ + if mode == "partial": + selected = cycles // 2 + elif mode == "full": + selected = cycles + else: + raise ValueError("unknown rescued-graph mode: " + mode) + rescuer = Rescuer.holder[0] + if type(rescuer) is not Rescuer: + raise RuntimeError("finalizer did not retain its Rescuer") + roots = rescuer.payload + if type(roots) is not list or len(roots) != selected: + raise RuntimeError("rescued payload has an unexpected type or size") + if rescuer.next.prev is not rescuer or rescuer.prev.next is not rescuer: + raise RuntimeError("rescued graph lost a link beside its Rescuer") + + if complete: + rescuer_visits = 0 + for index, root in enumerate(roots): + current = root + ring_size = RING_SIZE + 1 if index == 0 else RING_SIZE + for _ in range(ring_size): + if current.next.prev is not current or current.prev.next is not current: + raise RuntimeError("rescued graph lost a ring link") + rescuer_visits += current is rescuer + current = current.next + if current is not root: + raise RuntimeError("rescued ring has an unexpected size") + if rescuer_visits != 1: + raise RuntimeError("rescued graph has an unexpected Rescuer count") + return selected * RING_SIZE + + +def collect_population(cycles, mode, complete): + """Measure reclamation of one population and release its rescued objects. + + cycles: Even number of rings, at least two. + mode: weakref, partial, or full. + complete: Whether to inspect every rescued ring outside timing. + Return: Wall time spent in collections needed to reclaim unrescued garbage. + Setup, validation, and final cleanup are excluded. Automatic GC is restored. + """ + get_debug = getattr(gc, "get_debug", None) + if get_debug is not None and get_debug(): + raise RuntimeError("benchmark requires GC debug flags to be disabled") + was_enabled = gc.isenabled() + roots = None + reference = None + needs_cleanup = True + try: + gc.disable() + gc.collect() + if gc.garbage: + raise RuntimeError("benchmark starts with uncollectable garbage") + roots, reference = prepare_collection(cycles, mode) + roots = None + started = pyperf.perf_counter() + collected = gc.collect() + total_time = pyperf.perf_counter() - started + if type(collected) is not int: + raise RuntimeError("benchmark requires gc.collect() to report an integer reclamation count") + check_events(mode, reference) + + nodes = cycles * RING_SIZE + if mode == "weakref": + expected_collected = nodes + expected_cleanup = 0 + else: + assert mode in ("partial", "full") + rescued_nodes = check_rescued_graph(cycles, mode, complete) + expected_collected = nodes - rescued_nodes + # The Rescuer and its payload list add two tracked objects. + expected_cleanup = rescued_nodes + 2 + + if mode == "partial" and collected == 0: + # Older collectors defer the whole set when any object resurrects. + # Include the collection needed to finish the same reclamation work. + started = pyperf.perf_counter() + collected = gc.collect() + total_time += pyperf.perf_counter() - started + check_events(mode, reference) + check_rescued_graph(cycles, mode, False) + if type(collected) is not int or collected != expected_collected: + raise RuntimeError(f"collection reclaimed {collected} objects, expected {expected_collected}") + + Rescuer.holder[0] = None + cleanup = gc.collect() + if type(cleanup) is not int or cleanup != expected_cleanup: + raise RuntimeError(f"cleanup reclaimed {cleanup} objects, expected {expected_cleanup}") + check_events(mode, reference) + if Rescuer.holder[0] is not None or gc.garbage: + raise RuntimeError("cleanup retained a graph or uncollectable garbage") + needs_cleanup = False + return total_time + finally: + try: + roots = None + reference = None + Rescuer.holder[0] = None + if needs_cleanup: + gc.collect() + finally: + Rescuer.holder[0] = None + if was_enabled: + gc.enable() + else: + gc.disable() + + +def benchmark_collection(loops, cycles, mode): + """Time repeated collections with the selected callback outcome. + + loops: Number of measured populations. + cycles: Even number of rings in each population, at least two. + mode: weakref, partial, or full. + Return: Total collection wall time in seconds. + """ + total_time = 0.0 + for _ in range(loops): + total_time += collect_population(cycles, mode, False) + return total_time + + +def add_cmdline_args(command, args): + """Pass the selected graph size and callback mode to each worker. + + command: Worker command to extend. + args: Parsed benchmark arguments. + Return: None. + """ + command.extend(("--cycles", str(args.cycles), "--mode", args.mode)) + + +if __name__ == "__main__": + runner = pyperf.Runner(add_cmdline_args=add_cmdline_args) + runner.argparser.add_argument("--cycles", type=int, default=CYCLES, help="Even number of independent reference cycles.") + runner.argparser.add_argument("--mode", choices=("weakref", "partial", "full"), required=True) + args = runner.parse_args() + if args.cycles < 2 or args.cycles % 2: + runner.argparser.error("--cycles must be an even integer of at least two") + names = {"weakref": "gc_weakref_recheck", "partial": "gc_resurrection_partial", "full": "gc_resurrection_full"} + runner.metadata["description"] = "GC callback and resurrection benchmark" + runner.metadata["gc_cycles"] = args.cycles + runner.metadata["gc_nodes"] = args.cycles * RING_SIZE + runner.metadata["gc_callback_mode"] = args.mode + # Each worker proves the lifecycle on a small graph before reporting timings. + collect_population(2, args.mode, True) + runner.bench_time_func(names[args.mode], benchmark_collection, args.cycles, args.mode) diff --git a/pyperformance/data-files/benchmarks/bm_gc_tuple_survivors/pyproject.toml b/pyperformance/data-files/benchmarks/bm_gc_tuple_survivors/pyproject.toml new file mode 100644 index 00000000..c3e8ce56 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_gc_tuple_survivors/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "pyperformance_bm_gc_tuple_survivors" +requires-python = ">=3.8" +dependencies = ["pyperf"] +urls = {repository = "https://github.com/python/pyperformance"} +dynamic = ["version"] + +[tool.pyperformance] +name = "gc_tuple_survivors" diff --git a/pyperformance/data-files/benchmarks/bm_gc_tuple_survivors/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_gc_tuple_survivors/run_benchmark.py new file mode 100644 index 00000000..0f89e11b --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_gc_tuple_survivors/run_benchmark.py @@ -0,0 +1,91 @@ +"""Measure collection of fresh retained tuples containing atomic items.""" + +import gc + +import pyperf + + +ENTRIES = 200_000 +WIDTH = 16 + + +def benchmark_collection(loops, entries, width): + """Time collections of fresh tuples retained across the collection. + + loops: Number of measured collections. + entries: Positive number of tuples in each collection. + width: Positive number of None items in each tuple. + Return: Total collection wall time in seconds, excluding setup and checks. + The caller's automatic-GC state is restored on return or failure. + """ + assert entries > 0 and width > 0 + get_debug = getattr(gc, "get_debug", None) + if get_debug is not None and get_debug(): + raise RuntimeError("benchmark requires GC debug flags to be disabled") + + was_enabled = gc.isenabled() + roots = [] + seed = (None,) * width + total_time = 0.0 + try: + gc.disable() + # With the GIL collector, the retained root must be older than the fresh tuples. + gc.collect() + if gc.garbage: + raise RuntimeError("benchmark starts with uncollectable garbage") + + for _ in range(loops): + try: + # Going through an iterator creates fresh tuples instead of reusing seed. + for _ in range(entries): + roots.append(tuple(iter(seed))) + + started = pyperf.perf_counter() + collected = gc.collect(0) + total_time += pyperf.perf_counter() - started + + if collected is not None and collected != 0: + raise RuntimeError(f"survivor collection reclaimed {collected} objects") + if len(roots) != entries or any(roots[index] != seed for index in (0, entries // 2, entries - 1)): + raise RuntimeError("collection changed the retained tuples") + if gc.garbage: + raise RuntimeError("collection produced uncollectable garbage") + finally: + roots.clear() + # Discarded tuples must not carry into the next measured collection. + gc.collect() + if gc.garbage: + raise RuntimeError("cleanup produced uncollectable garbage") + finally: + if was_enabled: + gc.enable() + else: + gc.disable() + return total_time + + +def add_cmdline_args(command, args): + """Pass the selected tuple population to each worker. + + command: Worker command to extend. + args: Parsed benchmark arguments. + Return: None. + """ + command.extend(("--entries", str(args.entries), "--width", str(args.width))) + + +if __name__ == "__main__": + runner = pyperf.Runner(add_cmdline_args=add_cmdline_args) + runner.argparser.add_argument("--entries", type=int, default=ENTRIES, help="Number of retained tuples.") + runner.argparser.add_argument("--width", type=int, default=WIDTH, help="Number of None items in each tuple.") + args = runner.parse_args() + if args.entries < 1: + runner.argparser.error("--entries must be positive") + if args.width < 1: + runner.argparser.error("--width must be positive") + runner.metadata["description"] = "Collection of fresh retained tuples containing only None" + runner.metadata["gc_tuple_entries"] = args.entries + runner.metadata["gc_tuple_width"] = args.width + runner.metadata["gc_tuple_items"] = "none" + runner.metadata["gc_requested_generation"] = 0 + runner.bench_time_func("gc_tuple_survivors", benchmark_collection, args.entries, args.width) diff --git a/pyperformance/tests/test_benchmark_compatibility.py b/pyperformance/tests/test_benchmark_compatibility.py index b3f1190c..35c46e10 100644 --- a/pyperformance/tests/test_benchmark_compatibility.py +++ b/pyperformance/tests/test_benchmark_compatibility.py @@ -203,6 +203,30 @@ def test_empty_run_stops_before_benchmark_preparation(self): self.assertIn("ERROR: No benchmark was run", stdout.getvalue()) run_benchmarks.assert_not_called() + def test_registered_gc_benchmarks_use_supported_implementations(self): + """Normal and all selections should retain portable GC coverage on PyPy.""" + manifest = _manifest.load_manifest(None) + portable = {"gc_collect_large", "gc_collect_self_lists", "gc_tuple_survivors"} + restricted = { + "btree_gc_only_frozen", + "gc_weakref_recheck", + "gc_resurrection_partial", + "gc_resurrection_full", + } + for selection in ("default", "all"): + with self.subTest(selection=selection, implementation="cpython"): + names, _ = self.select(selection, manifest, self.cpython) + self.assertTrue(portable | restricted <= set(names)) + with self.subTest(selection=selection, implementation="pypy"): + names, stderr = self.select(selection, manifest, self.pypy) + self.assertTrue(portable <= set(names)) + self.assertTrue(restricted.isdisjoint(names)) + for name in restricted: + self.assertIn( + f"Skipping {name}: target implementation pypy is not supported", + stderr, + ) + def test_dispatch_selects_for_the_effective_interpreter(self): """Commands should select for the interpreter they pass to execution.""" _, manifest_path = self.make_manifest()