From e476f93649c4350679e699abae2e3563838bf0cc Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:31:45 -0700 Subject: [PATCH 1/2] gh-160535: Keep the ZstdDict alive until the zstd context is freed (GH-160536) The garbage collector could release a ZstdCompressor's dictionary, or the digested dictionaries cached by the ZstdDict, while worker threads of the compression context were still reading them. They are now only released on deallocation, after the context has been freed. ANT-2026-CT3P1CSJ + Visit the type in the compressor and decompressor traverse; move NEWS to Library (cherry picked from commit b949033ae8b22ff77e798a08f4ecc6eaf4cf0d8a) Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> --- Lib/test/test_zstd.py | 38 +++++++++++++++++++ ...-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst | 4 ++ Modules/_zstd/compressor.c | 11 ++---- Modules/_zstd/decompressor.c | 12 ++---- Modules/_zstd/zstddict.c | 16 +++----- 5 files changed, 53 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 9225c42e0e711a..d9fc21817748b2 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -7,9 +7,11 @@ import os import unittest import tempfile +import textwrap import threading from test.support.import_helper import import_module +from test.support.script_helper import assert_python_ok from test.support import threading_helper from test.support import _1M @@ -1548,6 +1550,42 @@ def test_len(self): self.assertEqual(len(TRAINED_DICT), len(TRAINED_DICT.dict_content)) self.assertIn(str(len(TRAINED_DICT)), str(TRAINED_DICT)) + @unittest.skipUnless(SUPPORT_MULTITHREADING, + "zstd build doesn't support multi-threaded compression") + @support.requires_subprocess() + def test_collect_cycle_with_busy_compressor(self): + # The dictionary must outlive the compression context when both are + # collected by the garbage collector while a worker thread is still + # using the dictionary. Whether the failure is detected depends on + # the memory allocator, so use a fresh process for each case. + code = textwrap.dedent(""" + import gc, random, sys + from compression.zstd import (CompressionParameter, + ZstdCompressor, ZstdDict) + + class Cycle: + pass + + job_size = 1 << 20 + content = random.randbytes(1 << 10) * (1 << 8) + zd = ZstdDict(content, is_raw=True) + options = {CompressionParameter.compression_level: 10, + CompressionParameter.nb_workers: 1, + CompressionParameter.job_size: job_size} + c = ZstdCompressor(options=options, + zstd_dict=getattr(zd, sys.argv[1])) + cycle = Cycle() + cycle.cycle = cycle + cycle.compressor = c + # Returns while the worker thread is compressing the first job. + c.compress(random.randbytes(1 << 10) * ((job_size >> 10) + 64)) + del zd, c, cycle + gc.collect() + """) + for attr in 'as_digested_dict', 'as_prefix': + with self.subTest(attr=attr): + assert_python_ok('-c', code, attr) + class FileTestCase(unittest.TestCase): def setUp(self): self.DECOMPRESSED_42 = b'a'*42 diff --git a/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst b/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst new file mode 100644 index 00000000000000..ea2082716d64ac --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst @@ -0,0 +1,4 @@ +Fix a crash in :mod:`compression.zstd` when the garbage collector frees a +:class:`~compression.zstd.ZstdCompressor` that uses worker threads together +with its :class:`~compression.zstd.ZstdDict` before the compression of +already submitted data has finished. diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 894568fce28d0d..19e12ed4af7479 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -766,17 +766,13 @@ static int ZstdCompressor_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdCompressor *self = ZstdCompressor_CAST(ob); + Py_VISIT(Py_TYPE(ob)); Py_VISIT(self->dict); return 0; } -static int -ZstdCompressor_clear(PyObject *ob) -{ - ZstdCompressor *self = ZstdCompressor_CAST(ob); - Py_CLEAR(self->dict); - return 0; -} +/* No tp_clear: the compression context (and its worker threads) uses the + dictionary until ZstdCompressor_dealloc() has freed the context. */ static PyType_Slot zstdcompressor_slots[] = { {Py_tp_new, _zstd_ZstdCompressor_new}, @@ -785,7 +781,6 @@ static PyType_Slot zstdcompressor_slots[] = { {Py_tp_members, ZstdCompressor_members}, {Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__}, {Py_tp_traverse, ZstdCompressor_traverse}, - {Py_tp_clear, ZstdCompressor_clear}, {0, 0} }; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index ca9ea7e3b8f99b..454736d687a430 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -689,18 +689,13 @@ static int ZstdDecompressor_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdDecompressor *self = ZstdDecompressor_CAST(ob); + Py_VISIT(Py_TYPE(ob)); Py_VISIT(self->dict); return 0; } -static int -ZstdDecompressor_clear(PyObject *ob) -{ - ZstdDecompressor *self = ZstdDecompressor_CAST(ob); - Py_CLEAR(self->dict); - Py_CLEAR(self->unused_data); - return 0; -} +/* No tp_clear: the decompression context references the dictionary until + ZstdDecompressor_dealloc() has freed the context. */ static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_new, _zstd_ZstdDecompressor_new}, @@ -710,7 +705,6 @@ static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_getset, ZstdDecompressor_getset}, {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__}, {Py_tp_traverse, ZstdDecompressor_traverse}, - {Py_tp_clear, ZstdDecompressor_clear}, {0, 0} }; diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index e1b9d998e697fb..11ea7619976bf9 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -240,16 +240,11 @@ ZstdDict_length(PyObject *ob) static int ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg) { - ZstdDict *self = ZstdDict_CAST(ob); - Py_VISIT(self->c_dicts); - return 0; -} - -static int -ZstdDict_clear(PyObject *ob) -{ - ZstdDict *self = ZstdDict_CAST(ob); - Py_CLEAR(self->c_dicts); + /* c_dicts is not visited and there is no tp_clear: it only maps ints + to capsules, and the ZSTD_CDict instances they own must stay alive + until ZstdDict_dealloc() as a compression context being collected + together with this object may still use them. */ + Py_VISIT(Py_TYPE(ob)); return 0; } @@ -262,7 +257,6 @@ static PyType_Slot zstddict_slots[] = { {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__}, {Py_sq_length, ZstdDict_length}, {Py_tp_traverse, ZstdDict_traverse}, - {Py_tp_clear, ZstdDict_clear}, {0, 0} }; From f9fb98c96e81a535dfb58b3460bf0ebab1d9264b Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sun, 25 Oct 2026 05:40:22 +0000 Subject: [PATCH 2/2] Import test.support for requires_subprocess() --- Lib/test/test_zstd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index d9fc21817748b2..b133597bcd532c 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -10,6 +10,7 @@ import textwrap import threading +from test import support from test.support.import_helper import import_module from test.support.script_helper import assert_python_ok from test.support import threading_helper