From ae18daedc75f40fdca7921bee3ce61dd1192c475 Mon Sep 17 00:00:00 2001 From: Oleksandr Baltian Date: Wed, 29 Jul 2026 15:56:07 +0200 Subject: [PATCH 1/3] gh-64192: Add *buffersize* to `imap()`/`imap_unordered()` in `multiprocessing.pool` (GH-136871) The new argument allows consuming the input iterator lazily, potentially saving memory, and allowing long/infinite iterators. It mirrors the *buffersize* argument to `concurrent.futures.Executor.map` that was added in 3.14. Co-authored-by: Oleksandr Baltian Co-authored-by: Petr Viktorin Co-authored-by: Sasha Baltian --- Doc/library/multiprocessing.rst | 19 +- Doc/whatsnew/3.16.rst | 18 +- Lib/multiprocessing/pool.py | 147 ++++++++------ Lib/test/_test_multiprocessing.py | 186 ++++++++++++++++-- ...5-07-28-11-00-49.gh-issue-64192.7htLtg.rst | 9 + 5 files changed, 305 insertions(+), 74 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-28-11-00-49.gh-issue-64192.7htLtg.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 2d13053915830b0..bedea46cb16d60d 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -2536,7 +2536,7 @@ with the :class:`Pool` class. Callbacks should complete immediately since otherwise the thread which handles the results will get blocked. - .. method:: imap(func, iterable[, chunksize]) + .. method:: imap(func, iterable, chunksize=1, *, buffersize=None) A lazier version of :meth:`.map`. @@ -2550,12 +2550,27 @@ with the :class:`Pool` class. ``next(timeout)`` will raise :exc:`multiprocessing.TimeoutError` if the result cannot be returned within *timeout* seconds. - .. method:: imap_unordered(func, iterable[, chunksize]) + The *iterable* is collected immediately rather than lazily, unless a + *buffersize* is specified to limit the number of submitted tasks whose + results have not yet been yielded. If the buffer is full, iteration over + the *iterables* pauses until a result is yielded from the buffer. + To fully utilize pool's capacity when using this feature, + set *buffersize* at least to the number of processes in pool + (to consume *iterable* as you go), or even higher + (to prefetch the next ``N=buffersize-processes`` arguments). + + .. versionchanged:: next + Added the *buffersize* parameter. + + .. method:: imap_unordered(func, iterable, chunksize=1, *, buffersize=None) The same as :meth:`imap` except that the ordering of the results from the returned iterator should be considered arbitrary. (Only when there is only one worker process is the order guaranteed to be "correct".) + .. versionchanged:: next + Added the *buffersize* parameter. + .. method:: starmap(func, iterable[, chunksize]) Like :meth:`~multiprocessing.pool.Pool.map` except that the diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 06e831ea1e34631..6e69737768d5e15 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -376,6 +376,22 @@ math (Contributed by Jeff Epler in :gh:`150534`.) +multiprocessing +--------------- + +* Add the optional ``buffersize`` parameter to + :meth:`multiprocessing.pool.Pool.imap` and + :meth:`multiprocessing.pool.Pool.imap_unordered` to limit the number of + submitted tasks whose results have not yet been yielded. If the buffer is + full, iteration over the *iterables* pauses until a result is yielded from + the buffer. To fully utilize pool's capacity when using this feature, set + *buffersize* at least to the number of processes in pool (to consume + *iterable* as you go), or even higher (to prefetch the next + ``N=buffersize-processes`` arguments). + + (Contributed by Oleksandr Baltian in :gh:`136871`.) + + os -- @@ -606,7 +622,7 @@ module_name Removed -======= +======== annotationlib ------------- diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index f979890170b1a1f..8fd0f98a02dd3a6 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -190,6 +190,11 @@ def __init__(self, processes=None, initializer=None, initargs=(), self._ctx = context or get_context() self._setup_queues() self._taskqueue = queue.SimpleQueue() + # The _taskqueue_buffersize_semaphores exist to allow calling .release() + # on every active semaphore when the pool is terminating to let task_handler + # wake up to stop. It's a set so that each iterator object can efficiently + # deregister its semaphore when iterator finishes. + self._taskqueue_buffersize_semaphores = set() # The _change_notifier queue exist to wake up self._handle_workers() # when the cache (self._cache) is empty or when there is a change in # the _state variable of the thread that runs _handle_workers. @@ -256,7 +261,8 @@ def __init__(self, processes=None, initializer=None, initargs=(), self, self._terminate_pool, args=(self._taskqueue, self._inqueue, self._outqueue, self._pool, self._change_notifier, self._worker_handler, self._task_handler, - self._result_handler, self._cache), + self._result_handler, self._cache, + self._taskqueue_buffersize_semaphores), exitpriority=15 ) self._state = RUN @@ -382,73 +388,43 @@ def starmap_async(self, func, iterable, chunksize=None, callback=None, return self._map_async(func, iterable, starmapstar, chunksize, callback, error_callback) - def _guarded_task_generation(self, result_job, func, iterable): + def _guarded_task_generation(self, result_job, func, iterable, sema=None): '''Provides a generator of tasks for imap and imap_unordered with appropriate handling for iterables which throw exceptions during iteration.''' try: i = -1 - for i, x in enumerate(iterable): - yield (result_job, i, func, (x,), {}) + + if sema is None: + for i, x in enumerate(iterable): + yield (result_job, i, func, (x,), {}) + + else: + enumerated_iter = iter(enumerate(iterable)) + while True: + sema.acquire() + try: + i, x = next(enumerated_iter) + except StopIteration: + break + yield (result_job, i, func, (x,), {}) + except Exception as e: yield (result_job, i+1, _helper_reraises_exception, (e,), {}) - def imap(self, func, iterable, chunksize=1): + def imap(self, func, iterable, chunksize=1, *, buffersize=None): ''' Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. ''' - self._check_running() - if chunksize == 1: - result = IMapIterator(self) - self._taskqueue.put( - ( - self._guarded_task_generation(result._job, func, iterable), - result._set_length - )) - return result - else: - if chunksize < 1: - raise ValueError( - "Chunksize must be 1+, not {0:n}".format( - chunksize)) - task_batches = Pool._get_tasks(func, iterable, chunksize) - result = IMapIterator(self) - self._taskqueue.put( - ( - self._guarded_task_generation(result._job, - mapstar, - task_batches), - result._set_length - )) - return (item for chunk in result for item in chunk) + return self._imap(IMapIterator, func, iterable, chunksize, + buffersize=buffersize) - def imap_unordered(self, func, iterable, chunksize=1): + def imap_unordered(self, func, iterable, chunksize=1, *, buffersize=None): ''' Like `imap()` method but ordering of results is arbitrary. ''' - self._check_running() - if chunksize == 1: - result = IMapUnorderedIterator(self) - self._taskqueue.put( - ( - self._guarded_task_generation(result._job, func, iterable), - result._set_length - )) - return result - else: - if chunksize < 1: - raise ValueError( - "Chunksize must be 1+, not {0!r}".format(chunksize)) - task_batches = Pool._get_tasks(func, iterable, chunksize) - result = IMapUnorderedIterator(self) - self._taskqueue.put( - ( - self._guarded_task_generation(result._job, - mapstar, - task_batches), - result._set_length - )) - return (item for chunk in result for item in chunk) + return self._imap(IMapUnorderedIterator, func, iterable, chunksize, + buffersize=buffersize) def apply_async(self, func, args=(), kwds={}, callback=None, error_callback=None): @@ -497,6 +473,41 @@ def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, ) return result + def _imap(self, iterator_cls, func, iterable, chunksize=1, + *, buffersize=None): + self._check_running() + if chunksize < 1: + raise ValueError( + f"Chunksize must be 1+, not {chunksize}" + ) + if buffersize is not None: + if not isinstance(buffersize, int): + raise TypeError("buffersize must be an integer or None") + if buffersize < 1: + raise ValueError("buffersize must be None or > 0") + + result = iterator_cls(self, buffersize=buffersize) + if chunksize == 1: + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable, + result._buffersize_sema), + result._set_length, + ) + ) + return result + else: + task_batches = Pool._get_tasks(func, iterable, chunksize) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, mapstar, + task_batches, + result._buffersize_sema), + result._set_length, + ) + ) + return (item for chunk in result for item in chunk) + @staticmethod def _wait_for_updates(sentinels, change_notifier, timeout=None): wait(sentinels, timeout=timeout) @@ -679,7 +690,8 @@ def _help_stuff_finish(inqueue, task_handler, size): @classmethod def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, change_notifier, - worker_handler, task_handler, result_handler, cache): + worker_handler, task_handler, result_handler, cache, + taskqueue_buffersize_semaphores): # this is guaranteed to only be called once util.debug('finalizing pool') @@ -690,6 +702,10 @@ def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, change_notifier, change_notifier.put(None) task_handler._state = TERMINATE + # Release all semaphores to wake up task_handler to stop. + for buffersize_sema in tuple(taskqueue_buffersize_semaphores): + buffersize_sema.release() + taskqueue_buffersize_semaphores.discard(buffersize_sema) util.debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) @@ -836,7 +852,7 @@ def _set(self, i, success_result): class IMapIterator(object): - def __init__(self, pool): + def __init__(self, pool, *, buffersize=None): self._pool = pool self._cond = threading.Condition(threading.Lock()) self._job = next(job_counter) @@ -846,6 +862,11 @@ def __init__(self, pool): self._length = None self._unsorted = {} self._cache[self._job] = self + if buffersize is None: + self._buffersize_sema = None + else: + self._buffersize_sema = threading.Semaphore(buffersize) + self._pool._taskqueue_buffersize_semaphores.add(self._buffersize_sema) def __iter__(self): return self @@ -856,22 +877,30 @@ def next(self, timeout=None): item = self._items.popleft() except IndexError: if self._index == self._length: - self._pool = None - raise StopIteration from None + self._stop_iterator() self._cond.wait(timeout) try: item = self._items.popleft() except IndexError: if self._index == self._length: - self._pool = None - raise StopIteration from None + self._stop_iterator() raise TimeoutError from None + if self._buffersize_sema is not None: + self._buffersize_sema.release() + success, value = item if success: return value raise value + def _stop_iterator(self): + if self._pool is not None: + # `self._pool` could be set to `None` in previous `.next()` calls + self._pool._taskqueue_buffersize_semaphores.discard(self._buffersize_sema) + self._pool = None + raise StopIteration from None + __next__ = next # XXX def _set(self, i, obj): diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 8f665b3a98a0371..36e0880bc088189 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2904,11 +2904,13 @@ def exception_throwing_generator(total, when): class _TestPool(BaseTestCase): + _POOL_SIZE = 4 + @classmethod def setUpClass(cls): with warnings_helper.ignore_fork_in_thread_deprecation_warnings(): super().setUpClass() - cls.pool = cls.Pool(4) + cls.pool = cls.Pool(cls._POOL_SIZE) @classmethod def tearDownClass(cls): @@ -3022,18 +3024,36 @@ def test_async_timeout(self): p.terminate() p.join() - def test_imap(self): - it = self.pool.imap(sqr, list(range(10))) - self.assertEqual(list(it), list(map(sqr, list(range(10))))) - - it = self.pool.imap(sqr, list(range(10))) + @support.subTests('buffersize', ( + None, + 1, + _POOL_SIZE, + _POOL_SIZE * 2, + )) + def test_imap(self, buffersize): + iterable = range(10) + if self.TYPE != "threads": + iterable = list(iterable) + it = self.pool.imap(sqr, iterable, buffersize=buffersize) for i in range(10): - self.assertEqual(next(it), i*i) + self.assertEqual(next(it), i * i) + self.assertRaises(StopIteration, it.__next__) + # again, verify that it's truly exhausted self.assertRaises(StopIteration, it.__next__) - it = self.pool.imap(sqr, list(range(1000)), chunksize=100) + @support.subTests(('chunksize', 'buffersize'), ( + (100, None), + (100, _POOL_SIZE), + )) + def test_imap_with_chunksize(self, chunksize, buffersize): + iterable = range(1000) + if self.TYPE != "threads": + iterable = list(iterable) + it = self.pool.imap(sqr, iterable, chunksize=chunksize, buffersize=buffersize) for i in range(1000): - self.assertEqual(next(it), i*i) + self.assertEqual(next(it), i * i) + self.assertRaises(StopIteration, it.__next__) + # again, verify that it's truly exhausted self.assertRaises(StopIteration, it.__next__) def test_imap_handle_iterable_exception(self): @@ -3062,11 +3082,29 @@ def test_imap_handle_iterable_exception(self): self.assertEqual(next(it), i*i) self.assertRaises(SayWhenError, it.__next__) - def test_imap_unordered(self): - it = self.pool.imap_unordered(sqr, list(range(10))) + @support.subTests('buffersize', ( + None, + 1, + _POOL_SIZE, + _POOL_SIZE * 2, + )) + def test_imap_unordered(self, buffersize): + iterable = range(10) + if self.TYPE != "threads": + iterable = list(iterable) + it = self.pool.imap(sqr, iterable, buffersize=buffersize) self.assertEqual(sorted(it), list(map(sqr, list(range(10))))) - it = self.pool.imap_unordered(sqr, list(range(1000)), chunksize=100) + @support.subTests(('chunksize', 'buffersize'), ( + (100, None), + (100, _POOL_SIZE), + )) + def test_imap_unordered_with_chunksize(self, chunksize, buffersize): + iterable = range(1000) + if self.TYPE != "threads": + iterable = list(iterable) + it = self.pool.imap_unordered(sqr, iterable, chunksize=chunksize, + buffersize=buffersize) self.assertEqual(sorted(it), list(map(sqr, list(range(1000))))) def test_imap_unordered_handle_iterable_exception(self): @@ -3105,6 +3143,130 @@ def test_imap_unordered_handle_iterable_exception(self): self.assertIn(value, expected_values) expected_values.remove(value) + @support.subTests('method_name', ("imap", "imap_unordered")) + @support.subTests(('buffersize', 'expected_exception', 'expected_regex'), ( + ("foo", TypeError, "buffersize must be an integer or None"), + (2.0, TypeError, "buffersize must be an integer or None"), + (0, ValueError, "buffersize must be None or > 0"), + (-1, ValueError, "buffersize must be None or > 0"), + )) + def test_imap_and_imap_unordered_with_buffersize_type_validation( + self, method_name, buffersize, expected_exception, expected_regex + ): + method = getattr(self.pool, method_name) + with self.assertRaisesRegex(expected_exception, expected_regex): + method(str, range(4), buffersize=buffersize) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + @support.subTests('method_name', ("imap", "imap_unordered")) + def test_imap_and_imap_unordered_when_buffer_is_full(self, method_name): + if self.TYPE != "threads": + self.skipTest("test not appropriate for {}".format(self.TYPE)) + + processes = 4 + p = self.Pool(processes) + last_produced_task_arg = Value("i") + + def produce_args(): + for arg in itertools.count(1): + last_produced_task_arg.value = arg + yield arg + + method = getattr(p, method_name) + it = method(functools.partial(sqr, wait=0.2), produce_args()) + + time.sleep(0.2) + # `iterable` could've been advanced only `processes` times, + # but in fact it advances further (`> processes`) because of + # not waiting for workers or user code to catch up. + self.assertGreater(last_produced_task_arg.value, processes) + + next(it) + time.sleep(0.2) + self.assertGreater(last_produced_task_arg.value, processes + 1) + + next(it) + time.sleep(0.2) + self.assertGreater(last_produced_task_arg.value, processes + 2) + + p.terminate() + p.join() + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + @support.subTests('method_name', ("imap", "imap_unordered")) + def test_imap_and_imap_unordered_with_buffersize_when_buffer_is_full( + self, method_name + ): + if self.TYPE != "threads": + self.skipTest("test not appropriate for {}".format(self.TYPE)) + + processes = 4 + p = self.Pool(processes) + last_produced_task_arg = Value("i") + + def produce_args(): + for arg in itertools.count(1): + last_produced_task_arg.value = arg + yield arg + + method = getattr(p, method_name) + it = method(functools.partial(sqr, wait=0.2), produce_args(), + buffersize=processes) + + time.sleep(0.2) + self.assertEqual(last_produced_task_arg.value, processes) + + next(it) + time.sleep(0.2) + self.assertEqual(last_produced_task_arg.value, processes + 1) + + next(it) + time.sleep(0.2) + self.assertEqual(last_produced_task_arg.value, processes + 2) + + p.terminate() + p.join() + + @support.subTests('method_name', ("imap", "imap_unordered")) + def test_imap_and_imap_unordered_with_buffersize_on_empty_iterable( + self, method_name + ): + method = getattr(self.pool, method_name) + res = method(str, [], buffersize=2) + self.assertIsNone(next(res, None)) + self.assertIsNone(next(res, None)) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_imap_with_buffersize_on_infinite_iterable(self): + if self.TYPE != "threads": + self.skipTest("test not appropriate for {}".format(self.TYPE)) + + p = self.Pool(4) + res = p.imap(str, itertools.count(), buffersize=2) + + self.assertEqual(next(res, None), "0") + self.assertEqual(next(res, None), "1") + self.assertEqual(next(res, None), "2") + + p.terminate() + p.join() + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_imap_unordered_with_buffersize_on_infinite_iterable(self): + if self.TYPE != "threads": + self.skipTest("test not appropriate for {}".format(self.TYPE)) + + p = self.Pool(4) + res = p.imap_unordered(str, itertools.count(), buffersize=2) + + # (4, 5, ...) can also be submitted to the pool, so assert just 3 unique results + first_three_results = [next(res, None) for _ in range(3)] + self.assertEqual(len(first_three_results), 3) + self.assertEqual(len(set(first_three_results)), 3) + + p.terminate() + p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_make_pool(self): expected_error = (RemoteError if self.TYPE == 'manager' diff --git a/Misc/NEWS.d/next/Library/2025-07-28-11-00-49.gh-issue-64192.7htLtg.rst b/Misc/NEWS.d/next/Library/2025-07-28-11-00-49.gh-issue-64192.7htLtg.rst new file mode 100644 index 000000000000000..ca40bc111176e07 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-28-11-00-49.gh-issue-64192.7htLtg.rst @@ -0,0 +1,9 @@ +Add the optional ``buffersize`` parameter to +:meth:`multiprocessing.pool.Pool.imap` and +:meth:`multiprocessing.pool.Pool.imap_unordered` to limit the number of +submitted tasks whose results have not yet been yielded. If the buffer is +full, iteration over the *iterables* pauses until a result is yielded from +the buffer. To fully utilize pool's capacity when using this feature, set +*buffersize* at least to the number of processes in pool (to consume +*iterable* as you go), or even higher (to prefetch the next +``N=buffersize-processes`` arguments). From bfc16a71cf18015c47c1b9cae9196e279d8f60d7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 29 Jul 2026 17:47:52 +0300 Subject: [PATCH 2/3] gh-131565: Implement ctypes.util.dllist() in the _ctypes extension (GH-154255) On NetBSD dl_iterate_phdr() reports only the link-map group of the calling object. Called through ctypes, the caller is libffi's closure trampoline, which belongs to no object, so only the main executable was reported. Calling dl_iterate_phdr() from the _ctypes extension module makes _ctypes the caller and reports all loaded shared libraries. Co-Authored-By: Claude Fable 5 --- Lib/ctypes/util.py | 53 +++---------------- ...-07-20-17-15-00.gh-issue-131565.dllist.rst | 4 ++ Modules/_ctypes/callproc.c | 40 ++++++++++++++ configure | 9 ++++ configure.ac | 3 ++ pyconfig.h.in | 3 ++ 6 files changed, 65 insertions(+), 47 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-20-17-15-00.gh-issue-131565.dllist.rst diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index c0a578c86549ec9..141d3fbe9382472 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -412,53 +412,12 @@ def find_library(name): _get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name)) -# Listing loaded libraries on other systems will try to use -# functions common to Linux and a few other Unix-like systems. -# See the following for several platforms' documentation of the same API: -# https://man7.org/linux/man-pages/man3/dl_iterate_phdr.3.html -# https://man.freebsd.org/cgi/man.cgi?query=dl_iterate_phdr -# https://man.openbsd.org/dl_iterate_phdr -# https://docs.oracle.com/cd/E88353_01/html/E37843/dl-iterate-phdr-3c.html -if (os.name == "posix" and - sys.platform not in {"darwin", "ios", "tvos", "watchos"}): - import ctypes - if hasattr((_libc := ctypes.CDLL(None)), "dl_iterate_phdr"): - - class _dl_phdr_info(ctypes.Structure): - _fields_ = [ - ("dlpi_addr", ctypes.c_void_p), - ("dlpi_name", ctypes.c_char_p), - ("dlpi_phdr", ctypes.c_void_p), - ("dlpi_phnum", ctypes.c_ushort), - ] - - _dl_phdr_callback = ctypes.CFUNCTYPE( - ctypes.c_int, - ctypes.POINTER(_dl_phdr_info), - ctypes.c_size_t, - ctypes.POINTER(ctypes.py_object), - ) - - @_dl_phdr_callback - def _info_callback(info, _size, data): - libraries = data.contents.value - name = os.fsdecode(info.contents.dlpi_name) - libraries.append(name) - return 0 - - _dl_iterate_phdr = _libc["dl_iterate_phdr"] - _dl_iterate_phdr.argtypes = [ - _dl_phdr_callback, - ctypes.POINTER(ctypes.py_object), - ] - _dl_iterate_phdr.restype = ctypes.c_int - - def dllist(): - """Return a list of loaded shared libraries in the current process.""" - libraries = [] - _dl_iterate_phdr(_info_callback, - ctypes.byref(ctypes.py_object(libraries))) - return libraries +# On platforms which provide dl_iterate_phdr(), dllist() is implemented +# in _ctypes. +try: + from _ctypes import dllist +except ImportError: + pass @dataclass(slots=True, frozen=True) diff --git a/Misc/NEWS.d/next/Library/2026-07-20-17-15-00.gh-issue-131565.dllist.rst b/Misc/NEWS.d/next/Library/2026-07-20-17-15-00.gh-issue-131565.dllist.rst new file mode 100644 index 000000000000000..5fb2dd099bd419f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-17-15-00.gh-issue-131565.dllist.rst @@ -0,0 +1,4 @@ +:func:`ctypes.util.dllist` now works on NetBSD. It is implemented in the +:mod:`!_ctypes` extension module so that ``dl_iterate_phdr()`` reports all +loaded shared libraries: on NetBSD it only reports the link-map group of +the calling object, which excluded them when called through ctypes. diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index ccc57e347b07acf..746034c8004c5c4 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1660,6 +1660,42 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) PyErr_Format(PyExc_OSError, "symbol '%s' not found", name); return NULL; } + +// Apple platforms use the dyld API in ctypes.util instead. +#if defined(HAVE_DL_ITERATE_PHDR) && !defined(__APPLE__) +#include + +static int +_dllist_callback(struct dl_phdr_info *info, size_t size, void *data) +{ + PyObject *list = (PyObject *)data; + PyObject *name = PyUnicode_DecodeFSDefault(info->dlpi_name); + if (name == NULL) { + return -1; + } + int res = PyList_Append(list, name); + Py_DECREF(name); + return res; +} + +static PyObject * +dllist(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + // On NetBSD dl_iterate_phdr() only reports the link-map group of the + // caller, so it cannot be called via a libffi trampoline. + PyObject *list = PyList_New(0); + if (list == NULL) { + return NULL; + } + // The return value only echoes the callback result. + dl_iterate_phdr(_dllist_callback, list); + if (PyErr_Occurred()) { + Py_DECREF(list); + return NULL; + } + return list; +} +#endif #endif /* @@ -2036,6 +2072,10 @@ PyMethodDef _ctypes_module_methods[] = { "dlopen(name, flag={RTLD_GLOBAL|RTLD_LOCAL}) open a shared library"}, {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, +#if defined(HAVE_DL_ITERATE_PHDR) && !defined(__APPLE__) + {"dllist", dllist, METH_NOARGS, + "dllist() return a list of loaded shared libraries"}, +#endif #endif #ifdef __APPLE__ {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, diff --git a/configure b/configure index 8e7accb4bc793da..2c82da923f68d4a 100755 --- a/configure +++ b/configure @@ -20229,6 +20229,15 @@ then : fi +# Used by ctypes.util.dllist(). +ac_fn_c_check_func "$LINENO" "dl_iterate_phdr" "ac_cv_func_dl_iterate_phdr" +if test "x$ac_cv_func_dl_iterate_phdr" = xyes +then : + printf "%s\n" "#define HAVE_DL_ITERATE_PHDR 1" >>confdefs.h + +fi + + # DYNLOADFILE specifies which dynload_*.o file we will use for dynamic # loading of modules. diff --git a/configure.ac b/configure.ac index ce146cfa1cbc4cd..771260c76b7f094 100644 --- a/configure.ac +++ b/configure.ac @@ -5421,6 +5421,9 @@ DLINCLDIR=. # platforms have dlopen(), but don't want to use it. AC_CHECK_FUNCS([dlopen]) +# Used by ctypes.util.dllist(). +AC_CHECK_FUNCS([dl_iterate_phdr]) + # DYNLOADFILE specifies which dynload_*.o file we will use for dynamic # loading of modules. AC_SUBST([DYNLOADFILE]) diff --git a/pyconfig.h.in b/pyconfig.h.in index 2658fe8116781db..691c6c0d9feb6d0 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -398,6 +398,9 @@ /* Define to 1 if you have the 'dlopen' function. */ #undef HAVE_DLOPEN +/* Define to 1 if you have the 'dl_iterate_phdr' function. */ +#undef HAVE_DL_ITERATE_PHDR + /* Define to 1 if you have the 'dup' function. */ #undef HAVE_DUP From a528a24f9d739126d790d4430050eed85ba60946 Mon Sep 17 00:00:00 2001 From: coroma01 Date: Wed, 29 Jul 2026 18:06:03 +0100 Subject: [PATCH 3/3] gh-151321: Fix incorrect opcode metadata flags for LOAD_DEREF opcode (GH-154778) --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_metadata.h | 2 +- Tools/cases_generator/analyzer.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 31d2e31573d8c53..457e5c5bf20d2b9 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1245,7 +1245,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 990706c0b329223..e52233b21277591 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -199,7 +199,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 4eed4d8d60e4856..78dfdc5d6a89d57 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -971,6 +971,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: or variable_used(op, "PyCell_GetRef") or variable_used(op, "PyCell_SetTakeRef") or variable_used(op, "PyCell_SwapTakeRef") + or variable_used(op, "_PyCell_GetStackRef") ) deopts_if = variable_used(op, "DEOPT_IF") exits_if = variable_used(op, "EXIT_IF")