From 7cd5807fcd6066eacb59aa4e76bc0070ccb8b1e9 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 30 Jul 2026 02:31:11 +0000 Subject: [PATCH 1/2] gh-154904: Speed up import of shutil by probing compression extensions shutil imported bz2, lzma and compression.zstd at module scope only to set the _*_SUPPORTED flags, then discarded the bindings. Executing those pure Python wrappers also pulled in compression._common[._streams] and compression.zstd._zstdfile, a cost paid by every program that imports shutil even if it never touches an archive. Their only importable dependency that may be missing is the extension module each one wraps, so probe _bz2, _lzma and _zstd directly instead. This keeps the exact ImportError semantics of the previous code -- unlike a find_spec() check, which resolves the always-present wrapper and would report success on builds where the extension fails to load. The wrappers are still imported by tarfile/zipfile when an archive is actually created or extracted. Cuts the net cost of "import shutil" by ~2.5 ms (-19.6%) and of "import urllib.request" by ~2.3 ms (-6.4%). --- Lib/shutil.py | 17 +++++---- Lib/test/test_shutil.py | 36 +++++++++++++++++++ ...-07-30-02-15-00.gh-issue-154904.Kx9mAq.rst | 4 +++ 3 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-30-02-15-00.gh-issue-154904.Kx9mAq.rst diff --git a/Lib/shutil.py b/Lib/shutil.py index 6a2e2b2ffdae2c0..ab143ef24480652 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -18,23 +18,28 @@ except ImportError: _ZLIB_SUPPORTED = False +# bz2, lzma and compression.zstd are pure Python wrappers whose only +# importable dependency that may be missing is the extension module they +# wrap. Probe those extensions directly instead: it gives the same answer +# without executing the wrappers, which shutil only needs when an archive +# is actually created or extracted. try: - import bz2 - del bz2 + import _bz2 + del _bz2 _BZ2_SUPPORTED = True except ImportError: _BZ2_SUPPORTED = False try: - import lzma - del lzma + import _lzma + del _lzma _LZMA_SUPPORTED = True except ImportError: _LZMA_SUPPORTED = False try: - from compression import zstd - del zstd + import _zstd + del _zstd _ZSTD_SUPPORTED = True except ImportError: _ZSTD_SUPPORTED = False diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 6832bea094fc1dc..a6ff0f11f6a58f9 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -10,6 +10,7 @@ import os.path import errno import functools +import importlib import socket import subprocess import random @@ -32,6 +33,7 @@ from test import support from test.support import os_helper, socket_helper from test.support.os_helper import TESTFN, FakePath +from test.support.script_helper import assert_python_ok TESTFN2 = TESTFN + "2" TESTFN_SRC = TESTFN + "_SRC" @@ -2321,6 +2323,40 @@ def _boo(filename, extract_dir, extra): unregister_unpack_format('Boo2') self.assertEqual(get_unpack_formats(), formats) + def test_compression_supported_flags(self): + # shutil determines compression support by probing the extension + # modules (_bz2, _lzma, _zstd) rather than importing the pure Python + # wrappers. The answer must match what importing the wrapper does, + # including on builds where the extension is missing or fails to load. + for wrapper, supported in ( + ('zlib', shutil._ZLIB_SUPPORTED), + ('bz2', shutil._BZ2_SUPPORTED), + ('lzma', shutil._LZMA_SUPPORTED), + ('compression.zstd', shutil._ZSTD_SUPPORTED), + ): + with self.subTest(wrapper=wrapper): + try: + importlib.import_module(wrapper) + except ImportError: + importable = False + else: + importable = True + self.assertEqual(supported, importable) + + def test_compression_wrappers_not_imported_by_shutil(self): + # Importing shutil must not pull in the compression wrappers: they are + # only needed once an archive is actually created or extracted, and + # importing them measurably slows down every process that uses shutil. + wrappers = ('bz2', 'lzma', 'compression', 'compression.zstd') + script = ( + 'import sys, shutil; ' + f'print([m for m in {wrappers!r} if m in sys.modules])' + ) + # -I so that a sitecustomize/usercustomize importing one of these + # cannot make the test fail spuriously. + rc, stdout, stderr = assert_python_ok('-I', '-c', script) + self.assertEqual(stdout.decode().strip(), '[]', stderr) + class TestMisc(BaseTest, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2026-07-30-02-15-00.gh-issue-154904.Kx9mAq.rst b/Misc/NEWS.d/next/Library/2026-07-30-02-15-00.gh-issue-154904.Kx9mAq.rst new file mode 100644 index 000000000000000..fa3f930dc21b863 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-30-02-15-00.gh-issue-154904.Kx9mAq.rst @@ -0,0 +1,4 @@ +Speed up :mod:`shutil` import by probing the ``_bz2``, ``_lzma`` and +``_zstd`` extension modules instead of importing the :mod:`bz2`, +:mod:`lzma` and :mod:`compression.zstd` wrappers, which are now only +imported when an archive is actually created or extracted. From 076562acf3ab246d9b0b2e253bdf5af5b5d3dc7e Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 30 Jul 2026 09:18:00 +0000 Subject: [PATCH 2/2] Address review: use ensure_lazy_imports, drop redundant flags test Per review feedback on GH-154908: - Rewrite test_compression_wrappers_not_imported_by_shutil on top of test.support.import_helper.ensure_lazy_imports instead of hand-rolling an assert_python_ok subprocess. - Remove test_compression_supported_flags: since each wrapper imports its extension module at the top level, the assertion is near-tautological, and the zlib subtest compared importing zlib with itself. --- Lib/test/test_shutil.py | 41 +++++++---------------------------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index a6ff0f11f6a58f9..a2ade3f3b824e70 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -10,7 +10,6 @@ import os.path import errno import functools -import importlib import socket import subprocess import random @@ -33,7 +32,7 @@ from test import support from test.support import os_helper, socket_helper from test.support.os_helper import TESTFN, FakePath -from test.support.script_helper import assert_python_ok +from test.support.import_helper import ensure_lazy_imports TESTFN2 = TESTFN + "2" TESTFN_SRC = TESTFN + "_SRC" @@ -2323,39 +2322,13 @@ def _boo(filename, extract_dir, extra): unregister_unpack_format('Boo2') self.assertEqual(get_unpack_formats(), formats) - def test_compression_supported_flags(self): - # shutil determines compression support by probing the extension - # modules (_bz2, _lzma, _zstd) rather than importing the pure Python - # wrappers. The answer must match what importing the wrapper does, - # including on builds where the extension is missing or fails to load. - for wrapper, supported in ( - ('zlib', shutil._ZLIB_SUPPORTED), - ('bz2', shutil._BZ2_SUPPORTED), - ('lzma', shutil._LZMA_SUPPORTED), - ('compression.zstd', shutil._ZSTD_SUPPORTED), - ): - with self.subTest(wrapper=wrapper): - try: - importlib.import_module(wrapper) - except ImportError: - importable = False - else: - importable = True - self.assertEqual(supported, importable) - def test_compression_wrappers_not_imported_by_shutil(self): - # Importing shutil must not pull in the compression wrappers: they are - # only needed once an archive is actually created or extracted, and - # importing them measurably slows down every process that uses shutil. - wrappers = ('bz2', 'lzma', 'compression', 'compression.zstd') - script = ( - 'import sys, shutil; ' - f'print([m for m in {wrappers!r} if m in sys.modules])' - ) - # -I so that a sitecustomize/usercustomize importing one of these - # cannot make the test fail spuriously. - rc, stdout, stderr = assert_python_ok('-I', '-c', script) - self.assertEqual(stdout.decode().strip(), '[]', stderr) + # gh-154904: Importing shutil must not pull in the compression + # wrappers: they are only needed once an archive is actually created + # or extracted, and importing them measurably slows down every + # process that uses shutil. + ensure_lazy_imports("shutil", + {"bz2", "lzma", "compression", "compression.zstd"}) class TestMisc(BaseTest, unittest.TestCase):