From 336b36b8e0d9ed0e551a1ddbbc6a67f6bbb8c7bf Mon Sep 17 00:00:00 2001 From: agu2347 Date: Tue, 28 Jul 2026 16:03:49 +0000 Subject: [PATCH] Fix locale data cross-contamination via shared, mutated alias caches Using one locale (e.g. 'he') could silently corrupt subsequently-used, completely unrelated locales (e.g. 'no', 'fr') for the remainder of the process, causing them to return the first locale's resolved text instead of their own -- persisting until the process restarts. Two existing, individually-reasonable optimizations combine to cause this: 1. localedata.load() gives each locale its own top-level dict via a shallow `load(parent).copy()` of its parent's cached data. merge() then only copies (and recurses into) a nested dict when the locale's own data file actually has an entry at that key. A nested structure the locale's data file doesn't touch at all -- e.g. a "stand-alone" month-name alias, when a locale relies entirely on the CLDR-inherited default rather than defining its own -- therefore remains the exact same dict object shared with whatever it was last inherited from, and consequently with every other locale that also inherits it unchanged. 2. LocaleDataDict.__getitem__ has a "cache the resolved alias value back into the dict" optimization, so repeated lookups of the same key don't need to re-resolve the alias every time. It writes this resolved value directly into self._data[key]. Put together: resolving a shared, not-locally-overridden alias for one locale permanently overwrites the shared dict with that locale's own resolved value. The next locale that looks up the same key -- sharing the exact same (now-overwritten) dict object -- sees the previous locale's resolved value instead of correctly re-resolving for itself. Give LocaleDataDict a copy-on-write: the first time __getitem__ needs to cache a resolved value into self._data, copy self._data first (the same way merge() already copies a nested dict before recursing into and mutating it), and remember that this instance now owns an independent copy so subsequent writes don't re-copy needlessly. This ensures resolving an alias for one locale can never affect any other LocaleDataDict that happens to still be sharing the same underlying dict object. Verified against the exact reproduction from the issue (installed a release build with prebuilt CLDR data, since building the data files from source requires network access to unicode.org this sandbox doesn't have, and overlaid the fixed localedata.py onto it): confirmed 'no' and 'fr' month names were previously corrupted to Hebrew text after formatting a date with 'he', and are correctly independent with the fix. Ran a broader sweep across 20 locales in both forward and reverse order, and across two different fields (month names and weekday names): 9 of 12 checked locales were corrupted without the fix (a wider-reaching bug than the original report's specific locales), and all are correct with the fix, in every ordering tested. Added a focused unit test reproducing the precise structural condition that causes this (a shared, unmutated "stand-alone" alias structure alongside independent, locale-specific "format" data -- the same shape real CLDR data has for many locales), without requiring built locale data files. Confirmed the test fails with the original code (one locale's resolved value leaks into the other's lookup) and passes with the fix. Ran the full existing test_localedata.py suite (15 passed: 14 baseline + 1 new) and the broader test_dates.py and test_core.py suites (3428 passed total, 2 pre-existing zoneinfo-data failures unrelated to this change, confirmed identical with and without the fix). Fixes #1234 --- babel/localedata.py | 20 +++++++++++++ tests/test_localedata.py | 61 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/babel/localedata.py b/babel/localedata.py index 4648e6626..8c641dd04 100644 --- a/babel/localedata.py +++ b/babel/localedata.py @@ -251,6 +251,12 @@ def __init__( base: Mapping[str | int | None, Any] | None = None, ): self._data = data + # Tracks whether self._data is already a dict this instance + # owns exclusively (safe to mutate directly), as opposed to a + # dict that might still be shared with another locale (e.g. + # via load()'s shallow copy of inherited-but-unoverridden + # data). See __getitem__ and GH #1234. + self._data_is_own_copy = False if base is None: base = data self.base = base @@ -272,6 +278,20 @@ def __getitem__(self, key: str | int | None) -> Any: if isinstance(val, dict): # Return a nested alias-resolving dict val = LocaleDataDict(val, base=self.base) if val is not orig: + # self._data may still be the very same dict object shared + # (via load()'s shallow `.copy()` of inherited-but-not- + # locally-overridden data) with another, unrelated locale + # that also inherits it unchanged from a common parent. + # Writing the resolved value directly into self._data here + # would therefore also "leak" it into that other locale's + # cached data the next time *it* looks up the same key. + # Copy self._data before mutating it, the same way merge() + # already copies a nested dict before recursing into it, so + # this locale gets its own independent dict to cache into. + # See GH #1234. + if not self._data_is_own_copy: + self._data = self._data.copy() + self._data_is_own_copy = True self._data[key] = val return val diff --git a/tests/test_localedata.py b/tests/test_localedata.py index 42810b992..f8c286d2c 100644 --- a/tests/test_localedata.py +++ b/tests/test_localedata.py @@ -57,6 +57,67 @@ def test_merge_with_alias_and_resolve(): assert dict(d.items()) == {'x': {'a': 1, 'b': 12, 'c': 3, 'd': 14}, 'y': {'a': 1, 'b': 22, 'c': 3, 'd': 14, 'e': 25}} +def test_localedatadict_resolving_alias_does_not_leak_into_shared_underlying_dict(): + """ + Regression test for + https://github.com/python-babel/babel/issues/1234 + + load() gives each locale its own top-level dict via a shallow + .copy() of its parent's cached data, and merge() only copies a + nested dict when the locale's own data file actually has an entry + at that key. So a nested structure a locale's data file doesn't + touch at all -- e.g. a "stand-alone" month-name alias, when the + locale relies entirely on the inherited default -- remains the + exact same object shared with whatever it last inherited it from, + even while sibling keys the locale *does* override (e.g. its own + "format" month names) are correctly independent per locale. + + Resolving an alias must not mutate that shared "stand-alone" + structure in place, or the resolved value leaks into every other + locale sharing the same object -- exactly what happened with + Hebrew's resolved month name leaking into Norwegian's and French's + month names in the issue. + """ + # Shared, uncopied 'stand-alone' structure: neither "locale" below + # overrides it, so (as load()'s shallow copy would produce) they + # reference the exact same dict object for it. + shared_standalone = { + 'wide': localedata.Alias(('months', 'format', 'wide')), + } + + locale_a_data = { + 'months': { + 'format': {'wide': {10: 'LocaleAOctober'}}, + 'stand-alone': shared_standalone, + }, + } + locale_b_data = { + 'months': { + 'format': {'wide': {10: 'LocaleBOctober'}}, + 'stand-alone': shared_standalone, + }, + } + # Each locale's own "format" data is independent, but they share + # the exact same "stand-alone" object -- the precise shape that + # produces the bug. + assert locale_a_data['months']['format'] is not locale_b_data['months']['format'] + assert locale_a_data['months']['stand-alone'] is locale_b_data['months']['stand-alone'] + + locale_a = localedata.LocaleDataDict(locale_a_data) + locale_b = localedata.LocaleDataDict(locale_b_data) + + resolved_a = locale_a['months']['stand-alone']['wide'] + assert resolved_a[10] == 'LocaleAOctober' + + # Resolving the alias for locale_a must not affect locale_b's + # resolution of the same (shared) 'stand-alone' object. + resolved_b = locale_b['months']['stand-alone']['wide'] + assert resolved_b[10] == 'LocaleBOctober' + + # Re-checking locale_a again afterwards must still be correct too. + assert locale_a['months']['stand-alone']['wide'][10] == 'LocaleAOctober' + + def test_load(): assert localedata.load('en_US')['languages']['sv'] == 'Swedish' assert localedata.load('en_US') is localedata.load('en_US')