Memoize resolved locale data per instance and share LocaleDataDicts per locale

Fixes #1234

diff --git a/babel/core.py b/babel/core.py
index 4210b46bba9cee294a53ebeaaee5f32118725304..f16bf24bf02a5f640ca2376c5a11a925f5dd85a5 100644 (file)
--- a/babel/core.py
+++ b/babel/core.py
@@ -471,7 +471,7 @@ class Locale:
     @property
     def _data(self) -> localedata.LocaleDataDict:
         if self.__data is None:
-            self.__data = localedata.LocaleDataDict(localedata.load(self.__data_identifier))
+            self.__data = localedata.get_locale_data(self.__data_identifier)
         return self.__data
 
     def get_display_name(self, locale: Locale | str | None = None) -> str | None:

diff --git a/babel/localedata.py b/babel/localedata.py
index 0536bc852241af0028615a927abb6c5a489abb90..9989316d95ca87c0d852d4628e230e9ab709a3aa 100644 (file)
--- a/babel/localedata.py
+++ b/babel/localedata.py
@@ -25,6 +25,7 @@ from itertools import chain
 from typing import Any
 
 _cache: dict[str, Any] = {}
+_dict_cache: dict[str, LocaleDataDict] = {}
 _cache_lock = threading.RLock()
 _dirname = os.path.join(os.path.dirname(__file__), 'locale-data')
 _windows_reserved_name_re = re.compile("^(con|prn|aux|nul|com[0-9]|lpt[0-9])$", re.I)
@@ -180,6 +181,21 @@ def clear_caches() -> None:
     """
     with _cache_lock:
         _cache.clear()
+        _dict_cache.clear()
+
+
+def get_locale_data(name: str) -> LocaleDataDict:
+    """Return an alias-resolving `LocaleDataDict` over the merged data for
+    the given locale.
+
+    The wrapper is cached and shared: repeated requests for the same locale
+    return the same object. Alias resolutions memoized within it are
+    locale-specific.
+    """
+    try:
+        return _dict_cache[name]
+    except KeyError:
+        return _dict_cache.setdefault(name, LocaleDataDict(load(name)))
 
 
 def merge(dict1: MutableMapping[Any, Any], dict2: Mapping[Any, Any]) -> None:
@@ -248,6 +264,9 @@ class Alias:
         return data
 
 
+_sentinel = object()
+
+
 class LocaleDataDict(abc.MutableMapping):
     """Dictionary wrapper that automatically resolves aliases to the actual
     values.
@@ -258,7 +277,10 @@ class LocaleDataDict(abc.MutableMapping):
         data: MutableMapping[str | int | None, Any],
         base: Mapping[str | int | None, Any] | None = None,
     ):
+        # May be shared between locales.
         self._data = data
+        # Per-instance memoization of resolved values.
+        self._resolved: dict[str | int | None, Any] = {}
         if base is None:
             base = data
         self.base = base
@@ -270,6 +292,9 @@ class LocaleDataDict(abc.MutableMapping):
         return iter(self._data)
 
     def __getitem__(self, key: str | int | None) -> Any:
+        val = self._resolved.get(key, _sentinel)
+        if val is not _sentinel:
+            return val
         orig = val = self._data[key]
         if isinstance(val, Alias):  # resolve an alias
             val = val.resolve(self.base)
@@ -280,13 +305,19 @@ class LocaleDataDict(abc.MutableMapping):
         if isinstance(val, dict):  # Return a nested alias-resolving dict
             val = LocaleDataDict(val, base=self.base)
         if val is not orig:
-            self._data[key] = val
+            # Only resolved/wrapped values are memoized.
+            # Scalars are always read from `self._data`, so that
+            # manual writes into the backing data (possibly shared)
+            # stay visible.
+            self._resolved[key] = val
         return val
 
     def __setitem__(self, key: str | int | None, value: Any) -> None:
+        self._resolved.pop(key, None)
         self._data[key] = value
 
     def __delitem__(self, key: str | int | None) -> None:
+        self._resolved.pop(key, None)
         del self._data[key]
 
     def copy(self) -> LocaleDataDict:

diff --git a/tests/test_localedata.py b/tests/test_localedata.py
index da93aa91a5f5e913b7dd7a258c46b989be3a3463..20093c7f489ad79662e664a9d0f4eeb6a173486e 100644 (file)
--- a/tests/test_localedata.py
+++ b/tests/test_localedata.py
@@ -55,6 +55,8 @@ def test_merge_with_alias_and_resolve():
     assert d1 == {'x': {'a': 1, 'b': 12, 'c': 3, 'd': 14}, 'y': (alias, {'b': 22, 'e': 25})}
     d = localedata.LocaleDataDict(d1)
     assert dict(d.items()) == {'x': {'a': 1, 'b': 12, 'c': 3, 'd': 14}, 'y': {'a': 1, 'b': 22, 'c': 3, 'd': 14, 'e': 25}}
+    # Resolving the partial alias must not have written the result back into the underlying data (GH-1234)
+    assert d1['y'] == (alias, {'b': 22, 'e': 25})
 
 
 def test_load():
@@ -62,6 +64,43 @@ def test_load():
     assert localedata.load('en_US') is localedata.load('en_US')
 
 
+def _calendar_snapshot(name):
+    locale = Locale.parse(name)
+    return {
+        'months': dict(locale.months['stand-alone']['wide']),
+        'months_abbr': dict(locale.months['format']['abbreviated']),
+        'days': dict(locale.days['stand-alone']['wide']),
+        'quarters': dict(locale.quarters['stand-alone']['wide']),
+        'eras': dict(locale.eras['wide']),
+    }
+
+
+@pytest.mark.parametrize('reverse', (False, True))
+def test_no_cross_locale_contamination(reverse):
+    """
+    Alias-heavy calendar data (e.g. what `format_date(..., 'LLLL')` reads)
+    must be identical whether a locale is loaded into a cold cache or after
+    other locales have already been read: e.g.
+    * reading 'he' must not turn Norwegian month names Hebrew
+    * reading 'aa' must not turn everyone else's stand-alone quarters into root's 'Q1' placeholders.
+
+    Regression test for https://github.com/python-babel/babel/issues/1234
+    """
+    locales = ('aa', 'de', 'fr', 'he', 'ja', 'no')
+    cold = {}
+    for name in locales:
+        localedata.clear_caches()
+        cold[name] = _calendar_snapshot(name)
+
+    assert cold['he']['months'] != cold['de']['months']  # Sanity check
+
+    localedata.clear_caches()
+    warm = {name: _calendar_snapshot(name) for name in (reversed(locales) if reverse else locales)}
+    assert warm == cold
+    # Repeated reads in the warmed-up state must stay correct too
+    assert {name: _calendar_snapshot(name) for name in locales} == cold
+
+
 def test_manual_locale_data_writes(request):
     """Writes into `Locale(...)._data` (misguided as they may be) must be
     visible to subsequent reads."""