[pull] main from python:main

Doc/deprecations/c-api-pending-removal-in-3.19.rst

@@ -0,0 +1,4 @@
+Pending removal in Python 3.19
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* :pep:`456` embedders support for the string hashing scheme definition.

Doc/deprecations/index.rst

@@ -20,8 +20,12 @@ C API deprecations
 
 .. include:: c-api-pending-removal-in-3.15.rst
 
+.. include:: c-api-pending-removal-in-3.16.rst
+
 .. include:: c-api-pending-removal-in-3.18.rst
 
+.. include:: c-api-pending-removal-in-3.19.rst
+
 .. include:: c-api-pending-removal-in-3.20.rst
 
 .. include:: c-api-pending-removal-in-future.rst

Doc/whatsnew/3.15.rst

@@ -403,7 +403,7 @@ Improved error messages
      File "/home/pablogsal/github/python/main/lel.py", line 42, in <module>
         print(container.area)
               ^^^^^^^^^^^^^^
-     AttributeError: 'Container' object has no attribute 'area'. Did you mean: 'inner.area'?
+     AttributeError: 'Container' object has no attribute 'area'. Did you mean '.inner.area' instead of '.area'?
 
 
 Other language changes
@@ -1723,6 +1723,16 @@ on Python 3.13 and older.
 Deprecated C APIs
 -----------------
 
+* Deprecate :pep:`456` support for providing an external definition
+  of the string hashing scheme. Removal is scheduled for Python 3.19.
+
+  Previously, embedders could define :c:macro:`Py_HASH_ALGORITHM` to be
+  ``Py_HASH_EXTERNAL`` to indicate that the hashing scheme was provided
+  externally but this feature was undocumented, untested and most likely
+  unused.
+
+  (Contributed by Bénédikt Tran in :gh:`141226`.)
+
 * For unsigned integer formats in :c:func:`PyArg_ParseTuple`,
   accepting Python integers with value that is larger than the maximal value
   for the C type or less than the minimal value for the corresponding

Lib/collections/__init__.py

@@ -32,6 +32,8 @@
 _sys.modules['collections.abc'] = _collections_abc
 abc = _collections_abc
 
+lazy from copy import copy as _copy
+lazy from heapq import nlargest as _nlargest
 from itertools import chain as _chain
 from itertools import repeat as _repeat
 from itertools import starmap as _starmap
@@ -59,8 +61,6 @@
 except ImportError:
     pass
 
-heapq = None  # Lazily imported
-
 
 ################################################################################
 ### OrderedDict
@@ -634,12 +634,7 @@ def most_common(self, n=None):
         if n is None:
             return sorted(self.items(), key=_itemgetter(1), reverse=True)
 
-        # Lazy import to speedup Python startup time
-        global heapq
-        if heapq is None:
-            import heapq
-
-        return heapq.nlargest(n, self.items(), key=_itemgetter(1))
+        return _nlargest(n, self.items(), key=_itemgetter(1))
 
     def elements(self):
         '''Iterator over elements repeating each as many times as its count.
@@ -1249,11 +1244,10 @@ def __copy__(self):
     def copy(self):
         if self.__class__ is UserDict:
             return UserDict(self.data.copy())
-        import copy
         data = self.data
         try:
             self.data = {}
-            c = copy.copy(self)
+            c = _copy(self)
         finally:
             self.data = data
         c.update(self)

Lib/copy.py

@@ -204,7 +204,17 @@ def _deepcopy_dict(x, memo, deepcopy=deepcopy):
 d[dict] = _deepcopy_dict
 
 def _deepcopy_frozendict(x, memo, deepcopy=deepcopy):
-    y = _deepcopy_dict(x, memo, deepcopy)
+    y = {}
+    for key, value in x.items():
+        y[deepcopy(key, memo)] = deepcopy(value, memo)
+
+    # We're not going to put the frozendict in the memo, but it's still
+    # important we check for it, in case the frozendict contains recursive
+    # mutable structures.
+    try:
+        return memo[id(x)]
+    except KeyError:
+        pass
     return frozendict(y)
 d[frozendict] = _deepcopy_frozendict
 

Lib/pprint.py

@@ -235,6 +235,20 @@ def _pprint_dict(self, object, stream, indent, allowance, context, level):
 
     _dispatch[dict.__repr__] = _pprint_dict
 
+    def _pprint_frozendict(self, object, stream, indent, allowance, context, level):
+        write = stream.write
+        cls = object.__class__
+        stream.write(cls.__name__ + '(')
+        length = len(object)
+        if length:
+            self._pprint_dict(object, stream,
+                              indent + len(cls.__name__) + 1,
+                              allowance + 1,
+                              context, level)
+        write(')')
+
+    _dispatch[frozendict.__repr__] = _pprint_frozendict
+
     def _pprint_ordered_dict(self, object, stream, indent, allowance, context, level):
         if not len(object):
             stream.write(repr(object))
@@ -623,12 +637,21 @@ def _safe_repr(self, object, context, maxlevels, level):
             else:
                 return repr(object), True, False
 
-        if issubclass(typ, dict) and r is dict.__repr__:
+        if ((issubclass(typ, dict) and r is dict.__repr__)
+            or (issubclass(typ, frozendict) and r is frozendict.__repr__)):
+            is_frozendict = issubclass(typ, frozendict)
             if not object:
-                return "{}", True, False
+                if is_frozendict:
+                    rep = f"{object.__class__.__name__}()"
+                else:
+                    rep = "{}"
+                return rep, True, False
             objid = id(object)
             if maxlevels and level >= maxlevels:
-                return "{...}", False, objid in context
+                rep = "{...}"
+                if is_frozendict:
+                    rep = f"{object.__class__.__name__}({rep})"
+                return rep, False, objid in context
             if objid in context:
                 return _recursion(object), False, True
             context[objid] = 1
@@ -651,7 +674,10 @@ def _safe_repr(self, object, context, maxlevels, level):
                 if krecur or vrecur:
                     recursive = True
             del context[objid]
-            return "{%s}" % ", ".join(components), readable, recursive
+            rep = "{%s}" % ", ".join(components)
+            if is_frozendict:
+                rep = f"{object.__class__.__name__}({rep})"
+            return rep, readable, recursive
 
         if (issubclass(typ, list) and r is list.__repr__) or \
            (issubclass(typ, tuple) and r is tuple.__repr__):

Lib/test/pickletester.py

@@ -2925,9 +2925,13 @@ def _test_recursive_collection_in_key(self, factory, minprotocol=0):
                 self.assertIs(keys[0].attr, x)
 
     def test_recursive_frozendict_in_key(self):
+        if self.py_version < (3, 15):
+            self.skipTest('need frozendict')
         self._test_recursive_collection_in_key(frozendict, minprotocol=2)
 
     def test_recursive_frozendict_subclass_in_key(self):
+        if self.py_version < (3, 15):
+            self.skipTest('need frozendict')
         self._test_recursive_collection_in_key(MyFrozenDict)
 
     def _test_recursive_collection_in_value(self, factory, minprotocol=0):
@@ -2942,9 +2946,13 @@ def _test_recursive_collection_in_value(self, factory, minprotocol=0):
                 self.assertIs(x['key'][0], x)
 
     def test_recursive_frozendict_in_value(self):
+        if self.py_version < (3, 15):
+            self.skipTest('need frozendict')
         self._test_recursive_collection_in_value(frozendict, minprotocol=2)
 
     def test_recursive_frozendict_subclass_in_value(self):
+        if self.py_version < (3, 15):
+            self.skipTest('need frozendict')
         self._test_recursive_collection_in_value(MyFrozenDict)
 
     def test_recursive_inst_state(self):
@@ -3437,6 +3445,8 @@ def test_newobj_generic(self):
                         self.skipTest('int and str subclasses are not interoperable with Python 2')
                     if (3, 0) <= self.py_version < (3, 4) and proto < 2 and C in (MyStr, MyUnicode):
                         self.skipTest('str subclasses are not interoperable with Python < 3.4')
+                    if self.py_version < (3, 15) and C == MyFrozenDict:
+                        self.skipTest('frozendict is not available on Python < 3.15')
                     B = C.__base__
                     x = C(C.sample)
                     x.foo = 42
@@ -3458,6 +3468,8 @@ def test_newobj_proxies(self):
                 with self.subTest(proto=proto, C=C):
                     if self.py_version < (3, 4) and proto < 3 and C in (MyStr, MyUnicode):
                         self.skipTest('str subclasses are not interoperable with Python < 3.4')
+                    if self.py_version < (3, 15) and C == MyFrozenDict:
+                        self.skipTest('frozendict is not available on Python < 3.15')
                     B = C.__base__
                     x = C(C.sample)
                     x.foo = 42

Lib/test/test_capi/test_dict.py

@@ -26,6 +26,19 @@ def gen():
     yield 'c'
 
 
+class FrozenDictSubclass(frozendict):
+    pass
+
+
+DICT_TYPES = (dict, DictSubclass, OrderedDict)
+FROZENDICT_TYPES = (frozendict, FrozenDictSubclass)
+ANYDICT_TYPES = DICT_TYPES + FROZENDICT_TYPES
+MAPPING_TYPES = (UserDict,)
+NOT_FROZENDICT_TYPES = DICT_TYPES + MAPPING_TYPES
+NOT_ANYDICT_TYPES = MAPPING_TYPES
+OTHER_TYPES = (lambda: [1], lambda: 42, object)  # (list, int, object)
+
+
 class CAPITest(unittest.TestCase):
 
     def test_dict_check(self):
@@ -406,6 +419,7 @@ def test_dict_next(self):
         # CRASHES dict_next(NULL, 0)
 
     def test_dict_update(self):
+        # Test PyDict_Update()
         update = _testlimitedcapi.dict_update
         for cls1 in dict, DictSubclass:
             for cls2 in dict, DictSubclass, UserDict:
@@ -416,11 +430,13 @@ def test_dict_update(self):
         self.assertRaises(AttributeError, update, {}, [])
         self.assertRaises(AttributeError, update, {}, 42)
         self.assertRaises(SystemError, update, UserDict(), {})
+        self.assertRaises(SystemError, update, frozendict(), {})
         self.assertRaises(SystemError, update, 42, {})
         self.assertRaises(SystemError, update, {}, NULL)
         self.assertRaises(SystemError, update, NULL, {})
 
     def test_dict_merge(self):
+        # Test PyDict_Merge()
         merge = _testlimitedcapi.dict_merge
         for cls1 in dict, DictSubclass:
             for cls2 in dict, DictSubclass, UserDict:
@@ -434,11 +450,13 @@ def test_dict_merge(self):
         self.assertRaises(AttributeError, merge, {}, [], 0)
         self.assertRaises(AttributeError, merge, {}, 42, 0)
         self.assertRaises(SystemError, merge, UserDict(), {}, 0)
+        self.assertRaises(SystemError, merge, frozendict(), {}, 0)
         self.assertRaises(SystemError, merge, 42, {}, 0)
         self.assertRaises(SystemError, merge, {}, NULL, 0)
         self.assertRaises(SystemError, merge, NULL, {}, 0)
 
     def test_dict_mergefromseq2(self):
+        # Test PyDict_MergeFromSeq2()
         mergefromseq2 = _testlimitedcapi.dict_mergefromseq2
         for cls1 in dict, DictSubclass:
             for cls2 in list, iter:
@@ -453,8 +471,8 @@ def test_dict_mergefromseq2(self):
         self.assertRaises(ValueError, mergefromseq2, {}, [(1, 2, 3)], 0)
         self.assertRaises(TypeError, mergefromseq2, {}, [1], 0)
         self.assertRaises(TypeError, mergefromseq2, {}, 42, 0)
-        # CRASHES mergefromseq2(UserDict(), [], 0)
-        # CRASHES mergefromseq2(42, [], 0)
+        self.assertRaises(SystemError, mergefromseq2, UserDict(), [], 0)
+        self.assertRaises(SystemError, mergefromseq2, 42, [], 0)
         # CRASHES mergefromseq2({}, NULL, 0)
         # CRASHES mergefromseq2(NULL, {}, 0)
 
@@ -545,6 +563,61 @@ def test_dict_popstring(self):
         # CRASHES dict_popstring({}, NULL)
         # CRASHES dict_popstring({"a": 1}, NULL)
 
+    def test_frozendict_check(self):
+        # Test PyFrozenDict_Check()
+        check = _testcapi.frozendict_check
+        for dict_type in FROZENDICT_TYPES:
+            self.assertTrue(check(dict_type(x=1)))
+        for dict_type in NOT_FROZENDICT_TYPES + OTHER_TYPES:
+            self.assertFalse(check(dict_type()))
+        # CRASHES check(NULL)
+
+    def test_frozendict_checkexact(self):
+        # Test PyFrozenDict_CheckExact()
+        check = _testcapi.frozendict_checkexact
+        for dict_type in FROZENDICT_TYPES:
+            self.assertEqual(check(dict_type(x=1)), dict_type == frozendict)
+        for dict_type in NOT_FROZENDICT_TYPES + OTHER_TYPES:
+            self.assertFalse(check(dict_type()))
+        # CRASHES check(NULL)
+
+    def test_anydict_check(self):
+        # Test PyAnyDict_Check()
+        check = _testcapi.anydict_check
+        for dict_type in ANYDICT_TYPES:
+            self.assertTrue(check(dict_type({1: 2})))
+        for test_type in NOT_ANYDICT_TYPES + OTHER_TYPES:
+            self.assertFalse(check(test_type()))
+        # CRASHES check(NULL)
+
+    def test_anydict_checkexact(self):
+        # Test PyAnyDict_CheckExact()
+        check = _testcapi.anydict_checkexact
+        for dict_type in ANYDICT_TYPES:
+            self.assertEqual(check(dict_type(x=1)),
+                             dict_type in (dict, frozendict))
+        for test_type in NOT_ANYDICT_TYPES + OTHER_TYPES:
+            self.assertFalse(check(test_type()))
+        # CRASHES check(NULL)
+
+    def test_frozendict_new(self):
+        # Test PyFrozenDict_New()
+        frozendict_new = _testcapi.frozendict_new
+
+        for dict_type in ANYDICT_TYPES:
+            dct = frozendict_new(dict_type({'x': 1}))
+            self.assertEqual(dct, frozendict(x=1))
+            self.assertIs(type(dct), frozendict)
+
+        dct = frozendict_new([('x', 1), ('y', 2)])
+        self.assertEqual(dct, frozendict(x=1, y=2))
+        self.assertIs(type(dct), frozendict)
+
+        # PyFrozenDict_New(NULL) creates an empty dictionary
+        dct = frozendict_new(NULL)
+        self.assertEqual(dct, frozendict())
+        self.assertIs(type(dct), frozendict)
+
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_copy.py

@@ -432,6 +432,23 @@ def test_deepcopy_frozendict(self):
         self.assertIsNot(x, y)
         self.assertIsNot(x["foo"], y["foo"])
 
+        # recursive frozendict
+        x = frozendict(foo=[])
+        x['foo'].append(x)
+        y = copy.deepcopy(x)
+        self.assertEqual(y.keys(), x.keys())
+        self.assertIsNot(x, y)
+        self.assertIsNot(x["foo"], y["foo"])
+        self.assertIs(y['foo'][0], y)
+
+        x = frozendict(foo=[])
+        x['foo'].append(x)
+        x = x['foo']
+        y = copy.deepcopy(x)
+        self.assertIsNot(x, y)
+        self.assertIsNot(x[0], y[0])
+        self.assertIs(y[0]['foo'], y)
+
     @support.skip_emscripten_stack_overflow()
     @support.skip_wasi_stack_overflow()
     def test_deepcopy_reflexive_dict(self):

Lib/test/test_import/test_lazy_imports.py

@@ -393,6 +393,15 @@ def test_dunder_lazy_import_used(self):
         import test.test_import.data.lazy_imports.dunder_lazy_import_used
         self.assertIn("test.test_import.data.lazy_imports.basic2", sys.modules)
 
+    def test_dunder_lazy_import_invalid_arguments(self):
+        """__lazy_import__ should reject invalid arguments."""
+        for invalid_name in (b"", 123, None):
+            with self.assertRaises(TypeError):
+                __lazy_import__(invalid_name)
+
+        with self.assertRaises(ValueError):
+            __lazy_import__("sys", level=-1)
+
     def test_dunder_lazy_import_builtins(self):
         """__lazy_import__ should use module's __builtins__ for __import__."""
         from test.test_import.data.lazy_imports import dunder_lazy_import_builtins