bpo-43224: Implement substitution of unpacked TypeVarTuple in C (GH-31828)

Co-authored-by: Matthew Rahtz <mrahtz@gmail.com>

diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h
index cc94662256e87e147260e777c5b7a080dfe3ae21..28fffa95071a04a1b9352cde0536f35dec5f518d 100644 (file)
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -202,6 +202,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(__truediv__)
         STRUCT_FOR_ID(__trunc__)
         STRUCT_FOR_ID(__typing_subst__)
+        STRUCT_FOR_ID(__typing_unpacked__)
         STRUCT_FOR_ID(__warningregistry__)
         STRUCT_FOR_ID(__weakref__)
         STRUCT_FOR_ID(__xor__)

diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h
index 9a3a9d04324ba839bc4f1d5411d600df3ac29af3..941badfc8cb6a87ece9e9f628193642ce6beb881 100644 (file)
--- a/Include/internal/pycore_runtime_init.h
+++ b/Include/internal/pycore_runtime_init.h
@@ -825,6 +825,7 @@ extern "C" {
                 INIT_ID(__truediv__), \
                 INIT_ID(__trunc__), \
                 INIT_ID(__typing_subst__), \
+                INIT_ID(__typing_unpacked__), \
                 INIT_ID(__warningregistry__), \
                 INIT_ID(__weakref__), \
                 INIT_ID(__xor__), \

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index 412d117c238da0cd764ac828f59db146b75aed76..88be2850acb1cc8aaeb55b9b1d5aecd50347c889 100644 (file)
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -754,89 +754,89 @@ class GenericAliasSubstitutionTests(BaseTestCase):
         tests = [
             # Alias                                    # Args                                            # Expected result
             ('C[*Ts]',                                 '[()]',                                           'C[()]'),
-            ('tuple[*Ts]',                             '[()]',                                           'TypeError'),  # Should be tuple[()]
+            ('tuple[*Ts]',                             '[()]',                                           'tuple[()]'),
             ('Tuple[*Ts]',                             '[()]',                                           'Tuple[()]'),
 
             ('C[*Ts]',                                 '[int]',                                          'C[int]'),
-            ('tuple[*Ts]',                             '[int]',                                          'tuple[(int,),]'),  # Should be tuple[int]
+            ('tuple[*Ts]',                             '[int]',                                          'tuple[int]'),
             ('Tuple[*Ts]',                             '[int]',                                          'Tuple[int]'),
 
             ('C[*Ts]',                                 '[int, str]',                                     'C[int, str]'),
-            ('tuple[*Ts]',                             '[int, str]',                                     'TypeError'),  # Should be tuple[int, str]
+            ('tuple[*Ts]',                             '[int, str]',                                     'tuple[int, str]'),
             ('Tuple[*Ts]',                             '[int, str]',                                     'Tuple[int, str]'),
 
             ('C[*Ts]',                                 '[*tuple_type[int]]',                             'C[*tuple_type[int]]'),  # Should be C[int]
-            ('tuple[*Ts]',                             '[*tuple_type[int]]',                             'tuple[(*tuple_type[int],),]'),  # Should be tuple[int]
+            ('tuple[*Ts]',                             '[*tuple_type[int]]',                             'tuple[*tuple_type[int]]'),  # Should be tuple[int]
             ('Tuple[*Ts]',                             '[*tuple_type[int]]',                             'Tuple[*tuple_type[int]]'),  # Should be Tuple[int]
 
             ('C[*Ts]',                                 '[*tuple_type[*Ts]]',                             'C[*tuple_type[*Ts]]'),  # Should be C[*Ts]
-            ('tuple[*Ts]',                             '[*tuple_type[*Ts]]',                             'tuple[(*tuple_type[*Ts],),]'),  # Should be tuple[*Ts]
+            ('tuple[*Ts]',                             '[*tuple_type[*Ts]]',                             'tuple[*tuple_type[*Ts]]'),  # Should be tuple[*Ts]
             ('Tuple[*Ts]',                             '[*tuple_type[*Ts]]',                             'Tuple[*tuple_type[*Ts]]'),  # Should be Tuple[*Ts]
 
             ('C[*Ts]',                                 '[*tuple_type[int, str]]',                        'C[*tuple_type[int, str]]'),  # Should be C[int, str]
-            ('tuple[*Ts]',                             '[*tuple_type[int, str]]',                        'tuple[(*tuple_type[int, str],),]'),  # Should be tuple[int, str]
+            ('tuple[*Ts]',                             '[*tuple_type[int, str]]',                        'tuple[*tuple_type[int, str]]'),  # Should be tuple[int, str]
             ('Tuple[*Ts]',                             '[*tuple_type[int, str]]',                        'Tuple[*tuple_type[int, str]]'),  # Should be Tuple[int, str]
 
             ('C[*Ts]',                                 '[tuple_type[int, ...]]',                         'C[tuple_type[int, ...]]'),
-            ('tuple[*Ts]',                             '[tuple_type[int, ...]]',                         'tuple[(tuple_type[int, ...],),]'),  # Should be tuple[tuple_type[int, ...]]
+            ('tuple[*Ts]',                             '[tuple_type[int, ...]]',                         'tuple[tuple_type[int, ...]]'),
             ('Tuple[*Ts]',                             '[tuple_type[int, ...]]',                         'Tuple[tuple_type[int, ...]]'),
 
             ('C[*Ts]',                                 '[tuple_type[int, ...], tuple_type[str, ...]]',   'C[tuple_type[int, ...], tuple_type[str, ...]]'),
-            ('tuple[*Ts]',                             '[tuple_type[int, ...], tuple_type[str, ...]]',   'TypeError'),  # Should be tuple[tuple_type[int, ...], tuple_type[str, ...]]
+            ('tuple[*Ts]',                             '[tuple_type[int, ...], tuple_type[str, ...]]',   'tuple[tuple_type[int, ...], tuple_type[str, ...]]'),
             ('Tuple[*Ts]',                             '[tuple_type[int, ...], tuple_type[str, ...]]',   'Tuple[tuple_type[int, ...], tuple_type[str, ...]]'),
 
             ('C[*Ts]',                                 '[*tuple_type[int, ...]]',                        'C[*tuple_type[int, ...]]'),
-            ('tuple[*Ts]',                             '[*tuple_type[int, ...]]',                        'tuple[(*tuple_type[int, ...],),]'),  # Should be tuple[*tuple_type[int, ...]]
+            ('tuple[*Ts]',                             '[*tuple_type[int, ...]]',                        'tuple[*tuple_type[int, ...]]'),
             ('Tuple[*Ts]',                             '[*tuple_type[int, ...]]',                        'Tuple[*tuple_type[int, ...]]'),
 
             # Technically, multiple unpackings are forbidden by PEP 646, but we
             # choose to be less restrictive at runtime, to allow folks room
             # to experiment. So all three of these should be valid.
             ('C[*Ts]',                                 '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'C[*tuple_type[int, ...], *tuple_type[str, ...]]'),
-            # Should be tuple[*tuple_type[int, ...], *tuple_type[str, ...]], to match the other two.
-            ('tuple[*Ts]',                             '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'),
+            ('tuple[*Ts]',                             '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'),
             ('Tuple[*Ts]',                             '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'Tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'),
 
             ('C[*Ts]',                                 '[*Ts]',                                          'C[*Ts]'),
-            ('tuple[*Ts]',                             '[*Ts]',                                          'tuple[(*Ts,),]'),  # Should be tuple[*Ts]
+            ('tuple[*Ts]',                             '[*Ts]',                                          'tuple[*Ts]'),
             ('Tuple[*Ts]',                             '[*Ts]',                                          'Tuple[*Ts]'),
 
             ('C[*Ts]',                                 '[T, *Ts]',                                       'C[T, *Ts]'),
-            ('tuple[*Ts]',                             '[T, *Ts]',                                       'TypeError'),  # Should be tuple[T, *Ts]
+            ('tuple[*Ts]',                             '[T, *Ts]',                                       'tuple[T, *Ts]'),
             ('Tuple[*Ts]',                             '[T, *Ts]',                                       'Tuple[T, *Ts]'),
 
             ('C[*Ts]',                                 '[*Ts, T]',                                       'C[*Ts, T]'),
-            ('tuple[*Ts]',                             '[*Ts, T]',                                       'TypeError'),  # Should be tuple[*Ts, T]
+            ('tuple[*Ts]',                             '[*Ts, T]',                                       'tuple[*Ts, T]'),
             ('Tuple[*Ts]',                             '[*Ts, T]',                                       'Tuple[*Ts, T]'),
 
             ('C[T, *Ts]',                              '[int]',                                          'C[int]'),
-            ('tuple[T, *Ts]',                          '[int]',                                          'TypeError'),  # Should be tuple[int]
+            ('tuple[T, *Ts]',                          '[int]',                                          'tuple[int]'),
             ('Tuple[T, *Ts]',                          '[int]',                                          'Tuple[int]'),
 
             ('C[T, *Ts]',                              '[int, str]',                                     'C[int, str]'),
-            ('tuple[T, *Ts]',                          '[int, str]',                                     'tuple[int, (str,)]'),  # Should be tuple[int, str]
+            ('tuple[T, *Ts]',                          '[int, str]',                                     'tuple[int, str]'),
             ('Tuple[T, *Ts]',                          '[int, str]',                                     'Tuple[int, str]'),
 
             ('C[T, *Ts]',                              '[int, str, bool]',                               'C[int, str, bool]'),
-            ('tuple[T, *Ts]',                          '[int, str, bool]',                               'TypeError'),  # Should be tuple[int, str, bool]
+            ('tuple[T, *Ts]',                          '[int, str, bool]',                               'tuple[int, str, bool]'),
             ('Tuple[T, *Ts]',                          '[int, str, bool]',                               'Tuple[int, str, bool]'),
 
             ('C[T, *Ts]',                              '[*tuple[int, ...]]',                             'C[*tuple[int, ...]]'),  # Should be C[int, *tuple[int, ...]]
             ('C[T, *Ts]',                              '[*Tuple[int, ...]]',                             'TypeError'),  # Ditto
-            ('tuple[T, *Ts]',                          '[*tuple_type[int, ...]]',                        'TypeError'),  # Should be tuple[int, *tuple[int, ...]]
-            ('Tuple[T, *Ts]',                          '[*tuple[int, ...]]',                             'Tuple[*tuple[int, ...]]'),  # Ditto
-            ('Tuple[T, *Ts]',                          '[*Tuple[int, ...]]',                             'TypeError'),  # Ditto
+            ('tuple[T, *Ts]',                          '[*tuple[int, ...]]',                             'tuple[*tuple[int, ...]]'),  # Should be tuple[int, *tuple[int, ...]]
+            ('tuple[T, *Ts]',                          '[*Tuple[int, ...]]',                             'TypeError'),  # Should be tuple[int, *Tuple[int, ...]]
+            ('Tuple[T, *Ts]',                          '[*tuple[int, ...]]',                             'Tuple[*tuple[int, ...]]'),  # Should be Tuple[int, *tuple[int, ...]]
+            ('Tuple[T, *Ts]',                          '[*Tuple[int, ...]]',                             'TypeError'),  # Should be Tuple[int, *Tuple[int, ...]]
 
             ('C[*Ts, T]',                              '[int]',                                          'C[int]'),
-            ('tuple[*Ts, T]',                          '[int]',                                          'TypeError'),  # Should be tuple[int]
+            ('tuple[*Ts, T]',                          '[int]',                                          'tuple[int]'),
             ('Tuple[*Ts, T]',                          '[int]',                                          'Tuple[int]'),
 
             ('C[*Ts, T]',                              '[int, str]',                                     'C[int, str]'),
-            ('tuple[*Ts, T]',                          '[int, str]',                                     'tuple[(int,), str]'),  # Should be tuple[int, str]
+            ('tuple[*Ts, T]',                          '[int, str]',                                     'tuple[int, str]'),
             ('Tuple[*Ts, T]',                          '[int, str]',                                     'Tuple[int, str]'),
 
             ('C[*Ts, T]',                              '[int, str, bool]',                               'C[int, str, bool]'),
-            ('tuple[*Ts, T]',                          '[int, str, bool]',                               'TypeError'),  # Should be tuple[int, str, bool]
+            ('tuple[*Ts, T]',                          '[int, str, bool]',                               'tuple[int, str, bool]'),
             ('Tuple[*Ts, T]',                          '[int, str, bool]',                               'Tuple[int, str, bool]'),
 
             ('generic[T, *tuple_type[int, ...]]',      '[str]',                                          'generic[str, *tuple_type[int, ...]]'),
@@ -945,7 +945,7 @@ class TypeVarTupleTests(BaseTestCase):
         T2 = TypeVar('T2')
         class G(Generic[Unpack[Ts]]): pass
 
-        for A in G, Tuple:
+        for A in G, Tuple, tuple:
             B = A[Unpack[Ts]]
             self.assertEqual(B[()], A[()])
             self.assertEqual(B[float], A[float])
@@ -984,6 +984,21 @@ class TypeVarTupleTests(BaseTestCase):
         T2 = TypeVar('T2')
         class G(Generic[Unpack[Ts]]): pass
 
+        for A in G, Tuple, tuple:
+            B = A[Ts]
+            with self.assertRaises(TypeError):
+                B[int, str]
+
+            C = A[T, T2]
+            with self.assertRaises(TypeError):
+                C[Unpack[Ts]]
+
+    def test_repr_is_correct(self):
+        Ts = TypeVarTuple('Ts')
+        T = TypeVar('T')
+        T2 = TypeVar('T2')
+        class G(Generic[Unpack[Ts]]): pass
+
         for A in G, Tuple:
             B = A[T, Unpack[Ts], str, T2]
             with self.assertRaises(TypeError):

diff --git a/Lib/typing.py b/Lib/typing.py
index b250f2992dfff38d6761bc81047dc6781ebcbf47..35deb32220eb8877f2c69c343bcb45d4a860a196 100644 (file)
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -1019,7 +1019,7 @@ class TypeVarTuple(_Final, _Immutable, _PickleUsingNameMixin, _root=True):
         return self.__name__
 
     def __typing_subst__(self, arg):
-        raise AssertionError
+        raise TypeError("Substitution of bare TypeVarTuple is not supported")
 
 
 class ParamSpecArgs(_Final, _Immutable, _root=True):
@@ -1686,11 +1686,15 @@ class _UnpackGenericAlias(_GenericAlias, _root=True):
         return '*' + repr(self.__args__[0])
 
     def __getitem__(self, args):
-        if (len(self.__parameters__) == 1 and
-                isinstance(self.__parameters__[0], TypeVarTuple)):
+        if self.__typing_unpacked__():
             return args
         return super().__getitem__(args)
 
+    def __typing_unpacked__(self):
+        # If x is Unpack[tuple[...]], __parameters__ will be empty.
+        return bool(self.__parameters__ and
+                    isinstance(self.__parameters__[0], TypeVarTuple))
+
 
 class Generic:
     """Abstract base class for generic types.

diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c
index 7059c4018303e056d4613227b65b2daf8d7673f9..7b689912dffc1b428b1a77a685d604d50cb28bce 100644 (file)
--- a/Objects/genericaliasobject.c
+++ b/Objects/genericaliasobject.c
@@ -190,6 +190,23 @@ tuple_add(PyObject *self, Py_ssize_t len, PyObject *item)
     return 0;
 }
 
+static Py_ssize_t
+tuple_extend(PyObject **dst, Py_ssize_t dstindex,
+             PyObject **src, Py_ssize_t count)
+{
+    assert(count >= 0);
+    if (_PyTuple_Resize(dst, PyTuple_GET_SIZE(*dst) + count - 1) != 0) {
+        return -1;
+    }
+    assert(dstindex + count <= PyTuple_GET_SIZE(*dst));
+    for (Py_ssize_t i = 0; i < count; ++i) {
+        PyObject *item = src[i];
+        Py_INCREF(item);
+        PyTuple_SET_ITEM(*dst, dstindex + i, item);
+    }
+    return dstindex + count;
+}
+
 PyObject *
 _Py_make_parameters(PyObject *args)
 {
@@ -251,7 +268,8 @@ _Py_make_parameters(PyObject *args)
    If obj doesn't have a __parameters__ attribute or that's not
    a non-empty tuple, return a new reference to obj. */
 static PyObject *
-subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems)
+subs_tvars(PyObject *obj, PyObject *params,
+           PyObject **argitems, Py_ssize_t nargs, Py_ssize_t varparam)
 {
     PyObject *subparams;
     if (_PyObject_LookupAttr(obj, &_Py_ID(__parameters__), &subparams) < 0) {
@@ -265,14 +283,27 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems)
             Py_DECREF(subparams);
             return NULL;
         }
-        for (Py_ssize_t i = 0; i < nsubargs; ++i) {
+        for (Py_ssize_t i = 0, j = 0; i < nsubargs; ++i) {
             PyObject *arg = PyTuple_GET_ITEM(subparams, i);
             Py_ssize_t iparam = tuple_index(params, nparams, arg);
-            if (iparam >= 0) {
-                arg = argitems[iparam];
+            if (iparam == varparam) {
+                j = tuple_extend(&subargs, j,
+                                 argitems + iparam, nargs - nparams + 1);
+                if (j < 0) {
+                    return NULL;
+                }
+            }
+            else {
+                if (iparam >= 0) {
+                    if (iparam > varparam) {
+                        iparam += nargs - nsubargs;
+                    }
+                    arg = argitems[iparam];
+                }
+                Py_INCREF(arg);
+                PyTuple_SET_ITEM(subargs, j, arg);
+                j++;
             }
-            Py_INCREF(arg);
-            PyTuple_SET_ITEM(subargs, i, arg);
         }
 
         obj = PyObject_GetItem(obj, subargs);
@@ -286,6 +317,23 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems)
     return obj;
 }
 
+static int
+_is_unpacked_typevartuple(PyObject *arg)
+{
+    PyObject *meth;
+    int res = _PyObject_LookupAttr(arg, &_Py_ID(__typing_unpacked__), &meth);
+    if (res > 0) {
+        PyObject *tmp = PyObject_CallNoArgs(meth);
+        Py_DECREF(meth);
+        if (tmp == NULL) {
+            return -1;
+        }
+        res = PyObject_IsTrue(tmp);
+        Py_DECREF(tmp);
+    }
+    return res;
+}
+
 PyObject *
 _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObject *item)
 {
@@ -298,11 +346,27 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
     int is_tuple = PyTuple_Check(item);
     Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1;
     PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item;
-    if (nitems != nparams) {
-        return PyErr_Format(PyExc_TypeError,
-                            "Too %s arguments for %R",
-                            nitems > nparams ? "many" : "few",
-                            self);
+    Py_ssize_t varparam = 0;
+    for (; varparam < nparams; varparam++) {
+        PyObject *param = PyTuple_GET_ITEM(parameters, varparam);
+        if (Py_TYPE(param)->tp_iter) {  // TypeVarTuple
+            break;
+        }
+    }
+    if (varparam < nparams) {
+        if (nitems < nparams - 1) {
+            return PyErr_Format(PyExc_TypeError,
+                                "Too few arguments for %R",
+                                self);
+        }
+    }
+    else {
+        if (nitems != nparams) {
+            return PyErr_Format(PyExc_TypeError,
+                                "Too %s arguments for %R",
+                                nitems > nparams ? "many" : "few",
+                                self);
+        }
     }
     /* Replace all type variables (specified by parameters)
        with corresponding values specified by argitems.
@@ -315,8 +379,13 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
     if (newargs == NULL) {
         return NULL;
     }
-    for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) {
+    for (Py_ssize_t iarg = 0, jarg = 0; iarg < nargs; iarg++) {
         PyObject *arg = PyTuple_GET_ITEM(args, iarg);
+        int unpack = _is_unpacked_typevartuple(arg);
+        if (unpack < 0) {
+            Py_DECREF(newargs);
+            return NULL;
+        }
         PyObject *subst;
         if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_subst__), &subst) < 0) {
             Py_DECREF(newargs);
@@ -325,17 +394,38 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
         if (subst) {
             Py_ssize_t iparam = tuple_index(parameters, nparams, arg);
             assert(iparam >= 0);
+            if (iparam == varparam) {
+                Py_DECREF(subst);
+                Py_DECREF(newargs);
+                PyErr_SetString(PyExc_TypeError,
+                        "Substitution of bare TypeVarTuple is not supported");
+                return NULL;
+            }
+            if (iparam > varparam) {
+                iparam += nitems - nparams;
+            }
             arg = PyObject_CallOneArg(subst, argitems[iparam]);
             Py_DECREF(subst);
         }
         else {
-            arg = subs_tvars(arg, parameters, argitems);
+            arg = subs_tvars(arg, parameters, argitems, nitems, varparam);
         }
         if (arg == NULL) {
             Py_DECREF(newargs);
             return NULL;
         }
-        PyTuple_SET_ITEM(newargs, iarg, arg);
+        if (unpack) {
+            jarg = tuple_extend(&newargs, jarg,
+                    &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg));
+            Py_DECREF(arg);
+            if (jarg < 0) {
+                return NULL;
+            }
+        }
+        else {
+            PyTuple_SET_ITEM(newargs, jarg, arg);
+            jarg++;
+        }
     }
 
     return newargs;