[3.11] gh-99952: fix refcount issues in ctypes.Structure from_param() result (#101339)

[3.11] gh-99952: [ctypes] fix refcount issues in from_param() result. (GH-100169)

Fixes a reference counting issue with `ctypes.Structure` when a `from_param()` method call is used and the structure size is larger than a C pointer `sizeof(void*)`.

This problem existed for a very long time, but became more apparent in 3.8+ by change likely due to garbage collection cleanup timing changes..
(cherry picked from commit dfad678d7024ab86d265d84ed45999e031a03691)

Co-authored-by: Yukihiro Nakadaira <yukihiro.nakadaira@gmail.com>

diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py
index 38af7ac13d756c225af7fbc514668476f66d0184..3fdc994e9078ce84a2c78ca694001183030c6235 100644 (file)
--- a/Lib/ctypes/test/test_parameters.py
+++ b/Lib/ctypes/test/test_parameters.py
@@ -244,6 +244,58 @@ class SimpleTypesTestCase(unittest.TestCase):
         self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^<cparam 'Z' \(0x[A-Fa-f0-9]+\)>$")
         self.assertRegex(repr(c_void_p.from_param(0x12)), r"^<cparam 'P' \(0x0*12\)>$")
 
+    @test.support.cpython_only
+    def test_from_param_result_refcount(self):
+        # Issue #99952
+        import _ctypes_test
+        from ctypes import PyDLL, c_int, c_void_p, py_object, Structure
+
+        class X(Structure):
+            """This struct size is <= sizeof(void*)."""
+            _fields_ = [("a", c_void_p)]
+
+            def __del__(self):
+                trace.append(4)
+
+            @classmethod
+            def from_param(cls, value):
+                trace.append(2)
+                return cls()
+
+        PyList_Append = PyDLL(_ctypes_test.__file__)._testfunc_pylist_append
+        PyList_Append.restype = c_int
+        PyList_Append.argtypes = [py_object, py_object, X]
+
+        trace = []
+        trace.append(1)
+        PyList_Append(trace, 3, "dummy")
+        trace.append(5)
+
+        self.assertEqual(trace, [1, 2, 3, 4, 5])
+
+        class Y(Structure):
+            """This struct size is > sizeof(void*)."""
+            _fields_ = [("a", c_void_p), ("b", c_void_p)]
+
+            def __del__(self):
+                trace.append(4)
+
+            @classmethod
+            def from_param(cls, value):
+                trace.append(2)
+                return cls()
+
+        PyList_Append = PyDLL(_ctypes_test.__file__)._testfunc_pylist_append
+        PyList_Append.restype = c_int
+        PyList_Append.argtypes = [py_object, py_object, Y]
+
+        trace = []
+        trace.append(1)
+        PyList_Append(trace, 3, "dummy")
+        trace.append(5)
+
+        self.assertEqual(trace, [1, 2, 3, 4, 5])
+
 ################################################################
 
 if __name__ == '__main__':

diff --git a/Misc/NEWS.d/next/Library/2022-12-11-14-38-59.gh-issue-99952.IYGLzr.rst b/Misc/NEWS.d/next/Library/2022-12-11-14-38-59.gh-issue-99952.IYGLzr.rst
new file mode 100644 (file)
index 0000000..09ec961
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-12-11-14-38-59.gh-issue-99952.IYGLzr.rst
@@ -0,0 +1,2 @@
+Fix a reference undercounting issue in :class:`ctypes.Structure` with ``from_param()``\r
+results larger than a C pointer.\r

diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index b5eafe49e4ee086c48650cce9cf7dc240539398e..fc73264ff5a32c8b5ef85682ca6ab64d7b45f9c3 100644 (file)
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -416,6 +416,7 @@ _ctypes_alloc_format_string_with_shape(int ndim, const Py_ssize_t *shape,
 typedef struct {
     PyObject_HEAD
     void *ptr;
+    PyObject *keep;  // If set, a reference to the original CDataObject.
 } StructParamObject;
 
 
@@ -423,6 +424,7 @@ static void
 StructParam_dealloc(PyObject *myself)
 {
     StructParamObject *self = (StructParamObject *)myself;
+    Py_XDECREF(self->keep);
     PyMem_Free(self->ptr);
     Py_TYPE(self)->tp_free(myself);
 }
@@ -470,6 +472,7 @@ StructUnionType_paramfunc(CDataObject *self)
 
         StructParamObject *struct_param = (StructParamObject *)obj;
         struct_param->ptr = ptr;
+        struct_param->keep = Py_NewRef(self);
     } else {
         ptr = self->b_ptr;
         obj = (PyObject *)self;

diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c
index 770c96c60d1f7d2ab125c0d8f3452658238f6595..8a6a11670ba140746946669b4f9dbe6c6de31579 100644 (file)
--- a/Modules/_ctypes/_ctypes_test.c
+++ b/Modules/_ctypes/_ctypes_test.c
@@ -1034,6 +1034,12 @@ EXPORT (HRESULT) KeepObject(IUnknown *punk)
 
 #endif
 
+EXPORT(int)
+_testfunc_pylist_append(PyObject *list, PyObject *item)
+{
+    return PyList_Append(list, item);
+}
+
 static struct PyModuleDef_Slot _ctypes_test_slots[] = {
     {0, NULL}
 };