GH-81061: Fix refcount issue when returning `None` from a `ctypes.py_object` callback...

(cherry picked from commit 837ba052672d1a5f85a46c1b6d4b6e7d192af6f3)

Co-authored-by: dgelessus <dgelessus@users.noreply.github.com>

diff --git a/Lib/ctypes/test/test_refcounts.py b/Lib/ctypes/test/test_refcounts.py
index f2edfa6400ef2477b2bf1aab8a86300b007fe392..48958cd2a60196bd1f5e010883fe1e465cbe02d4 100644 (file)
--- a/Lib/ctypes/test/test_refcounts.py
+++ b/Lib/ctypes/test/test_refcounts.py
@@ -97,5 +97,20 @@ class AnotherLeak(unittest.TestCase):
         f(1, 2)
         self.assertEqual(sys.getrefcount(ctypes.c_int), a)
 
+    @support.refcount_test
+    def test_callback_py_object_none_return(self):
+        # bpo-36880: test that returning None from a py_object callback
+        # does not decrement the refcount of None.
+
+        for FUNCTYPE in (ctypes.CFUNCTYPE, ctypes.PYFUNCTYPE):
+            with self.subTest(FUNCTYPE=FUNCTYPE):
+                @FUNCTYPE(ctypes.py_object)
+                def func():
+                    return None
+
+                # Check that calling func does not affect None's refcount.
+                for _ in range(10000):
+                    func()
+
 if __name__ == '__main__':
     unittest.main()

diff --git a/Misc/NEWS.d/next/Library/2019-05-13-11-37-30.bpo-36880.ZgBgH0.rst b/Misc/NEWS.d/next/Library/2019-05-13-11-37-30.bpo-36880.ZgBgH0.rst
new file mode 100644 (file)
index 0000000..f653238
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-13-11-37-30.bpo-36880.ZgBgH0.rst
@@ -0,0 +1,2 @@
+Fix a reference counting issue when a :mod:`ctypes` callback with return
+type :class:`~ctypes.py_object` returns ``None``, which could cause crashes.

diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c
index c28762d49ba4970f70967679a0fddc1d0c452245..b46067bf2fa6556239cbfd99b708aecbcbef0f9e 100644 (file)
--- a/Modules/_ctypes/callbacks.c
+++ b/Modules/_ctypes/callbacks.c
@@ -276,15 +276,14 @@ static void _CallPythonObject(void *mem,
                                       "of ctypes callback function",
                                       callable);
         }
-        else if (keep == Py_None) {
-            /* Nothing to keep */
-            Py_DECREF(keep);
-        }
         else if (setfunc != _ctypes_get_fielddesc("O")->setfunc) {
-            if (-1 == PyErr_WarnEx(PyExc_RuntimeWarning,
-                                   "memory leak in callback function.",
-                                   1))
-            {
+            if (keep == Py_None) {
+                /* Nothing to keep */
+                Py_DECREF(keep);
+            }
+            else if (PyErr_WarnEx(PyExc_RuntimeWarning,
+                                  "memory leak in callback function.",
+                                  1) == -1) {
                 _PyErr_WriteUnraisableMsg("on converting result "
                                           "of ctypes callback function",
                                           callable);