gh-117376: Make `Py_DECREF` a macro in ceval.c in free-threaded build (#122975)

`Py_DECREF` and `PyStackRef_CLOSE` are now implemented as macros in the
free-threaded build in ceval.c. There are two motivations;

 * MSVC has problems inlining functions in ceval.c in the PGO build.

 * We will want to mark escaping calls in order to spill the stack
   pointer in ceval.c and we will want to do this around `_Py_Dealloc`
   (or `_Py_MergeZeroLocalRefcount` or `_Py_DecRefShared`), not around
   the entire `Py_DECREF` or `PyStackRef_CLOSE` call.

diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h
index 8c9bb1ae8c49086da8e54be64ddf1317a1ca5edc..7c106577671b5f05d50cddb39286eb4ff1b434e5 100644 (file)
--- a/Include/internal/pycore_stackref.h
+++ b/Include/internal/pycore_stackref.h
@@ -190,18 +190,15 @@ PyStackRef_FromPyObjectImmortal(PyObject *obj)
     } while (0)
 
 #ifdef Py_GIL_DISABLED
-static inline void
-PyStackRef_CLOSE(_PyStackRef stackref)
-{
-    if (PyStackRef_IsDeferred(stackref)) {
-        // No assert for being immortal or deferred here.
-        // The GC unsets deferred objects right before clearing.
-        return;
-    }
-    Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
-}
+#   define PyStackRef_CLOSE(REF)                                        \
+        do {                                                            \
+            _PyStackRef _close_tmp = (REF);                             \
+            if (!PyStackRef_IsDeferred(_close_tmp)) {                   \
+                Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp));     \
+            }                                                           \
+        } while (0)
 #else
-#   define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
+#   define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref))
 #endif
 
 #define PyStackRef_XCLOSE(stackref) \
@@ -227,7 +224,7 @@ PyStackRef_DUP(_PyStackRef stackref)
     return stackref;
 }
 #else
-#   define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)));
+#   define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)))
 #endif
 
 static inline void

diff --git a/Python/ceval.c b/Python/ceval.c
index b615deec3c42427533f1c5da44c8a25a696362b8..2a5c16aa1019853930c4be7ae97e4da8bdf0f15e 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -52,13 +52,27 @@
 #  error "ceval.c must be build with Py_BUILD_CORE define for best performance"
 #endif
 
-#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_GIL_DISABLED)
+#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
 // GH-89279: The MSVC compiler does not inline these static inline functions
 // in PGO build in _PyEval_EvalFrameDefault(), because this function is over
 // the limit of PGO, and that limit cannot be configured.
 // Define them as macros to make sure that they are always inlined by the
 // preprocessor.
-// TODO: implement Py_DECREF macro for Py_GIL_DISABLED
+
+#undef Py_IS_TYPE
+#define Py_IS_TYPE(ob, type) \
+    (_PyObject_CAST(ob)->ob_type == (type))
+
+#undef Py_XDECREF
+#define Py_XDECREF(arg) \
+    do { \
+        PyObject *xop = _PyObject_CAST(arg); \
+        if (xop != NULL) { \
+            Py_DECREF(xop); \
+        } \
+    } while (0)
+
+#ifndef Py_GIL_DISABLED
 
 #undef Py_DECREF
 #define Py_DECREF(arg) \
@@ -74,19 +88,6 @@
         } \
     } while (0)
 
-#undef Py_XDECREF
-#define Py_XDECREF(arg) \
-    do { \
-        PyObject *xop = _PyObject_CAST(arg); \
-        if (xop != NULL) { \
-            Py_DECREF(xop); \
-        } \
-    } while (0)
-
-#undef Py_IS_TYPE
-#define Py_IS_TYPE(ob, type) \
-    (_PyObject_CAST(ob)->ob_type == (type))
-
 #undef _Py_DECREF_SPECIALIZED
 #define _Py_DECREF_SPECIALIZED(arg, dealloc) \
     do { \
@@ -100,6 +101,34 @@
             d(op); \
         } \
     } while (0)
+
+#else // Py_GIL_DISABLED
+
+#undef Py_DECREF
+#define Py_DECREF(arg) \
+    do { \
+        PyObject *op = _PyObject_CAST(arg); \
+        uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \
+        if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \
+            break; \
+        } \
+        _Py_DECREF_STAT_INC(); \
+        if (_Py_IsOwnedByCurrentThread(op)) { \
+            local--; \
+            _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \
+            if (local == 0) { \
+                _Py_MergeZeroLocalRefcount(op); \
+            } \
+        } \
+        else { \
+            _Py_DecRefShared(op); \
+        } \
+    } while (0)
+
+#undef _Py_DECREF_SPECIALIZED
+#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg)
+
+#endif
 #endif