gh-115999: Stop the world when invalidating function versions (#124997)

Stop the world when invalidating function versions

The tier1 interpreter specializes `CALL` instructions based on the values
of certain function attributes (e.g. `__code__`, `__defaults__`). The tier1
interpreter uses function versions to verify that the attributes of a function
during execution of a specialization match those seen during specialization.
A function's version is initialized in `MAKE_FUNCTION` and is invalidated when
any of the critical function attributes are changed. The tier1 interpreter stores
the function version in the inline cache during specialization. A guard is used by
the specialized instruction to verify that the version of the function on the operand
stack matches the cached version (and therefore has all of the expected attributes).
It is assumed that once the guard passes, all attributes will remain unchanged
while executing the rest of the specialized instruction.

Stopping the world when invalidating function versions ensures that all critical
function attributes will remain unchanged after the function version guard passes
in free-threaded builds. It's important to note that this is only true if the remainder
of the specialized instruction does not enter and exit a stop-the-world point.

We will stop the world the first time any of the following function attributes
are mutated:

- defaults
- vectorcall
- kwdefaults
- closure
- code

This should happen rarely and only happens once per function, so the performance
impact on majority of code should be minimal.

Additionally, refactor the API for manipulating function versions to more clearly
match the stated semantics.

diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h
index 6d44e933e8a8cb3ad03291f47a891f60b1938277..c45d281125febb230e710c1ca627ba1599743291 100644 (file)
--- a/Include/internal/pycore_function.h
+++ b/Include/internal/pycore_function.h
@@ -18,6 +18,10 @@ extern PyObject* _PyFunction_Vectorcall(
 
 #define FUNC_MAX_WATCHERS 8
 
+#define FUNC_VERSION_UNSET 0
+#define FUNC_VERSION_CLEARED 1
+#define FUNC_VERSION_FIRST_VALID 2
+
 #define FUNC_VERSION_CACHE_SIZE (1<<12)  /* Must be a power of 2 */
 
 struct _func_version_cache_item {
@@ -41,6 +45,12 @@ struct _py_func_state {
 
 extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr);
 
+static inline int
+_PyFunction_IsVersionValid(uint32_t version)
+{
+    return version >= FUNC_VERSION_FIRST_VALID;
+}
+
 extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func);
 PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version);
 void _PyFunction_ClearCodeByVersion(uint32_t version);

diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h
index e6adb98eb1913095516eea8a694e81a463b2f074..a17ba46966daa129db498ebbaf0365c402a6e563 100644 (file)
--- a/Include/internal/pycore_runtime_init.h
+++ b/Include/internal/pycore_runtime_init.h
@@ -11,6 +11,7 @@ extern "C" {
 #include "pycore_ceval_state.h"   // _PyEval_RUNTIME_PERF_INIT
 #include "pycore_faulthandler.h"  // _faulthandler_runtime_state_INIT
 #include "pycore_floatobject.h"   // _py_float_format_unknown
+#include "pycore_function.h"
 #include "pycore_object.h"        // _PyObject_HEAD_INIT
 #include "pycore_obmalloc_init.h" // _obmalloc_global_state_INIT
 #include "pycore_parser.h"        // _parser_runtime_state_INIT
@@ -243,7 +244,7 @@ extern PyTypeObject _PyExc_MemoryError;
         .dict_state = _dict_state_INIT, \
         .mem_free_queue = _Py_mem_free_queue_INIT(INTERP.mem_free_queue), \
         .func_state = { \
-            .next_version = 1, \
+            .next_version = FUNC_VERSION_FIRST_VALID, \
         }, \
         .types = { \
             .next_version_tag = _Py_TYPE_BASE_VERSION_TAG, \

diff --git a/Objects/funcobject.c b/Objects/funcobject.c
index 98e6766d0ca0d8d340d403216b704c71d79e251a..855d1a2eeca819400c259f71d082dd96c92d6b12 100644 (file)
--- a/Objects/funcobject.c
+++ b/Objects/funcobject.c
@@ -128,7 +128,7 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr)
     op->func_annotate = NULL;
     op->func_typeparams = NULL;
     op->vectorcall = _PyFunction_Vectorcall;
-    op->func_version = 0;
+    op->func_version = FUNC_VERSION_UNSET;
     // NOTE: functions created via FrameConstructor do not use deferred
     // reference counting because they are typically not part of cycles
     // nor accessed by multiple threads.
@@ -207,7 +207,7 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname
     op->func_annotate = NULL;
     op->func_typeparams = NULL;
     op->vectorcall = _PyFunction_Vectorcall;
-    op->func_version = 0;
+    op->func_version = FUNC_VERSION_UNSET;
     if ((code_obj->co_flags & CO_NESTED) == 0) {
         // Use deferred reference counting for top-level functions, but not
         // nested functions because they are more likely to capture variables,
@@ -287,31 +287,59 @@ functions is running.
 
 */
 
+static inline struct _func_version_cache_item *
+get_cache_item(PyInterpreterState *interp, uint32_t version)
+{
+    return interp->func_state.func_version_cache +
+           (version % FUNC_VERSION_CACHE_SIZE);
+}
+
 void
 _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version)
 {
+    assert(func->func_version == FUNC_VERSION_UNSET);
+    assert(version >= FUNC_VERSION_FIRST_VALID);
+    // This should only be called from MAKE_FUNCTION. No code is specialized
+    // based on the version, so we do not need to stop the world to set it.
+    func->func_version = version;
 #ifndef Py_GIL_DISABLED
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    if (func->func_version != 0) {
-        struct _func_version_cache_item *slot =
-            interp->func_state.func_version_cache
-            + (func->func_version % FUNC_VERSION_CACHE_SIZE);
-        if (slot->func == func) {
-            slot->func = NULL;
-            // Leave slot->code alone, there may be use for it.
-        }
-    }
+    struct _func_version_cache_item *slot = get_cache_item(interp, version);
+    slot->func = func;
+    slot->code = func->func_code;
 #endif
-    func->func_version = version;
+}
+
+static void
+func_clear_version(PyInterpreterState *interp, PyFunctionObject *func)
+{
+    if (func->func_version < FUNC_VERSION_FIRST_VALID) {
+        // Version was never set or has already been cleared.
+        return;
+    }
 #ifndef Py_GIL_DISABLED
-    if (version != 0) {
-        struct _func_version_cache_item *slot =
-            interp->func_state.func_version_cache
-            + (version % FUNC_VERSION_CACHE_SIZE);
-        slot->func = func;
-        slot->code = func->func_code;
+    struct _func_version_cache_item *slot =
+        get_cache_item(interp, func->func_version);
+    if (slot->func == func) {
+        slot->func = NULL;
+        // Leave slot->code alone, there may be use for it.
     }
 #endif
+    func->func_version = FUNC_VERSION_CLEARED;
+}
+
+// Called when any of the critical function attributes are changed
+static void
+_PyFunction_ClearVersion(PyFunctionObject *func)
+{
+    if (func->func_version < FUNC_VERSION_FIRST_VALID) {
+        // Version was never set or has already been cleared.
+        return;
+    }
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    _PyEval_StopTheWorld(interp);
+    func_clear_version(interp, func);
+    _PyEval_StartTheWorld(interp);
 }
 
 void
@@ -319,9 +347,7 @@ _PyFunction_ClearCodeByVersion(uint32_t version)
 {
 #ifndef Py_GIL_DISABLED
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    struct _func_version_cache_item *slot =
-        interp->func_state.func_version_cache
-        + (version % FUNC_VERSION_CACHE_SIZE);
+    struct _func_version_cache_item *slot = get_cache_item(interp, version);
     if (slot->code) {
         assert(PyCode_Check(slot->code));
         PyCodeObject *code = (PyCodeObject *)slot->code;
@@ -340,9 +366,7 @@ _PyFunction_LookupByVersion(uint32_t version, PyObject **p_code)
     return NULL;
 #else
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    struct _func_version_cache_item *slot =
-        interp->func_state.func_version_cache
-        + (version % FUNC_VERSION_CACHE_SIZE);
+    struct _func_version_cache_item *slot = get_cache_item(interp, version);
     if (slot->code) {
         assert(PyCode_Check(slot->code));
         PyCodeObject *code = (PyCodeObject *)slot->code;
@@ -431,7 +455,7 @@ PyFunction_SetDefaults(PyObject *op, PyObject *defaults)
     }
     handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS,
                       (PyFunctionObject *) op, defaults);
-    _PyFunction_SetVersion((PyFunctionObject *)op, 0);
+    _PyFunction_ClearVersion((PyFunctionObject *)op);
     Py_XSETREF(((PyFunctionObject *)op)->func_defaults, defaults);
     return 0;
 }
@@ -440,7 +464,7 @@ void
 PyFunction_SetVectorcall(PyFunctionObject *func, vectorcallfunc vectorcall)
 {
     assert(func != NULL);
-    _PyFunction_SetVersion(func, 0);
+    _PyFunction_ClearVersion(func);
     func->vectorcall = vectorcall;
 }
 
@@ -473,7 +497,7 @@ PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults)
     }
     handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS,
                       (PyFunctionObject *) op, defaults);
-    _PyFunction_SetVersion((PyFunctionObject *)op, 0);
+    _PyFunction_ClearVersion((PyFunctionObject *)op);
     Py_XSETREF(((PyFunctionObject *)op)->func_kwdefaults, defaults);
     return 0;
 }
@@ -506,7 +530,7 @@ PyFunction_SetClosure(PyObject *op, PyObject *closure)
                      Py_TYPE(closure)->tp_name);
         return -1;
     }
-    _PyFunction_SetVersion((PyFunctionObject *)op, 0);
+    _PyFunction_ClearVersion((PyFunctionObject *)op);
     Py_XSETREF(((PyFunctionObject *)op)->func_closure, closure);
     return 0;
 }
@@ -658,7 +682,7 @@ func_set_code(PyObject *self, PyObject *value, void *Py_UNUSED(ignored))
     }
 
     handle_func_event(PyFunction_EVENT_MODIFY_CODE, op, value);
-    _PyFunction_SetVersion(op, 0);
+    _PyFunction_ClearVersion(op);
     Py_XSETREF(op->func_code, Py_NewRef(value));
     return 0;
 }
@@ -744,7 +768,7 @@ func_set_defaults(PyObject *self, PyObject *value, void *Py_UNUSED(ignored))
     }
 
     handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS, op, value);
-    _PyFunction_SetVersion(op, 0);
+    _PyFunction_ClearVersion(op);
     Py_XSETREF(op->func_defaults, Py_XNewRef(value));
     return 0;
 }
@@ -787,7 +811,7 @@ func_set_kwdefaults(PyObject *self, PyObject *value, void *Py_UNUSED(ignored))
     }
 
     handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS, op, value);
-    _PyFunction_SetVersion(op, 0);
+    _PyFunction_ClearVersion(op);
     Py_XSETREF(op->func_kwdefaults, Py_XNewRef(value));
     return 0;
 }
@@ -1030,7 +1054,7 @@ static int
 func_clear(PyObject *self)
 {
     PyFunctionObject *op = _PyFunction_CAST(self);
-    _PyFunction_SetVersion(op, 0);
+    func_clear_version(_PyInterpreterState_GET(), op);
     Py_CLEAR(op->func_globals);
     Py_CLEAR(op->func_builtins);
     Py_CLEAR(op->func_module);
@@ -1068,7 +1092,6 @@ func_dealloc(PyObject *self)
     if (op->func_weakreflist != NULL) {
         PyObject_ClearWeakRefs((PyObject *) op);
     }
-    _PyFunction_SetVersion(op, 0);
     (void)func_clear((PyObject*)op);
     // These aren't cleared by func_clear().
     Py_DECREF(op->func_code);

diff --git a/Python/specialize.c b/Python/specialize.c
index d8bff39511cf127d99cc257357f5020025f541fd..4b33a468733d6badfd0d3d31f2b228c9a64dc2d4 100644 (file)
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -1599,7 +1599,7 @@ function_get_version(PyObject *o, int opcode)
     assert(Py_IS_TYPE(o, &PyFunction_Type));
     PyFunctionObject *func = (PyFunctionObject *)o;
     uint32_t version = _PyFunction_GetVersionForCurrentState(func);
-    if (version == 0) {
+    if (!_PyFunction_IsVersionValid(version)) {
         SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
         return 0;
     }
@@ -1692,7 +1692,7 @@ _Py_Specialize_BinarySubscr(
             goto fail;
         }
         uint32_t version = _PyFunction_GetVersionForCurrentState(func);
-        if (version == 0) {
+        if (!_PyFunction_IsVersionValid(version)) {
             SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_VERSIONS);
             goto fail;
         }
@@ -1977,7 +1977,7 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs,
         argcount = code->co_argcount;
     }
     int version = _PyFunction_GetVersionForCurrentState(func);
-    if (version == 0) {
+    if (!_PyFunction_IsVersionValid(version)) {
         SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS);
         return -1;
     }
@@ -2009,7 +2009,7 @@ specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs,
         return -1;
     }
     int version = _PyFunction_GetVersionForCurrentState(func);
-    if (version == 0) {
+    if (!_PyFunction_IsVersionValid(version)) {
         SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS);
         return -1;
     }