From: Victor Stinner Date: Wed, 13 Oct 2021 12:09:13 +0000 (+0200) Subject: pycore_pystate.h no longer redefines PyThreadState_GET() (GH-28921) X-Git-Tag: v3.11.0a2~220 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=7cdc2a0f4b785327ad9d55312409a06e554df3d5;p=thirdparty%2FPython%2Fcpython.git pycore_pystate.h no longer redefines PyThreadState_GET() (GH-28921) Redefining the PyThreadState_GET() macro in pycore_pystate.h is useless since it doesn't affect files not including it. Either use _PyThreadState_GET() directly, or don't use pycore_pystate.h internal C API. For example, the _testcapi extension don't use the internal C API, but use the public PyThreadState_Get() function instead. Replace PyThreadState_Get() with _PyThreadState_GET(). The _PyThreadState_GET() macro is more efficient than PyThreadState_Get() and PyThreadState_GET() function calls which call fail with a fatal Python error. posixmodule.c and _ctypes extension now include before pycore header files (like pycore_call.h). _PyTraceback_Add() now uses _PyErr_Fetch()/_PyErr_Restore() instead of PyErr_Fetch()/PyErr_Restore(). The _decimal and _xxsubinterpreters extensions are now built with the Py_BUILD_CORE_MODULE macro defined to get access to the internal C API. --- diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 28a4194db748..933d714f1337 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_pystate.h" // _PyThreadState_GET() + PyAPI_FUNC(PyObject *) _PyObject_Call_Prepend( PyThreadState *tstate, PyObject *callable, @@ -36,7 +38,7 @@ _PyObject_CallNoArgsTstate(PyThreadState *tstate, PyObject *func) { // Private static inline function variant of public PyObject_CallNoArgs() static inline PyObject * _PyObject_CallNoArgs(PyObject *func) { - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_GET(); return _PyObject_VectorcallTstate(tstate, func, NULL, 0, NULL); } diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index f2acf10df8c2..53580b99d33a 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -12,7 +12,8 @@ extern "C" { struct pyruntimestate; struct _ceval_runtime_state; -#include "pycore_interp.h" /* PyInterpreterState.eval_frame */ +#include "pycore_interp.h" // PyInterpreterState.eval_frame +#include "pycore_pystate.h" // _PyThreadState_GET() extern void _Py_FinishPendingCalls(PyThreadState *tstate); extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *); @@ -93,7 +94,7 @@ static inline int _Py_EnterRecursiveCall(PyThreadState *tstate, } static inline int _Py_EnterRecursiveCall_inline(const char *where) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); return _Py_EnterRecursiveCall(tstate, where); } @@ -104,7 +105,7 @@ static inline void _Py_LeaveRecursiveCall(PyThreadState *tstate) { } static inline void _Py_LeaveRecursiveCall_inline(void) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); _Py_LeaveRecursiveCall(tstate); } diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index aef318989aa6..1b74e5ffb047 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -82,7 +82,7 @@ _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) The caller must hold the GIL. - See also PyThreadState_Get() and PyThreadState_GET(). */ + See also PyThreadState_Get() and _PyThreadState_UncheckedGet(). */ static inline PyThreadState* _PyThreadState_GET(void) { @@ -93,10 +93,6 @@ _PyThreadState_GET(void) #endif } -/* Redefine PyThreadState_GET() as an alias to _PyThreadState_GET() */ -#undef PyThreadState_GET -#define PyThreadState_GET() _PyThreadState_GET() - PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalError_TstateNULL(const char *func); static inline void diff --git a/Include/pystate.h b/Include/pystate.h index bae440778b26..d81f42a3efe3 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -66,18 +66,10 @@ PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *); The caller must hold the GIL. - See also PyThreadState_GET() and _PyThreadState_GET(). */ + See also _PyThreadState_UncheckedGet() and _PyThreadState_GET(). */ PyAPI_FUNC(PyThreadState *) PyThreadState_Get(void); -/* Get the current Python thread state. - - Macro using PyThreadState_Get() or _PyThreadState_GET() depending if - pycore_pystate.h is included or not (this header redefines the macro). - - If PyThreadState_Get() is used, issue a fatal error if the current thread - state is NULL. - - See also PyThreadState_Get() and _PyThreadState_GET(). */ +// Alias to PyThreadState_Get() #define PyThreadState_GET() PyThreadState_Get() PyAPI_FUNC(PyThreadState *) PyThreadState_Swap(PyThreadState *); diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 56079b0277d1..adc5ff9f796c 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_pyerrors.h" // _PyErr_ClearExcState() +#include "pycore_pystate.h" // _PyThreadState_GET() #include // offsetof() @@ -225,7 +226,7 @@ get_running_loop(PyObject **loop) { PyObject *rl; - PyThreadState *ts = PyThreadState_Get(); + PyThreadState *ts = _PyThreadState_GET(); uint64_t ts_id = PyThreadState_GetID(ts); if (ts_id == cached_running_holder_tsid && cached_running_holder != NULL) { // Fast path, check the cache. @@ -287,7 +288,7 @@ set_running_loop(PyObject *loop) { PyObject *ts_dict = NULL; - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_GET(); if (tstate != NULL) { ts_dict = _PyThreadState_GetDict(tstate); // borrowed } diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 2567067f0b39..f8940fdbed24 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -102,12 +102,16 @@ bytes(cdata) #define PY_SSIZE_T_CLEAN #include "Python.h" +// windows.h must be included before pycore internal headers +#ifdef MS_WIN32 +# include +#endif + #include "pycore_call.h" // _PyObject_CallNoArgs() #include "structmember.h" // PyMemberDef #include #ifdef MS_WIN32 -#include #include #ifndef IS_INTRESOURCE #define IS_INTRESOURCE(x) (((size_t)(x) >> 16) == 0) diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 18b010493628..c24f04ce3a0f 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -1,13 +1,15 @@ #include "Python.h" +// windows.h must be included before pycore internal headers +#ifdef MS_WIN32 +# include +#endif + #include "pycore_call.h" // _PyObject_CallNoArgs() #include "frameobject.h" #include #include -#ifdef MS_WIN32 -#include -#endif #include "ctypes.h" /**************************************************************/ diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 6788aee78a2d..b7585bc8a9ff 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -1,11 +1,13 @@ #include "Python.h" +// windows.h must be included before pycore internal headers +#ifdef MS_WIN32 +# include +#endif + #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_call.h" // _PyObject_CallNoArgs() #include -#ifdef MS_WIN32 -#include -#endif #include "ctypes.h" diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index ea3c58b88e70..43669d7152a7 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -1,9 +1,13 @@ #include "Python.h" +// windows.h must be included before pycore internal headers +#ifdef MS_WIN32 +# include +#endif + #include "pycore_call.h" // _PyObject_CallNoArgs() #include #ifdef MS_WIN32 -#include -#include +# include #endif #include "ctypes.h" diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index e2979a577c96..dd876f200365 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -27,6 +27,7 @@ #include +#include "pycore_pystate.h" // _PyThreadState_GET() #include "longintrepr.h" #include "complexobject.h" #include "mpdecimal.h" @@ -1511,18 +1512,20 @@ static PyGetSetDef context_getsets [] = static PyObject * current_context_from_dict(void) { - PyObject *dict; - PyObject *tl_context; - PyThreadState *tstate; + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + // The caller must hold the GIL + _Py_EnsureTstateNotNULL(tstate); +#endif - dict = PyThreadState_GetDict(); + PyObject *dict = _PyThreadState_GetDict(tstate); if (dict == NULL) { PyErr_SetString(PyExc_RuntimeError, "cannot get thread state"); return NULL; } - tl_context = PyDict_GetItemWithError(dict, tls_context_key); + PyObject *tl_context = PyDict_GetItemWithError(dict, tls_context_key); if (tl_context != NULL) { /* We already have a thread local context. */ CONTEXT_CHECK(tl_context); @@ -1548,11 +1551,8 @@ current_context_from_dict(void) /* Cache the context of the current thread, assuming that it * will be accessed several times before a thread switch. */ - tstate = PyThreadState_GET(); - if (tstate) { - cached_context = (PyDecContextObject *)tl_context; - cached_context->tstate = tstate; - } + cached_context = (PyDecContextObject *)tl_context; + cached_context->tstate = tstate; /* Borrowed reference with refcount==1 */ return tl_context; @@ -1562,9 +1562,7 @@ current_context_from_dict(void) static PyObject * current_context(void) { - PyThreadState *tstate; - - tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); if (cached_context && cached_context->tstate == tstate) { return (PyObject *)cached_context; } diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 097d0eff2602..2e27afcea1b7 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_pystate.h" // _PyThreadState_GET() #include "rotatingtree.h" /************************************************************/ @@ -672,7 +673,7 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); if (_PyEval_SetProfile(tstate, profiler_callback, (PyObject*)self) < 0) { return NULL; } @@ -706,7 +707,7 @@ Stop collecting profiling information.\n\ static PyObject* profiler_disable(ProfilerObject *self, PyObject* noarg) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { return NULL; } @@ -743,7 +744,7 @@ static void profiler_dealloc(ProfilerObject *op) { if (op->flags & POF_ENABLED) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { PyErr_WriteUnraisable((PyObject *)op); } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index d5616fd59c6e..3ba939651a41 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -18,7 +18,8 @@ #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() -#include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() +#include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() +#include "pycore_pystate.h" // _PyThreadState_GET() static PyObject * @@ -31,7 +32,7 @@ get_configs(PyObject *self, PyObject *Py_UNUSED(args)) static PyObject* get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args)) { - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_GET(); /* subtract one to ignore the frame of the get_recursion_depth() call */ return PyLong_FromLong(tstate->recursion_depth - 1); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index ff858386b426..39b116afcaa3 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1319,7 +1319,7 @@ static PyObject * thread__set_sentinel(PyObject *module, PyObject *Py_UNUSED(ignored)) { PyObject *wr; - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_GET(); lockobject *lock; if (tstate->on_delete_data != NULL) { diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 11b55f7f0c7d..939114e60cae 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -5,6 +5,7 @@ #include "Python.h" #include "frameobject.h" #include "pycore_frame.h" +#include "pycore_pystate.h" // _PyThreadState_GET() #include "interpreteridobject.h" @@ -2017,7 +2018,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) } // Create and initialize the new interpreter. - PyThreadState *save_tstate = PyThreadState_Get(); + PyThreadState *save_tstate = _PyThreadState_GET(); // XXX Possible GILState issues? PyThreadState *tstate = _Py_NewInterpreter(isolated); PyThreadState_Swap(save_tstate); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ada1a5865aae..9c174ee6b18d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10,31 +10,25 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_fileutils.h" // _Py_closerange() -#include "pycore_moduleobject.h" // _PyModule_GetState() +// Include before pycore internal headers. FSCTL_GET_REPARSE_POINT +// is not exported by if the WIN32_LEAN_AND_MEAN macro is defined, +// whereas pycore_condvar.h defines the WIN32_LEAN_AND_MEAN macro. #ifdef MS_WINDOWS - /* include early to avoid conflict with pycore_condvar.h: - - #define WIN32_LEAN_AND_MEAN - #include - - FSCTL_GET_REPARSE_POINT is not exported with WIN32_LEAN_AND_MEAN. */ # include # include #endif -#if !defined(EX_OK) && defined(EXIT_SUCCESS) -#define EX_OK EXIT_SUCCESS -#endif - #ifdef __VXWORKS__ # include "pycore_bitutils.h" // _Py_popcount32() #endif +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_fileutils.h" // _Py_closerange() +#include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_ceval.h" // _PyEval_ReInitThreads() #include "pycore_import.h" // _PyImport_ReInitLock() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_pystate.h" // _PyInterpreterState_GET() + #include "structmember.h" // PyMemberDef #ifndef MS_WINDOWS # include "posixmodule.h" @@ -42,6 +36,10 @@ # include "winreparse.h" #endif +#if !defined(EX_OK) && defined(EXIT_SUCCESS) +# define EX_OK EXIT_SUCCESS +#endif + /* On android API level 21, 'AT_EACCESS' is not declared although * HAVE_FACCESSAT is defined. */ #ifdef __ANDROID__ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0f56e59bc518..544e8a4aa221 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3332,7 +3332,7 @@ type_vectorcall(PyObject *metatype, PyObject *const *args, } /* In other (much less common) cases, fall back to more flexible calling conventions. */ - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); return _PyObject_MakeTpCall(tstate, metatype, args, nargs, kwnames); } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d07ba38b698c..9e3b25c59a55 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -213,7 +213,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, goto error; } PyFrameConstructor *f = PyFunction_AS_FRAME_CONSTRUCTOR(func); - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); cell = _PyEval_Vector(tstate, f, ns, NULL, 0, NULL); if (cell != NULL) { if (bases != orig_bases) { diff --git a/Python/ceval.c b/Python/ceval.c index 2d617a636488..f4cacd84b66c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1064,7 +1064,7 @@ static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **); PyObject * PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); if (locals == NULL) { locals = globals; } diff --git a/Python/context.c b/Python/context.c index 9560fb3f6676..ad47992d9e3c 100644 --- a/Python/context.c +++ b/Python/context.c @@ -728,7 +728,7 @@ static int contextvar_set(PyContextVar *var, PyObject *val) { var->var_cached = NULL; - PyThreadState *ts = PyThreadState_Get(); + PyThreadState *ts = _PyThreadState_GET(); PyContext *ctx = context_get(); if (ctx == NULL) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index bfddc1922d25..c67a9b7fbfa2 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -472,7 +472,7 @@ interpreter_update_config(PyThreadState *tstate, int only_update_path_config) int _PyInterpreterState_SetConfig(const PyConfig *src_config) { - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_GET(); int res = -1; PyConfig config; diff --git a/Python/pystate.c b/Python/pystate.c index f02de926c6f0..ee9507c7dd80 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1671,8 +1671,11 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) int _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) { - // PyThreadState_Get() aborts if tstate is NULL. - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + // The caller must hold the GIL + _Py_EnsureTstateNotNULL(tstate); +#endif PyInterpreterState *interp = tstate->interp; // Reset data before re-populating. diff --git a/Python/traceback.c b/Python/traceback.c index 06b40bbbdc9f..3ea1db578b43 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -6,11 +6,13 @@ #include "code.h" // PyCode_Addr2Line etc #include "pycore_interp.h" // PyInterpreterState.gc #include "frameobject.h" // PyFrame_GetBack() -#include "pycore_frame.h" // _PyFrame_GetCode() -#include "pycore_pyarena.h" // _PyArena_Free() #include "pycore_ast.h" // asdl_seq_* #include "pycore_compile.h" // _PyAST_Optimize +#include "pycore_frame.h" // _PyFrame_GetCode() #include "pycore_parser.h" // _PyParser_ASTFromString +#include "pycore_pyarena.h" // _PyArena_Free() +#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pystate.h" // _PyThreadState_GET() #include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() #include "structmember.h" // PyMemberDef #include "osdefs.h" // SEP @@ -267,11 +269,12 @@ void _PyTraceback_Add(const char *funcname, const char *filename, int lineno) PyCodeObject *code; PyFrameObject *frame; PyObject *exc, *val, *tb; + PyThreadState *tstate = _PyThreadState_GET(); /* Save and clear the current exception. Python functions must not be called with an exception set. Calling Python functions happens when the codec of the filesystem encoding is implemented in pure Python. */ - PyErr_Fetch(&exc, &val, &tb); + _PyErr_Fetch(tstate, &exc, &val, &tb); globals = PyDict_New(); if (!globals) @@ -281,14 +284,14 @@ void _PyTraceback_Add(const char *funcname, const char *filename, int lineno) Py_DECREF(globals); goto error; } - frame = PyFrame_New(PyThreadState_Get(), code, globals, NULL); + frame = PyFrame_New(tstate, code, globals, NULL); Py_DECREF(globals); Py_DECREF(code); if (!frame) goto error; frame->f_lineno = lineno; - PyErr_Restore(exc, val, tb); + _PyErr_Restore(tstate, exc, val, tb); PyTraceBack_Here(frame); Py_DECREF(frame); return; diff --git a/setup.py b/setup.py index c6290eebed30..039c96b0e4d7 100644 --- a/setup.py +++ b/setup.py @@ -1000,7 +1000,8 @@ class PyBuildExt(build_ext): self.add(Extension('syslog', ['syslogmodule.c'])) # Python interface to subinterpreter C-API. - self.add(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'])) + self.add(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) # # Here ends the simple stuff. From here on, modules need certain @@ -2310,7 +2311,7 @@ class PyBuildExt(build_ext): def detect_decimal(self): # Stefan Krah's _decimal module - extra_compile_args = [] + extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] undef_macros = [] if '--with-system-libmpdec' in sysconfig.get_config_var("CONFIG_ARGS"): include_dirs = []