From: Eric Snow Date: Fri, 15 Sep 2017 22:35:20 +0000 (-0600) Subject: bpo-28411: Support other mappings in PyInterpreterState.modules. (#3593) X-Git-Tag: v3.7.0a1~18 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=3f9eee6eb4b25fe1926eaa5f00e02344b126f54d;p=thirdparty%2FPython%2Fcpython.git bpo-28411: Support other mappings in PyInterpreterState.modules. (#3593) The concrete PyDict_* API is used to interact with PyInterpreterState.modules in a number of places. This isn't compatible with all dict subclasses, nor with other Mapping implementations. This patch switches the concrete API usage to the corresponding abstract API calls. We also add a PyImport_GetModule() function (and some other helpers) to reduce a bunch of code duplication. --- diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 7c16ece05866..8cdc256e7c9e 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -204,6 +204,13 @@ Importing Modules Return the dictionary used for the module administration (a.k.a. ``sys.modules``). Note that this is a per-interpreter variable. +.. c:function:: PyObject* PyImport_GetModule(PyObject *name) + + Return the already imported module with the given name. If the + module has not been imported yet then returns NULL but does not set + an error. Returns NULL and sets an error if the lookup failed. + + .. versionadded:: 3.7 .. c:function:: PyObject* PyImport_GetImporter(PyObject *path) diff --git a/Include/import.h b/Include/import.h index 95c52b0bfba1..c30f3ea33942 100644 --- a/Include/import.h +++ b/Include/import.h @@ -38,8 +38,14 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( ); #endif PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); +PyAPI_FUNC(PyObject *) PyImport_GetModule(PyObject *name); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *); +PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(struct _Py_Identifier *name); +PyAPI_FUNC(PyObject *) _PyImport_AddModuleObject(PyObject *name, + PyObject *modules); +PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); +PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-11-09-11-20.bpo-28411.Ax91lz.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-11-09-11-20.bpo-28411.Ax91lz.rst new file mode 100644 index 000000000000..a45f9d86248e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-11-09-11-20.bpo-28411.Ax91lz.rst @@ -0,0 +1,4 @@ +Switch to the abstract API when dealing with ``PyInterpreterState.modules``. +This allows later support for all dict subclasses and other Mapping +implementations. Also add a ``PyImport_GetModule()`` function to reduce +a bunch of duplicated code. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index fb69f14b53db..d531dee2565f 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1649,13 +1649,40 @@ getattribute(PyObject *obj, PyObject *name, int allow_qualname) return attr; } +static int +_checkmodule(PyObject *module_name, PyObject *module, + PyObject *global, PyObject *dotted_path) +{ + if (module == Py_None) { + return -1; + } + if (PyUnicode_Check(module_name) && + _PyUnicode_EqualToASCIIString(module_name, "__main__")) { + return -1; + } + + PyObject *candidate = get_deep_attribute(module, dotted_path, NULL); + if (candidate == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + return -1; + } + if (candidate != global) { + Py_DECREF(candidate); + return -1; + } + Py_DECREF(candidate); + return 0; +} + static PyObject * whichmodule(PyObject *global, PyObject *dotted_path) { PyObject *module_name; - PyObject *modules_dict; - PyObject *module; + PyObject *module = NULL; Py_ssize_t i; + PyObject *modules; _Py_IDENTIFIER(__module__); _Py_IDENTIFIER(modules); _Py_IDENTIFIER(__main__); @@ -1678,35 +1705,48 @@ whichmodule(PyObject *global, PyObject *dotted_path) assert(module_name == NULL); /* Fallback on walking sys.modules */ - modules_dict = _PySys_GetObjectId(&PyId_modules); - if (modules_dict == NULL) { + modules = _PySys_GetObjectId(&PyId_modules); + if (modules == NULL) { PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); return NULL; } - - i = 0; - while (PyDict_Next(modules_dict, &i, &module_name, &module)) { - PyObject *candidate; - if (PyUnicode_Check(module_name) && - _PyUnicode_EqualToASCIIString(module_name, "__main__")) - continue; - if (module == Py_None) - continue; - - candidate = get_deep_attribute(module, dotted_path, NULL); - if (candidate == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) + if (PyDict_CheckExact(modules)) { + i = 0; + while (PyDict_Next(modules, &i, &module_name, &module)) { + if (_checkmodule(module_name, module, global, dotted_path) == 0) { + Py_INCREF(module_name); + return module_name; + } + if (PyErr_Occurred()) { return NULL; - PyErr_Clear(); - continue; + } } - - if (candidate == global) { - Py_INCREF(module_name); - Py_DECREF(candidate); - return module_name; + } + else { + PyObject *iterator = PyObject_GetIter(modules); + if (iterator == NULL) { + return NULL; } - Py_DECREF(candidate); + while ((module_name = PyIter_Next(iterator))) { + module = PyObject_GetItem(modules, module_name); + if (module == NULL) { + Py_DECREF(module_name); + Py_DECREF(iterator); + return NULL; + } + if (_checkmodule(module_name, module, global, dotted_path) == 0) { + Py_DECREF(module); + Py_DECREF(iterator); + return module_name; + } + Py_DECREF(module); + Py_DECREF(module_name); + if (PyErr_Occurred()) { + Py_DECREF(iterator); + return NULL; + } + } + Py_DECREF(iterator); } /* If no module is found, use __main__. */ @@ -6424,9 +6464,7 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, /*[clinic end generated code: output=becc08d7f9ed41e3 input=e2e6a865de093ef4]*/ { PyObject *global; - PyObject *modules_dict; PyObject *module; - _Py_IDENTIFIER(modules); /* Try to map the old names used in Python 2.x to the new ones used in Python 3.x. We do this only with old pickle protocols and when the @@ -6483,25 +6521,16 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, } } - modules_dict = _PySys_GetObjectId(&PyId_modules); - if (modules_dict == NULL) { - PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); - return NULL; - } - - module = PyDict_GetItemWithError(modules_dict, module_name); + module = PyImport_GetModule(module_name); if (module == NULL) { if (PyErr_Occurred()) return NULL; module = PyImport_Import(module_name); if (module == NULL) return NULL; - global = getattribute(module, global_name, self->proto >= 4); - Py_DECREF(module); - } - else { - global = getattribute(module, global_name, self->proto >= 4); } + global = getattribute(module, global_name, self->proto >= 4); + Py_DECREF(module); return global; } diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index d9cfa3e20859..c8a01d4e088e 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1643,7 +1643,6 @@ MODULE_INITFUNC(void) PyObject *errors_module; PyObject *modelmod_name; PyObject *model_module; - PyObject *sys_modules; PyObject *tmpnum, *tmpstr; PyObject *codes_dict; PyObject *rev_codes_dict; @@ -1693,11 +1692,6 @@ MODULE_INITFUNC(void) */ PyModule_AddStringConstant(m, "native_encoding", "UTF-8"); - sys_modules = PySys_GetObject("modules"); - if (sys_modules == NULL) { - Py_DECREF(m); - return NULL; - } d = PyModule_GetDict(m); if (d == NULL) { Py_DECREF(m); @@ -1707,7 +1701,7 @@ MODULE_INITFUNC(void) if (errors_module == NULL) { errors_module = PyModule_New(MODULE_NAME ".errors"); if (errors_module != NULL) { - PyDict_SetItem(sys_modules, errmod_name, errors_module); + _PyImport_SetModule(errmod_name, errors_module); /* gives away the reference to errors_module */ PyModule_AddObject(m, "errors", errors_module); } @@ -1717,7 +1711,7 @@ MODULE_INITFUNC(void) if (model_module == NULL) { model_module = PyModule_New(MODULE_NAME ".model"); if (model_module != NULL) { - PyDict_SetItem(sys_modules, modelmod_name, model_module); + _PyImport_SetModule(modelmod_name, model_module); /* gives away the reference to model_module */ PyModule_AddObject(m, "model", model_module); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9ebbb21ef8d3..662c493ff2dd 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3913,10 +3913,8 @@ import_copyreg(void) by storing a reference to the cached module in a static variable, but this broke when multiple embedded interpreters were in use (see issue #17408 and #19088). */ - PyObject *modules = PyImport_GetModuleDict(); - copyreg_module = PyDict_GetItemWithError(modules, copyreg_str); + copyreg_module = PyImport_GetModule(copyreg_str); if (copyreg_module != NULL) { - Py_INCREF(copyreg_module); return copyreg_module; } if (PyErr_Occurred()) { diff --git a/Python/_warnings.c b/Python/_warnings.c index ba004859df82..f6688b040679 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -38,7 +38,6 @@ static PyObject * get_warnings_attr(const char *attr, int try_import) { static PyObject *warnings_str = NULL; - PyObject *all_modules; PyObject *warnings_module, *obj; if (warnings_str == NULL) { @@ -58,13 +57,9 @@ get_warnings_attr(const char *attr, int try_import) } } else { - all_modules = PyImport_GetModuleDict(); - - warnings_module = PyDict_GetItem(all_modules, warnings_str); + warnings_module = PyImport_GetModule(warnings_str); if (warnings_module == NULL) return NULL; - - Py_INCREF(warnings_module); } if (!PyObject_HasAttrString(warnings_module, attr)) { diff --git a/Python/ceval.c b/Python/ceval.c index 5b810f29d122..8cc5094a3f46 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4935,13 +4935,12 @@ import_from(PyObject *v, PyObject *name) Py_DECREF(pkgname); return NULL; } - x = PyDict_GetItem(PyImport_GetModuleDict(), fullmodname); + x = PyImport_GetModule(fullmodname); Py_DECREF(fullmodname); if (x == NULL) { goto error; } Py_DECREF(pkgname); - Py_INCREF(x); return x; error: pkgpath = PyModule_GetFilenameObject(v); diff --git a/Python/import.c b/Python/import.c index 7aa7a1bdf798..5e841ca782d2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -291,8 +291,9 @@ PyObject * PyImport_GetModuleDict(void) { PyInterpreterState *interp = PyThreadState_GET()->interp; - if (interp->modules == NULL) + if (interp->modules == NULL) { Py_FatalError("PyImport_GetModuleDict: no module dictionary!"); + } return interp->modules; } @@ -308,6 +309,55 @@ _PyImport_IsInitialized(PyInterpreterState *interp) return 1; } +PyObject * +_PyImport_GetModuleId(struct _Py_Identifier *nameid) +{ + PyObject *name = _PyUnicode_FromId(nameid); /* borrowed */ + if (name == NULL) { + return NULL; + } + return PyImport_GetModule(name); +} + +int +_PyImport_SetModule(PyObject *name, PyObject *m) +{ + PyObject *modules = PyImport_GetModuleDict(); + return PyObject_SetItem(modules, name, m); +} + +int +_PyImport_SetModuleString(const char *name, PyObject *m) +{ + PyObject *modules = PyImport_GetModuleDict(); + return PyMapping_SetItemString(modules, name, m); +} + +PyObject * +PyImport_GetModule(PyObject *name) +{ + PyObject *m; + PyObject *modules = PyImport_GetModuleDict(); + if (modules == NULL) { + PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); + return NULL; + } + Py_INCREF(modules); + if (PyDict_CheckExact(modules)) { + m = PyDict_GetItemWithError(modules, name); /* borrowed */ + Py_XINCREF(m); + } + else { + m = PyObject_GetItem(modules, name); + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + } + } + Py_DECREF(modules); + return m; +} + + /* List of names to clear in sys */ static const char * const sys_deletes[] = { "path", "argv", "ps1", "ps2", @@ -387,21 +437,51 @@ PyImport_Cleanup(void) if (PyErr_Occurred()) \ PyErr_Clear(); \ } +#define CLEAR_MODULE(name, mod) \ + if (PyModule_Check(mod)) { \ + if (Py_VerboseFlag && PyUnicode_Check(name)) \ + PySys_FormatStderr("# cleanup[2] removing %U\n", name); \ + STORE_MODULE_WEAKREF(name, mod); \ + PyObject_SetItem(modules, name, Py_None); \ + } /* Remove all modules from sys.modules, hoping that garbage collection can reclaim most of them. */ - pos = 0; - while (PyDict_Next(modules, &pos, &key, &value)) { - if (PyModule_Check(value)) { - if (Py_VerboseFlag && PyUnicode_Check(key)) - PySys_FormatStderr("# cleanup[2] removing %U\n", key); - STORE_MODULE_WEAKREF(key, value); - PyDict_SetItem(modules, key, Py_None); + if (PyDict_CheckExact(modules)) { + pos = 0; + while (PyDict_Next(modules, &pos, &key, &value)) { + CLEAR_MODULE(key, value); + } + } + else { + PyObject *iterator = PyObject_GetIter(modules); + if (iterator == NULL) { + PyErr_Clear(); + } + else { + while ((key = PyIter_Next(iterator))) { + value = PyObject_GetItem(modules, key); + if (value == NULL) { + PyErr_Clear(); + continue; + } + CLEAR_MODULE(key, value); + Py_DECREF(value); + Py_DECREF(key); + } + Py_DECREF(iterator); } } /* Clear the modules dict. */ - PyDict_Clear(modules); + if (PyDict_CheckExact(modules)) { + PyDict_Clear(modules); + } + else { + _Py_IDENTIFIER(clear); + if (_PyObject_CallMethodId(modules, &PyId_clear, "") == NULL) + PyErr_Clear(); + } /* Restore the original builtins dict, to ensure that any user data gets cleared. */ dict = PyDict_Copy(interp->builtins); @@ -541,10 +621,10 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyErr_BadInternalCall(); return -1; } - if (PyDict_SetItem(modules, name, mod) < 0) + if (PyObject_SetItem(modules, name, mod) < 0) return -1; if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItem(modules, name); + PyMapping_DelItem(modules, name); return -1; } if (def->m_size == -1) { @@ -625,14 +705,14 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename, mod = def->m_base.m_init(); if (mod == NULL) return NULL; - if (PyDict_SetItem(modules, name, mod) == -1) { + if (PyObject_SetItem(modules, name, mod) == -1) { Py_DECREF(mod); return NULL; } Py_DECREF(mod); } if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItem(modules, name); + PyMapping_DelItem(modules, name); Py_DECREF(mod); return NULL; } @@ -672,18 +752,27 @@ PyObject * _PyImport_AddModuleObject(PyObject *name, PyObject *modules) { PyObject *m; - - if ((m = PyDict_GetItemWithError(modules, name)) != NULL && - PyModule_Check(m)) { - return m; + if (PyDict_CheckExact(modules)) { + m = PyDict_GetItemWithError(modules, name); + } + else { + m = PyObject_GetItem(modules, name); + // For backward-comaptibility we copy the behavior + // of PyDict_GetItemWithError(). + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + } } if (PyErr_Occurred()) { return NULL; } + if (m != NULL && PyModule_Check(m)) { + return m; + } m = PyModule_NewObject(name); if (m == NULL) return NULL; - if (PyDict_SetItem(modules, name, m) != 0) { + if (PyObject_SetItem(modules, name, m) != 0) { Py_DECREF(m); return NULL; } @@ -710,11 +799,13 @@ static void remove_module(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_GetItem(modules, name) == NULL) - return; - if (PyDict_DelItem(modules, name) < 0) + if (PyMapping_DelItem(modules, name) < 0) { + if (!PyMapping_HasKey(modules, name)) { + return; + } Py_FatalError("import: deleting existing key in" "sys.modules failed"); + } } @@ -823,7 +914,6 @@ module_dict_for_exec(PyObject *name) static PyObject * exec_code_in_module(PyObject *name, PyObject *module_dict, PyObject *code_object) { - PyObject *modules = PyImport_GetModuleDict(); PyObject *v, *m; v = PyEval_EvalCode(code_object, module_dict, module_dict); @@ -833,15 +923,14 @@ exec_code_in_module(PyObject *name, PyObject *module_dict, PyObject *code_object } Py_DECREF(v); - if ((m = PyDict_GetItem(modules, name)) == NULL) { + m = PyImport_GetModule(name); + if (m == NULL) { PyErr_Format(PyExc_ImportError, "Loaded module %R not found in sys.modules", name); return NULL; } - Py_INCREF(m); - return m; } @@ -1540,8 +1629,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, Py_INCREF(abs_name); } - PyObject *modules = PyImport_GetModuleDict(); - mod = PyDict_GetItem(modules, abs_name); + mod = PyImport_GetModule(abs_name); if (mod != NULL && mod != Py_None) { _Py_IDENTIFIER(__spec__); _Py_IDENTIFIER(_initializing); @@ -1550,7 +1638,6 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *spec; int initializing = 0; - Py_INCREF(mod); /* Optimization: only call _bootstrap._lock_unlock_module() if __spec__._initializing is true. NOTE: because of this, initializing must be set *before* @@ -1579,6 +1666,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } } else { + Py_XDECREF(mod); mod = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__find_and_load, abs_name, interp->import_func, NULL); @@ -1628,8 +1716,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, goto error; } - PyObject *modules = PyImport_GetModuleDict(); - final_mod = PyDict_GetItem(modules, to_return); + final_mod = PyImport_GetModule(to_return); Py_DECREF(to_return); if (final_mod == NULL) { PyErr_Format(PyExc_KeyError, @@ -1637,7 +1724,6 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, to_return); goto error; } - Py_INCREF(final_mod); } } else { @@ -1682,19 +1768,16 @@ PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals PyObject * PyImport_ReloadModule(PyObject *m) { + _Py_IDENTIFIER(imp); _Py_IDENTIFIER(reload); PyObject *reloaded_module = NULL; - PyObject *modules = PyImport_GetModuleDict(); - PyObject *imp = PyDict_GetItemString(modules, "imp"); + PyObject *imp = _PyImport_GetModuleId(&PyId_imp); if (imp == NULL) { imp = PyImport_ImportModule("imp"); if (imp == NULL) { return NULL; } } - else { - Py_INCREF(imp); - } reloaded_module = _PyObject_CallMethodIdObjArgs(imp, &PyId_reload, m, NULL); Py_DECREF(imp); @@ -1720,7 +1803,6 @@ PyImport_Import(PyObject *module_name) PyObject *globals = NULL; PyObject *import = NULL; PyObject *builtins = NULL; - PyObject *modules = NULL; PyObject *r = NULL; /* Initialize constant string objects */ @@ -1775,12 +1857,8 @@ PyImport_Import(PyObject *module_name) goto err; Py_DECREF(r); - modules = PyImport_GetModuleDict(); - r = PyDict_GetItemWithError(modules, module_name); - if (r != NULL) { - Py_INCREF(r); - } - else if (!PyErr_Occurred()) { + r = PyImport_GetModule(module_name); + if (r == NULL && !PyErr_Occurred()) { PyErr_SetObject(PyExc_KeyError, module_name); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 5c8cf5b9bd5f..7adbc29406e0 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -42,6 +42,7 @@ _Py_IDENTIFIER(name); _Py_IDENTIFIER(stdin); _Py_IDENTIFIER(stdout); _Py_IDENTIFIER(stderr); +_Py_IDENTIFIER(threading); #ifdef __cplusplus extern "C" { @@ -283,7 +284,6 @@ initimport(PyInterpreterState *interp, PyObject *sysmod) { PyObject *importlib; PyObject *impmod; - PyObject *sys_modules; PyObject *value; /* Import _importlib through its frozen version, _frozen_importlib. */ @@ -314,11 +314,7 @@ initimport(PyInterpreterState *interp, PyObject *sysmod) else if (Py_VerboseFlag) { PySys_FormatStderr("import _imp # builtin\n"); } - sys_modules = PyImport_GetModuleDict(); - if (Py_VerboseFlag) { - PySys_FormatStderr("import sys # builtin\n"); - } - if (PyDict_SetItemString(sys_modules, "_imp", impmod) < 0) { + if (_PyImport_SetModuleString("_imp", impmod) < 0) { Py_FatalError("Py_Initialize: can't save _imp to sys.modules"); } @@ -1916,8 +1912,7 @@ wait_for_thread_shutdown(void) { _Py_IDENTIFIER(_shutdown); PyObject *result; - PyObject *modules = PyImport_GetModuleDict(); - PyObject *threading = PyMapping_GetItemString(modules, "threading"); + PyObject *threading = _PyImport_GetModuleId(&PyId_threading); if (threading == NULL) { /* threading not imported */ PyErr_Clear(); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d463683df1df..6d2cc96b5ebb 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -162,18 +162,16 @@ static PyObject * sys_displayhook(PyObject *self, PyObject *o) { PyObject *outf; - PyObject *modules = PyImport_GetModuleDict(); - if (modules == NULL) - return NULL; PyObject *builtins; static PyObject *newline = NULL; int err; - builtins = _PyDict_GetItemId(modules, &PyId_builtins); + builtins = _PyImport_GetModuleId(&PyId_builtins); if (builtins == NULL) { PyErr_SetString(PyExc_RuntimeError, "lost builtins module"); return NULL; } + Py_DECREF(builtins); /* Print value except if None */ /* After printing, also assign to '_' */