From: Michael Tremer Date: Wed, 15 Mar 2023 13:28:46 +0000 (+0000) Subject: python: Release and acquire the GIL when we need it X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a853ec56d498c0875cf7dd67ba4ff2f44fef0d91;p=people%2Fstevee%2Fpakfire.git python: Release and acquire the GIL when we need it Signed-off-by: Michael Tremer --- diff --git a/src/_pakfire/archive.c b/src/_pakfire/archive.c index 4c89c4a4..aedf67cd 100644 --- a/src/_pakfire/archive.c +++ b/src/_pakfire/archive.c @@ -87,12 +87,17 @@ static PyObject* Archive_read(ArchiveObject* self, PyObject* args) { char* data = NULL; size_t data_size = 0; + Py_BEGIN_ALLOW_THREADS + int r = pakfire_archive_read(self->archive, filename, &data, &data_size); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + // Return None if there was no data if (!data) Py_RETURN_NONE; @@ -108,13 +113,18 @@ static PyObject* Archive_read(ArchiveObject* self, PyObject* args) { static PyObject* Archive_verify(ArchiveObject* self) { int status; + Py_BEGIN_ALLOW_THREADS + // Verify this archive int r = pakfire_archive_verify(self->archive, &status); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + if (status) Py_RETURN_TRUE; else @@ -135,13 +145,18 @@ static PyObject* Archive_extract(ArchiveObject* self) { static PyObject* Archive_get_package(ArchiveObject* self) { struct pakfire_package* package = NULL; + Py_BEGIN_ALLOW_THREADS + // Make the package int r = pakfire_archive_make_package(self->archive, NULL, &package); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + // Make the Python object PyObject* ret = new_package(&PackageType, package); @@ -160,15 +175,22 @@ static PyObject* Archive_get_path(ArchiveObject* self) { } static PyObject* Archive_get_filelist(ArchiveObject* self) { + struct pakfire_filelist* filelist = NULL; + + Py_BEGIN_ALLOW_THREADS + // Fetch the filelist - struct pakfire_filelist* filelist = pakfire_archive_get_filelist(self->archive); + filelist = pakfire_archive_get_filelist(self->archive); if (!filelist) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + // Convert to filelist - PyObject* _filelist = PyList_FromFileList(filelist); + PyObject* _filelist = PyList_FromFileList(filelist); // Cleanup pakfire_filelist_unref(filelist); diff --git a/src/_pakfire/pakfire.c b/src/_pakfire/pakfire.c index 49222e63..ee74847b 100644 --- a/src/_pakfire/pakfire.c +++ b/src/_pakfire/pakfire.c @@ -67,6 +67,8 @@ static void Pakfire_log_callback(void* data, int priority, const char* file, int if (!callback) return; + PyGILState_STATE state = PyGILState_Ensure(); + // Translate priority to Python logging priorities switch (priority) { case LOG_DEBUG: @@ -119,22 +121,31 @@ ERROR: free(buffer); Py_XDECREF(tuple); Py_XDECREF(result); + + // Release the GIL + PyGILState_Release(state); } static int Pakfire_confirm_callback(struct pakfire* pakfire, void* data, const char* message, const char* question) { PyObject* callback = (PyObject*)data; + PyObject* result = NULL; int r = 0; // Do nothing if callback isn't set if (!callback) return 0; + // Acquire GIL + PyGILState_STATE state = PyGILState_Ensure(); + PyObject* args = Py_BuildValue("(ss)", message, question); - if (!args) - return -1; + if (!args) { + r = -1; + goto ERROR; + } - PyObject* result = PyObject_CallObject(callback, args); + result = PyObject_CallObject(callback, args); // If the callback raised an exception, we will ignore it and indicate // that an error happened, but we cannot re-raise the exception @@ -153,6 +164,9 @@ ERROR: Py_DECREF(args); Py_DECREF(result); + // Release the GIL + PyGILState_Release(state); + return r; } @@ -170,6 +184,7 @@ static int Pakfire_init(PakfireObject* self, PyObject* args, PyObject* kwds) { const char* arch = NULL; const char* conf = NULL; int offline = 0; + int r; if (!PyArg_ParseTupleAndKeywords(args, kwds, "|zzOpzO", kwlist, &path, &arch, &self->callbacks.log, &offline, &conf, &self->callbacks.confirm)) @@ -197,9 +212,14 @@ static int Pakfire_init(PakfireObject* self, PyObject* args, PyObject* kwds) { if (self->callbacks.log) Py_INCREF(self->callbacks.log); + Py_BEGIN_ALLOW_THREADS + // Create a new Pakfire instance - int r = pakfire_create(&self->pakfire, path, arch, conf, flags, + r = pakfire_create(&self->pakfire, path, arch, conf, flags, LOG_DEBUG, Pakfire_log_callback, self->callbacks.log); + + Py_END_ALLOW_THREADS + if (r) { switch (errno) { // Invalid architecture or path @@ -240,29 +260,33 @@ static void Pakfire_dealloc(PakfireObject* self) { Py_DECREF(self->callbacks.confirm); } + Py_BEGIN_ALLOW_THREADS + pakfire_unref(self->pakfire); + + Py_END_ALLOW_THREADS } Py_TYPE(self)->tp_free((PyObject *)self); } static PyObject* Pakfire_repr(PakfireObject* self) { - const char* path = pakfire_get_path(self->pakfire); - const char* arch = pakfire_get_arch(self->pakfire); + const char* path = pakfire_get_path(self->pakfire); + const char* arch = pakfire_get_arch(self->pakfire); return PyUnicode_FromFormat("<_pakfire.Pakfire %s (%s)>", path, arch); } static PyObject* Pakfire_get_path(PakfireObject* self) { - const char* path = pakfire_get_path(self->pakfire); + const char* path = pakfire_get_path(self->pakfire); - return PyUnicode_FromString(path); + return PyUnicode_FromString(path); } static PyObject* Pakfire_get_arch(PakfireObject* self) { - const char* arch = pakfire_get_arch(self->pakfire); + const char* arch = pakfire_get_arch(self->pakfire); - return PyUnicode_FromString(arch); + return PyUnicode_FromString(arch); } static PyObject* Pakfire_get_repo(PakfireObject* self, PyObject* args) { @@ -289,16 +313,21 @@ static void Pakfire_status_callback(struct pakfire* pakfire, void* data, if (!callback) return; + // Acquire GIL + PyGILState_STATE state = PyGILState_Ensure(); + // Compile arguments PyObject* args = Py_BuildValue("(is)", progress, status); - if (!args) - return; + if (args) { + // Call the callback + PyObject* result = PyObject_CallObject(callback, args); - // Call the callback - PyObject* result = PyObject_CallObject(callback, args); + Py_XDECREF(result); + Py_DECREF(args); + } - Py_XDECREF(result); - Py_DECREF(args); + // Release the GIL + PyGILState_Release(state); } static int convert_packages(PyObject* object, void* address) { @@ -387,6 +416,7 @@ static PyObject* Pakfire_install(PakfireObject* self, PyObject* args, PyObject* int solver_flags = 0; int transaction_flags = 0; PyObject* status_callback = NULL; + int r; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|$ppppO", kwlist, convert_packages, &packages, &dryrun, &without_recommended, &allow_uninstall, @@ -415,9 +445,14 @@ static PyObject* Pakfire_install(PakfireObject* self, PyObject* args, PyObject* if (allow_downgrade) solver_flags |= PAKFIRE_REQUEST_ALLOW_DOWNGRADE; + Py_BEGIN_ALLOW_THREADS + // Run pakfire_install - int r = pakfire_install(self->pakfire, transaction_flags, solver_flags, + r = pakfire_install(self->pakfire, transaction_flags, solver_flags, (const char**)packages, NULL, 0, NULL, Pakfire_status_callback, status_callback); + + Py_END_ALLOW_THREADS + if (r) PyErr_SetFromErrno(PyExc_OSError); @@ -447,6 +482,7 @@ static PyObject* Pakfire_erase(PakfireObject* self, PyObject* args, PyObject* kw int transaction_flags = 0; int flags = 0; PyObject* status_callback = NULL; + int r; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|$ppO", kwlist, convert_packages, &packages, &dryrun, &keep_dependencies, &status_callback)) @@ -464,9 +500,14 @@ static PyObject* Pakfire_erase(PakfireObject* self, PyObject* args, PyObject* kw if (keep_dependencies) flags |= PAKFIRE_REQUEST_KEEP_DEPS; + Py_BEGIN_ALLOW_THREADS + // Run pakfire_erase - int r = pakfire_erase(self->pakfire, transaction_flags, 0, (const char**)packages, + r = pakfire_erase(self->pakfire, transaction_flags, 0, (const char**)packages, NULL, flags, NULL, Pakfire_status_callback, status_callback); + + Py_END_ALLOW_THREADS + if (r) PyErr_SetFromErrno(PyExc_OSError); @@ -500,6 +541,7 @@ static PyObject* Pakfire_update(PakfireObject* self, PyObject* args, PyObject* k int solver_flags = 0; int transaction_flags = 0; PyObject* status_callback = NULL; + int r; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|$pO&ppO", kwlist, convert_packages, &packages, &dryrun, convert_packages, &excludes, @@ -523,10 +565,15 @@ static PyObject* Pakfire_update(PakfireObject* self, PyObject* args, PyObject* k if (allow_downgrade) solver_flags |= PAKFIRE_REQUEST_ALLOW_DOWNGRADE; + Py_BEGIN_ALLOW_THREADS + // Run pakfire_update - int r = pakfire_update(self->pakfire, transaction_flags, solver_flags, + r = pakfire_update(self->pakfire, transaction_flags, solver_flags, (const char**)packages, (const char**)excludes, 0, NULL, Pakfire_status_callback, status_callback); + + Py_END_ALLOW_THREADS + if (r) PyErr_SetFromErrno(PyExc_OSError); @@ -739,19 +786,25 @@ static PyObject* Pakfire_whatrequires(PakfireObject* self, PyObject* args) { if (!PyArg_ParseTuple(args, "s", &requires)) return NULL; + Py_BEGIN_ALLOW_THREADS + // Create a new list r = pakfire_packagelist_create(&list, self->pakfire); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); goto ERROR; } r = pakfire_whatrequires(self->pakfire, requires, 0, list); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); goto ERROR; } + Py_END_ALLOW_THREADS + // Create a Python list from the package list ret = PyList_FromPackageList(list); @@ -999,10 +1052,14 @@ static PyObject* Pakfire_execute(PakfireObject* self, PyObject* args, PyObject* Py_DECREF(b); } + Py_BEGIN_ALLOW_THREADS + // Execute command r = pakfire_jail_exec(jail, argv, NULL, Pakfire_execute_output_callback, callback); + Py_END_ALLOW_THREADS + // If the return code was negative, we had some internal error if (r < 0) { PyErr_SetFromErrno(PyExc_OSError); @@ -1038,16 +1095,22 @@ static PyObject* Pakfire_dist(PakfireObject* self, PyObject* args) { const char* path = NULL; const char* target = NULL; char* result = NULL; + int r; if (!PyArg_ParseTuple(args, "s|z", &path, &target)) return NULL; - int r = pakfire_dist(self->pakfire, path, target, &result); + Py_BEGIN_ALLOW_THREADS + + r = pakfire_dist(self->pakfire, path, target, &result); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + PyObject* ret = PyUnicode_FromString(result); free(result); @@ -1061,12 +1124,17 @@ static PyObject* Pakfire_copy_in(PakfireObject* self, PyObject* args) { if (!PyArg_ParseTuple(args, "ss", &src, &dst)) return NULL; + Py_BEGIN_ALLOW_THREADS + int r = pakfire_copy_in(self->pakfire, src, dst); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + Py_RETURN_NONE; } @@ -1077,12 +1145,17 @@ static PyObject* Pakfire_copy_out(PakfireObject* self, PyObject* args) { if (!PyArg_ParseTuple(args, "ss", &src, &dst)) return NULL; + Py_BEGIN_ALLOW_THREADS + int r = pakfire_copy_out(self->pakfire, src, dst); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + Py_RETURN_NONE; } @@ -1148,7 +1221,6 @@ static PyObject* Pakfire_build(PakfireObject* self, PyObject* args, PyObject* kw "disable_tests", NULL, }; - const char* path = NULL; const char* target = NULL; const char* build_id = NULL; @@ -1156,6 +1228,7 @@ static PyObject* Pakfire_build(PakfireObject* self, PyObject* args, PyObject* kw int disable_snapshot = 0; int disable_ccache = 0; int disable_tests = 0; + int r; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|zzpppp", kwlist, &path, &target, &build_id, &interactive, &disable_snapshot, &disable_ccache, &disable_tests)) @@ -1178,8 +1251,12 @@ static PyObject* Pakfire_build(PakfireObject* self, PyObject* args, PyObject* kw if (disable_tests) flags |= PAKFIRE_BUILD_DISABLE_TESTS; + Py_BEGIN_ALLOW_THREADS + // Run build - int r = pakfire_build(self->pakfire, path, target, build_id, flags); + r = pakfire_build(self->pakfire, path, target, build_id, flags); + + Py_END_ALLOW_THREADS return execute_return_value(r); } @@ -1192,6 +1269,7 @@ static PyObject* Pakfire_shell(PakfireObject* self, PyObject* args, PyObject* kw }; char** packages = NULL; int disable_snapshot = 0; + int r; // Parse everything if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O&p", kwlist, @@ -1203,33 +1281,50 @@ static PyObject* Pakfire_shell(PakfireObject* self, PyObject* args, PyObject* kw if (disable_snapshot) flags |= PAKFIRE_BUILD_DISABLE_SNAPSHOT; - int r = pakfire_shell(self->pakfire, (const char**)packages, flags); + Py_BEGIN_ALLOW_THREADS + + r = pakfire_shell(self->pakfire, (const char**)packages, flags); + + Py_END_ALLOW_THREADS return execute_return_value(r); } static PyObject* Pakfire_clean(PakfireObject* self) { - int r = pakfire_clean(self->pakfire, 0); + int r; + + Py_BEGIN_ALLOW_THREADS + + r = pakfire_clean(self->pakfire, 0); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + Py_RETURN_NONE; } static PyObject* Pakfire_refresh(PakfireObject* self, PyObject* args) { int force = 0; + int r; if (!PyArg_ParseTuple(args, "|p", &force)) return NULL; - int r = pakfire_refresh(self->pakfire, force); + Py_BEGIN_ALLOW_THREADS + + r = pakfire_refresh(self->pakfire, force); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + Py_RETURN_NONE; } @@ -1242,8 +1337,13 @@ static PyObject* Pakfire_check(PakfireObject* self) { if (r) goto ERROR; + Py_BEGIN_ALLOW_THREADS + // Perform check r = pakfire_check(self->pakfire, errors); + + Py_END_ALLOW_THREADS + if (r) goto ERROR; @@ -1285,6 +1385,7 @@ static PyObject* Pakfire_sync(PakfireObject* self, PyObject* args, PyObject* kwa int keep_orphaned = 0; int flags = 0; PyObject* status_callback = NULL; + int r; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|$pO", kwlist, &keep_orphaned, &status_callback)) @@ -1293,13 +1394,18 @@ static PyObject* Pakfire_sync(PakfireObject* self, PyObject* args, PyObject* kwa if (keep_orphaned) flags |= PAKFIRE_REQUEST_KEEP_ORPHANED; - int r = pakfire_sync(self->pakfire, 0, flags, NULL, + Py_BEGIN_ALLOW_THREADS + + r = pakfire_sync(self->pakfire, 0, flags, NULL, Pakfire_status_callback, status_callback); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + Py_RETURN_NONE; } @@ -1310,12 +1416,17 @@ static PyObject* Pakfire_open(PakfireObject* self, PyObject* args) { if (!PyArg_ParseTuple(args, "s", &path)) return NULL; + Py_BEGIN_ALLOW_THREADS + int r = pakfire_archive_open(&archive, self->pakfire, path); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + // Create Python object PyObject* object = new_archive(&ArchiveType, archive); pakfire_archive_unref(archive); @@ -1372,12 +1483,17 @@ static PyObject* Pakfire_repo_compose(PakfireObject* self, PyObject* args, PyObj Py_DECREF(file); } + Py_BEGIN_ALLOW_THREADS + int r = pakfire_repo_compose(self->pakfire, path, flags, files); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + // Return None on success ret = Py_None; Py_INCREF(ret); diff --git a/src/_pakfire/repo.c b/src/_pakfire/repo.c index 5f5cf339..4b7024bc 100644 --- a/src/_pakfire/repo.c +++ b/src/_pakfire/repo.c @@ -222,23 +222,32 @@ static PyObject* Repo_refresh(RepoObject* self, PyObject* args, PyObject* kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "|p", kwlist, &force)) return NULL; + Py_BEGIN_ALLOW_THREADS + int r = pakfire_repo_refresh(self->repo, force); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + Py_RETURN_NONE; } static PyObject* Repo_clean(RepoObject* self, PyObject* args) { - int r = pakfire_repo_clean(self->repo, 0); + Py_BEGIN_ALLOW_THREADS + int r = pakfire_repo_clean(self->repo, 0); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + Py_RETURN_NONE; } @@ -248,12 +257,17 @@ static PyObject* Repo_scan(RepoObject* self, PyObject* args) { if (!PyArg_ParseTuple(args, "|i", &flags)) return NULL; + Py_BEGIN_ALLOW_THREADS + int r = pakfire_repo_scan(self->repo, flags); if (r) { + Py_BLOCK_THREADS PyErr_SetFromErrno(PyExc_OSError); return NULL; } + Py_END_ALLOW_THREADS + Py_RETURN_NONE; }