bpo-41094: Fix decoding errors with audit when open files. (GH-21095)

(cherry picked from commit 6c6810d98979add7a89391c3c38990d0859f7a29)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index da79d7af0502372c7fc6230a5698f0aace679696..886ccc5a125a94e4f356d6b4ded99174d1fe2bb3 100644 (file)
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -1309,7 +1309,7 @@ class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
         self.run_embedded_interpreter("test_audit_run_file", timeout=3, returncode=1)
 
     def test_audit_run_interactivehook(self):
-        startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
+        startup = os.path.join(self.oldcwd, support.TESTFN) + (support.TESTFN or '') + ".py"
         with open(startup, "w", encoding="utf-8") as f:
             print("import sys", file=f)
             print("sys.__interactivehook__ = lambda: None", file=f)
@@ -1321,7 +1321,7 @@ class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
             os.unlink(startup)
 
     def test_audit_run_startup(self):
-        startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
+        startup = os.path.join(self.oldcwd, support.TESTFN) + (support.TESTFN or '') + ".py"
         with open(startup, "w", encoding="utf-8") as f:
             print("pass", file=f)
         try:

diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-23-23-26-42.bpo-41094.zEIJse.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-23-23-26-42.bpo-41094.zEIJse.rst
new file mode 100644 (file)
index 0000000..6dd45e2
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2020-06-23-23-26-42.bpo-41094.zEIJse.rst
@@ -0,0 +1,2 @@
+Fix decoding errors with audit when open files with non-ASCII names on non-UTF-8
+locale.

diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
index 55fc226ca125821b73a91e99d6d32b5d46104870..a9b8675cd951b51670d64fbe622ab4a1b76e3338 100644 (file)
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -1416,15 +1416,12 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args)
     if (name != Py_None) {
         if (PyUnicode_FSConverter(name, &name2) == 0)
             return NULL;
-        if (PyBytes_Check(name2))
-            name_str = PyBytes_AS_STRING(name2);
-        else
-            name_str = PyByteArray_AS_STRING(name2);
+        name_str = PyBytes_AS_STRING(name2);
     } else {
         name_str = NULL;
         name2 = NULL;
     }
-    if (PySys_Audit("ctypes.dlopen", "s", name_str) < 0) {
+    if (PySys_Audit("ctypes.dlopen", "O", name) < 0) {
         return NULL;
     }
     handle = ctypes_dlopen(name_str, mode);

diff --git a/Modules/main.c b/Modules/main.c
index 2a360b58efa8318e51ddcabdc82af1abde042c25..788bc119095c0f235ff94da020af9858b6d226d1 100644 (file)
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -391,13 +391,20 @@ pymain_run_startup(PyConfig *config, PyCompilerFlags *cf, int *exitcode)
     if (startup == NULL) {
         return 0;
     }
-    if (PySys_Audit("cpython.run_startup", "s", startup) < 0) {
+    PyObject *startup_obj = PyUnicode_DecodeFSDefault(startup);
+    if (startup_obj == NULL) {
         return pymain_err_print(exitcode);
     }
+    if (PySys_Audit("cpython.run_startup", "O", startup_obj) < 0) {
+        Py_DECREF(startup_obj);
+        return pymain_err_print(exitcode);
+    }
+    Py_DECREF(startup_obj);
 
     FILE *fp = _Py_fopen(startup, "r");
     if (fp == NULL) {
         int save_errno = errno;
+        PyErr_Clear();
         PySys_WriteStderr("Could not open PYTHONSTARTUP\n");
 
         errno = save_errno;

diff --git a/Python/fileutils.c b/Python/fileutils.c
index 1021ddb58853e0aa993327dd6c23b4cbca1c9572..b274116745efe1744a71c65a6ffa0c80441da8e7 100644 (file)
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -1274,7 +1274,12 @@ _Py_open_impl(const char *pathname, int flags, int gil_held)
 #endif
 
     if (gil_held) {
-        if (PySys_Audit("open", "sOi", pathname, Py_None, flags) < 0) {
+        PyObject *pathname_obj = PyUnicode_DecodeFSDefault(pathname);
+        if (pathname_obj == NULL) {
+            return -1;
+        }
+        if (PySys_Audit("open", "OOi", pathname_obj, Py_None, flags) < 0) {
+            Py_DECREF(pathname_obj);
             return -1;
         }
 
@@ -1284,12 +1289,16 @@ _Py_open_impl(const char *pathname, int flags, int gil_held)
             Py_END_ALLOW_THREADS
         } while (fd < 0
                  && errno == EINTR && !(async_err = PyErr_CheckSignals()));
-        if (async_err)
+        if (async_err) {
+            Py_DECREF(pathname_obj);
             return -1;
+        }
         if (fd < 0) {
-            PyErr_SetFromErrnoWithFilename(PyExc_OSError, pathname);
+            PyErr_SetFromErrnoWithFilenameObjects(PyExc_OSError, pathname_obj, NULL);
+            Py_DECREF(pathname_obj);
             return -1;
         }
+        Py_DECREF(pathname_obj);
     }
     else {
         fd = open(pathname, flags);
@@ -1385,9 +1394,15 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode)
 FILE*
 _Py_fopen(const char *pathname, const char *mode)
 {
-    if (PySys_Audit("open", "ssi", pathname, mode, 0) < 0) {
+    PyObject *pathname_obj = PyUnicode_DecodeFSDefault(pathname);
+    if (pathname_obj == NULL) {
+        return NULL;
+    }
+    if (PySys_Audit("open", "Osi", pathname_obj, mode, 0) < 0) {
+        Py_DECREF(pathname_obj);
         return NULL;
     }
+    Py_DECREF(pathname_obj);
 
     FILE *f = fopen(pathname, mode);
     if (f == NULL)