[3.14] gh-152132: Test Py_CompileString() in test_capi.test_run (#153470) (#153473)

gh-152132: Test Py_CompileString() in test_capi.test_run (#153470)

PyRun functions now raises ValueError if the start argument is
invalid.

Add Modules/_testlimitedcapi/run.c.

(cherry picked from commit d83d26724b8c3be5f535e56ded88593cc6803b86)

diff --git a/Include/cpython/pythonrun.h b/Include/cpython/pythonrun.h
index edc40952254029f8c8f2927f4e3324a14d40fba6..b0bf527f4f0ec19c2c4b4e4e8089ffd8ab5def64 100644 (file)
--- a/Include/cpython/pythonrun.h
+++ b/Include/cpython/pythonrun.h
@@ -40,6 +40,11 @@ PyAPI_FUNC(PyObject *) PyRun_FileExFlags(
     PyCompilerFlags *flags);
 
 
+PyAPI_FUNC(PyObject *) Py_CompileStringFlags(
+    const char *str,
+    const char *filename,
+    int start,
+    PyCompilerFlags *flags);
 PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(
     const char *str,
     const char *filename,       /* decoded from the filesystem encoding */

diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py
index d0418cdee6286776262be8a402cd2005aeaca2ee..14f9833f63a4fb8cf0e7e2b95d7c4f4e1077887a 100644 (file)
--- a/Lib/test/test_capi/test_run.py
+++ b/Lib/test/test_capi/test_run.py
@@ -1,20 +1,27 @@
+import ast
 import os
 import sys
 import tempfile
+import textwrap
 import unittest
 from collections import UserDict
 from test import support
 from test.support import import_helper
 from test.support.os_helper import unlink, TESTFN, TESTFN_ASCII, TESTFN_UNDECODABLE
 
+_testcapi = import_helper.import_module('_testcapi')
+_testlimitedcapi = import_helper.import_module('_testlimitedcapi')
+
 
 NULL = None
-_testcapi = import_helper.import_module('_testcapi')
 Py_single_input = _testcapi.Py_single_input
 Py_file_input = _testcapi.Py_file_input
 Py_eval_input = _testcapi.Py_eval_input
+INVALID_START = Py_single_input - 1
 STDIN = '<stdin>'
 STDERR_FD = 2
+PyCF_ONLY_AST = _testcapi.PyCF_ONLY_AST
+PyCF_IGNORE_COOKIE = _testcapi.PyCF_IGNORE_COOKIE
 
 # Code raising a SyntaxError
 SYNTAX_ERROR = 'True = 1'
@@ -68,6 +75,9 @@ class CAPITest(unittest.TestCase):
             run(b'raise ValueError("BUG")', {})
         self.assertEqual(str(cm.exception), 'BUG')
 
+        with self.assertRaises(ValueError):
+            func(b'x = 1', INVALID_START, {})
+
         self.assertIsNone(run(b'a\n', dict(a=1)))
         self.assertIsNone(run(b'a\n', dict(a=1), {}))
         self.assertIsNone(run(b'a\n', {}, dict(a=1)))
@@ -118,6 +128,9 @@ class CAPITest(unittest.TestCase):
             closeit = 1
             self.assertIsNone(run(dict(a=1), {}, closeit))
 
+        with self.assertRaises(ValueError):
+            func(filename, INVALID_START, {})
+
         self.assertRaises(NameError, run, {})
         self.assertRaises(NameError, run, {}, {})
         self.assertRaises(TypeError, run, dict(a=1), [])
@@ -357,6 +370,65 @@ class CAPITest(unittest.TestCase):
         # Test PyRun_SimpleStringFlags()
         self.check_run_simplestring(_testcapi.run_simplestringflags)
 
+    def check_compilestring(self, compilestring, has_flags, encode_filename=True):
+        filename_str = TESTFN
+        if encode_filename:
+            filename = os.fsencode(filename_str)
+        else:
+            filename = filename_str
+
+        def check_code(co, name, value):
+            ns = {}
+            exec(co, ns, ns)
+            self.assertEqual(ns[name], value)
+
+        co = compilestring(b'x = 1', filename, Py_file_input)
+        self.assertEqual(co.co_filename, filename_str)
+        check_code(co, 'x', 1)
+
+        if has_flags:
+            code = textwrap.dedent("""
+                # encoding: latin1
+                x = 'a\xe9'
+            """)
+            co = compilestring(code.encode(), filename, Py_file_input, PyCF_IGNORE_COOKIE)
+            self.assertEqual(co.co_filename, filename_str)
+            check_code(co, 'x', 'a\xe9')
+
+            co = compilestring(code.encode(), filename, Py_file_input)
+            self.assertEqual(co.co_filename, filename_str)
+            check_code(co, 'x', 'a\xc3\xa9')
+
+            tree = compilestring(b'x = 1', filename, Py_file_input, PyCF_ONLY_AST)
+            self.assertIsInstance(tree, ast.AST)
+
+        co = compilestring(b'raise ValueError("BUG")', filename, Py_file_input)
+        with self.assertRaises(ValueError):
+            exec(co, {})
+
+        with self.assertRaises(SyntaxError) as cm:
+            compilestring(SYNTAX_ERROR.encode(), filename, Py_file_input)
+
+        with self.assertRaises(ValueError):
+            compilestring(b'x = 1', filename, INVALID_START)
+
+    def test_compilestring(self):
+        # Test Py_CompileString()
+        self.check_compilestring(_testlimitedcapi.run_compilestring, False)
+
+    def test_compilestringflags(self):
+        # Test Py_CompileStringFlags()
+        self.check_compilestring(_testcapi.run_compilestringflags, True)
+
+    def test_compilestringexflags(self):
+        # Test Py_CompileStringExFlags()
+        self.check_compilestring(_testcapi.run_compilestringexflags, True)
+
+    def test_compilestringobject(self):
+        # Test Py_CompileStringObject()
+        self.check_compilestring(_testcapi.run_compilestringobject, True,
+                                 encode_filename=False)
+
 
 if __name__ == '__main__':
     unittest.main()

diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in
index 323716870aabe82cda1d10392d7c82464d82c362..74e77f19f52ba062a90ee1c3a5d068d005d75b58 100644 (file)
--- a/Modules/Setup.stdlib.in
+++ b/Modules/Setup.stdlib.in
@@ -176,7 +176,7 @@
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
 @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c
 @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/weakref.c
-@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c
+@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c
 @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c
 @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c
 

diff --git a/Modules/_testcapi/run.c b/Modules/_testcapi/run.c
index 7fc180e136559bab7d17c1b624871c044cf2bcb4..6812af994a433023abc78b03e1ca52cabea8895e 100644 (file)
--- a/Modules/_testcapi/run.c
+++ b/Modules/_testcapi/run.c
@@ -24,34 +24,16 @@
 #undef PyRun_InteractiveLoop
 
 
-// Some PyRun functions crash if start is invalid,
-// so validate the start argument.
-static int
-check_start(int start)
-{
-    if (start == Py_single_input || start == Py_file_input
-        || start == Py_eval_input || start == Py_func_type_input)
-    {
-        return 0;
-    }
-    PyErr_SetString(PyExc_ValueError, "invalid start argument");
-    return -1;
-}
-
-
 // Test PyRun_String()
 static PyObject*
 run_string(PyObject *mod, PyObject *args)
 {
     const char *str;
-    int start = 0;
+    int start;
     PyObject *globals = NULL;
     PyObject *locals = NULL;
 
-    if (!PyArg_ParseTuple(args, "y|iOO", &str, &start, &globals, &locals)) {
-        return NULL;
-    }
-    if (check_start(start) < 0) {
+    if (!PyArg_ParseTuple(args, "yi|OO", &str, &start, &globals, &locals)) {
         return NULL;
     }
     NULLABLE(globals);
@@ -79,9 +61,6 @@ run_stringflags(PyObject *mod, PyObject *args)
                           &cf_flags, &cf_feature_version)) {
         return NULL;
     }
-    if (check_start(start) < 0) {
-        return NULL;
-    }
     NULLABLE(globals);
     NULLABLE(locals);
     if (cf_flags || cf_feature_version) {
@@ -481,9 +460,6 @@ run_file(PyObject *mod, PyObject *args)
                           &globals, &locals)) {
         return NULL;
     }
-    if (check_start(start) < 0) {
-        return NULL;
-    }
     NULLABLE(globals);
     NULLABLE(locals);
 
@@ -518,9 +494,6 @@ run_fileex(PyObject *mod, PyObject *args)
                           &closeit)) {
         return NULL;
     }
-    if (check_start(start) < 0) {
-        return NULL;
-    }
     NULLABLE(globals);
     NULLABLE(locals);
 
@@ -560,9 +533,6 @@ run_fileflags(PyObject *mod, PyObject *args)
                           &cf_flags, &cf_feature_version)) {
         return NULL;
     }
-    if (check_start(start) < 0) {
-        return NULL;
-    }
     NULLABLE(globals);
     NULLABLE(locals);
     if (cf_flags || cf_feature_version) {
@@ -606,9 +576,6 @@ run_fileexflags(PyObject *mod, PyObject *args)
                           &closeit, &cf_flags, &cf_feature_version)) {
         return NULL;
     }
-    if (check_start(start) < 0) {
-        return NULL;
-    }
     NULLABLE(globals);
     NULLABLE(locals);
     if (cf_flags || cf_feature_version) {
@@ -733,6 +700,86 @@ run_interactiveloopflags(PyObject *mod, PyObject *args)
 }
 
 
+// Test Py_CompileStringFlags()
+static PyObject*
+run_compilestringflags(PyObject *mod, PyObject *args)
+{
+    const char *str;
+    const char *filename;
+    int start;
+    PyCompilerFlags flags = _PyCompilerFlags_INIT;
+    PyCompilerFlags *pflags = NULL;
+    int cf_flags = 0;
+    int cf_feature_version = 0;
+
+    if (!PyArg_ParseTuple(args, "yyi|ii", &str, &filename, &start,
+                          &cf_flags, &cf_feature_version)) {
+        return NULL;
+    }
+    if (cf_flags || cf_feature_version) {
+        flags.cf_flags = cf_flags;
+        flags.cf_feature_version = cf_feature_version;
+        pflags = &flags;
+    }
+
+    return Py_CompileStringFlags(str, filename, start, pflags);
+}
+
+
+// Test Py_CompileStringExFlags()
+static PyObject*
+run_compilestringexflags(PyObject *mod, PyObject *args)
+{
+    const char *str;
+    const char *filename;
+    int start;
+    PyCompilerFlags flags = _PyCompilerFlags_INIT;
+    PyCompilerFlags *pflags = NULL;
+    int cf_flags = 0;
+    int cf_feature_version = 0;
+    int optimize = -1;
+
+    if (!PyArg_ParseTuple(args, "yyi|iii", &str, &filename, &start,
+                          &cf_flags, &cf_feature_version, &optimize)) {
+        return NULL;
+    }
+    if (cf_flags || cf_feature_version) {
+        flags.cf_flags = cf_flags;
+        flags.cf_feature_version = cf_feature_version;
+        pflags = &flags;
+    }
+
+    return Py_CompileStringExFlags(str, filename, start, pflags, optimize);
+}
+
+
+// Test Py_CompileStringObject()
+static PyObject*
+run_compilestringobject(PyObject *mod, PyObject *args)
+{
+    const char *str;
+    PyObject *filename;
+    int start;
+    PyCompilerFlags flags = _PyCompilerFlags_INIT;
+    PyCompilerFlags *pflags = NULL;
+    int cf_flags = 0;
+    int cf_feature_version = 0;
+    int optimize = -1;
+
+    if (!PyArg_ParseTuple(args, "yOi|iii", &str, &filename, &start,
+                          &cf_flags, &cf_feature_version, &optimize)) {
+        return NULL;
+    }
+    if (cf_flags || cf_feature_version) {
+        flags.cf_flags = cf_flags;
+        flags.cf_feature_version = cf_feature_version;
+        pflags = &flags;
+    }
+
+    return Py_CompileStringObject(str, filename, start, pflags, optimize);
+}
+
+
 static PyMethodDef test_methods[] = {
     {"run_string", run_string, METH_VARARGS},
     {"run_stringflags", run_stringflags, METH_VARARGS},
@@ -754,6 +801,9 @@ static PyMethodDef test_methods[] = {
     {"run_simplestringflags", run_simplestringflags, METH_VARARGS},
     {"run_interactiveloop", run_interactiveloop, METH_VARARGS},
     {"run_interactiveloopflags", run_interactiveloopflags, METH_VARARGS},
+    {"run_compilestringflags", run_compilestringflags, METH_VARARGS},
+    {"run_compilestringexflags", run_compilestringexflags, METH_VARARGS},
+    {"run_compilestringobject", run_compilestringobject, METH_VARARGS},
     {NULL},
 };
 
@@ -763,5 +813,11 @@ _PyTestCapi_Init_Run(PyObject *mod)
     if (PyModule_AddFunctions(mod, test_methods) < 0) {
         return -1;
     }
+    if (PyModule_AddIntMacro(mod, PyCF_ONLY_AST) < 0) {
+        return -1;
+    }
+    if (PyModule_AddIntMacro(mod, PyCF_IGNORE_COOKIE) < 0) {
+        return -1;
+    }
     return 0;
 }

diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c
index ab504ee617b8982ef5d020bc3b9481ca626b33c8..99e5d8ce341fbd00333490abbac1a318ab44edb2 100644 (file)
--- a/Modules/_testlimitedcapi.c
+++ b/Modules/_testlimitedcapi.c
@@ -95,5 +95,8 @@ PyInit__testlimitedcapi(void)
     if (_PyTestLimitedCAPI_Init_Weakref(mod) < 0) {
         return NULL;
     }
+    if (_PyTestLimitedCAPI_Init_Run(mod) < 0) {
+        return NULL;
+    }
     return mod;
 }

diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h
index 011b16bdabd357950ef722b8c39073b7cbeebdb1..df023640ed55f841473ffd062f2274ccb14d877b 100644 (file)
--- a/Modules/_testlimitedcapi/parts.h
+++ b/Modules/_testlimitedcapi/parts.h
@@ -44,5 +44,6 @@ int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module);
 int _PyTestLimitedCAPI_Init_Version(PyObject *module);
 int _PyTestLimitedCAPI_Init_File(PyObject *module);
 int _PyTestLimitedCAPI_Init_Weakref(PyObject *module);
+int _PyTestLimitedCAPI_Init_Run(PyObject *module);
 
 #endif // Py_TESTLIMITEDCAPI_PARTS_H

diff --git a/Modules/_testlimitedcapi/run.c b/Modules/_testlimitedcapi/run.c
new file mode 100644 (file)
index 0000000..ca12e5f
--- /dev/null
+++ b/Modules/_testlimitedcapi/run.c
@@ -0,0 +1,35 @@
+#include "parts.h"
+#include "util.h"
+
+
+// Test functions, not macros
+#undef Py_CompileString
+
+
+// Test Py_CompileString()
+static PyObject*
+run_compilestring(PyObject *mod, PyObject *args)
+{
+    const char *str;
+    const char *filename;
+    int start;
+
+    if (!PyArg_ParseTuple(args, "yyi", &str, &filename, &start)) {
+        return NULL;
+    }
+
+    return Py_CompileString(str, filename, start);
+}
+
+
+static PyMethodDef test_methods[] = {
+    {"run_compilestring", run_compilestring, METH_VARARGS},
+    {NULL},
+};
+
+int
+_PyTestLimitedCAPI_Init_Run(PyObject *m)
+{
+    return PyModule_AddFunctions(m, test_methods);
+}
+

diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj
index 8bc963ba65774a7e2f25caacfd46da1c60e49e89..3872ad0dc979ad9053c29ca367ea655d3c8074c2 100644 (file)
--- a/PCbuild/_testlimitedcapi.vcxproj
+++ b/PCbuild/_testlimitedcapi.vcxproj
@@ -116,6 +116,7 @@
     <ClCompile Include="..\Modules\_testlimitedcapi\version.c" />
     <ClCompile Include="..\Modules\_testlimitedcapi\file.c" />
     <ClCompile Include="..\Modules\_testlimitedcapi\weakref.c" />
+    <ClCompile Include="..\Modules\_testlimitedcapi\run.c" />
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="..\PC\python_nt.rc" />

diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters
index d84b634722d169e3d1f3638594a8bbd01461278e..ac16423163cf4d60120e785a6466da7f842e8f51 100644 (file)
--- a/PCbuild/_testlimitedcapi.vcxproj.filters
+++ b/PCbuild/_testlimitedcapi.vcxproj.filters
@@ -32,6 +32,7 @@
     <ClCompile Include="..\Modules\_testlimitedcapi\version.c" />
     <ClCompile Include="..\Modules\_testlimitedcapi\file.c" />
     <ClCompile Include="..\Modules\_testlimitedcapi\weakref.c" />
+    <ClCompile Include="..\Modules\_testlimitedcapi\run.c" />
     <ClCompile Include="..\Modules\_testlimitedcapi.c" />
   </ItemGroup>
   <ItemGroup>

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 74cec29f6f0ccb86e454822151c4835c0da0bcae..a97bc7dbb8e5a1e6399c577aa4bd10a329b72485 100644 (file)
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -1238,11 +1238,27 @@ void PyErr_DisplayException(PyObject *exc)
     PyErr_Display(NULL, exc, NULL);
 }
 
+static int
+check_start(int start)
+{
+    if (start == Py_single_input || start == Py_file_input
+        || start == Py_eval_input || start == Py_func_type_input)
+    {
+        return 0;
+    }
+    PyErr_SetString(PyExc_ValueError, "invalid start argument");
+    return -1;
+}
+
 static PyObject *
 _PyRun_String(const char *str, PyObject* name, int start,
               PyObject *globals, PyObject *locals, PyCompilerFlags *flags,
               int generate_new_source)
 {
+    if (check_start(start) < 0) {
+        return NULL;
+    }
+
     PyObject *ret = NULL;
     mod_ty mod;
     PyArena *arena;
@@ -1285,6 +1301,10 @@ static PyObject *
 _PyRun_File(FILE *fp, PyObject *filename, int start, PyObject *globals,
             PyObject *locals, int closeit, PyCompilerFlags *flags)
 {
+    if (check_start(start) < 0) {
+        return NULL;
+    }
+
     PyArena *arena = _PyArena_New();
     if (arena == NULL) {
         return NULL;
@@ -1519,6 +1539,10 @@ PyObject *
 Py_CompileStringObject(const char *str, PyObject *filename, int start,
                        PyCompilerFlags *flags, int optimize)
 {
+    if (check_start(start) < 0) {
+        return NULL;
+    }
+
     PyCodeObject *co;
     mod_ty mod;
     PyArena *arena = _PyArena_New();
@@ -1725,7 +1749,7 @@ Py_CompileString(const char *str, const char *p, int s)
 }
 
 #undef Py_CompileStringFlags
-PyAPI_FUNC(PyObject *)
+PyObject*
 Py_CompileStringFlags(const char *str, const char *p, int s,
                       PyCompilerFlags *flags)
 {