gh-118335: Configure Tier 2 interpreter at build time (#118339)

The code for Tier 2 is now only compiled when configured
with `--enable-experimental-jit[=yes|interpreter]`.

We drop support for `PYTHON_UOPS` and -`Xuops`,
but you can disable the interpreter or JIT
at runtime by setting `PYTHON_JIT=0`.
You can also build it without enabling it by default
using `--enable-experimental-jit=yes-off`;
enable with `PYTHON_JIT=1`.

On Windows, the `build.bat` script supports
`--experimental-jit`, `--experimental-jit-off`,
`--experimental-interpreter`.

In the C code, `_Py_JIT` is defined as before
when the JIT is enabled; the new variable
`_Py_TIER2` is defined when the JIT *or* the
interpreter is enabled. It is actually a bitmask:
1: JIT; 2: default-off; 4: interpreter.

diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst
index a9a7e485dd2bac4b9f04f0ba0eb18ce748d2162f..ee50effd662f12e1a79f8f4a25bad412b8c8d4d7 100644 (file)
--- a/Doc/whatsnew/3.13.rst
+++ b/Doc/whatsnew/3.13.rst
@@ -888,7 +888,7 @@ Experimental JIT Compiler
 =========================
 
 When CPython is configured using the ``--enable-experimental-jit`` option,
-a just-in-time compiler is added which can speed up some Python programs.
+a just-in-time compiler is added which may speed up some Python programs.
 
 The internal architecture is roughly as follows.
 
@@ -905,19 +905,35 @@ The internal architecture is roughly as follows.
   before it is interpreted or translated to machine code.
 
 * There is a Tier 2 interpreter, but it is mostly intended for debugging
-  the earlier stages of the optimization pipeline. If the JIT is not
-  enabled, the Tier 2 interpreter can be invoked by passing Python the
-  ``-X uops`` option or by setting the ``PYTHON_UOPS`` environment
-  variable to ``1``.
+  the earlier stages of the optimization pipeline.
+  The Tier 2 interpreter can be enabled by configuring Python
+  with ``--enable-experimental-jit=interpreter``.
 
-* When the ``--enable-experimental-jit`` option is used, the optimized
+* When the JIT is enabled, the optimized
   Tier 2 IR is translated to machine code, which is then executed.
-  This does not require additional runtime options.
 
 * The machine code translation process uses an architecture called
   *copy-and-patch*. It has no runtime dependencies, but there is a new
   build-time dependency on LLVM.
 
+The ``--enable-experimental-jit`` flag has the following optional values:
+
+* ``no`` (default) -- Disable the entire Tier 2 and JIT pipeline.
+
+* ``yes`` (default if the flag is present without optional value)
+  -- Enable the JIT. To disable the JIT at runtime,
+  pass the environment variable ``PYTHON_JIT=0``.
+
+* ``yes-off`` -- Build the JIT but disable it by default.
+  To enable the JIT at runtime, pass the environment variable
+  ``PYTHON_JIT=1``.
+
+* ``interpreter`` -- Enable the Tier 2 interpreter but disable the JIT.
+  The interpreter can be disabled by running with
+  ``PYTHON_JIT=0``.
+
+(On Windows, use ``PCbuild/build.bat --enable-jit`` to enable the JIT.)
+
 See :pep:`744` for more details.
 
 (JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad.

diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h
index 2ccc548ca6c5fd95172840ded6b79b33d7e8ffe2..808badea90d8eda38788c8bdee02fcbeff66c84a 100644 (file)
--- a/Include/internal/pycore_opcode_metadata.h
+++ b/Include/internal/pycore_opcode_metadata.h
@@ -981,7 +981,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
     [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
-    [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
+    [CACHE] = { true, INSTR_FMT_IX, 0 },
     [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
     [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
     [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
@@ -1121,7 +1121,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
     [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
     [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
     [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
-    [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
+    [RESERVED] = { true, INSTR_FMT_IX, 0 },
     [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
     [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG },
     [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG },

diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h
index a84212c1ec0b69e1df361edd8a6e0a79db6a427f..5697b13d1fc51d24b649e3d9c1e4ad09d01f8eb8 100644 (file)
--- a/Include/internal/pycore_uop_metadata.h
+++ b/Include/internal/pycore_uop_metadata.h
@@ -249,7 +249,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
     [_DYNAMIC_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
     [_START_EXECUTOR] = HAS_DEOPT_FLAG,
-    [_FATAL_ERROR] = HAS_ESCAPES_FLAG,
+    [_FATAL_ERROR] = 0,
     [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG,
     [_DEOPT] = 0,
     [_SIDE_EXIT] = 0,

diff --git a/Lib/dis.py b/Lib/dis.py
index 111d624fc259c55f3e4fa393f07205f8e30a3d8d..76934eb00e63f0a6bb4fce87e31d4bef849e899e 100644 (file)
--- a/Lib/dis.py
+++ b/Lib/dis.py
@@ -216,7 +216,7 @@ def _get_code_array(co, adaptive):
             if op == ENTER_EXECUTOR:
                 try:
                     ex = get_executor(co, i)
-                except ValueError:
+                except (ValueError, RuntimeError):
                     ex = None
 
                 if ex:

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 15f654302b1793511bda4aeeb1cacec1f099ca03..52573e665a1273f2d1c731bfc54ae082ef116cd8 100644 (file)
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -2539,17 +2539,17 @@ Py_TRACE_REFS = hasattr(sys, 'getobjects')
 # Decorator to disable optimizer while a function run
 def without_optimizer(func):
     try:
-        import _testinternalcapi
+        from _testinternalcapi import get_optimizer, set_optimizer
     except ImportError:
         return func
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
-        save_opt = _testinternalcapi.get_optimizer()
+        save_opt = get_optimizer()
         try:
-            _testinternalcapi.set_optimizer(None)
+            set_optimizer(None)
             return func(*args, **kwargs)
         finally:
-            _testinternalcapi.set_optimizer(save_opt)
+            set_optimizer(save_opt)
     return wrapper
 
 

diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py
index c798b343626677778feecca67b1b4506db43892c..6e5b626e93291ae7024898c31853aded042e76d2 100644 (file)
--- a/Lib/test/test_capi/test_opt.py
+++ b/Lib/test/test_capi/test_opt.py
@@ -34,6 +34,8 @@ def clear_executors(func):
 
 
 @requires_specialization
+@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"),
+                     "Requires optimizer infrastructure")
 class TestOptimizerAPI(unittest.TestCase):
 
     def test_new_counter_optimizer_dealloc(self):
@@ -136,6 +138,8 @@ def get_opnames(ex):
 
 
 @requires_specialization
+@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"),
+                     "Requires optimizer infrastructure")
 class TestExecutorInvalidation(unittest.TestCase):
 
     def setUp(self):
@@ -215,6 +219,8 @@ class TestExecutorInvalidation(unittest.TestCase):
 
 
 @requires_specialization
+@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"),
+                     "Requires optimizer infrastructure")
 @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.")
 class TestUops(unittest.TestCase):
 
@@ -579,6 +585,8 @@ class TestUops(unittest.TestCase):
 
 
 @requires_specialization
+@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"),
+                     "Requires optimizer infrastructure")
 @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.")
 class TestUopsOptimization(unittest.TestCase):
 

diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py
index 11c61bc2e0688d3bf51b837385ba523365c16244..a9140d4d3dd7434f71e12dc6fe9d87c2b9ede648 100644 (file)
--- a/Lib/test/test_monitoring.py
+++ b/Lib/test/test_monitoring.py
@@ -1831,15 +1831,17 @@ class TestOptimizer(MonitoringTestBase, unittest.TestCase):
 
     def setUp(self):
         _testinternalcapi = import_module("_testinternalcapi")
-        self.old_opt = _testinternalcapi.get_optimizer()
-        opt = _testinternalcapi.new_counter_optimizer()
-        _testinternalcapi.set_optimizer(opt)
+        if hasattr(_testinternalcapi, "get_optimizer"):
+            self.old_opt = _testinternalcapi.get_optimizer()
+            opt = _testinternalcapi.new_counter_optimizer()
+            _testinternalcapi.set_optimizer(opt)
         super(TestOptimizer, self).setUp()
 
     def tearDown(self):
         super(TestOptimizer, self).tearDown()
         import _testinternalcapi
-        _testinternalcapi.set_optimizer(self.old_opt)
+        if hasattr(_testinternalcapi, "get_optimizer"):
+            _testinternalcapi.set_optimizer(self.old_opt)
 
     def test_for_loop(self):
         def test_func(x):

diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py
index f4e954fd02148dddb233466bdec22458a2f7f8e7..92a34113bc0383396c7d73b3fe09aa3acda66c28 100644 (file)
--- a/Lib/test/test_opcache.py
+++ b/Lib/test/test_opcache.py
@@ -16,6 +16,8 @@ _testinternalcapi = import_module("_testinternalcapi")
 
 def disabling_optimizer(func):
     def wrapper(*args, **kwargs):
+        if not hasattr(_testinternalcapi, "get_optimizer"):
+            return func(*args, **kwargs)
         old_opt = _testinternalcapi.get_optimizer()
         _testinternalcapi.set_optimizer(None)
         try:

diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py
index 899a450731733475bf25ea7a57af4058b9fbc2bf..fac4d1a4ab44c47bf566376b081c8d15df19dc88 100644 (file)
--- a/Lib/test/test_optimizer.py
+++ b/Lib/test/test_optimizer.py
@@ -80,6 +80,8 @@ class TestRareEventCounters(unittest.TestCase):
 
 class TestOptimizerSymbols(unittest.TestCase):
 
+    @unittest.skipUnless(hasattr(_testinternalcapi, "uop_symbols_test"),
+                "requires _testinternalcapi.uop_symbols_test")
     def test_optimizer_symbols(self):
         _testinternalcapi.uop_symbols_test()
 

diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py
index df90647baee31e6a9c304b48800d8f800774efa3..16da24d7805b566c09f6eb24d5d62d4bbfb3f324 100644 (file)
--- a/Lib/test/test_weakref.py
+++ b/Lib/test/test_weakref.py
@@ -17,6 +17,7 @@ from test.support import script_helper, ALWAYS_EQ, suppress_immortalization
 from test.support import gc_collect
 from test.support import import_helper
 from test.support import threading_helper
+from test.support import is_wasi, Py_DEBUG
 
 # Used in ReferencesTestCase.test_ref_created_during_del() .
 ref_from_del = None
@@ -960,6 +961,7 @@ class ReferencesTestCase(TestBase):
         self.assertEqual(hash(a), hash(42))
         self.assertRaises(TypeError, hash, b)
 
+    @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack")
     def test_trashcan_16602(self):
         # Issue #16602: when a weakref's target was part of a long
         # deallocation chain, the trashcan mechanism could delay clearing

diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst
new file mode 100644 (file)
index 0000000..e13edbb
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst
@@ -0,0 +1,4 @@
+Change how to use the tier 2 interpreter. Instead of running Python with
+``-X uops`` or setting the environment variable ``PYTHON_UOPS=1``, this
+choice is now made at build time by configuring with
+``--enable-experimental-jit=interpreter``.

diff --git a/Modules/_opcode.c b/Modules/_opcode.c
index 5350adb456b85951dbb9fbfbc87f4cdf4a916f70..85e0ffec900e89f9c2bf58fea63db9cb70815a12 100644 (file)
--- a/Modules/_opcode.c
+++ b/Modules/_opcode.c
@@ -367,7 +367,13 @@ _opcode_get_executor_impl(PyObject *module, PyObject *code, int offset)
                      Py_TYPE(code)->tp_name);
         return NULL;
     }
+#ifdef _Py_TIER2
     return (PyObject *)PyUnstable_GetExecutor((PyCodeObject *)code, offset);
+#else
+    PyErr_Format(PyExc_RuntimeError,
+                 "Executors are not available in this build");
+    return NULL;
+#endif
 }
 
 static PyMethodDef

diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index 99e80baa223e8f7acdd702504d76c1a51526acf3..f7952a119f0bd9c73c921f327c8170890bc69b28 100644 (file)
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -985,6 +985,8 @@ get_co_framesize(PyObject *self, PyObject *arg)
     return PyLong_FromLong(code->co_framesize);
 }
 
+#ifdef _Py_TIER2
+
 static PyObject *
 new_counter_optimizer(PyObject *self, PyObject *arg)
 {
@@ -1012,7 +1014,10 @@ set_optimizer(PyObject *self, PyObject *opt)
 static PyObject *
 get_optimizer(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
-    PyObject *opt = (PyObject *)PyUnstable_GetOptimizer();
+    PyObject *opt = NULL;
+#ifdef _Py_TIER2
+    opt = (PyObject *)PyUnstable_GetOptimizer();
+#endif
     if (opt == NULL) {
         Py_RETURN_NONE;
     }
@@ -1045,6 +1050,8 @@ invalidate_executors(PyObject *self, PyObject *obj)
     Py_RETURN_NONE;
 }
 
+#endif
+
 static int _pending_callback(void *arg)
 {
     /* we assume the argument is callable object to which we own a reference */
@@ -2020,12 +2027,14 @@ static PyMethodDef module_functions[] = {
     {"iframe_getline", iframe_getline, METH_O, NULL},
     {"iframe_getlasti", iframe_getlasti, METH_O, NULL},
     {"get_co_framesize", get_co_framesize, METH_O, NULL},
+#ifdef _Py_TIER2
     {"get_optimizer", get_optimizer,  METH_NOARGS, NULL},
     {"set_optimizer", set_optimizer,  METH_O, NULL},
     {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL},
     {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL},
     {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL},
     {"invalidate_executors", invalidate_executors, METH_O, NULL},
+#endif
     {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc),
      METH_VARARGS | METH_KEYWORDS},
     {"pending_identify", pending_identify, METH_VARARGS, NULL},
@@ -2072,7 +2081,9 @@ static PyMethodDef module_functions[] = {
     {"py_thread_id", get_py_thread_id, METH_NOARGS},
 #endif
     {"set_immortalize_deferred", set_immortalize_deferred, METH_VARARGS},
+#ifdef _Py_TIER2
     {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS},
+#endif
     {NULL, NULL} /* sentinel */
 };
 

diff --git a/Objects/codeobject.c b/Objects/codeobject.c
index 014632962bfcf36b48170af745b5564f0e7f8e5f..605167c5c0bd3aae2fd5bf3c88eb88900bb98de0 100644 (file)
--- a/Objects/codeobject.c
+++ b/Objects/codeobject.c
@@ -1496,6 +1496,8 @@ PyCode_GetFreevars(PyCodeObject *code)
     return _PyCode_GetFreevars(code);
 }
 
+#ifdef _Py_TIER2
+
 static void
 clear_executors(PyCodeObject *co)
 {
@@ -1515,6 +1517,8 @@ _PyCode_Clear_Executors(PyCodeObject *code)
     clear_executors(code);
 }
 
+#endif
+
 static void
 deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions)
 {
@@ -1739,9 +1743,11 @@ code_dealloc(PyCodeObject *co)
 
         PyMem_Free(co_extra);
     }
+#ifdef _Py_TIER2
     if (co->co_executors != NULL) {
         clear_executors(co);
     }
+#endif
 
     Py_XDECREF(co->co_consts);
     Py_XDECREF(co->co_names);

diff --git a/Objects/object.c b/Objects/object.c
index 8d856939254080f30fa43012c7682d5c9512f270..45310a6c22d6774d38609356ed2f0304c2a8d81c 100644 (file)
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -2281,9 +2281,11 @@ static PyTypeObject* static_types[] = {
     &_PyBufferWrapper_Type,
     &_PyContextTokenMissing_Type,
     &_PyCoroWrapper_Type,
+#ifdef _Py_TIER2
     &_PyCounterExecutor_Type,
     &_PyCounterOptimizer_Type,
     &_PyDefaultOptimizer_Type,
+#endif
     &_Py_GenericAliasIterType,
     &_PyHamtItems_Type,
     &_PyHamtKeys_Type,
@@ -2304,8 +2306,10 @@ static PyTypeObject* static_types[] = {
     &_PyPositionsIterator,
     &_PyUnicodeASCIIIter_Type,
     &_PyUnion_Type,
+#ifdef _Py_TIER2
     &_PyUOpExecutor_Type,
     &_PyUOpOptimizer_Type,
+#endif
     &_PyWeakref_CallableProxyType,
     &_PyWeakref_ProxyType,
     &_PyWeakref_RefType,

diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj
index a825cac9138674a02101d84800fa717c722fa69e..d4cd8ad1a46f24a9b46f6f7c28ec0a1111838765 100644 (file)
--- a/PCbuild/_testinternalcapi.vcxproj
+++ b/PCbuild/_testinternalcapi.vcxproj
@@ -108,6 +108,12 @@
       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
     </ProjectReference>
   </ItemGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <PreprocessorDefinitions Condition="'$(UseJIT)' == 'true'">_Py_JIT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions Condition="'$(UseTIER2)' != '0'">_Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>

diff --git a/PCbuild/build.bat b/PCbuild/build.bat
index 83b50db44670337d0e340cf2bd60247880d3dc35..43a99aa684483f3b250401437b6c0f4a23151dba 100644 (file)
--- a/PCbuild/build.bat
+++ b/PCbuild/build.bat
@@ -36,7 +36,9 @@ echo.                 overrides -c and -d
 echo.  --disable-gil  Enable experimental support for running without the GIL.
 echo.  --test-marker  Enable the test marker within the build.
 echo.  --regen        Regenerate all opcodes, grammar and tokens.
-echo.  --experimental-jit  Enable the experimental just-in-time compiler.
+echo.  --experimental-jit          Enable the experimental just-in-time compiler.
+echo.  --experimental-jit-off      Ditto but off by default (PYTHON_JIT=1 enables).
+echo.  --experimental-interpreter  Enable the experimental Tier 2 interpreter.
 echo.
 echo.Available flags to avoid building certain modules.
 echo.These flags have no effect if '-e' is not given:
@@ -66,6 +68,7 @@ set verbose=/nologo /v:m /clp:summary
 set kill=
 set do_pgo=
 set pgo_job=-m test --pgo
+set UseTIER2=0
 
 :CheckOpts
 if "%~1"=="-h" goto Usage
@@ -86,7 +89,10 @@ if "%~1"=="--disable-gil" (set UseDisableGil=true) & shift & goto CheckOpts
 if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts
 if "%~1"=="-V" shift & goto Version
 if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts
-if "%~1"=="--experimental-jit" (set UseJIT=true) & shift & goto CheckOpts
+if "%~1"=="--experimental-jit" (set UseJIT=true) & (set UseTIER2=1) & shift & goto CheckOpts
+if "%~1"=="--experimental-jit-off" (set UseJIT=true) & (set UseTIER2=3) & shift & goto CheckOpts
+if "%~1"=="--experimental-interpreter" (set UseTIER2=4) & shift & goto CheckOpts
+if "%~1"=="--experimental-interpreter-off" (set UseTIER2=6) & shift & goto CheckOpts
 rem These use the actual property names used by MSBuild.  We could just let
 rem them in through the environment, but we specify them on the command line
 rem anyway for visibility so set defaults after this
@@ -179,6 +185,7 @@ echo on
  /p:DisableGil=%UseDisableGil%^
  /p:UseTestMarker=%UseTestMarker% %GITProperty%^
  /p:UseJIT=%UseJIT%^
+ /p:UseTIER2=%UseTIER2%^
  %1 %2 %3 %4 %5 %6 %7 %8 %9
 
 @echo off

diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index 4cb3e0d32270927cedf49c50ca91883b9078f269..a24667dc74064aa6e336c447badb5075295fb376 100644 (file)
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -105,6 +105,7 @@
       <PreprocessorDefinitions>_USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="$(IncludeExternals)">_Py_HAVE_ZLIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(UseJIT)' == 'true'">_Py_JIT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions Condition="'$(UseTIER2)' != '0'">_Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
     <Link>
       <AdditionalDependencies>version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies)</AdditionalDependencies>

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 18837aef74d78e2f59f6544dbb63bd4ea61cfcd8..28766d6d97cc5f9349e48994c1b0ce640607a363 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -2363,6 +2363,7 @@ dummy_func(
             CHECK_EVAL_BREAKER();
             assert(oparg <= INSTR_OFFSET());
             JUMPBY(-oparg);
+            #ifdef _Py_TIER2
             #if ENABLE_SPECIALIZATION
             _Py_BackoffCounter counter = this_instr[1].counter;
             if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) {
@@ -2388,6 +2389,7 @@ dummy_func(
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
             }
             #endif  /* ENABLE_SPECIALIZATION */
+            #endif /* _Py_TIER2 */
         }
 
         pseudo(JUMP) = {
@@ -2401,6 +2403,7 @@ dummy_func(
         };
 
         tier1 inst(ENTER_EXECUTOR, (--)) {
+            #ifdef _Py_TIER2
             int prevoparg = oparg;
             CHECK_EVAL_BREAKER();
             if (this_instr->op.code != ENTER_EXECUTOR ||
@@ -2418,6 +2421,9 @@ dummy_func(
             tstate->previous_executor = Py_None;
             Py_INCREF(executor);
             GOTO_TIER_TWO(executor);
+            #else
+            Py_FatalError("ENTER_EXECUTOR is not supported in this build");
+            #endif /* _Py_TIER2 */
         }
 
         replaced op(_POP_JUMP_IF_FALSE, (cond -- )) {

diff --git a/Python/ceval.c b/Python/ceval.c
index 8b23bc6bcbeb3963f1ae6adeae1b94c7947054ed..59498bc826e9412ff40f3f6d329a8df3d01074ee 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -755,7 +755,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     _Py_CODEUNIT *next_instr;
     PyObject **stack_pointer;
 
-#ifndef _Py_JIT
+#if defined(_Py_TIER2) && !defined(_Py_JIT)
     /* Tier 2 interpreter state */
     _PyExecutorObject *current_executor = NULL;
     const _PyUOpInstruction *next_uop = NULL;
@@ -959,6 +959,7 @@ resume_with_error:
     goto error;
 
 
+#ifdef _Py_TIER2
 
 // Tier 2 is also here!
 enter_tier_two:
@@ -1113,6 +1114,8 @@ exit_to_trace:
 
 #endif  // _Py_JIT
 
+#endif // _Py_TIER2
+
 }
 
 #if defined(__GNUC__)

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 1444f5cdebba4b66b73a20c81a0e4abc59951ab0..602cce4beb1e28a20a648d67bf3f8ac11c7ae760 100644 (file)
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -2492,6 +2492,7 @@
             (void)this_instr;
             next_instr += 1;
             INSTRUCTION_STATS(ENTER_EXECUTOR);
+            #ifdef _Py_TIER2
             int prevoparg = oparg;
             CHECK_EVAL_BREAKER();
             if (this_instr->op.code != ENTER_EXECUTOR ||
@@ -2508,6 +2509,9 @@
             tstate->previous_executor = Py_None;
             Py_INCREF(executor);
             GOTO_TIER_TWO(executor);
+            #else
+            Py_FatalError("ENTER_EXECUTOR is not supported in this build");
+            #endif /* _Py_TIER2 */
             DISPATCH();
         }
 
@@ -3432,6 +3436,7 @@
             CHECK_EVAL_BREAKER();
             assert(oparg <= INSTR_OFFSET());
             JUMPBY(-oparg);
+            #ifdef _Py_TIER2
             #if ENABLE_SPECIALIZATION
             _Py_BackoffCounter counter = this_instr[1].counter;
             if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) {
@@ -3457,6 +3462,7 @@
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
             }
             #endif  /* ENABLE_SPECIALIZATION */
+            #endif /* _Py_TIER2 */
             DISPATCH();
         }
 

diff --git a/Python/instrumentation.c b/Python/instrumentation.c
index ce97c3add7d0ea3d408dcdd593a0db50fb0f35db..5614f4867a390d4aefd3b8624a176d2e338838dc 100644 (file)
--- a/Python/instrumentation.c
+++ b/Python/instrumentation.c
@@ -1705,10 +1705,12 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
         );
         return 0;
     }
+#ifdef _Py_TIER2
     if (code->co_executors != NULL) {
         _PyCode_Clear_Executors(code);
     }
     _Py_Executors_InvalidateDependency(interp, code, 1);
+#endif
     int code_len = (int)Py_SIZE(code);
     /* Exit early to avoid creating instrumentation
      * data for potential statically allocated code
@@ -1946,7 +1948,9 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events)
         goto done;
     }
     set_global_version(tstate, new_version);
+#ifdef _Py_TIER2
     _Py_Executors_InvalidateAll(interp, 1);
+#endif
     res = instrument_all_executing_code_objects(interp);
 done:
     _PyEval_StartTheWorld(interp);
@@ -1986,7 +1990,9 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent
         code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT;
     }
 
+#ifdef _Py_TIER2
     _Py_Executors_InvalidateDependency(interp, code, 1);
+#endif
 
     res = instrument_lock_held(code, interp);
 

diff --git a/Python/optimizer.c b/Python/optimizer.c
index a9a35fc902018ab735146a8457c477233ec83583..a5e7430c46450ef19d675260c146fb9f90b061c0 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -1,3 +1,5 @@
+#ifdef _Py_TIER2
+
 #include "Python.h"
 #include "opcode.h"
 #include "pycore_interp.h"
@@ -1622,3 +1624,5 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation)
         }
     }
 }
+
+#endif /* _Py_TIER2 */

diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c
index 9315d7228b57323eee969aa530cf99c9fe80c822..842b2e489239af07a0ad0d70b7d989c022599900 100644 (file)
--- a/Python/optimizer_analysis.c
+++ b/Python/optimizer_analysis.c
@@ -1,3 +1,5 @@
+#ifdef _Py_TIER2
+
 /*
  * This file contains the support code for CPython's uops optimizer.
  * It also performs some simple optimizations.
@@ -603,3 +605,5 @@ _Py_uop_analyze_and_optimize(
     OPT_STAT_INC(optimizer_successes);
     return length;
 }
+
+#endif /* _Py_TIER2 */

diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c
index 204599b08766c3a8873b59d0690345714273a424..d52f490853c006a3edebebbd7823bd15bbe9eca9 100644 (file)
--- a/Python/optimizer_symbols.c
+++ b/Python/optimizer_symbols.c
@@ -1,3 +1,4 @@
+#ifdef _Py_TIER2
 
 #include "Python.h"
 
@@ -506,3 +507,5 @@ fail:
     Py_XDECREF(val_43);
     return NULL;
 }
+
+#endif /* _Py_TIER2 */

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 790398e944f58e807527c5e57384413edb31e169..7726ccc979dbcc093a97fd95db1d13eafd1e2884 100644 (file)
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -624,9 +624,11 @@ static int
 builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value)
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
+#ifdef _Py_TIER2
     if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) {
         _Py_Executors_InvalidateAll(interp, 1);
     }
+#endif
     RARE_EVENT_INTERP_INC(interp, builtin_dict);
     return 0;
 }
@@ -1272,30 +1274,30 @@ init_interp_main(PyThreadState *tstate)
     }
 
     // Turn on experimental tier 2 (uops-based) optimizer
+    // This is also needed when the JIT is enabled
+#ifdef _Py_TIER2
     if (is_main_interp) {
-#ifndef _Py_JIT
-        // No JIT, maybe use the tier two interpreter:
-        char *envvar = Py_GETENV("PYTHON_UOPS");
-        int enabled = envvar != NULL && *envvar > '0';
-        if (_Py_get_xoption(&config->xoptions, L"uops") != NULL) {
-            enabled = 1;
+        int enabled = 1;
+#if _Py_TIER2 & 2
+        enabled = 0;
+#endif
+        char *env = Py_GETENV("PYTHON_JIT");
+        if (env && *env != '\0') {
+            // PYTHON_JIT=0|1 overrides the default
+            enabled = *env != '0';
         }
         if (enabled) {
-#else
-        // Always enable tier two for JIT builds (ignoring the environment
-        // variable and command-line option above):
-        if (true) {
-#endif
             PyObject *opt = PyUnstable_Optimizer_NewUOpOptimizer();
             if (opt == NULL) {
                 return _PyStatus_ERR("can't initialize optimizer");
             }
             if (PyUnstable_SetOptimizer((_PyOptimizerObject *)opt)) {
-                return _PyStatus_ERR("can't initialize optimizer");
+                return _PyStatus_ERR("can't install optimizer");
             }
             Py_DECREF(opt);
         }
     }
+#endif
 
     if (!is_main_interp) {
         // The main interpreter is handled in Py_Main(), for now.
@@ -1655,10 +1657,12 @@ finalize_modules(PyThreadState *tstate)
 {
     PyInterpreterState *interp = tstate->interp;
 
+#ifdef _Py_TIER2
     // Invalidate all executors and turn off tier 2 optimizer
     _Py_Executors_InvalidateAll(interp, 0);
     _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL);
     Py_XDECREF(old);
+#endif
 
     // Stop watching __builtin__ modifications
     PyDict_Unwatch(0, interp->builtins);

diff --git a/Python/pystate.c b/Python/pystate.c
index 9d7b73b3a071e124e555f2f798a4d9caf4f026e8..3d6e76e88bd73109fc0fe4941bb76016711df61e 100644 (file)
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -653,8 +653,10 @@ init_interpreter(PyInterpreterState *interp,
     }
     interp->sys_profile_initialized = false;
     interp->sys_trace_initialized = false;
+#ifdef _Py_TIER2
     (void)_Py_SetOptimizer(interp, NULL);
     interp->executor_list_head = NULL;
+#endif
     if (interp != &runtime->_main_interpreter) {
         /* Fix the self-referential, statically initialized fields. */
         interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp);
@@ -806,9 +808,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
         tstate->_status.cleared = 0;
     }
 
+#ifdef _Py_TIER2
     _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL);
     assert(old != NULL);
     Py_DECREF(old);
+#endif
 
     /* It is possible that any of the objects below have a finalizer
        that runs Python code or otherwise relies on a thread state
@@ -2821,9 +2825,11 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp,
     if (eval_frame == interp->eval_frame) {
         return;
     }
+#ifdef _Py_TIER2
     if (eval_frame != NULL) {
         _Py_Executors_InvalidateAll(interp, 1);
     }
+#endif
     RARE_EVENT_INC(set_eval_frame_func);
     interp->eval_frame = eval_frame;
 }

diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 7af363678e8e867f37c2d176f1e5d5a20ee2a752..726051521cf574edbba5b14461b4ae479dca956e 100644 (file)
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -2165,8 +2165,10 @@ static PyObject *
 sys__clear_internal_caches_impl(PyObject *module)
 /*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/
 {
+#ifdef _Py_TIER2
     PyInterpreterState *interp = _PyInterpreterState_GET();
     _Py_Executors_InvalidateAll(interp, 0);
+#endif
     PyType_ClearCache();
     Py_RETURN_NONE;
 }

diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py
index 18cefa083288042522c24dd68a55e51b4a776c56..fdb635486b9531715a67696da6688ec81535925e 100644 (file)
--- a/Tools/cases_generator/analyzer.py
+++ b/Tools/cases_generator/analyzer.py
@@ -411,6 +411,7 @@ NON_ESCAPING_FUNCTIONS = (
     "PyCell_New",
     "PyFloat_AS_DOUBLE",
     "_PyFrame_PushUnchecked",
+    "Py_FatalError",
 )
 
 ESCAPING_FUNCTIONS = (

diff --git a/Tools/jit/README.md b/Tools/jit/README.md
index 7b33f99d23f75d01d77f2156dcf7661aa15b2b6c..0f5aa9ce656bc98a66a6799d05597c3a82ec16b0 100644 (file)
--- a/Tools/jit/README.md
+++ b/Tools/jit/README.md
@@ -7,18 +7,18 @@ This version of CPython can be built with an experimental just-in-time compiler.
 
 The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon).
 
-LLVM version 16 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-16`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code.
+LLVM version 18 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-18`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code.
 
 It's easy to install all of the required tools:
 
 ### Linux
 
-Install LLVM 16 on Ubuntu/Debian:
+Install LLVM 18 on Ubuntu/Debian:
 
 ```sh
 wget https://apt.llvm.org/llvm.sh
 chmod +x llvm.sh
-sudo ./llvm.sh 16
+sudo ./llvm.sh 18
 ```
 
 ### macOS

diff --git a/configure b/configure
index e1d0f36463c44b0e8cf9a620453ed5bf2fa29831..01c00d2198548f5d9f590d5db6a8fc138dcda38e 100755 (executable)
--- a/configure
+++ b/configure
@@ -1818,7 +1818,7 @@ Optional Features:
   --disable-gil           enable experimental support for running without the
                           GIL (default is no)
   --enable-pystats        enable internal statistics gathering (default is no)
-  --enable-experimental-jit
+  --enable-experimental-jit[=no|yes|yes-off|interpreter]
                           build the experimental just-in-time compiler
                           (default is no)
   --enable-optimizations  enable expensive, stable optimizations (PGO, etc.)
@@ -8229,11 +8229,19 @@ else $as_nop
   enable_experimental_jit=no
 fi
 
+case $enable_experimental_jit in
+  no) enable_experimental_jit=no ;;
+  yes) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=1" ;;
+  yes-off) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=3" ;;
+  interpreter) enable_experimental_jit="-D_Py_TIER2=4" ;;
+  interpreter-off) enable_experimental_jit="-D_Py_TIER2=6" ;;  # Secret option
+  *) as_fn_error $? "invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter" "$LINENO" 5 ;;
+esac
 if test "x$enable_experimental_jit" = xno
 then :
 
 else $as_nop
-  as_fn_append CFLAGS_NODIST " -D_Py_JIT"
+  as_fn_append CFLAGS_NODIST " $enable_experimental_jit"
            REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"
            JIT_STENCILS_H="jit_stencils.h"
            if test "x$Py_DEBUG" = xtrue

diff --git a/configure.ac b/configure.ac
index 7681ea3d3fe3c69a3a0634732f5511465328480a..ae65efeac55366a9663688df49908d70f33bcc2e 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -1768,14 +1768,23 @@ fi
 # Check for --enable-experimental-jit:
 AC_MSG_CHECKING([for --enable-experimental-jit])
 AC_ARG_ENABLE([experimental-jit],
-              [AS_HELP_STRING([--enable-experimental-jit],
+              [AS_HELP_STRING([--enable-experimental-jit@<:@=no|yes|yes-off|interpreter@:>@],
                               [build the experimental just-in-time compiler (default is no)])],
               [],
               [enable_experimental_jit=no])
+case $enable_experimental_jit in
+  no) enable_experimental_jit=no ;;
+  yes) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=1" ;;
+  yes-off) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=3" ;;
+  interpreter) enable_experimental_jit="-D_Py_TIER2=4" ;;
+  interpreter-off) enable_experimental_jit="-D_Py_TIER2=6" ;;  # Secret option
+  *) AC_MSG_ERROR(
+      [invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter]) ;;
+esac
 AS_VAR_IF([enable_experimental_jit],
           [no],
           [],
-          [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"])
+          [AS_VAR_APPEND([CFLAGS_NODIST], [" $enable_experimental_jit"])
            AS_VAR_SET([REGEN_JIT_COMMAND],
                       ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"])
            AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"])