Clean up labels

diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h
index 825321739c40c5bde1d66a69fe5039318fa3bb82..548627dc7982ec9c92bda5dea95d97c50891a3bd 100644 (file)
--- a/Include/internal/pycore_opcode_metadata.h
+++ b/Include/internal/pycore_opcode_metadata.h
@@ -1150,7 +1150,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = {
     [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG },
     [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG },
     [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG },
-    [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+    [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
     [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
     [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
     [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@@ -1189,7 +1189,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = {
     [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
     [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG },
     [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG },
-    [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+    [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
     [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG },
     [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
     [JUMP_BACKWARD_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },

diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
index e4219ebc86ae29058a4c43fdc59d1fc1a0b44d9f..653285a2c6b79b0cf935a814f5ba393e619cbb29 100644 (file)
--- a/Include/internal/pycore_optimizer.h
+++ b/Include/internal/pycore_optimizer.h
@@ -362,7 +362,7 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out);
 extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp);
 #endif
 
-int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr);
+int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, bool stop_tracing);
 
 int
 _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame,

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 283a18424715b607e93fbe975b04274b24b35950..5bfd9ea8a5a03082684bf350a53b5e56990cef75 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -1218,33 +1218,19 @@ dummy_func(
             tstate->current_frame = frame->previous;
             assert(!_PyErr_Occurred(tstate));
             PyObject *result = PyStackRef_AsPyObjectSteal(retval);
-            if (IS_JIT_TRACING()) {
-#if _Py_TIER2
-                _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL);
-                LEAVE_TRACING();
-                int err = bail_tracing_and_jit(tstate, frame);
-                if (err < 0) {
-                    Py_DECREF(result);
-                    ERROR_IF(true);
-                }
-                return result;
-#endif
-            }
-            else {
 #if !_Py_TAIL_CALL_INTERP
-                assert(frame == &entry.frame);
+            assert(frame == &entry.frame);
 #endif
 #ifdef _Py_TIER2
-                _PyStackRef executor = frame->localsplus[0];
-                assert(tstate->current_executor == NULL);
-                if (!PyStackRef_IsNull(executor)) {
-                    tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
-                    PyStackRef_CLOSE(executor);
-                }
-#endif
-                LLTRACE_RESUME_FRAME();
-                return result;
+            _PyStackRef executor = frame->localsplus[0];
+            assert(tstate->current_executor == NULL);
+            if (!PyStackRef_IsNull(executor)) {
+                tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
+                PyStackRef_CLOSE(executor);
             }
+#endif
+            LLTRACE_RESUME_FRAME();
+            return result;
         }
 
         // The stack effect here is a bit misleading.
@@ -3028,10 +3014,8 @@ dummy_func(
         tier1 inst(ENTER_EXECUTOR, (--)) {
             #ifdef _Py_TIER2
             if (IS_JIT_TRACING()) {
-                _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
-                LEAVE_TRACING();
-                int err = bail_tracing_and_jit(tstate, frame);
-                ERROR_IF(err < 0);
+                next_instr = this_instr;
+                goto stop_tracing;
             }
             PyCodeObject *code = _PyFrame_GetCode(frame);
             _PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
@@ -5657,7 +5641,10 @@ dummy_func(
 #if _Py_TIER2
             assert(IS_JIT_TRACING());
             int opcode = next_instr->op.code;
-            int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
+            bool stop_tracing = (opcode == WITH_EXCEPT_START ||
+                opcode == RERAISE || opcode == CLEANUP_THROW ||
+                opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT);
+            int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing);
             if (full) {
                 LEAVE_TRACING();
                 int err = bail_tracing_and_jit(tstate, frame);
@@ -5677,9 +5664,9 @@ dummy_func(
                 _tstate->jit_state.prev_state.instr = next_instr;
             }
             _tstate->jit_state.prev_state.specialize_counter = 0;
-            PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable));
-            if (_tstate->jit_state.prev_state.instr_code != prev_code) {
-                Py_SETREF(_tstate->jit_state.prev_state.instr_code, prev_code);
+            PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable);
+            if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) {
+                Py_SETREF(_tstate->jit_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code)));
             }
 
             _tstate->jit_state.prev_state.instr_frame = frame;
@@ -5691,6 +5678,19 @@ dummy_func(
 #endif
         }
 
+        label(stop_tracing) {
+#if _Py_TIER2
+            assert(IS_JIT_TRACING());
+            _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true);
+            LEAVE_TRACING();
+            int err = bail_tracing_and_jit(tstate, frame);
+            ERROR_IF(err < 0);
+            DISPATCH_GOTO_NON_TRACING();
+#else
+            Py_FatalError("JIT label executed in non-jit build.");
+#endif
+        }
+
 
 // END BYTECODES //
 

diff --git a/Python/ceval.c b/Python/ceval.c
index e6424503e88988067c07ec8201c4144db65da61d..aea353603b545f8211ff419b42c58ca133b0a48a 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1017,13 +1017,7 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame)
         // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed
         // to be valid to access.
         if (err <= 0) {
-            // Some opcodes will forever be unchanged. Don't ever bother specializing for them ever again.
-            if (_tstate->jit_state.prev_state.instr->op.code == INTERPRETER_EXIT) {
-                exit->temperature = initial_unreachable_backoff_counter();
-            }
-            else {
-                exit->temperature = restart_backoff_counter(exit->temperature);
-            }
+            exit->temperature = restart_backoff_counter(exit->temperature);
         }
         else {
             exit->temperature = initial_temperature_backoff_counter();

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 284d90869aea80820517a2b6cf6e6ab8dea98d28..29b294f0f6e9cb8aa795217cc89e422cfa97c2e6 100644 (file)
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -5492,16 +5492,8 @@
             opcode = ENTER_EXECUTOR;
             #ifdef _Py_TIER2
             if (IS_JIT_TRACING()) {
-                _PyFrame_SetStackPointer(frame, stack_pointer);
-                _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
-                stack_pointer = _PyFrame_GetStackPointer(frame);
-                LEAVE_TRACING();
-                _PyFrame_SetStackPointer(frame, stack_pointer);
-                int err = bail_tracing_and_jit(tstate, frame);
-                stack_pointer = _PyFrame_GetStackPointer(frame);
-                if (err < 0) {
-                    JUMP_TO_LABEL(error);
-                }
+                next_instr = this_instr;
+                JUMP_TO_LABEL(stop_tracing);
             }
             PyCodeObject *code = _PyFrame_GetCode(frame);
             _PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
@@ -7552,46 +7544,24 @@
             tstate->current_frame = frame->previous;
             assert(!_PyErr_Occurred(tstate));
             PyObject *result = PyStackRef_AsPyObjectSteal(retval);
-            if (IS_JIT_TRACING()) {
-                #if _Py_TIER2
+            #if !_Py_TAIL_CALL_INTERP
+            assert(frame == &entry.frame);
+            #endif
+            #ifdef _Py_TIER2
+            _PyStackRef executor = frame->localsplus[0];
+            assert(tstate->current_executor == NULL);
+            if (!PyStackRef_IsNull(executor)) {
+                tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
                 stack_pointer += -1;
                 assert(WITHIN_STACK_BOUNDS());
                 _PyFrame_SetStackPointer(frame, stack_pointer);
-                _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL);
-                stack_pointer = _PyFrame_GetStackPointer(frame);
-                LEAVE_TRACING();
-                _PyFrame_SetStackPointer(frame, stack_pointer);
-                int err = bail_tracing_and_jit(tstate, frame);
+                PyStackRef_CLOSE(executor);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
-                if (err < 0) {
-                    _PyFrame_SetStackPointer(frame, stack_pointer);
-                    Py_DECREF(result);
-                    stack_pointer = _PyFrame_GetStackPointer(frame);
-                    JUMP_TO_LABEL(error);
-                }
-                return result;
-                #endif
-            }
-            else {
-                #if !_Py_TAIL_CALL_INTERP
-                assert(frame == &entry.frame);
-                #endif
-                #ifdef _Py_TIER2
-                _PyStackRef executor = frame->localsplus[0];
-                assert(tstate->current_executor == NULL);
-                if (!PyStackRef_IsNull(executor)) {
-                    tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
-                    stack_pointer += -1;
-                    assert(WITHIN_STACK_BOUNDS());
-                    _PyFrame_SetStackPointer(frame, stack_pointer);
-                    PyStackRef_CLOSE(executor);
-                    stack_pointer = _PyFrame_GetStackPointer(frame);
-                    stack_pointer += 1;
-                }
-                #endif
-                LLTRACE_RESUME_FRAME();
-                return result;
+                stack_pointer += 1;
             }
+            #endif
+            LLTRACE_RESUME_FRAME();
+            return result;
         }
 
         TARGET(IS_OP) {
@@ -12339,8 +12309,11 @@ JUMP_TO_LABEL(error);
             #if _Py_TIER2
             assert(IS_JIT_TRACING());
             int opcode = next_instr->op.code;
+            bool stop_tracing = (opcode == WITH_EXCEPT_START ||
+                                 opcode == RERAISE || opcode == CLEANUP_THROW ||
+                                 opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT);
             _PyFrame_SetStackPointer(frame, stack_pointer);
-            int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
+            int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing);
             stack_pointer = _PyFrame_GetStackPointer(frame);
             if (full) {
                 LEAVE_TRACING();
@@ -12363,10 +12336,10 @@ JUMP_TO_LABEL(error);
                 _tstate->jit_state.prev_state.instr = next_instr;
             }
             _tstate->jit_state.prev_state.specialize_counter = 0;
-            PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable));
-            if (_tstate->jit_state.prev_state.instr_code != prev_code) {
+            PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable);
+            if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) {
                 _PyFrame_SetStackPointer(frame, stack_pointer);
-                Py_SETREF(_tstate->jit_state.prev_state.instr_code, prev_code);
+                Py_SETREF(_tstate->jit_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code)));
                 stack_pointer = _PyFrame_GetStackPointer(frame);
             }
             _tstate->jit_state.prev_state.instr_frame = frame;
@@ -12378,5 +12351,25 @@ JUMP_TO_LABEL(error);
             #endif
         }
 
+        LABEL(stop_tracing)
+        {
+            #if _Py_TIER2
+            assert(IS_JIT_TRACING());
+            _PyFrame_SetStackPointer(frame, stack_pointer);
+            _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true);
+            stack_pointer = _PyFrame_GetStackPointer(frame);
+            LEAVE_TRACING();
+            _PyFrame_SetStackPointer(frame, stack_pointer);
+            int err = bail_tracing_and_jit(tstate, frame);
+            stack_pointer = _PyFrame_GetStackPointer(frame);
+            if (err < 0) {
+                JUMP_TO_LABEL(error);
+            }
+            DISPATCH_GOTO_NON_TRACING();
+            #else
+            Py_FatalError("JIT label executed in non-jit build.");
+            #endif
+        }
+
 /* END LABELS */
 #undef TIER_ONE

diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index 288e45ce8bc8439c4a9e5a888ed3604472eb41a0..1b9196503b570b7c21b07349ef503c2a67bd0bbb 100644 (file)
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -529,6 +529,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM
 Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS);
 Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS);
 Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_record_previous_inst(TAIL_CALL_PARAMS);
+Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS);
 
 Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS);
 Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS);

diff --git a/Python/optimizer.c b/Python/optimizer.c
index 5a7de32f4607d94a1e10d543975063108046d9fd..64e57f26094d4fb4ccafe08a13850e5684ce41f4 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -552,7 +552,8 @@ int
 _PyJit_translate_single_bytecode_to_trace(
     PyThreadState *tstate,
     _PyInterpreterFrame *frame,
-    _Py_CODEUNIT *next_instr)
+    _Py_CODEUNIT *next_instr,
+    bool stop_tracing)
 {
 
 #ifdef Py_DEBUG
@@ -571,11 +572,6 @@ _PyJit_translate_single_bytecode_to_trace(
     _PyUOpInstruction *trace = _tstate->jit_state.code_buffer;
     int max_length = _tstate->jit_state.prev_state.code_max_size;
 
-    int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL);
-    if (is_sys_tracing) {
-        goto full;
-    }
-
     _Py_CODEUNIT *this_instr =  _tstate->jit_state.prev_state.instr;
     _Py_CODEUNIT *target_instr = this_instr;
     uint32_t target = 0;
@@ -619,6 +615,17 @@ _PyJit_translate_single_bytecode_to_trace(
         DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]);
         goto unsupported;
     }
+
+    int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL);
+    if (is_sys_tracing) {
+        goto full;
+    }
+
+    if (stop_tracing) {
+        ADD_TO_TRACE(_DEOPT, 0, 0, target);
+        goto done;
+    }
+
     DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level);
 
 #ifdef Py_DEBUG
@@ -655,11 +662,6 @@ _PyJit_translate_single_bytecode_to_trace(
 
     // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom.
     if (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize) {
-        goto unsupported;
-    }
-
-    if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) {
-        DPRINTF(2, "Unsupported: strange control-flow\n");
         unsupported:
         {
             // Rewind to previous instruction and replace with _EXIT_TRACE.
@@ -773,9 +775,6 @@ _PyJit_translate_single_bytecode_to_trace(
              *  start with RESUME_CHECK */
             ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target);
             break;
-        case INTERPRETER_EXIT:
-            ADD_TO_TRACE(_DEOPT, 0, 0, target);
-            goto done;;
         default:
         {
             const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode];