Fix dispatch_inlined

diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
index ac509b890fd31deb2571add7878883d6aefa626e..5dc0a34268696e52d8323e3101fc3b8fa292471b 100644 (file)
--- a/Include/internal/pycore_optimizer.h
+++ b/Include/internal/pycore_optimizer.h
@@ -45,7 +45,6 @@ typedef struct _PyExitData {
 typedef struct _PyExecutorObject {
     PyObject_VAR_HEAD
     const _PyUOpInstruction *trace;
-    _Py_CODEUNIT *expected_entrypoint;
     _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */
     uint32_t exit_count;
     uint32_t code_size;

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index a3854fbe0cc190d2f46ab6b58d5e1da077dba286..43f81bfdf746cc05d6a7ceeb76ff6247c01bb2ef 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -5438,33 +5438,26 @@ dummy_func(
             assert(exit != NULL);
             bool is_dynamic = exit->is_dynamic;
             _Py_CODEUNIT *target = is_dynamic ? frame->instr_ptr : (_PyFrame_GetBytecode(frame) + exit->target);
-            GOTO_TIER_ONE(target, 0);
-            // _Py_BackoffCounter temperature = exit->temperature;
-            // if (target->op.code == ENTER_EXECUTOR) {
-            //     PyCodeObject *code = _PyFrame_GetCode(frame);
-            //     _PyExecutorObject *executor = code->co_executors->executors[target->op.arg];
-            //     if (is_dynamic && executor->expected_entrypoint != target) {
-            //         GOTO_TIER_ONE(target, 0);
-            //     }
-            //     Py_INCREF(executor);
-            //     assert(tstate->jit_exit == exit);
-            //     exit->executor = executor;
-            //     TIER2_TO_TIER2(exit->executor);
-            // }
-            // else {
-            //     if (!backoff_counter_triggers(temperature)) {
-            //         exit->temperature = advance_backoff_counter(temperature);
-            //         GOTO_TIER_ONE(target, 0);
-            //     }
-            //     if (is_dynamic) {
-            //         GOTO_TIER_ONE(target, 0);
-            //     }
-            //     _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
-            //     assert(tstate->current_executor == (PyObject *)previous_executor);
-            //     int chain_depth = 0;
-            //     _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth);
-            //     GOTO_TIER_ONE(target, 1);
-            // }
+            _Py_BackoffCounter temperature = exit->temperature;
+            if (target->op.code == ENTER_EXECUTOR) {
+                PyCodeObject *code = _PyFrame_GetCode(frame);
+                _PyExecutorObject *executor = code->co_executors->executors[target->op.arg];
+                Py_INCREF(executor);
+                assert(tstate->jit_exit == exit);
+                exit->executor = executor;
+                TIER2_TO_TIER2(exit->executor);
+            }
+            else {
+                if (!backoff_counter_triggers(temperature)) {
+                    exit->temperature = advance_backoff_counter(temperature);
+                    GOTO_TIER_ONE(target, 0);
+                }
+                _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
+                assert(tstate->current_executor == (PyObject *)previous_executor);
+                int chain_depth = 0;
+                _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth);
+                GOTO_TIER_ONE(target, 1);
+            }
         }
 
         tier2 op(_GUARD_IP, (ip/4 --)) {

diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index 2f53366915dc72ff860bbc184241fd9540ff31a5..540785414e7334380721c74a7890c34e5d6944c0 100644 (file)
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -217,6 +217,10 @@ do { \
         JUMP_TO_LABEL(start_frame);                      \
     } while (0)
 
+#define TRACING_DISPATCH_INLINED(NEW_FRAME) \
+    RECORD_TRACE_NO_DISPATCH(); \
+    DISPATCH_INLINED(NEW_FRAME);
+
 #define TRACING_DISPATCH() \
     { \
         assert(frame->stackpointer == NULL); \
@@ -421,7 +425,6 @@ do {                                                   \
         JUMP_TO_LABEL(error);                          \
     }                                                  \
     if (keep_tracing_bit) { \
-        assert(next_instr == frame->instr_ptr); \
         assert(next_instr->op.code != ENTER_EXECUTOR); \
         assert(tstate->interp->jit_tracer_code_curr_size == 2); \
         ENTER_TRACING(); \

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 0a94baa0557d37d00548b6f4eac2f569de3b51c1..f2a100210d4b86b213fac01569691343482533f2 100644 (file)
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -7522,7 +7522,26 @@
             assert(exit != NULL);
             bool is_dynamic = exit->is_dynamic;
             _Py_CODEUNIT *target = is_dynamic ? frame->instr_ptr : (_PyFrame_GetBytecode(frame) + exit->target);
-            GOTO_TIER_ONE(target, 0);
+            _Py_BackoffCounter temperature = exit->temperature;
+            if (target->op.code == ENTER_EXECUTOR) {
+                PyCodeObject *code = _PyFrame_GetCode(frame);
+                _PyExecutorObject *executor = code->co_executors->executors[target->op.arg];
+                Py_INCREF(executor);
+                assert(tstate->jit_exit == exit);
+                exit->executor = executor;
+                TIER2_TO_TIER2(exit->executor);
+            }
+            else {
+                if (!backoff_counter_triggers(temperature)) {
+                    exit->temperature = advance_backoff_counter(temperature);
+                    GOTO_TIER_ONE(target, 0);
+                }
+                _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
+                assert(tstate->current_executor == (PyObject *)previous_executor);
+                int chain_depth = 0;
+                _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth);
+                GOTO_TIER_ONE(target, 1);
+            }
             break;
         }
 

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 78a458b4899eff274bb9cc78622b1ad049f28ab5..1f77215f582219d60ff036cbb9eaddaef59d50e1 100644 (file)
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -15256,7 +15256,7 @@
                         TRACING_JUMP_TO_LABEL(error);
                     }
                     frame->return_offset = 4u ;
-                    DISPATCH_INLINED(new_frame);
+                    TRACING_DISPATCH_INLINED(new_frame);
                 }
                 STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
                 if (CONVERSION_FAILED(args_o)) {
@@ -16329,7 +16329,7 @@
                         }
                         assert( 1u == 1);
                         frame->return_offset = 1;
-                        DISPATCH_INLINED(new_frame);
+                        TRACING_DISPATCH_INLINED(new_frame);
                     }
                     PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
                     assert(PyTuple_CheckExact(callargs));
@@ -16641,7 +16641,7 @@
                     }
                     assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW);
                     frame->return_offset = 4u ;
-                    DISPATCH_INLINED(new_frame);
+                    TRACING_DISPATCH_INLINED(new_frame);
                 }
                 STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
                 if (CONVERSION_FAILED(args_o)) {
@@ -20564,7 +20564,7 @@
                         TRACING_JUMP_TO_LABEL(error);
                     }
                     frame->return_offset = 4u ;
-                    DISPATCH_INLINED(new_frame);
+                    TRACING_DISPATCH_INLINED(new_frame);
                 }
                 STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
                 if (CONVERSION_FAILED(args_o)) {
@@ -20785,7 +20785,7 @@
                         }
                         assert( 1u == 1);
                         frame->return_offset = 1;
-                        DISPATCH_INLINED(new_frame);
+                        TRACING_DISPATCH_INLINED(new_frame);
                     }
                     PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
                     assert(PyTuple_CheckExact(callargs));
@@ -20935,7 +20935,7 @@
                     }
                     assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW);
                     frame->return_offset = 4u ;
-                    DISPATCH_INLINED(new_frame);
+                    TRACING_DISPATCH_INLINED(new_frame);
                 }
                 STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
                 if (CONVERSION_FAILED(args_o)) {
@@ -22544,7 +22544,7 @@
             assert(WITHIN_STACK_BOUNDS());
             new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name);
             frame->return_offset = 10u ;
-            DISPATCH_INLINED(new_frame);
+            TRACING_DISPATCH_INLINED(new_frame);
         }
 
         TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) {
@@ -25424,7 +25424,7 @@
                     frame->return_offset = (uint16_t)( 2u + oparg);
                     assert(gen_frame->previous == NULL);
                     gen_frame->previous = frame;
-                    DISPATCH_INLINED(gen_frame);
+                    TRACING_DISPATCH_INLINED(gen_frame);
                 }
                 if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) {
                     _PyFrame_SetStackPointer(frame, stack_pointer);

diff --git a/Python/optimizer.c b/Python/optimizer.c
index 2f6f48271710f9c0370513f5374f213f878a3359..8c69bd10662aa6cdce88d6d6c28bb995747c1959 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -584,6 +584,10 @@ _PyJIT_translate_single_bytecode_to_trace(
         return 1;
     }
 
+    if (opcode == JUMP_BACKWARD_NO_INTERRUPT) {
+        return 1;
+    }
+
     if (opcode == ENTER_EXECUTOR) {
         ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target);
         ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target);
@@ -899,9 +903,13 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length)
                 exit_op = _HANDLE_PENDING_AND_DEOPT;
             }
             int32_t jump_target = target;
-            if (opcode == _FOR_ITER_TIER_TWO || opcode == _GUARD_IP) {
+            bool unique_target = false;
+            if (opcode == _FOR_ITER_TIER_TWO) {
+                exit_op = _DYNAMIC_EXIT;
+            }
+            else if (opcode == _GUARD_IP) {
                 exit_op = _DYNAMIC_EXIT;
-                jump_target = current_jump_target + 1;
+                unique_target = true;
             }
             if (is_for_iter_test[opcode]) {
                 /* Target the POP_TOP immediately after the END_FOR,
@@ -910,7 +918,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length)
                 int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg;
                 jump_target = next_inst + inst->oparg + 1;
             }
-            if (jump_target != current_jump_target || current_exit_op != exit_op) {
+            if (unique_target || jump_target != current_jump_target || current_exit_op != exit_op) {
                 make_exit(&buffer[next_spare], exit_op, jump_target);
                 current_exit_op = exit_op;
                 current_jump_target = jump_target;

diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py
index 690c555b8c083b792daaa4c9d83a4e25a713a34e..61088ff185e2d56b208d99827ae9ca7ab6faa7e4 100644 (file)
--- a/Tools/cases_generator/tracer_generator.py
+++ b/Tools/cases_generator/tracer_generator.py
@@ -40,6 +40,7 @@ class TracerEmitter(Emitter):
         self._replacers = {
             **self._replacers,
             "DISPATCH": self.dispatch,
+            "DISPATCH_INLINED": self.dispatch_inlined,
         }
 
     def dispatch(
@@ -57,6 +58,20 @@ class TracerEmitter(Emitter):
         self.emit("TRACING_DISPATCH")
         return False
 
+    def dispatch_inlined(
+        self,
+        tkn: Token,
+        tkn_iter: TokenIterator,
+        uop: CodeSection,
+        storage: Storage,
+        inst: Instruction | None,
+    ) -> bool:
+        if storage.spilled:
+            raise analysis_error("stack_pointer needs reloading before dispatch", tkn)
+        storage.stack.flush(self.out)
+        self.out.start_line()
+        self.emit("TRACING_DISPATCH_INLINED")
+        return False
     def record_jump_taken(
         self,
         tkn: Token,