GH-122294: Burn in the addresses of side exits (GH-122295)

diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h
index fd0d4a67d9353853a469684bc9d459ce436febab..d23a4e2ea143457c0f6e6769cadff8c8dcbe19c0 100644 (file)
--- a/Include/internal/pycore_uop_metadata.h
+++ b/Include/internal/pycore_uop_metadata.h
@@ -252,7 +252,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_SET_IP] = 0,
     [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG,
     [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG,
-    [_EXIT_TRACE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
+    [_EXIT_TRACE] = HAS_ESCAPES_FLAG,
     [_CHECK_VALIDITY] = HAS_DEOPT_FLAG,
     [_LOAD_CONST_INLINE] = HAS_PURE_FLAG,
     [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG,
@@ -261,7 +261,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG,
     [_CHECK_FUNCTION] = HAS_DEOPT_FLAG,
     [_INTERNAL_INCREMENT_OPT_COUNTER] = 0,
-    [_DYNAMIC_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
+    [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG,
     [_START_EXECUTOR] = 0,
     [_FATAL_ERROR] = 0,
     [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG,

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 871e2dbf35841857774cf1e75239814a11c676ab..d74f2aae0483ce169fa3c2f3514751a8a35ddd1c 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -4609,8 +4609,8 @@ dummy_func(
             #endif
         }
 
-        tier2 op(_EXIT_TRACE, (--)) {
-            _PyExitData *exit = &current_executor->exits[oparg];
+        tier2 op(_EXIT_TRACE, (exit_p/4 --)) {
+            _PyExitData *exit = (_PyExitData *)exit_p;
             PyCodeObject *code = _PyFrame_GetCode(frame);
             _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target;
         #if defined(Py_DEBUG) && !defined(_Py_JIT)
@@ -4619,7 +4619,7 @@ dummy_func(
                 printf("SIDE EXIT: [UOp ");
                 _PyUOpPrint(&next_uop[-1]);
                 printf(", exit %u, temp %d, target %d -> %s]\n",
-                    oparg, exit->temperature.as_counter,
+                    exit - current_executor->exits, exit->temperature.as_counter,
                     (int)(target - _PyCode_CODE(code)),
                     _PyOpcode_OpName[target->op.code]);
             }
@@ -4698,9 +4698,9 @@ dummy_func(
             exe->count++;
         }
 
-        tier2 op(_DYNAMIC_EXIT, (--)) {
+        tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) {
             tstate->previous_executor = (PyObject *)current_executor;
-            _PyExitData *exit = (_PyExitData *)&current_executor->exits[oparg];
+            _PyExitData *exit = (_PyExitData *)exit_p;
             _Py_CODEUNIT *target = frame->instr_ptr;
         #if defined(Py_DEBUG) && !defined(_Py_JIT)
             OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
@@ -4708,7 +4708,7 @@ dummy_func(
                 printf("DYNAMIC EXIT: [UOp ");
                 _PyUOpPrint(&next_uop[-1]);
                 printf(", exit %u, temp %d, target %d -> %s]\n",
-                    oparg, exit->temperature.as_counter,
+                    exit - current_executor->exits, exit->temperature.as_counter,
                     (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))),
                     _PyOpcode_OpName[target->op.code]);
             }

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 1ced8b951b5ce9002fc94cc01c09cf90df9dde63..6e3f6cc62fe11fd72d2a0ffb20554ba45012bcd8 100644 (file)
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -5044,8 +5044,8 @@
         }
 
         case _EXIT_TRACE: {
-            oparg = CURRENT_OPARG();
-            _PyExitData *exit = &current_executor->exits[oparg];
+            PyObject *exit_p = (PyObject *)CURRENT_OPERAND();
+            _PyExitData *exit = (_PyExitData *)exit_p;
             PyCodeObject *code = _PyFrame_GetCode(frame);
             _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target;
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
@@ -5054,7 +5054,7 @@
                 printf("SIDE EXIT: [UOp ");
                 _PyUOpPrint(&next_uop[-1]);
                 printf(", exit %u, temp %d, target %d -> %s]\n",
-                       oparg, exit->temperature.as_counter,
+                       exit - current_executor->exits, exit->temperature.as_counter,
                        (int)(target - _PyCode_CODE(code)),
                        _PyOpcode_OpName[target->op.code]);
             }
@@ -5182,9 +5182,9 @@
         }
 
         case _DYNAMIC_EXIT: {
-            oparg = CURRENT_OPARG();
+            PyObject *exit_p = (PyObject *)CURRENT_OPERAND();
             tstate->previous_executor = (PyObject *)current_executor;
-            _PyExitData *exit = (_PyExitData *)&current_executor->exits[oparg];
+            _PyExitData *exit = (_PyExitData *)exit_p;
             _Py_CODEUNIT *target = frame->instr_ptr;
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
             OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
@@ -5192,7 +5192,7 @@
                 printf("DYNAMIC EXIT: [UOp ");
                 _PyUOpPrint(&next_uop[-1]);
                 printf(", exit %u, temp %d, target %d -> %s]\n",
-                       oparg, exit->temperature.as_counter,
+                       exit - current_executor->exits, exit->temperature.as_counter,
                        (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))),
                        _PyOpcode_OpName[target->op.code]);
             }

diff --git a/Python/optimizer.c b/Python/optimizer.c
index f0793b8c8f20882c78fe49435608260c26cee736..7b875af2aae898a84e34433d1a0eeaeb1b46de8f 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -1153,13 +1153,15 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil
         *dest = buffer[i];
         assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE);
         if (opcode == _EXIT_TRACE) {
-            executor->exits[next_exit].target = buffer[i].target;
-            dest->oparg = next_exit;
+            _PyExitData *exit = &executor->exits[next_exit];
+            exit->target = buffer[i].target;
+            dest->operand = (uint64_t)exit;
             next_exit--;
         }
         if (opcode == _DYNAMIC_EXIT) {
-            executor->exits[next_exit].target = 0;
-            dest->oparg = next_exit;
+            _PyExitData *exit = &executor->exits[next_exit];
+            exit->target = 0;
+            dest->operand = (uint64_t)exit;
             next_exit--;
         }
     }

diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c
index a506f9948fd9aecf80321bcb9f27de94d43f92b0..4d4f89301c747570f23539056c0225d63c301a1a 100644 (file)
--- a/Python/optimizer_bytecodes.c
+++ b/Python/optimizer_bytecodes.c
@@ -788,7 +788,8 @@ dummy_func(void) {
         ctx->done = true;
     }
 
-    op(_EXIT_TRACE, (--)) {
+    op(_EXIT_TRACE, (exit_p/4 --)) {
+        (void)exit_p;
         ctx->done = true;
     }
 

diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h
index 166b1674bc3334681b0858292321ab2c93855852..fae93ce89e82e508376fda5133bcfb0e6d454395 100644 (file)
--- a/Python/optimizer_cases.c.h
+++ b/Python/optimizer_cases.c.h
@@ -2163,6 +2163,8 @@
         }
 
         case _EXIT_TRACE: {
+            PyObject *exit_p = (PyObject *)this_instr->operand;
+            (void)exit_p;
             ctx->done = true;
             break;
         }