Remove bytecode object for ceval.c

diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index 61917b01b7cb1351520f1653880f3adaa82249eb..3039a141a0dc13c30d5bff01309b42d30ce76eec 100644 (file)
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -391,8 +391,7 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame
 #define SPECIAL___AEXIT__   3
 #define SPECIAL_MAX   3
 
-struct _PyCode12 _PyCode_DEF(12);
-PyAPI_DATA(const struct _PyCode12) _PyEntryFrameCode;
+PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR;
 
 #ifdef __cplusplus
 }

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index d62a7515963ff03a06c652ffdfdac10a330ed9c6..b5d4468fe94fcae973dbb785598a372589782a2a 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -44,6 +44,7 @@
 
 #define USE_COMPUTED_GOTOS 0
 #include "ceval_macros.h"
+#include "../Include/internal/pycore_stackref.h"
 
 /* Flow control macros */
 
@@ -170,7 +171,6 @@ dummy_func(
 
         op(_QUICKEN_RESUME, (--)) {
             #if ENABLE_SPECIALIZATION_FT
-            PyCodeObject *code = _PyFrame_GetCode(frame);
             if (tstate->tracing == 0 && this_instr->op.code == RESUME) {
                 FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
             }
@@ -5266,7 +5266,9 @@ dummy_func(
         tier2 op(_EXIT_TRACE, (exit_p/4 --)) {
             _PyExitData *exit = (_PyExitData *)exit_p;
         #if defined(Py_DEBUG) && !defined(_Py_JIT)
-            _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
+            const _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER)
+                ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame))
+                + exit->target;
             OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
             if (frame->lltrace >= 2) {
                 printf("SIDE EXIT: [UOp ");
@@ -5420,7 +5422,8 @@ dummy_func(
         tier2 op(_COLD_EXIT, ( -- )) {
             _PyExitData *exit = tstate->jit_exit;
             assert(exit != NULL);
-            _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
+            _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER)
+                ? (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + exit->target;
             _Py_BackoffCounter temperature = exit->temperature;
             _PyExecutorObject *executor;
             if (target->op.code == ENTER_EXECUTOR) {
@@ -5429,6 +5432,9 @@ dummy_func(
                 Py_INCREF(executor);
             }
             else {
+                if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
+                    GOTO_TIER_ONE(target);
+                }
                 if (!backoff_counter_triggers(temperature)) {
                     exit->temperature = advance_backoff_counter(temperature);
                     GOTO_TIER_ONE(target);
@@ -5664,14 +5670,14 @@ dummy_func(
                 tstate->interp->jit_state.prev_instr = next_instr;
             }
             tstate->interp->jit_state.specialize_counter = 0;
-            PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame));
+            PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable));
             if (tstate->interp->jit_state.prev_instr_code != prev_code) {
                 Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code);
             }
 
             tstate->interp->jit_state.prev_instr_frame = frame;
             tstate->interp->jit_state.prev_instr_oparg = oparg;
-            tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL();
+            tstate->interp->jit_state.prev_instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL();
             DISPATCH_GOTO_NON_TRACING();
 #else
             Py_FatalError("JIT label executed in non-jit build.");

diff --git a/Python/ceval.c b/Python/ceval.c
index 09fd30b0c8a030d523805ffe5b320a8ff1efa3d1..6abb9872037097b65ab840df10b54a888a836011 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -931,44 +931,17 @@ int _Py_CheckRecursiveCallPy(
     return 0;
 }
 
-static const PyBytesObject no_location = {
-    PyVarObject_HEAD_INIT(&PyBytes_Type, 1)
-    .ob_sval = { NO_LOC_4 }
+static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = {
+    /* Put a NOP at the start, so that the IP points into
+    * the code, rather than before it */
+    { .op.code = NOP, .op.arg = 0 },
+    { .op.code = INTERPRETER_EXIT, .op.arg = 0 },  /* reached on return */
+    { .op.code = NOP, .op.arg = 0 },
+    { .op.code = INTERPRETER_EXIT, .op.arg = 0 },  /* reached on yield */
+    { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START }
 };
 
-#ifdef Py_GIL_DISABLED
-static _PyCodeArray emtry_cleanup_tlbc = {
-    .size = 1,
-    .entries = {(char*) &_PyEntryFrameCode.co_code_adaptive},
-};
-#endif
-
-const struct _PyCode12 _PyEntryFrameCode = {
-    _PyVarObject_HEAD_INIT(&PyCode_Type, 5),
-    .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty),
-    .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty),
-    .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty),
-    .co_flags = CO_OPTIMIZED | CO_NO_MONITORING_EVENTS,
-    .co_localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty),
-    .co_localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty),
-    .co_filename = &_Py_ID(_PyEval_EvalFrameDefault),
-    .co_name = &_Py_ID(_PyEval_EvalFrameDefault),
-    .co_qualname = &_Py_ID(_PyEval_EvalFrameDefault),
-    .co_linetable = (PyObject *)&no_location,
-    ._co_firsttraceable = 4,
-    .co_stacksize = 2,
-    .co_framesize = 2 + FRAME_SPECIALS_SIZE,
-#ifdef Py_GIL_DISABLED
-    .co_tlbc = &emtry_cleanup_tlbc,
-#endif
-    .co_code_adaptive = {
-        NOP, 0,
-        INTERPRETER_EXIT, 0, /* reached on return */
-        NOP, 0,
-        INTERPRETER_EXIT, 0, /* reached on yield */
-        RESUME, RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START
-    }
-};
+const _Py_CODEUNIT *_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR = (_Py_CODEUNIT*)&_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS;
 
 #ifdef Py_DEBUG
 extern void _PyUOpPrint(const _PyUOpInstruction *uop);
@@ -1140,8 +1113,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     entry.frame.f_globals = (PyObject*)0xaaa3;
     entry.frame.f_builtins = (PyObject*)0xaaa4;
 #endif
-    entry.frame.f_executable = PyStackRef_FromPyObjectBorrow((PyObject *)&_PyEntryFrameCode);
-    entry.frame.instr_ptr = ((_Py_CODEUNIT *)_PyEntryFrameCode.co_code_adaptive) + 1;
+    entry.frame.f_executable = PyStackRef_None;
+    entry.frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1;
     entry.frame.stackpointer = entry.stack;
     entry.frame.owner = FRAME_OWNED_BY_INTERPRETER;
     entry.frame.visited = 0;

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 88ef5a8656e0098b51fcbcc3bb76fad33cbb4cfe..1b52a27621af5ed894c7a2e0ce11e1ab1cef29b3 100644 (file)
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -7110,7 +7110,9 @@
             PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
             _PyExitData *exit = (_PyExitData *)exit_p;
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
-            _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
+            const _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER)
+                ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame))
+            + exit->target;
             OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
             if (frame->lltrace >= 2) {
                 _PyFrame_SetStackPointer(frame, stack_pointer);
@@ -7462,7 +7464,8 @@
         case _COLD_EXIT: {
             _PyExitData *exit = tstate->jit_exit;
             assert(exit != NULL);
-            _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
+            _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER)
+                                    ? (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + exit->target;
             _Py_BackoffCounter temperature = exit->temperature;
             _PyExecutorObject *executor;
             if (target->op.code == ENTER_EXECUTOR) {
@@ -7471,6 +7474,9 @@
                 Py_INCREF(executor);
             }
             else {
+                if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
+                    GOTO_TIER_ONE(target);
+                }
                 if (!backoff_counter_triggers(temperature)) {
                     exit->temperature = advance_backoff_counter(temperature);
                     GOTO_TIER_ONE(target);

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 1c7b1035a1a8ce726fb76dd1f54865c499137130..ff5b47496e1124cc86b36d4c4c2c475833e5ce54 100644 (file)
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -10343,7 +10343,6 @@
             // _QUICKEN_RESUME
             {
                 #if ENABLE_SPECIALIZATION_FT
-                PyCodeObject *code = _PyFrame_GetCode(frame);
                 if (tstate->tracing == 0 && this_instr->op.code == RESUME) {
                     FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
                 }
@@ -12368,7 +12367,7 @@ JUMP_TO_LABEL(error);
                 tstate->interp->jit_state.prev_instr = next_instr;
             }
             tstate->interp->jit_state.specialize_counter = 0;
-            PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame));
+            PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable));
             if (tstate->interp->jit_state.prev_instr_code != prev_code) {
                 _PyFrame_SetStackPointer(frame, stack_pointer);
                 Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code);
@@ -12376,7 +12375,7 @@ JUMP_TO_LABEL(error);
             }
             tstate->interp->jit_state.prev_instr_frame = frame;
             tstate->interp->jit_state.prev_instr_oparg = oparg;
-            tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL();
+            tstate->interp->jit_state.prev_instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL();
             DISPATCH_GOTO_NON_TRACING();
             #else
             Py_FatalError("JIT label executed in non-jit build.");

diff --git a/Python/optimizer.c b/Python/optimizer.c
index 97b891ac97f3813be49c8f26a2cf7bdbb408621d..f0ea40b44ef1f5b9423b15436daf6d4a76df7e14 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -34,7 +34,7 @@
 static bool
 has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr)
 {
-    if (code == (PyCodeObject *)&_Py_InitCleanup || code == (PyCodeObject *)&_PyEntryFrameCode) {
+    if (code == (PyCodeObject *)&_Py_InitCleanup) {
         return false;
     }
     if (instr->op.code == ENTER_EXECUTOR) {
@@ -580,7 +580,9 @@ _PyJit_translate_single_bytecode_to_trace(
     _Py_CODEUNIT *target_instr = this_instr;
     uint32_t target = 0;
 
-    target = INSTR_IP(target_instr, old_code);
+    target = Py_IsNone((PyObject *)old_code)
+        ? (int)(target_instr - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR)
+        : INSTR_IP(target_instr, old_code);
 
     // Rewind EXTENDED_ARG so that we see the whole thing.
     // We must point to the first EXTENDED_ARG when deopting.
@@ -858,7 +860,7 @@ _PyJit_translate_single_bytecode_to_trace(
                         DPRINTF(2, "Adding %p func to op\n", (void *)operand);
                         _Py_BloomFilter_Add(dependencies, new_func);
                     }
-                    else if (new_code != NULL) {
+                    else if (new_code != NULL && !Py_IsNone((PyObject*)new_code)) {
                         operand = (uintptr_t)new_code | 1;
                         DPRINTF(2, "Adding %p code to op\n", (void *)operand);
                         _Py_BloomFilter_Add(dependencies, new_code);
@@ -867,7 +869,7 @@ _PyJit_translate_single_bytecode_to_trace(
                         operand = 0;
                     }
                     ADD_TO_TRACE(uop, oparg, operand, target);
-                    trace[trace_length - 1].operand1 = ((int)(frame->stackpointer - _PyFrame_Stackbase(frame)));
+                    trace[trace_length - 1].operand1 = PyStackRef_IsNone(frame->f_executable) ? 2 : ((int)(frame->stackpointer - _PyFrame_Stackbase(frame)));
                     break;
                 }
                 if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) {

diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c
index d57bea14c5ad76a7cc483be76f5b8df5e68e4505..98dabe989aed14a2c3cc491e1b22305d4ceefc78 100644 (file)
--- a/Python/optimizer_bytecodes.c
+++ b/Python/optimizer_bytecodes.c
@@ -812,6 +812,7 @@ dummy_func(void) {
         JitOptRef temp = PyJitRef_StripReferenceInfo(retval);
         DEAD(retval);
         SAVE_STACK();
+        ctx->frame->stack_pointer = stack_pointer;
         PyCodeObject *returning_code = get_code_with_logging(this_instr);
         if (returning_code == NULL) {
             ctx->done = true;

diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h
index d26803e97d3257a2cd1d4a26be52800bfde5d0e9..40bf95d3081bac5e9a4e75d4314e2a2d4747eca0 100644 (file)
--- a/Python/optimizer_cases.c.h
+++ b/Python/optimizer_cases.c.h
@@ -1175,6 +1175,7 @@
             JitOptRef temp = PyJitRef_StripReferenceInfo(retval);
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
+            ctx->frame->stack_pointer = stack_pointer;
             PyCodeObject *returning_code = get_code_with_logging(this_instr);
             if (returning_code == NULL) {
                 ctx->done = true;