From 13188a9423137527cd9479d701d82a0fd3f17a26 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 19 Sep 2025 22:31:24 +0100 Subject: [PATCH] refactor to translate on the go --- Include/internal/pycore_interp_structs.h | 4 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 20 +- Include/internal/pycore_optimizer.h | 19 +- Include/internal/pycore_uop.h | 11 +- Include/internal/pycore_uop_metadata.h | 2 +- Include/opcode_ids.h | 168 ++++--- Lib/_opcode_metadata.py | 168 ++++--- Python/bytecodes.c | 33 +- Python/ceval.c | 95 +--- Python/ceval_macros.h | 2 +- Python/executor_cases.c.h | 11 +- Python/generated_cases.c.h | 104 +---- Python/opcode_targets.h | 20 +- Python/optimizer.c | 510 ++++++++++------------ Python/pystate.c | 2 +- 16 files changed, 450 insertions(+), 722 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 2cd61c215ce0..5834a6f8edda 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -944,8 +944,10 @@ struct _is { struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; // JIT tracing state + int jit_tracer_code_max_size; int jit_tracer_code_curr_size; - _Py_CODEUNIT *jit_tracer_code_buffer; + _PyBloomFilter jit_tracer_dependencies; + _PyUOpInstruction *jit_tracer_code_buffer; _Py_CODEUNIT *jit_tracer_initial_instr; int jit_tracer_initial_stack_depth; int jit_tracer_initial_chain_depth; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index e1f5d29d2b8c..7ec7bd1c6955 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -286,7 +286,6 @@ Known values: Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) - Python 3.15a1 3656 (Add GUARD_IP instruction) Python 3.16 will start with 3700 @@ -300,7 +299,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3656 +#define PYC_MAGIC_NUMBER 3655 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 97642c2ff467..0c3e33df1191 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -474,10 +474,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 3; case SWAP: return 2 + (oparg-2); - case TIER1_GUARD_IP: - return 0; - case TIER1_SET_IP: - return 0; case TO_BOOL: return 1; case TO_BOOL_ALWAYS_TRUE: @@ -961,10 +957,6 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case SWAP: return 2 + (oparg-2); - case TIER1_GUARD_IP: - return 0; - case TIER1_SET_IP: - return 0; case TO_BOOL: return 1; case TO_BOOL_ALWAYS_TRUE: @@ -1284,8 +1276,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, - [TIER1_GUARD_IP] = { true, INSTR_FMT_IXC000, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, - [TIER1_SET_IP] = { true, INSTR_FMT_IXC000, HAS_ESCAPES_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, @@ -1736,8 +1726,6 @@ const char *_PyOpcode_OpName[267] = { [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [SWAP] = "SWAP", - [TIER1_GUARD_IP] = "TIER1_GUARD_IP", - [TIER1_SET_IP] = "TIER1_SET_IP", [TO_BOOL] = "TO_BOOL", [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", [TO_BOOL_BOOL] = "TO_BOOL_BOOL", @@ -1761,8 +1749,6 @@ const char *_PyOpcode_OpName[267] = { extern const uint8_t _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { - [TIER1_GUARD_IP] = 4, - [TIER1_SET_IP] = 4, [TO_BOOL] = 3, [STORE_SUBSCR] = 1, [SEND] = 1, @@ -1828,6 +1814,8 @@ const uint8_t _PyOpcode_NeedsGuardIp[256] = { extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { + [121] = 121, + [122] = 122, [123] = 123, [124] = 124, [125] = 125, @@ -2065,8 +2053,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [STORE_SUBSCR_DICT] = STORE_SUBSCR, [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, [SWAP] = SWAP, - [TIER1_GUARD_IP] = TIER1_GUARD_IP, - [TIER1_SET_IP] = TIER1_SET_IP, [TO_BOOL] = TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = TO_BOOL, [TO_BOOL_BOOL] = TO_BOOL, @@ -2089,6 +2075,8 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 121: \ + case 122: \ case 123: \ case 124: \ case 125: \ diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 9bfd301efecc..56224f8a73e4 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -21,14 +21,6 @@ typedef struct _PyExecutorLinkListNode { } _PyExecutorLinkListNode; -/* Bloom filter with m = 256 - * https://en.wikipedia.org/wiki/Bloom_filter */ -#define _Py_BLOOM_FILTER_WORDS 8 - -typedef struct { - uint32_t bits[_Py_BLOOM_FILTER_WORDS]; -} _PyBloomFilter; - typedef struct { uint8_t opcode; uint8_t oparg; @@ -364,6 +356,17 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif +int +_PyJIT_translate_single_bytecode_to_trace( + PyThreadState *tstate, + _Py_CODEUNIT *this_instr, + _Py_CODEUNIT *next_instr, + PyCodeObject *code, + PyFunctionObject *func, + int oparg); + +void +_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth); #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 301779905cf4..6effe1291542 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -39,10 +39,13 @@ typedef struct _PyUOpInstruction{ #define UOP_MAX_TRACE_LENGTH 1200 #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) -// This is the length of the trace we record. -// This includes the inline caches. -#define TRACE_MAX_TRACE_LENGTH 2000 -#define TRACER_BUFFER_SIZE ((int)(TRACE_MAX_TRACE_LENGTH * sizeof(_Py_CODEUNIT))) +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define _Py_BLOOM_FILTER_WORDS 8 + +typedef struct { + uint32_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f8099301b6b4..973093acf8aa 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -341,7 +341,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, [_COLD_EXIT] = HAS_ESCAPES_FLAG, - [_GUARD_IP] = HAS_ESCAPES_FLAG, + [_GUARD_IP] = 0, [_DYNAMIC_EXIT] = 0, }; diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 4021dc0ad72e..1d5c74adefcd 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -49,90 +49,88 @@ extern "C" { #define SETUP_ANNOTATIONS 36 #define STORE_SLICE 37 #define STORE_SUBSCR 38 -#define TIER1_GUARD_IP 39 -#define TIER1_SET_IP 40 -#define TO_BOOL 41 -#define UNARY_INVERT 42 -#define UNARY_NEGATIVE 43 -#define UNARY_NOT 44 -#define WITH_EXCEPT_START 45 -#define BINARY_OP 46 -#define BUILD_INTERPOLATION 47 -#define BUILD_LIST 48 -#define BUILD_MAP 49 -#define BUILD_SET 50 -#define BUILD_SLICE 51 -#define BUILD_STRING 52 -#define BUILD_TUPLE 53 -#define CALL 54 -#define CALL_INTRINSIC_1 55 -#define CALL_INTRINSIC_2 56 -#define CALL_KW 57 -#define COMPARE_OP 58 -#define CONTAINS_OP 59 -#define CONVERT_VALUE 60 -#define COPY 61 -#define COPY_FREE_VARS 62 -#define DELETE_ATTR 63 -#define DELETE_DEREF 64 -#define DELETE_FAST 65 -#define DELETE_GLOBAL 66 -#define DELETE_NAME 67 -#define DICT_MERGE 68 -#define DICT_UPDATE 69 -#define END_ASYNC_FOR 70 -#define EXTENDED_ARG 71 -#define FOR_ITER 72 -#define GET_AWAITABLE 73 -#define IMPORT_FROM 74 -#define IMPORT_NAME 75 -#define IS_OP 76 -#define JUMP_BACKWARD 77 -#define JUMP_BACKWARD_NO_INTERRUPT 78 -#define JUMP_FORWARD 79 -#define LIST_APPEND 80 -#define LIST_EXTEND 81 -#define LOAD_ATTR 82 -#define LOAD_COMMON_CONSTANT 83 -#define LOAD_CONST 84 -#define LOAD_DEREF 85 -#define LOAD_FAST 86 -#define LOAD_FAST_AND_CLEAR 87 -#define LOAD_FAST_BORROW 88 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 89 -#define LOAD_FAST_CHECK 90 -#define LOAD_FAST_LOAD_FAST 91 -#define LOAD_FROM_DICT_OR_DEREF 92 -#define LOAD_FROM_DICT_OR_GLOBALS 93 -#define LOAD_GLOBAL 94 -#define LOAD_NAME 95 -#define LOAD_SMALL_INT 96 -#define LOAD_SPECIAL 97 -#define LOAD_SUPER_ATTR 98 -#define MAKE_CELL 99 -#define MAP_ADD 100 -#define MATCH_CLASS 101 -#define POP_JUMP_IF_FALSE 102 -#define POP_JUMP_IF_NONE 103 -#define POP_JUMP_IF_NOT_NONE 104 -#define POP_JUMP_IF_TRUE 105 -#define RAISE_VARARGS 106 -#define RERAISE 107 -#define SEND 108 -#define SET_ADD 109 -#define SET_FUNCTION_ATTRIBUTE 110 -#define SET_UPDATE 111 -#define STORE_ATTR 112 -#define STORE_DEREF 113 -#define STORE_FAST 114 -#define STORE_FAST_LOAD_FAST 115 -#define STORE_FAST_STORE_FAST 116 -#define STORE_GLOBAL 117 -#define STORE_NAME 118 -#define SWAP 119 -#define UNPACK_EX 120 -#define UNPACK_SEQUENCE 121 -#define YIELD_VALUE 122 +#define TO_BOOL 39 +#define UNARY_INVERT 40 +#define UNARY_NEGATIVE 41 +#define UNARY_NOT 42 +#define WITH_EXCEPT_START 43 +#define BINARY_OP 44 +#define BUILD_INTERPOLATION 45 +#define BUILD_LIST 46 +#define BUILD_MAP 47 +#define BUILD_SET 48 +#define BUILD_SLICE 49 +#define BUILD_STRING 50 +#define BUILD_TUPLE 51 +#define CALL 52 +#define CALL_INTRINSIC_1 53 +#define CALL_INTRINSIC_2 54 +#define CALL_KW 55 +#define COMPARE_OP 56 +#define CONTAINS_OP 57 +#define CONVERT_VALUE 58 +#define COPY 59 +#define COPY_FREE_VARS 60 +#define DELETE_ATTR 61 +#define DELETE_DEREF 62 +#define DELETE_FAST 63 +#define DELETE_GLOBAL 64 +#define DELETE_NAME 65 +#define DICT_MERGE 66 +#define DICT_UPDATE 67 +#define END_ASYNC_FOR 68 +#define EXTENDED_ARG 69 +#define FOR_ITER 70 +#define GET_AWAITABLE 71 +#define IMPORT_FROM 72 +#define IMPORT_NAME 73 +#define IS_OP 74 +#define JUMP_BACKWARD 75 +#define JUMP_BACKWARD_NO_INTERRUPT 76 +#define JUMP_FORWARD 77 +#define LIST_APPEND 78 +#define LIST_EXTEND 79 +#define LOAD_ATTR 80 +#define LOAD_COMMON_CONSTANT 81 +#define LOAD_CONST 82 +#define LOAD_DEREF 83 +#define LOAD_FAST 84 +#define LOAD_FAST_AND_CLEAR 85 +#define LOAD_FAST_BORROW 86 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 87 +#define LOAD_FAST_CHECK 88 +#define LOAD_FAST_LOAD_FAST 89 +#define LOAD_FROM_DICT_OR_DEREF 90 +#define LOAD_FROM_DICT_OR_GLOBALS 91 +#define LOAD_GLOBAL 92 +#define LOAD_NAME 93 +#define LOAD_SMALL_INT 94 +#define LOAD_SPECIAL 95 +#define LOAD_SUPER_ATTR 96 +#define MAKE_CELL 97 +#define MAP_ADD 98 +#define MATCH_CLASS 99 +#define POP_JUMP_IF_FALSE 100 +#define POP_JUMP_IF_NONE 101 +#define POP_JUMP_IF_NOT_NONE 102 +#define POP_JUMP_IF_TRUE 103 +#define RAISE_VARARGS 104 +#define RERAISE 105 +#define SEND 106 +#define SET_ADD 107 +#define SET_FUNCTION_ATTRIBUTE 108 +#define SET_UPDATE 109 +#define STORE_ATTR 110 +#define STORE_DEREF 111 +#define STORE_FAST 112 +#define STORE_FAST_LOAD_FAST 113 +#define STORE_FAST_STORE_FAST 114 +#define STORE_GLOBAL 115 +#define STORE_NAME 116 +#define SWAP 117 +#define UNPACK_EX 118 +#define UNPACK_SEQUENCE 119 +#define YIELD_VALUE 120 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 @@ -249,7 +247,7 @@ extern "C" { #define SETUP_WITH 265 #define STORE_FAST_MAYBE_NULL 266 -#define HAVE_ARGUMENT 45 +#define HAVE_ARGUMENT 43 #define MIN_SPECIALIZED_OPCODE 129 #define MIN_INSTRUMENTED_OPCODE 234 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index ca7fc579b3aa..f168d169a329 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -246,90 +246,88 @@ opmap = { 'SETUP_ANNOTATIONS': 36, 'STORE_SLICE': 37, 'STORE_SUBSCR': 38, - 'TIER1_GUARD_IP': 39, - 'TIER1_SET_IP': 40, - 'TO_BOOL': 41, - 'UNARY_INVERT': 42, - 'UNARY_NEGATIVE': 43, - 'UNARY_NOT': 44, - 'WITH_EXCEPT_START': 45, - 'BINARY_OP': 46, - 'BUILD_INTERPOLATION': 47, - 'BUILD_LIST': 48, - 'BUILD_MAP': 49, - 'BUILD_SET': 50, - 'BUILD_SLICE': 51, - 'BUILD_STRING': 52, - 'BUILD_TUPLE': 53, - 'CALL': 54, - 'CALL_INTRINSIC_1': 55, - 'CALL_INTRINSIC_2': 56, - 'CALL_KW': 57, - 'COMPARE_OP': 58, - 'CONTAINS_OP': 59, - 'CONVERT_VALUE': 60, - 'COPY': 61, - 'COPY_FREE_VARS': 62, - 'DELETE_ATTR': 63, - 'DELETE_DEREF': 64, - 'DELETE_FAST': 65, - 'DELETE_GLOBAL': 66, - 'DELETE_NAME': 67, - 'DICT_MERGE': 68, - 'DICT_UPDATE': 69, - 'END_ASYNC_FOR': 70, - 'EXTENDED_ARG': 71, - 'FOR_ITER': 72, - 'GET_AWAITABLE': 73, - 'IMPORT_FROM': 74, - 'IMPORT_NAME': 75, - 'IS_OP': 76, - 'JUMP_BACKWARD': 77, - 'JUMP_BACKWARD_NO_INTERRUPT': 78, - 'JUMP_FORWARD': 79, - 'LIST_APPEND': 80, - 'LIST_EXTEND': 81, - 'LOAD_ATTR': 82, - 'LOAD_COMMON_CONSTANT': 83, - 'LOAD_CONST': 84, - 'LOAD_DEREF': 85, - 'LOAD_FAST': 86, - 'LOAD_FAST_AND_CLEAR': 87, - 'LOAD_FAST_BORROW': 88, - 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 89, - 'LOAD_FAST_CHECK': 90, - 'LOAD_FAST_LOAD_FAST': 91, - 'LOAD_FROM_DICT_OR_DEREF': 92, - 'LOAD_FROM_DICT_OR_GLOBALS': 93, - 'LOAD_GLOBAL': 94, - 'LOAD_NAME': 95, - 'LOAD_SMALL_INT': 96, - 'LOAD_SPECIAL': 97, - 'LOAD_SUPER_ATTR': 98, - 'MAKE_CELL': 99, - 'MAP_ADD': 100, - 'MATCH_CLASS': 101, - 'POP_JUMP_IF_FALSE': 102, - 'POP_JUMP_IF_NONE': 103, - 'POP_JUMP_IF_NOT_NONE': 104, - 'POP_JUMP_IF_TRUE': 105, - 'RAISE_VARARGS': 106, - 'RERAISE': 107, - 'SEND': 108, - 'SET_ADD': 109, - 'SET_FUNCTION_ATTRIBUTE': 110, - 'SET_UPDATE': 111, - 'STORE_ATTR': 112, - 'STORE_DEREF': 113, - 'STORE_FAST': 114, - 'STORE_FAST_LOAD_FAST': 115, - 'STORE_FAST_STORE_FAST': 116, - 'STORE_GLOBAL': 117, - 'STORE_NAME': 118, - 'SWAP': 119, - 'UNPACK_EX': 120, - 'UNPACK_SEQUENCE': 121, - 'YIELD_VALUE': 122, + 'TO_BOOL': 39, + 'UNARY_INVERT': 40, + 'UNARY_NEGATIVE': 41, + 'UNARY_NOT': 42, + 'WITH_EXCEPT_START': 43, + 'BINARY_OP': 44, + 'BUILD_INTERPOLATION': 45, + 'BUILD_LIST': 46, + 'BUILD_MAP': 47, + 'BUILD_SET': 48, + 'BUILD_SLICE': 49, + 'BUILD_STRING': 50, + 'BUILD_TUPLE': 51, + 'CALL': 52, + 'CALL_INTRINSIC_1': 53, + 'CALL_INTRINSIC_2': 54, + 'CALL_KW': 55, + 'COMPARE_OP': 56, + 'CONTAINS_OP': 57, + 'CONVERT_VALUE': 58, + 'COPY': 59, + 'COPY_FREE_VARS': 60, + 'DELETE_ATTR': 61, + 'DELETE_DEREF': 62, + 'DELETE_FAST': 63, + 'DELETE_GLOBAL': 64, + 'DELETE_NAME': 65, + 'DICT_MERGE': 66, + 'DICT_UPDATE': 67, + 'END_ASYNC_FOR': 68, + 'EXTENDED_ARG': 69, + 'FOR_ITER': 70, + 'GET_AWAITABLE': 71, + 'IMPORT_FROM': 72, + 'IMPORT_NAME': 73, + 'IS_OP': 74, + 'JUMP_BACKWARD': 75, + 'JUMP_BACKWARD_NO_INTERRUPT': 76, + 'JUMP_FORWARD': 77, + 'LIST_APPEND': 78, + 'LIST_EXTEND': 79, + 'LOAD_ATTR': 80, + 'LOAD_COMMON_CONSTANT': 81, + 'LOAD_CONST': 82, + 'LOAD_DEREF': 83, + 'LOAD_FAST': 84, + 'LOAD_FAST_AND_CLEAR': 85, + 'LOAD_FAST_BORROW': 86, + 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87, + 'LOAD_FAST_CHECK': 88, + 'LOAD_FAST_LOAD_FAST': 89, + 'LOAD_FROM_DICT_OR_DEREF': 90, + 'LOAD_FROM_DICT_OR_GLOBALS': 91, + 'LOAD_GLOBAL': 92, + 'LOAD_NAME': 93, + 'LOAD_SMALL_INT': 94, + 'LOAD_SPECIAL': 95, + 'LOAD_SUPER_ATTR': 96, + 'MAKE_CELL': 97, + 'MAP_ADD': 98, + 'MATCH_CLASS': 99, + 'POP_JUMP_IF_FALSE': 100, + 'POP_JUMP_IF_NONE': 101, + 'POP_JUMP_IF_NOT_NONE': 102, + 'POP_JUMP_IF_TRUE': 103, + 'RAISE_VARARGS': 104, + 'RERAISE': 105, + 'SEND': 106, + 'SET_ADD': 107, + 'SET_FUNCTION_ATTRIBUTE': 108, + 'SET_UPDATE': 109, + 'STORE_ATTR': 110, + 'STORE_DEREF': 111, + 'STORE_FAST': 112, + 'STORE_FAST_LOAD_FAST': 113, + 'STORE_FAST_STORE_FAST': 114, + 'STORE_GLOBAL': 115, + 'STORE_NAME': 116, + 'SWAP': 117, + 'UNPACK_EX': 118, + 'UNPACK_SEQUENCE': 119, + 'YIELD_VALUE': 120, 'INSTRUMENTED_END_FOR': 234, 'INSTRUMENTED_POP_ITER': 235, 'INSTRUMENTED_END_SEND': 236, @@ -363,5 +361,5 @@ opmap = { 'STORE_FAST_MAYBE_NULL': 266, } -HAVE_ARGUMENT = 45 +HAVE_ARGUMENT = 43 MIN_INSTRUMENTED_OPCODE = 234 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a495e332d978..66553c8c501f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2970,24 +2970,14 @@ dummy_func( start--; } if (tstate->interp->jit_tracer_code_buffer == NULL) { - tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); + tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); if (tstate->interp->jit_tracer_code_buffer == NULL) { // Don't error, just go to next instruction. DISPATCH(); } - tstate->interp->jit_tracer_code_curr_size = 0; } + _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); ENTER_TRACING(); - // Nothing in the buffer, begin tracing! - if (tstate->interp->jit_tracer_code_curr_size == 0) { - tstate->interp->jit_tracer_initial_instr = next_instr; - tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); - tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); - tstate->interp->jit_tracer_seen_initial_before = 0; - tstate->interp->jit_completed_loop = false; - tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL(); - tstate->interp->jit_tracer_initial_chain_depth = 0; - } // Don't add the JUMP_BACKWARD_JIT instruction to the trace. DISPATCH(); } @@ -5443,13 +5433,7 @@ dummy_func( _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - tstate->interp->jit_tracer_initial_chain_depth = chain_depth; - tstate->interp->jit_tracer_initial_instr = target; - tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); - tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); - tstate->interp->jit_tracer_seen_initial_before = 0; - tstate->interp->jit_completed_loop = false; - exit->temperature = restart_backoff_counter(temperature); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); GOTO_TIER_ONE(target, 1); } assert(tstate->jit_exit == exit); @@ -5457,14 +5441,8 @@ dummy_func( TIER2_TO_TIER2(exit->executor); } - - no_save_ip tier1 inst(TIER1_GUARD_IP, (func_ptr/4 --)) { - (void)(func_ptr); - } - tier2 op(_GUARD_IP, (ip/4 --)) { if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { - fprintf(stdout, "d:%p:%p\n", frame->instr_ptr, ip); GOTO_TIER_ONE(frame->instr_ptr, 1); } } @@ -5473,11 +5451,6 @@ dummy_func( GOTO_TIER_ONE(frame->instr_ptr, 1); } - tier1 inst(TIER1_SET_IP, (ip/4 --)) { - (void)(ip); - } - - label(pop_2_error) { stack_pointer -= 2; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/ceval.c b/Python/ceval.c index 4378afebc6ab..4029575853d6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -975,99 +975,12 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) // 1 for trace full, 0 for successful write. static int -add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, int oparg) +add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int oparg) { assert(frame != NULL); - assert(tstate->interp->jit_tracer_code_curr_size < TRACE_MAX_TRACE_LENGTH); - int curr_size = tstate->interp->jit_tracer_code_curr_size; - int opcode = this_instr->op.code; - if (opcode == ENTER_EXECUTOR) { - return 1; - // PyCodeObject *code = _PyFrame_GetCode(frame); - // assert(code != NULL); - // _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - // assert(executor->vm_data.code == code); - // assert(executor->vm_data.valid); - // assert(tstate->current_executor == NULL); - // opcode = executor->vm_data.opcode; - // oparg = (oparg & ~255) | executor->vm_data.oparg; - } - else { - oparg = this_instr->op.arg; - } - assert(opcode != 0); - // Check if we looped back to the start. - if (this_instr == tstate->interp->jit_tracer_initial_instr) { - if (tstate->interp->jit_tracer_seen_initial_before >= 1) { - tstate->interp->jit_completed_loop = true; - return 1; - } - tstate->interp->jit_tracer_seen_initial_before++; - } -#ifdef Py_DEBUG - char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); - int lltrace = 3; - if (python_lltrace != NULL && *python_lltrace >= '0') { - lltrace = *python_lltrace - '0'; - } - if (lltrace >= 3) { - printf("%d ADD_TO_BYTECODE_TRACE: %s %d\n", curr_size, _PyOpcode_OpName[opcode], oparg); - } -#endif - // JUMP_BACKWARD_NO_INTERRUPT is not registered with cache due to a bug in the compiler. - int caches = opcode == JUMP_BACKWARD_NO_INTERRUPT ? 1 : _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - int nsize = caches + 1; - int needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; - int total_size = nsize + (needs_guard_ip ? 5 : 0) + 5; - if (curr_size + total_size >= TRACE_MAX_TRACE_LENGTH) { -#ifdef Py_DEBUG - if (lltrace >= 3) { - printf("END TRACE LENGTH: %d\n", curr_size); - } -#endif - return 1; - } - tstate->interp->jit_tracer_code_buffer[curr_size].op.code = opcode; - tstate->interp->jit_tracer_code_buffer[curr_size].op.arg = oparg; - for (int i = 1; i < nsize; i++) { - tstate->interp->jit_tracer_code_buffer[curr_size + i] = *(this_instr + i); - } - if (needs_guard_ip) { -#ifdef Py_DEBUG - if (lltrace >= 3) { - printf("%d ADD_TO_BYTECODE_TRACE: %s %d\n", curr_size + nsize, _PyOpcode_OpName[TIER1_GUARD_IP], 0); - } -#endif - tstate->interp->jit_tracer_code_buffer[curr_size + nsize].op.code = TIER1_GUARD_IP; - PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - if (Py_IsNone(func)) { - func = NULL; // Trampoline frames don't set their func object field. - } - assert(func == NULL || PyFunction_Check(func)); - write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 1].cache, func); - -#ifdef Py_DEBUG - if (lltrace >= 3) { - printf("%d ADD_TO_BYTECODE_TRACE: %s %p\n", curr_size + nsize + 5, _PyOpcode_OpName[TIER1_SET_IP], frame->instr_ptr); - } -#endif - tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 5].op.code = TIER1_SET_IP; - write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 6].cache, frame->instr_ptr); - - } - else { -#ifdef Py_DEBUG - if (lltrace >= 3) { - printf("%d ADD_TO_BYTECODE_TRACE: %s %p\n", curr_size + nsize, _PyOpcode_OpName[TIER1_SET_IP], frame->instr_ptr); - } -#endif - tstate->interp->jit_tracer_code_buffer[curr_size + nsize].op.code = TIER1_SET_IP; - write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 1].cache, frame->instr_ptr); - - } - - tstate->interp->jit_tracer_code_curr_size += total_size; - return 0; + assert(tstate->interp->jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + return !_PyJIT_translate_single_bytecode_to_trace(tstate, this_instr, next_instr, _PyFrame_GetCode(frame), func, oparg); } /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c0f33dd5b31f..a47b0e524c70 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -144,7 +144,7 @@ DISPATCH(); # define RECORD_TRACE() do { \ frame->instr_ptr = next_instr; \ - if (add_to_code_trace(tstate, frame, this_instr, oparg)) { \ + if (add_to_code_trace(tstate, frame, this_instr, next_instr, oparg)) { \ BAIL_TRACING(); \ } \ } while (0); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0b5aee504012..145f010db479 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7560,15 +7560,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - tstate->interp->jit_tracer_initial_chain_depth = chain_depth; - tstate->interp->jit_tracer_initial_instr = target; - tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->interp->jit_tracer_seen_initial_before = 0; - tstate->interp->jit_completed_loop = false; - exit->temperature = restart_backoff_counter(temperature); GOTO_TIER_ONE(target, 1); } assert(tstate->jit_exit == exit); @@ -7580,9 +7574,6 @@ case _GUARD_IP: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { - _PyFrame_SetStackPointer(frame, stack_pointer); - fprintf(stdout, "d:%p:%p\n", frame->instr_ptr, ip); - stack_pointer = _PyFrame_GetStackPointer(frame); GOTO_TIER_ONE(frame->instr_ptr, 1); } break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a9749abb5fad..99fb53f26c5c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7663,25 +7663,16 @@ } if (tstate->interp->jit_tracer_code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); + tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); if (tstate->interp->jit_tracer_code_buffer == NULL) { DISPATCH(); } - tstate->interp->jit_tracer_code_curr_size = 0; } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); + stack_pointer = _PyFrame_GetStackPointer(frame); ENTER_TRACING(); - if (tstate->interp->jit_tracer_code_curr_size == 0) { - tstate->interp->jit_tracer_initial_instr = next_instr; - tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->interp->jit_tracer_seen_initial_before = 0; - tstate->interp->jit_completed_loop = false; - tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL(); - tstate->interp->jit_tracer_initial_chain_depth = 0; - } DISPATCH(); } else { @@ -11428,41 +11419,6 @@ DISPATCH(); } - TARGET(TIER1_GUARD_IP) { - #if _Py_TAIL_CALL_INTERP - int opcode = TIER1_GUARD_IP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 5; - INSTRUCTION_STATS(TIER1_GUARD_IP); - PyObject *func_ptr = read_obj(&this_instr[1].cache); - (void)func_ptr; - _PyFrame_SetStackPointer(frame, stack_pointer); - (void)(func_ptr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } - - TARGET(TIER1_SET_IP) { - #if _Py_TAIL_CALL_INTERP - int opcode = TIER1_SET_IP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(TIER1_SET_IP); - PyObject *ip = read_obj(&this_instr[1].cache); - (void)ip; - _PyFrame_SetStackPointer(frame, stack_pointer); - (void)(ip); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } - TARGET(TO_BOOL) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL; @@ -19877,25 +19833,16 @@ } if (tstate->interp->jit_tracer_code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); + tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); if (tstate->interp->jit_tracer_code_buffer == NULL) { TRACING_DISPATCH(); } - tstate->interp->jit_tracer_code_curr_size = 0; } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); + stack_pointer = _PyFrame_GetStackPointer(frame); ENTER_TRACING(); - if (tstate->interp->jit_tracer_code_curr_size == 0) { - tstate->interp->jit_tracer_initial_instr = next_instr; - tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->interp->jit_tracer_seen_initial_before = 0; - tstate->interp->jit_completed_loop = false; - tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL(); - tstate->interp->jit_tracer_initial_chain_depth = 0; - } TRACING_DISPATCH(); } else { @@ -23750,41 +23697,6 @@ TRACING_DISPATCH(); } - TRACING_TARGET(TIER1_GUARD_IP) { - #if _Py_TAIL_CALL_INTERP - int opcode = TIER1_GUARD_IP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 5; - INSTRUCTION_STATS(TIER1_GUARD_IP); - PyObject *func_ptr = read_obj(&this_instr[1].cache); - (void)func_ptr; - _PyFrame_SetStackPointer(frame, stack_pointer); - (void)(func_ptr); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TIER1_SET_IP) { - #if _Py_TAIL_CALL_INTERP - int opcode = TIER1_SET_IP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(TIER1_SET_IP); - PyObject *ip = read_obj(&this_instr[1].cache); - (void)ip; - _PyFrame_SetStackPointer(frame, stack_pointer); - (void)(ip); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - TRACING_TARGET(TO_BOOL) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 47d6290b0dc1..299739764919 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -39,8 +39,6 @@ static void *opcode_targets_table[256] = { &&TARGET_SETUP_ANNOTATIONS, &&TARGET_STORE_SLICE, &&TARGET_STORE_SUBSCR, - &&TARGET_TIER1_GUARD_IP, - &&TARGET_TIER1_SET_IP, &&TARGET_TO_BOOL, &&TARGET_UNARY_INVERT, &&TARGET_UNARY_NEGATIVE, @@ -128,6 +126,8 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -297,8 +297,6 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACING_SETUP_ANNOTATIONS, &&TARGET_TRACING_STORE_SLICE, &&TARGET_TRACING_STORE_SUBSCR, - &&TARGET_TRACING_TIER1_GUARD_IP, - &&TARGET_TRACING_TIER1_SET_IP, &&TARGET_TRACING_TO_BOOL, &&TARGET_TRACING_UNARY_INVERT, &&TARGET_TRACING_UNARY_NEGATIVE, @@ -386,6 +384,8 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_TRACING_RESUME, &&TARGET_TRACING_BINARY_OP_ADD_FLOAT, &&TARGET_TRACING_BINARY_OP_ADD_INT, @@ -943,10 +943,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SWAP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TIER1_GUARD_IP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TIER1_GUARD_IP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TIER1_SET_IP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TIER1_SET_IP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); @@ -1201,8 +1197,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, [SWAP] = _TAIL_CALL_SWAP, - [TIER1_GUARD_IP] = _TAIL_CALL_TIER1_GUARD_IP, - [TIER1_SET_IP] = _TAIL_CALL_TIER1_SET_IP, [TO_BOOL] = _TAIL_CALL_TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, [TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL, @@ -1220,6 +1214,8 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, [125] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1459,8 +1455,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACING_STORE_SUBSCR_DICT, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT, [SWAP] = _TAIL_CALL_TRACING_SWAP, - [TIER1_GUARD_IP] = _TAIL_CALL_TRACING_TIER1_GUARD_IP, - [TIER1_SET_IP] = _TAIL_CALL_TRACING_TIER1_SET_IP, [TO_BOOL] = _TAIL_CALL_TRACING_TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE, [TO_BOOL_BOOL] = _TAIL_CALL_TRACING_TO_BOOL_BOOL, @@ -1478,6 +1472,8 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_TRACING_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_TRACING_YIELD_VALUE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, [125] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer.c b/Python/optimizer.c index e4ccca924302..32aa4481c7e5 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -513,35 +513,34 @@ add_to_trace( DPRINTF(2, "No room for %s (need %d, got %d)\n", \ (opname), (n), max_length - trace_length); \ OPT_STAT_INC(trace_too_long); \ - goto done; \ + goto full; \ } // Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE #define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) -/* Returns the length of the trace on success, - * 0 if it failed to produce a worthwhile trace, - * and -1 on an error. +/* Returns 1 on success (added to trace), 0 on trace end. */ -static int -translate_bytecode_to_trace( - _PyInterpreterFrame *frame, +int +_PyJIT_translate_single_bytecode_to_trace( PyThreadState *tstate, - _PyUOpInstruction *trace, - int buffer_size, - _PyBloomFilter *dependencies, bool progress_needed) + _Py_CODEUNIT *this_instr, + _Py_CODEUNIT *next_instr, + PyCodeObject *code, + PyFunctionObject *func, + int oparg) { - bool first = true; - PyCodeObject *code = tstate->interp->jit_tracer_initial_code;; - PyFunctionObject *func = tstate->interp->jit_tracer_initial_func; - PyCodeObject *initial_code = code; - _Py_BloomFilter_Add(dependencies, initial_code); - _Py_CODEUNIT *target_instr = tstate->interp->jit_tracer_initial_instr; - _Py_CODEUNIT *initial_instr = tstate->interp->jit_tracer_code_buffer; - int trace_length = 0; - // Leave space for possible trailing _EXIT_TRACE - int max_length = buffer_size-2; + if (Py_IsNone((PyObject *)func)) { + func = NULL; + } + bool progress_needed = (tstate->interp->jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; + _PyBloomFilter *dependencies = &tstate->interp->jit_tracer_dependencies; + _Py_BloomFilter_Add(dependencies, code); + _Py_CODEUNIT *target_instr = this_instr; + int trace_length = tstate->interp->jit_tracer_code_curr_size; + _PyUOpInstruction *trace = tstate->interp->jit_tracer_code_buffer; + int max_length = tstate->interp->jit_tracer_code_max_size; #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -551,269 +550,233 @@ translate_bytecode_to_trace( } #endif - DPRINTF(2, - "Optimizing %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(target_instr, code)); - ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)target_instr, INSTR_IP(target_instr, code)); - ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); uint32_t target = 0; - bool preceded_by_for_iter = false; - - for (int x = 0; x < tstate->interp->jit_tracer_code_curr_size;) { - target = INSTR_IP(target_instr, code); - // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT - max_length-=2; - uint32_t opcode = initial_instr[x].op.code; - uint32_t oparg = initial_instr[x].op.arg; - - DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); - - if (opcode == EXTENDED_ARG) { - x++; - opcode = initial_instr[x].op.code; - oparg = (oparg << 8) | initial_instr[x].op.arg; - if (opcode == EXTENDED_ARG) { - x--; - goto done; - } - } - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - if (opcode != NOP) { - if (opcode != TIER1_GUARD_IP) { - RESERVE_RAW(1, "_CHECK_VALIDITY"); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - } - if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { - RESERVE_RAW(2, "_SET_IP"); - ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); - } - } - /* Special case the first instruction, - * so that we can guarantee forward progress */ - if (first && progress_needed) { - assert(first); - if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; - } - assert(!OPCODE_HAS_EXIT(opcode)); - assert(!OPCODE_HAS_DEOPT(opcode)); - } + target = INSTR_IP(target_instr, code); + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT + max_length-=2; + uint32_t opcode = this_instr->op.code; + if ((uint16_t)oparg != (uint64_t)oparg) { + goto full; + } - if (OPCODE_HAS_EXIT(opcode)) { - // Make space for side exit and final _EXIT_TRACE: - RESERVE_RAW(2, "_EXIT_TRACE"); - max_length--; - } - if (OPCODE_HAS_ERROR(opcode)) { - // Make space for error stub and final _EXIT_TRACE: - RESERVE_RAW(2, "_ERROR_POP_N"); - max_length--; - } - if (opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) { - preceded_by_for_iter = true; - // assert((initial_instr + x + 1 + _PyOpcode_Caches[FOR_ITER])->op.code == TIER1_GUARD_IP); - // assert((initial_instr + x + 6 + _PyOpcode_Caches[FOR_ITER])->op.code == TIER1_SET_IP); - // PyObject *temp = read_obj(&(initial_instr + x + 2 + _PyOpcode_Caches[FOR_ITER])->cache); - // if (temp != NULL) { - // code = ((PyFunctionObject*)temp)->func_code; - // } - // target_instr = (_Py_CODEUNIT *)(read_obj(&(initial_instr + x + 7 + _PyOpcode_Caches[FOR_ITER])->cache)); - // target = INSTR_IP(target_instr, code); - // - // (initial_instr + x + 1 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 2 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 3 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 4 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 5 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // - // (initial_instr + x + 6 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 7 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 8 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 9 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 10 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + + if (opcode == EXTENDED_ARG) { + return 1; + } + if (opcode == ENTER_EXECUTOR) { + goto full; + } + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + if (opcode == NOP) { + return 1; + } + + RESERVE_RAW(1, "_CHECK_VALIDITY"); + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + + if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { + RESERVE_RAW(2, "_SET_IP"); + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); + } + + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] && + !(opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) && + !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT); + + if (needs_guard_ip) { + RESERVE_RAW(1, "_GUARD_IP"); + } + + /* Special case the first instruction, + * so that we can guarantee forward progress */ + if (progress_needed && tstate->interp->jit_tracer_initial_instr == this_instr && tstate->interp->jit_tracer_code_curr_size == 0) { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; } - switch (opcode) { - case JUMP_BACKWARD_JIT: - case JUMP_BACKWARD: - ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); - _Py_FALLTHROUGH; - case JUMP_BACKWARD_NO_INTERRUPT: - ADD_TO_TRACE(_JUMP_BACKWARD_NO_INTERRUPT, oparg, 0, target); - break; - case TIER1_GUARD_IP: - func = (PyFunctionObject *)read_obj(&(initial_instr + x + 1)->cache); - assert(func == NULL || PyFunction_Check(func)); - if (func != NULL) { - code = (PyCodeObject *)func->func_code; - } - assert((initial_instr + x + 5)->op.code == TIER1_SET_IP); - target_instr = (_Py_CODEUNIT*)read_obj(&(initial_instr + x + 6)->cache); - if (preceded_by_for_iter) {; - preceded_by_for_iter = false; - } - else { - ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)target_instr, 0) - } - break; + assert(!OPCODE_HAS_EXIT(opcode)); + assert(!OPCODE_HAS_DEOPT(opcode)); + } - case TIER1_SET_IP: - target_instr = (_Py_CODEUNIT*)read_obj(&(initial_instr + x + 1)->cache); - break; + // Loop back to the start + if (tstate->interp->jit_tracer_initial_instr == this_instr && tstate->interp->jit_tracer_code_curr_size > 2) { + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + goto done; + } - case RESUME: - /* Use a special tier 2 version of RESUME_CHECK to allow traces to - * start with RESUME_CHECK */ - ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); - break; + if (OPCODE_HAS_EXIT(opcode)) { + // Make space for side exit and final _EXIT_TRACE: + RESERVE_RAW(2, "_EXIT_TRACE"); + max_length--; + } + if (OPCODE_HAS_ERROR(opcode)) { + // Make space for error stub and final _EXIT_TRACE: + RESERVE_RAW(2, "_ERROR_POP_N"); + max_length--; + } - default: - { - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - if (expansion->nuops > 0) { - // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) - int nuops = expansion->nuops; - RESERVE(nuops + 1); /* One extra for exit */ - uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM - for (int i = 0; i < nuops; i++) { - oparg = orig_oparg; - uint32_t uop = expansion->uops[i].uop; - uint64_t operand = 0; - // Add one to account for the actual opcode/oparg pair: - int offset = expansion->uops[i].offset + 1; - switch (expansion->uops[i].size) { - case OPARG_SIMPLE: - assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); - break; - case OPARG_CACHE_1: - operand = read_u16(&(initial_instr + x)[offset].cache); - break; - case OPARG_CACHE_2: - operand = read_u32(&(initial_instr + x)[offset].cache); - break; - case OPARG_CACHE_4: - operand = read_u64(&(initial_instr + x)[offset].cache); - break; - case OPARG_TOP: // First half of super-instr - oparg = orig_oparg >> 4; - break; - case OPARG_BOTTOM: // Second half of super-instr - oparg = orig_oparg & 0xF; - break; - case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset - oparg = offset; - assert(uop == _SAVE_RETURN_OFFSET); - break; - case OPARG_REPLACED: - uop = _PyUOp_Replacements[uop]; - assert(uop != 0); - - uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); - if (uop == _TIER2_RESUME_CHECK) { - target = next_inst; - } + switch (opcode) { + case JUMP_BACKWARD_JIT: + case JUMP_BACKWARD: + ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); + _Py_FALLTHROUGH; + case JUMP_BACKWARD_NO_INTERRUPT: + break; + + case RESUME: + /* Use a special tier 2 version of RESUME_CHECK to allow traces to + * start with RESUME_CHECK */ + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + break; + + default: + { + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + if (expansion->nuops > 0) { + // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) + int nuops = expansion->nuops; + RESERVE(nuops + 1); /* One extra for exit */ + uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM + for (int i = 0; i < nuops; i++) { + oparg = orig_oparg; + uint32_t uop = expansion->uops[i].uop; + uint64_t operand = 0; + // Add one to account for the actual opcode/oparg pair: + int offset = expansion->uops[i].offset + 1; + switch (expansion->uops[i].size) { + case OPARG_SIMPLE: + assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); + break; + case OPARG_CACHE_1: + operand = read_u16(&this_instr[offset].cache); + break; + case OPARG_CACHE_2: + operand = read_u32(&this_instr[offset].cache); + break; + case OPARG_CACHE_4: + operand = read_u64(&this_instr[offset].cache); + break; + case OPARG_TOP: // First half of super-instr + oparg = orig_oparg >> 4; + break; + case OPARG_BOTTOM: // Second half of super-instr + oparg = orig_oparg & 0xF; + break; + case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset + oparg = offset; + assert(uop == _SAVE_RETURN_OFFSET); + break; + case OPARG_REPLACED: + uop = _PyUOp_Replacements[uop]; + assert(uop != 0); + + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); + if (uop == _TIER2_RESUME_CHECK) { + target = next_inst; + } #ifdef Py_DEBUG - else { - uint32_t jump_target = next_inst + oparg; - assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); - assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); - } + else { + uint32_t jump_target = next_inst + oparg; + assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); + assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); + } #endif - break; - case OPERAND1_1: - assert(trace[trace_length-1].opcode == uop); - operand = read_u16(&(initial_instr + x)[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_2: - assert(trace[trace_length-1].opcode == uop); - operand = read_u32(&(initial_instr + x)[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_4: - assert(trace[trace_length-1].opcode == uop); - operand = read_u64(&(initial_instr + x)[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - default: - fprintf(stderr, - "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", - opcode, oparg, nuops, i, - expansion->uops[i].size, - expansion->uops[i].offset); - Py_FatalError("garbled expansion"); + break; + case OPERAND1_1: + assert(trace[trace_length-1].opcode == uop); + operand = read_u16(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_2: + assert(trace[trace_length-1].opcode == uop); + operand = read_u32(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_4: + assert(trace[trace_length-1].opcode == uop); + operand = read_u64(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + default: + fprintf(stderr, + "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", + opcode, oparg, nuops, i, + expansion->uops[i].size, + expansion->uops[i].offset); + Py_FatalError("garbled expansion"); + } + if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { + if (func != NULL) { + operand = (uintptr_t)func; } - - if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { - assert(i + 1 == nuops); - _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - assert(next_instr->op.code == STORE_FAST); - operand = next_instr->op.arg; - // Skip the STORE_FAST: - x++; + else if (code != NULL) { + operand = (uintptr_t)code | 1; + } + else { + operand = 0; } - - // All other instructions - ADD_TO_TRACE(uop, oparg, operand, target); } - break; + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); } - DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); - OPT_UNSUPPORTED_OPCODE(opcode); - goto done; // Break out of loop - } // End default - - } // End switch (opcode) - - x++; - // Add cache size for opcode - x += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - - if (opcode == CALL_LIST_APPEND) { - assert(initial_instr[x].op.code == POP_TOP); - x++; - } + break; + } + DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); + OPT_UNSUPPORTED_OPCODE(opcode); + } // End default - if (x > tstate->interp->jit_tracer_code_curr_size) { - goto done; - } - // Jump here after _PUSH_FRAME or likely branches. - first = false; - } // End for (;;) + } // End switch (opcode) -done: - // Looped back to top. - if (!first && tstate->interp->jit_completed_loop && target_instr == tstate->interp->jit_tracer_initial_instr) { - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + if (needs_guard_ip) { + ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)next_instr, 0); } - // Skip short traces where we can't even translate a single instruction: - if (first) { - OPT_STAT_INC(trace_too_short); - DPRINTF(2, - "No trace for %s (%s:%d) at byte offset %d (no progress)\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); - return 0; - } - if (!is_terminator(&trace[trace_length-1])) { - /* Allow space for _EXIT_TRACE */ - max_length += 2; + tstate->interp->jit_tracer_code_curr_size = trace_length; + tstate->interp->jit_tracer_code_max_size = max_length; + return 1; +done: + tstate->interp->jit_tracer_code_curr_size = trace_length; + tstate->interp->jit_tracer_code_max_size = max_length; + return 0; +full: + if (!is_terminator(&tstate->interp->jit_tracer_code_buffer[trace_length-1])) { + // Undo the last few instructions. + trace_length = tstate->interp->jit_tracer_code_curr_size; ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); } - DPRINTF(1, - "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(tstate->interp->jit_tracer_initial_instr, (PyCodeObject *)tstate->interp->jit_tracer_initial_func->func_code), - trace_length); - OPT_HIST(trace_length, trace_length_hist); - return trace_length; + tstate->interp->jit_tracer_code_curr_size = trace_length; + tstate->interp->jit_tracer_code_max_size = max_length; + return 0; +} + +void +_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth) +{ + PyCodeObject *code = _PyFrame_GetCode(frame); +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 3; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } + DPRINTF(2, + "Optimizing %s (%s:%d) at byte offset %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(next_instr, code)); +#endif + add_to_trace(tstate->interp->jit_tracer_code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)next_instr, INSTR_IP(next_instr, code)); + add_to_trace(tstate->interp->jit_tracer_code_buffer, 1, _MAKE_WARM, 0, 0, 0); + tstate->interp->jit_tracer_code_curr_size = 2; + tstate->interp->jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH; + tstate->interp->jit_tracer_initial_instr = next_instr; + tstate->interp->jit_tracer_initial_code = code; + tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); + tstate->interp->jit_tracer_seen_initial_before = 0; + tstate->interp->jit_completed_loop = false; + tstate->interp->jit_tracer_initial_stack_depth = curr_stackdepth; + tstate->interp->jit_tracer_initial_chain_depth = chain_depth; } #undef RESERVE @@ -1123,7 +1086,7 @@ uop_optimize( return 0; } } - _PyUOpInstruction *buffer = interp->jit_uop_buffer; + _PyUOpInstruction *buffer = interp->jit_tracer_code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; @@ -1131,19 +1094,8 @@ uop_optimize( is_noopt = false; } int curr_stackentries = tstate->interp->jit_tracer_initial_stack_depth; -// #ifdef Py_DEBUG -// for (int x = 0; x < tstate->interp->jit_tracer_code_curr_size;) { -// int opcode = tstate->interp->jit_tracer_code_buffer[x].op.code; -// int oparg = tstate->interp->jit_tracer_code_buffer[x].op.arg; -// fprintf(stdout, "%s(%d)\n", _PyOpcode_OpName[opcode], oparg); -// x += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; -// } -// #endif - int length = translate_bytecode_to_trace(frame, tstate, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); - if (length <= 0) { - // Error or nothing translated - return length; - } + int length = interp->jit_tracer_code_curr_size; + assert(length > 0); assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { diff --git a/Python/pystate.c b/Python/pystate.c index 33ffc034b441..671daabd1edb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -815,7 +815,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->jit_uop_buffer = NULL; } if (interp->jit_tracer_code_buffer != NULL) { - _PyObject_VirtualFree(interp->jit_tracer_code_buffer, TRACER_BUFFER_SIZE); + _PyObject_VirtualFree(interp->jit_tracer_code_buffer, UOP_BUFFER_SIZE); interp->jit_tracer_code_buffer = NULL; } #endif -- 2.47.3