struct callable_cache callable_cache;
PyObject *common_consts[NUM_COMMON_CONSTANTS];
// JIT tracing state
+ int jit_tracer_code_max_size;
int jit_tracer_code_curr_size;
- _Py_CODEUNIT *jit_tracer_code_buffer;
+ _PyBloomFilter jit_tracer_dependencies;
+ _PyUOpInstruction *jit_tracer_code_buffer;
_Py_CODEUNIT *jit_tracer_initial_instr;
int jit_tracer_initial_stack_depth;
int jit_tracer_initial_chain_depth;
Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST)
Python 3.15a1 3654 (Fix missing exception handlers in logical expression)
Python 3.15a1 3655 (Fix miscompilation of some module-level annotations)
- Python 3.15a1 3656 (Add GUARD_IP instruction)
Python 3.16 will start with 3700
*/
-#define PYC_MAGIC_NUMBER 3656
+#define PYC_MAGIC_NUMBER 3655
/* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
(little-endian) and then appending b'\r\n'. */
#define PYC_MAGIC_NUMBER_TOKEN \
return 3;
case SWAP:
return 2 + (oparg-2);
- case TIER1_GUARD_IP:
- return 0;
- case TIER1_SET_IP:
- return 0;
case TO_BOOL:
return 1;
case TO_BOOL_ALWAYS_TRUE:
return 0;
case SWAP:
return 2 + (oparg-2);
- case TIER1_GUARD_IP:
- return 0;
- case TIER1_SET_IP:
- return 0;
case TO_BOOL:
return 1;
case TO_BOOL_ALWAYS_TRUE:
[STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG },
- [TIER1_GUARD_IP] = { true, INSTR_FMT_IXC000, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG },
- [TIER1_SET_IP] = { true, INSTR_FMT_IXC000, HAS_ESCAPES_FLAG },
[TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
[STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT",
[STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT",
[SWAP] = "SWAP",
- [TIER1_GUARD_IP] = "TIER1_GUARD_IP",
- [TIER1_SET_IP] = "TIER1_SET_IP",
[TO_BOOL] = "TO_BOOL",
[TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE",
[TO_BOOL_BOOL] = "TO_BOOL_BOOL",
extern const uint8_t _PyOpcode_Caches[256];
#ifdef NEED_OPCODE_METADATA
const uint8_t _PyOpcode_Caches[256] = {
- [TIER1_GUARD_IP] = 4,
- [TIER1_SET_IP] = 4,
[TO_BOOL] = 3,
[STORE_SUBSCR] = 1,
[SEND] = 1,
extern const uint8_t _PyOpcode_Deopt[256];
#ifdef NEED_OPCODE_METADATA
const uint8_t _PyOpcode_Deopt[256] = {
+ [121] = 121,
+ [122] = 122,
[123] = 123,
[124] = 124,
[125] = 125,
[STORE_SUBSCR_DICT] = STORE_SUBSCR,
[STORE_SUBSCR_LIST_INT] = STORE_SUBSCR,
[SWAP] = SWAP,
- [TIER1_GUARD_IP] = TIER1_GUARD_IP,
- [TIER1_SET_IP] = TIER1_SET_IP,
[TO_BOOL] = TO_BOOL,
[TO_BOOL_ALWAYS_TRUE] = TO_BOOL,
[TO_BOOL_BOOL] = TO_BOOL,
#endif // NEED_OPCODE_METADATA
#define EXTRA_CASES \
+ case 121: \
+ case 122: \
case 123: \
case 124: \
case 125: \
} _PyExecutorLinkListNode;
-/* Bloom filter with m = 256
- * https://en.wikipedia.org/wiki/Bloom_filter */
-#define _Py_BLOOM_FILTER_WORDS 8
-
-typedef struct {
- uint32_t bits[_Py_BLOOM_FILTER_WORDS];
-} _PyBloomFilter;
-
typedef struct {
uint8_t opcode;
uint8_t oparg;
extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp);
#endif
+int
+_PyJIT_translate_single_bytecode_to_trace(
+ PyThreadState *tstate,
+ _Py_CODEUNIT *this_instr,
+ _Py_CODEUNIT *next_instr,
+ PyCodeObject *code,
+ PyFunctionObject *func,
+ int oparg);
+
+void
+_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth);
#ifdef __cplusplus
}
#endif
#define UOP_MAX_TRACE_LENGTH 1200
#define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction))
-// This is the length of the trace we record.
-// This includes the inline caches.
-#define TRACE_MAX_TRACE_LENGTH 2000
-#define TRACER_BUFFER_SIZE ((int)(TRACE_MAX_TRACE_LENGTH * sizeof(_Py_CODEUNIT)))
+/* Bloom filter with m = 256
+ * https://en.wikipedia.org/wiki/Bloom_filter */
+#define _Py_BLOOM_FILTER_WORDS 8
+
+typedef struct {
+ uint32_t bits[_Py_BLOOM_FILTER_WORDS];
+} _PyBloomFilter;
#ifdef __cplusplus
}
[_ERROR_POP_N] = HAS_ARG_FLAG,
[_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG,
[_COLD_EXIT] = HAS_ESCAPES_FLAG,
- [_GUARD_IP] = HAS_ESCAPES_FLAG,
+ [_GUARD_IP] = 0,
[_DYNAMIC_EXIT] = 0,
};
#define SETUP_ANNOTATIONS 36
#define STORE_SLICE 37
#define STORE_SUBSCR 38
-#define TIER1_GUARD_IP 39
-#define TIER1_SET_IP 40
-#define TO_BOOL 41
-#define UNARY_INVERT 42
-#define UNARY_NEGATIVE 43
-#define UNARY_NOT 44
-#define WITH_EXCEPT_START 45
-#define BINARY_OP 46
-#define BUILD_INTERPOLATION 47
-#define BUILD_LIST 48
-#define BUILD_MAP 49
-#define BUILD_SET 50
-#define BUILD_SLICE 51
-#define BUILD_STRING 52
-#define BUILD_TUPLE 53
-#define CALL 54
-#define CALL_INTRINSIC_1 55
-#define CALL_INTRINSIC_2 56
-#define CALL_KW 57
-#define COMPARE_OP 58
-#define CONTAINS_OP 59
-#define CONVERT_VALUE 60
-#define COPY 61
-#define COPY_FREE_VARS 62
-#define DELETE_ATTR 63
-#define DELETE_DEREF 64
-#define DELETE_FAST 65
-#define DELETE_GLOBAL 66
-#define DELETE_NAME 67
-#define DICT_MERGE 68
-#define DICT_UPDATE 69
-#define END_ASYNC_FOR 70
-#define EXTENDED_ARG 71
-#define FOR_ITER 72
-#define GET_AWAITABLE 73
-#define IMPORT_FROM 74
-#define IMPORT_NAME 75
-#define IS_OP 76
-#define JUMP_BACKWARD 77
-#define JUMP_BACKWARD_NO_INTERRUPT 78
-#define JUMP_FORWARD 79
-#define LIST_APPEND 80
-#define LIST_EXTEND 81
-#define LOAD_ATTR 82
-#define LOAD_COMMON_CONSTANT 83
-#define LOAD_CONST 84
-#define LOAD_DEREF 85
-#define LOAD_FAST 86
-#define LOAD_FAST_AND_CLEAR 87
-#define LOAD_FAST_BORROW 88
-#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 89
-#define LOAD_FAST_CHECK 90
-#define LOAD_FAST_LOAD_FAST 91
-#define LOAD_FROM_DICT_OR_DEREF 92
-#define LOAD_FROM_DICT_OR_GLOBALS 93
-#define LOAD_GLOBAL 94
-#define LOAD_NAME 95
-#define LOAD_SMALL_INT 96
-#define LOAD_SPECIAL 97
-#define LOAD_SUPER_ATTR 98
-#define MAKE_CELL 99
-#define MAP_ADD 100
-#define MATCH_CLASS 101
-#define POP_JUMP_IF_FALSE 102
-#define POP_JUMP_IF_NONE 103
-#define POP_JUMP_IF_NOT_NONE 104
-#define POP_JUMP_IF_TRUE 105
-#define RAISE_VARARGS 106
-#define RERAISE 107
-#define SEND 108
-#define SET_ADD 109
-#define SET_FUNCTION_ATTRIBUTE 110
-#define SET_UPDATE 111
-#define STORE_ATTR 112
-#define STORE_DEREF 113
-#define STORE_FAST 114
-#define STORE_FAST_LOAD_FAST 115
-#define STORE_FAST_STORE_FAST 116
-#define STORE_GLOBAL 117
-#define STORE_NAME 118
-#define SWAP 119
-#define UNPACK_EX 120
-#define UNPACK_SEQUENCE 121
-#define YIELD_VALUE 122
+#define TO_BOOL 39
+#define UNARY_INVERT 40
+#define UNARY_NEGATIVE 41
+#define UNARY_NOT 42
+#define WITH_EXCEPT_START 43
+#define BINARY_OP 44
+#define BUILD_INTERPOLATION 45
+#define BUILD_LIST 46
+#define BUILD_MAP 47
+#define BUILD_SET 48
+#define BUILD_SLICE 49
+#define BUILD_STRING 50
+#define BUILD_TUPLE 51
+#define CALL 52
+#define CALL_INTRINSIC_1 53
+#define CALL_INTRINSIC_2 54
+#define CALL_KW 55
+#define COMPARE_OP 56
+#define CONTAINS_OP 57
+#define CONVERT_VALUE 58
+#define COPY 59
+#define COPY_FREE_VARS 60
+#define DELETE_ATTR 61
+#define DELETE_DEREF 62
+#define DELETE_FAST 63
+#define DELETE_GLOBAL 64
+#define DELETE_NAME 65
+#define DICT_MERGE 66
+#define DICT_UPDATE 67
+#define END_ASYNC_FOR 68
+#define EXTENDED_ARG 69
+#define FOR_ITER 70
+#define GET_AWAITABLE 71
+#define IMPORT_FROM 72
+#define IMPORT_NAME 73
+#define IS_OP 74
+#define JUMP_BACKWARD 75
+#define JUMP_BACKWARD_NO_INTERRUPT 76
+#define JUMP_FORWARD 77
+#define LIST_APPEND 78
+#define LIST_EXTEND 79
+#define LOAD_ATTR 80
+#define LOAD_COMMON_CONSTANT 81
+#define LOAD_CONST 82
+#define LOAD_DEREF 83
+#define LOAD_FAST 84
+#define LOAD_FAST_AND_CLEAR 85
+#define LOAD_FAST_BORROW 86
+#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 87
+#define LOAD_FAST_CHECK 88
+#define LOAD_FAST_LOAD_FAST 89
+#define LOAD_FROM_DICT_OR_DEREF 90
+#define LOAD_FROM_DICT_OR_GLOBALS 91
+#define LOAD_GLOBAL 92
+#define LOAD_NAME 93
+#define LOAD_SMALL_INT 94
+#define LOAD_SPECIAL 95
+#define LOAD_SUPER_ATTR 96
+#define MAKE_CELL 97
+#define MAP_ADD 98
+#define MATCH_CLASS 99
+#define POP_JUMP_IF_FALSE 100
+#define POP_JUMP_IF_NONE 101
+#define POP_JUMP_IF_NOT_NONE 102
+#define POP_JUMP_IF_TRUE 103
+#define RAISE_VARARGS 104
+#define RERAISE 105
+#define SEND 106
+#define SET_ADD 107
+#define SET_FUNCTION_ATTRIBUTE 108
+#define SET_UPDATE 109
+#define STORE_ATTR 110
+#define STORE_DEREF 111
+#define STORE_FAST 112
+#define STORE_FAST_LOAD_FAST 113
+#define STORE_FAST_STORE_FAST 114
+#define STORE_GLOBAL 115
+#define STORE_NAME 116
+#define SWAP 117
+#define UNPACK_EX 118
+#define UNPACK_SEQUENCE 119
+#define YIELD_VALUE 120
#define RESUME 128
#define BINARY_OP_ADD_FLOAT 129
#define BINARY_OP_ADD_INT 130
#define SETUP_WITH 265
#define STORE_FAST_MAYBE_NULL 266
-#define HAVE_ARGUMENT 45
+#define HAVE_ARGUMENT 43
#define MIN_SPECIALIZED_OPCODE 129
#define MIN_INSTRUMENTED_OPCODE 234
'SETUP_ANNOTATIONS': 36,
'STORE_SLICE': 37,
'STORE_SUBSCR': 38,
- 'TIER1_GUARD_IP': 39,
- 'TIER1_SET_IP': 40,
- 'TO_BOOL': 41,
- 'UNARY_INVERT': 42,
- 'UNARY_NEGATIVE': 43,
- 'UNARY_NOT': 44,
- 'WITH_EXCEPT_START': 45,
- 'BINARY_OP': 46,
- 'BUILD_INTERPOLATION': 47,
- 'BUILD_LIST': 48,
- 'BUILD_MAP': 49,
- 'BUILD_SET': 50,
- 'BUILD_SLICE': 51,
- 'BUILD_STRING': 52,
- 'BUILD_TUPLE': 53,
- 'CALL': 54,
- 'CALL_INTRINSIC_1': 55,
- 'CALL_INTRINSIC_2': 56,
- 'CALL_KW': 57,
- 'COMPARE_OP': 58,
- 'CONTAINS_OP': 59,
- 'CONVERT_VALUE': 60,
- 'COPY': 61,
- 'COPY_FREE_VARS': 62,
- 'DELETE_ATTR': 63,
- 'DELETE_DEREF': 64,
- 'DELETE_FAST': 65,
- 'DELETE_GLOBAL': 66,
- 'DELETE_NAME': 67,
- 'DICT_MERGE': 68,
- 'DICT_UPDATE': 69,
- 'END_ASYNC_FOR': 70,
- 'EXTENDED_ARG': 71,
- 'FOR_ITER': 72,
- 'GET_AWAITABLE': 73,
- 'IMPORT_FROM': 74,
- 'IMPORT_NAME': 75,
- 'IS_OP': 76,
- 'JUMP_BACKWARD': 77,
- 'JUMP_BACKWARD_NO_INTERRUPT': 78,
- 'JUMP_FORWARD': 79,
- 'LIST_APPEND': 80,
- 'LIST_EXTEND': 81,
- 'LOAD_ATTR': 82,
- 'LOAD_COMMON_CONSTANT': 83,
- 'LOAD_CONST': 84,
- 'LOAD_DEREF': 85,
- 'LOAD_FAST': 86,
- 'LOAD_FAST_AND_CLEAR': 87,
- 'LOAD_FAST_BORROW': 88,
- 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 89,
- 'LOAD_FAST_CHECK': 90,
- 'LOAD_FAST_LOAD_FAST': 91,
- 'LOAD_FROM_DICT_OR_DEREF': 92,
- 'LOAD_FROM_DICT_OR_GLOBALS': 93,
- 'LOAD_GLOBAL': 94,
- 'LOAD_NAME': 95,
- 'LOAD_SMALL_INT': 96,
- 'LOAD_SPECIAL': 97,
- 'LOAD_SUPER_ATTR': 98,
- 'MAKE_CELL': 99,
- 'MAP_ADD': 100,
- 'MATCH_CLASS': 101,
- 'POP_JUMP_IF_FALSE': 102,
- 'POP_JUMP_IF_NONE': 103,
- 'POP_JUMP_IF_NOT_NONE': 104,
- 'POP_JUMP_IF_TRUE': 105,
- 'RAISE_VARARGS': 106,
- 'RERAISE': 107,
- 'SEND': 108,
- 'SET_ADD': 109,
- 'SET_FUNCTION_ATTRIBUTE': 110,
- 'SET_UPDATE': 111,
- 'STORE_ATTR': 112,
- 'STORE_DEREF': 113,
- 'STORE_FAST': 114,
- 'STORE_FAST_LOAD_FAST': 115,
- 'STORE_FAST_STORE_FAST': 116,
- 'STORE_GLOBAL': 117,
- 'STORE_NAME': 118,
- 'SWAP': 119,
- 'UNPACK_EX': 120,
- 'UNPACK_SEQUENCE': 121,
- 'YIELD_VALUE': 122,
+ 'TO_BOOL': 39,
+ 'UNARY_INVERT': 40,
+ 'UNARY_NEGATIVE': 41,
+ 'UNARY_NOT': 42,
+ 'WITH_EXCEPT_START': 43,
+ 'BINARY_OP': 44,
+ 'BUILD_INTERPOLATION': 45,
+ 'BUILD_LIST': 46,
+ 'BUILD_MAP': 47,
+ 'BUILD_SET': 48,
+ 'BUILD_SLICE': 49,
+ 'BUILD_STRING': 50,
+ 'BUILD_TUPLE': 51,
+ 'CALL': 52,
+ 'CALL_INTRINSIC_1': 53,
+ 'CALL_INTRINSIC_2': 54,
+ 'CALL_KW': 55,
+ 'COMPARE_OP': 56,
+ 'CONTAINS_OP': 57,
+ 'CONVERT_VALUE': 58,
+ 'COPY': 59,
+ 'COPY_FREE_VARS': 60,
+ 'DELETE_ATTR': 61,
+ 'DELETE_DEREF': 62,
+ 'DELETE_FAST': 63,
+ 'DELETE_GLOBAL': 64,
+ 'DELETE_NAME': 65,
+ 'DICT_MERGE': 66,
+ 'DICT_UPDATE': 67,
+ 'END_ASYNC_FOR': 68,
+ 'EXTENDED_ARG': 69,
+ 'FOR_ITER': 70,
+ 'GET_AWAITABLE': 71,
+ 'IMPORT_FROM': 72,
+ 'IMPORT_NAME': 73,
+ 'IS_OP': 74,
+ 'JUMP_BACKWARD': 75,
+ 'JUMP_BACKWARD_NO_INTERRUPT': 76,
+ 'JUMP_FORWARD': 77,
+ 'LIST_APPEND': 78,
+ 'LIST_EXTEND': 79,
+ 'LOAD_ATTR': 80,
+ 'LOAD_COMMON_CONSTANT': 81,
+ 'LOAD_CONST': 82,
+ 'LOAD_DEREF': 83,
+ 'LOAD_FAST': 84,
+ 'LOAD_FAST_AND_CLEAR': 85,
+ 'LOAD_FAST_BORROW': 86,
+ 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87,
+ 'LOAD_FAST_CHECK': 88,
+ 'LOAD_FAST_LOAD_FAST': 89,
+ 'LOAD_FROM_DICT_OR_DEREF': 90,
+ 'LOAD_FROM_DICT_OR_GLOBALS': 91,
+ 'LOAD_GLOBAL': 92,
+ 'LOAD_NAME': 93,
+ 'LOAD_SMALL_INT': 94,
+ 'LOAD_SPECIAL': 95,
+ 'LOAD_SUPER_ATTR': 96,
+ 'MAKE_CELL': 97,
+ 'MAP_ADD': 98,
+ 'MATCH_CLASS': 99,
+ 'POP_JUMP_IF_FALSE': 100,
+ 'POP_JUMP_IF_NONE': 101,
+ 'POP_JUMP_IF_NOT_NONE': 102,
+ 'POP_JUMP_IF_TRUE': 103,
+ 'RAISE_VARARGS': 104,
+ 'RERAISE': 105,
+ 'SEND': 106,
+ 'SET_ADD': 107,
+ 'SET_FUNCTION_ATTRIBUTE': 108,
+ 'SET_UPDATE': 109,
+ 'STORE_ATTR': 110,
+ 'STORE_DEREF': 111,
+ 'STORE_FAST': 112,
+ 'STORE_FAST_LOAD_FAST': 113,
+ 'STORE_FAST_STORE_FAST': 114,
+ 'STORE_GLOBAL': 115,
+ 'STORE_NAME': 116,
+ 'SWAP': 117,
+ 'UNPACK_EX': 118,
+ 'UNPACK_SEQUENCE': 119,
+ 'YIELD_VALUE': 120,
'INSTRUMENTED_END_FOR': 234,
'INSTRUMENTED_POP_ITER': 235,
'INSTRUMENTED_END_SEND': 236,
'STORE_FAST_MAYBE_NULL': 266,
}
-HAVE_ARGUMENT = 45
+HAVE_ARGUMENT = 43
MIN_INSTRUMENTED_OPCODE = 234
start--;
}
if (tstate->interp->jit_tracer_code_buffer == NULL) {
- tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE);
+ tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE);
if (tstate->interp->jit_tracer_code_buffer == NULL) {
// Don't error, just go to next instruction.
DISPATCH();
}
- tstate->interp->jit_tracer_code_curr_size = 0;
}
+ _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0);
ENTER_TRACING();
- // Nothing in the buffer, begin tracing!
- if (tstate->interp->jit_tracer_code_curr_size == 0) {
- tstate->interp->jit_tracer_initial_instr = next_instr;
- tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame);
- tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame);
- tstate->interp->jit_tracer_seen_initial_before = 0;
- tstate->interp->jit_completed_loop = false;
- tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL();
- tstate->interp->jit_tracer_initial_chain_depth = 0;
- }
// Don't add the JUMP_BACKWARD_JIT instruction to the trace.
DISPATCH();
}
_PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
assert(tstate->current_executor == (PyObject *)previous_executor);
int chain_depth = previous_executor->vm_data.chain_depth + 1;
- tstate->interp->jit_tracer_initial_chain_depth = chain_depth;
- tstate->interp->jit_tracer_initial_instr = target;
- tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame);
- tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame);
- tstate->interp->jit_tracer_seen_initial_before = 0;
- tstate->interp->jit_completed_loop = false;
- exit->temperature = restart_backoff_counter(temperature);
+ _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth);
GOTO_TIER_ONE(target, 1);
}
assert(tstate->jit_exit == exit);
TIER2_TO_TIER2(exit->executor);
}
-
- no_save_ip tier1 inst(TIER1_GUARD_IP, (func_ptr/4 --)) {
- (void)(func_ptr);
- }
-
tier2 op(_GUARD_IP, (ip/4 --)) {
if (frame->instr_ptr != (_Py_CODEUNIT *)ip) {
- fprintf(stdout, "d:%p:%p\n", frame->instr_ptr, ip);
GOTO_TIER_ONE(frame->instr_ptr, 1);
}
}
GOTO_TIER_ONE(frame->instr_ptr, 1);
}
- tier1 inst(TIER1_SET_IP, (ip/4 --)) {
- (void)(ip);
- }
-
-
label(pop_2_error) {
stack_pointer -= 2;
assert(WITHIN_STACK_BOUNDS());
// 1 for trace full, 0 for successful write.
static int
-add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, int oparg)
+add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int oparg)
{
assert(frame != NULL);
- assert(tstate->interp->jit_tracer_code_curr_size < TRACE_MAX_TRACE_LENGTH);
- int curr_size = tstate->interp->jit_tracer_code_curr_size;
- int opcode = this_instr->op.code;
- if (opcode == ENTER_EXECUTOR) {
- return 1;
- // PyCodeObject *code = _PyFrame_GetCode(frame);
- // assert(code != NULL);
- // _PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
- // assert(executor->vm_data.code == code);
- // assert(executor->vm_data.valid);
- // assert(tstate->current_executor == NULL);
- // opcode = executor->vm_data.opcode;
- // oparg = (oparg & ~255) | executor->vm_data.oparg;
- }
- else {
- oparg = this_instr->op.arg;
- }
- assert(opcode != 0);
- // Check if we looped back to the start.
- if (this_instr == tstate->interp->jit_tracer_initial_instr) {
- if (tstate->interp->jit_tracer_seen_initial_before >= 1) {
- tstate->interp->jit_completed_loop = true;
- return 1;
- }
- tstate->interp->jit_tracer_seen_initial_before++;
- }
-#ifdef Py_DEBUG
- char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
- int lltrace = 3;
- if (python_lltrace != NULL && *python_lltrace >= '0') {
- lltrace = *python_lltrace - '0';
- }
- if (lltrace >= 3) {
- printf("%d ADD_TO_BYTECODE_TRACE: %s %d\n", curr_size, _PyOpcode_OpName[opcode], oparg);
- }
-#endif
- // JUMP_BACKWARD_NO_INTERRUPT is not registered with cache due to a bug in the compiler.
- int caches = opcode == JUMP_BACKWARD_NO_INTERRUPT ? 1 : _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];
- int nsize = caches + 1;
- int needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode];
- int total_size = nsize + (needs_guard_ip ? 5 : 0) + 5;
- if (curr_size + total_size >= TRACE_MAX_TRACE_LENGTH) {
-#ifdef Py_DEBUG
- if (lltrace >= 3) {
- printf("END TRACE LENGTH: %d\n", curr_size);
- }
-#endif
- return 1;
- }
- tstate->interp->jit_tracer_code_buffer[curr_size].op.code = opcode;
- tstate->interp->jit_tracer_code_buffer[curr_size].op.arg = oparg;
- for (int i = 1; i < nsize; i++) {
- tstate->interp->jit_tracer_code_buffer[curr_size + i] = *(this_instr + i);
- }
- if (needs_guard_ip) {
-#ifdef Py_DEBUG
- if (lltrace >= 3) {
- printf("%d ADD_TO_BYTECODE_TRACE: %s %d\n", curr_size + nsize, _PyOpcode_OpName[TIER1_GUARD_IP], 0);
- }
-#endif
- tstate->interp->jit_tracer_code_buffer[curr_size + nsize].op.code = TIER1_GUARD_IP;
- PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
- if (Py_IsNone(func)) {
- func = NULL; // Trampoline frames don't set their func object field.
- }
- assert(func == NULL || PyFunction_Check(func));
- write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 1].cache, func);
-
-#ifdef Py_DEBUG
- if (lltrace >= 3) {
- printf("%d ADD_TO_BYTECODE_TRACE: %s %p\n", curr_size + nsize + 5, _PyOpcode_OpName[TIER1_SET_IP], frame->instr_ptr);
- }
-#endif
- tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 5].op.code = TIER1_SET_IP;
- write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 6].cache, frame->instr_ptr);
-
- }
- else {
-#ifdef Py_DEBUG
- if (lltrace >= 3) {
- printf("%d ADD_TO_BYTECODE_TRACE: %s %p\n", curr_size + nsize, _PyOpcode_OpName[TIER1_SET_IP], frame->instr_ptr);
- }
-#endif
- tstate->interp->jit_tracer_code_buffer[curr_size + nsize].op.code = TIER1_SET_IP;
- write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 1].cache, frame->instr_ptr);
-
- }
-
- tstate->interp->jit_tracer_code_curr_size += total_size;
- return 0;
+ assert(tstate->interp->jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH);
+ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
+ return !_PyJIT_translate_single_bytecode_to_trace(tstate, this_instr, next_instr, _PyFrame_GetCode(frame), func, oparg);
}
/* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC.
DISPATCH();
# define RECORD_TRACE() do { \
frame->instr_ptr = next_instr; \
- if (add_to_code_trace(tstate, frame, this_instr, oparg)) { \
+ if (add_to_code_trace(tstate, frame, this_instr, next_instr, oparg)) { \
BAIL_TRACING(); \
} \
} while (0);
stack_pointer = _PyFrame_GetStackPointer(frame);
assert(tstate->current_executor == (PyObject *)previous_executor);
int chain_depth = previous_executor->vm_data.chain_depth + 1;
- tstate->interp->jit_tracer_initial_chain_depth = chain_depth;
- tstate->interp->jit_tracer_initial_instr = target;
- tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
- tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame);
+ _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth);
stack_pointer = _PyFrame_GetStackPointer(frame);
- tstate->interp->jit_tracer_seen_initial_before = 0;
- tstate->interp->jit_completed_loop = false;
- exit->temperature = restart_backoff_counter(temperature);
GOTO_TIER_ONE(target, 1);
}
assert(tstate->jit_exit == exit);
case _GUARD_IP: {
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
if (frame->instr_ptr != (_Py_CODEUNIT *)ip) {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- fprintf(stdout, "d:%p:%p\n", frame->instr_ptr, ip);
- stack_pointer = _PyFrame_GetStackPointer(frame);
GOTO_TIER_ONE(frame->instr_ptr, 1);
}
break;
}
if (tstate->interp->jit_tracer_code_buffer == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
- tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE);
+ tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (tstate->interp->jit_tracer_code_buffer == NULL) {
DISPATCH();
}
- tstate->interp->jit_tracer_code_curr_size = 0;
}
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
ENTER_TRACING();
- if (tstate->interp->jit_tracer_code_curr_size == 0) {
- tstate->interp->jit_tracer_initial_instr = next_instr;
- tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- tstate->interp->jit_tracer_seen_initial_before = 0;
- tstate->interp->jit_completed_loop = false;
- tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL();
- tstate->interp->jit_tracer_initial_chain_depth = 0;
- }
DISPATCH();
}
else {
DISPATCH();
}
- TARGET(TIER1_GUARD_IP) {
- #if _Py_TAIL_CALL_INTERP
- int opcode = TIER1_GUARD_IP;
- (void)(opcode);
- #endif
- _Py_CODEUNIT* const this_instr = next_instr;
- (void)this_instr;
- next_instr += 5;
- INSTRUCTION_STATS(TIER1_GUARD_IP);
- PyObject *func_ptr = read_obj(&this_instr[1].cache);
- (void)func_ptr;
- _PyFrame_SetStackPointer(frame, stack_pointer);
- (void)(func_ptr);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- DISPATCH();
- }
-
- TARGET(TIER1_SET_IP) {
- #if _Py_TAIL_CALL_INTERP
- int opcode = TIER1_SET_IP;
- (void)(opcode);
- #endif
- _Py_CODEUNIT* const this_instr = next_instr;
- (void)this_instr;
- frame->instr_ptr = next_instr;
- next_instr += 5;
- INSTRUCTION_STATS(TIER1_SET_IP);
- PyObject *ip = read_obj(&this_instr[1].cache);
- (void)ip;
- _PyFrame_SetStackPointer(frame, stack_pointer);
- (void)(ip);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- DISPATCH();
- }
-
TARGET(TO_BOOL) {
#if _Py_TAIL_CALL_INTERP
int opcode = TO_BOOL;
}
if (tstate->interp->jit_tracer_code_buffer == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
- tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE);
+ tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (tstate->interp->jit_tracer_code_buffer == NULL) {
TRACING_DISPATCH();
}
- tstate->interp->jit_tracer_code_curr_size = 0;
}
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
ENTER_TRACING();
- if (tstate->interp->jit_tracer_code_curr_size == 0) {
- tstate->interp->jit_tracer_initial_instr = next_instr;
- tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- tstate->interp->jit_tracer_seen_initial_before = 0;
- tstate->interp->jit_completed_loop = false;
- tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL();
- tstate->interp->jit_tracer_initial_chain_depth = 0;
- }
TRACING_DISPATCH();
}
else {
TRACING_DISPATCH();
}
- TRACING_TARGET(TIER1_GUARD_IP) {
- #if _Py_TAIL_CALL_INTERP
- int opcode = TIER1_GUARD_IP;
- (void)(opcode);
- #endif
- _Py_CODEUNIT* const this_instr = next_instr;
- (void)this_instr;
- next_instr += 5;
- INSTRUCTION_STATS(TIER1_GUARD_IP);
- PyObject *func_ptr = read_obj(&this_instr[1].cache);
- (void)func_ptr;
- _PyFrame_SetStackPointer(frame, stack_pointer);
- (void)(func_ptr);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- TRACING_DISPATCH();
- }
-
- TRACING_TARGET(TIER1_SET_IP) {
- #if _Py_TAIL_CALL_INTERP
- int opcode = TIER1_SET_IP;
- (void)(opcode);
- #endif
- _Py_CODEUNIT* const this_instr = next_instr;
- (void)this_instr;
- frame->instr_ptr = next_instr;
- next_instr += 5;
- INSTRUCTION_STATS(TIER1_SET_IP);
- PyObject *ip = read_obj(&this_instr[1].cache);
- (void)ip;
- _PyFrame_SetStackPointer(frame, stack_pointer);
- (void)(ip);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- TRACING_DISPATCH();
- }
-
TRACING_TARGET(TO_BOOL) {
#if _Py_TAIL_CALL_INTERP
int opcode = TO_BOOL;
&&TARGET_SETUP_ANNOTATIONS,
&&TARGET_STORE_SLICE,
&&TARGET_STORE_SUBSCR,
- &&TARGET_TIER1_GUARD_IP,
- &&TARGET_TIER1_SET_IP,
&&TARGET_TO_BOOL,
&&TARGET_UNARY_INVERT,
&&TARGET_UNARY_NEGATIVE,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
&&TARGET_RESUME,
&&TARGET_BINARY_OP_ADD_FLOAT,
&&TARGET_BINARY_OP_ADD_INT,
&&TARGET_TRACING_SETUP_ANNOTATIONS,
&&TARGET_TRACING_STORE_SLICE,
&&TARGET_TRACING_STORE_SUBSCR,
- &&TARGET_TRACING_TIER1_GUARD_IP,
- &&TARGET_TRACING_TIER1_SET_IP,
&&TARGET_TRACING_TO_BOOL,
&&TARGET_TRACING_UNARY_INVERT,
&&TARGET_TRACING_UNARY_NEGATIVE,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
+ &&_unknown_opcode,
+ &&_unknown_opcode,
&&TARGET_TRACING_RESUME,
&&TARGET_TRACING_BINARY_OP_ADD_FLOAT,
&&TARGET_TRACING_BINARY_OP_ADD_INT,
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SWAP(TAIL_CALL_PARAMS);
-Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TIER1_GUARD_IP(TAIL_CALL_PARAMS);
-Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TIER1_GUARD_IP(TAIL_CALL_PARAMS);
-Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TIER1_SET_IP(TAIL_CALL_PARAMS);
-Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TIER1_SET_IP(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS);
[STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT,
[STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT,
[SWAP] = _TAIL_CALL_SWAP,
- [TIER1_GUARD_IP] = _TAIL_CALL_TIER1_GUARD_IP,
- [TIER1_SET_IP] = _TAIL_CALL_TIER1_SET_IP,
[TO_BOOL] = _TAIL_CALL_TO_BOOL,
[TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE,
[TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL,
[UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE,
[WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START,
[YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE,
+ [121] = _TAIL_CALL_UNKNOWN_OPCODE,
+ [122] = _TAIL_CALL_UNKNOWN_OPCODE,
[123] = _TAIL_CALL_UNKNOWN_OPCODE,
[124] = _TAIL_CALL_UNKNOWN_OPCODE,
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
[STORE_SUBSCR_DICT] = _TAIL_CALL_TRACING_STORE_SUBSCR_DICT,
[STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT,
[SWAP] = _TAIL_CALL_TRACING_SWAP,
- [TIER1_GUARD_IP] = _TAIL_CALL_TRACING_TIER1_GUARD_IP,
- [TIER1_SET_IP] = _TAIL_CALL_TRACING_TIER1_SET_IP,
[TO_BOOL] = _TAIL_CALL_TRACING_TO_BOOL,
[TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE,
[TO_BOOL_BOOL] = _TAIL_CALL_TRACING_TO_BOOL_BOOL,
[UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE,
[WITH_EXCEPT_START] = _TAIL_CALL_TRACING_WITH_EXCEPT_START,
[YIELD_VALUE] = _TAIL_CALL_TRACING_YIELD_VALUE,
+ [121] = _TAIL_CALL_UNKNOWN_OPCODE,
+ [122] = _TAIL_CALL_UNKNOWN_OPCODE,
[123] = _TAIL_CALL_UNKNOWN_OPCODE,
[124] = _TAIL_CALL_UNKNOWN_OPCODE,
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
DPRINTF(2, "No room for %s (need %d, got %d)\n", \
(opname), (n), max_length - trace_length); \
OPT_STAT_INC(trace_too_long); \
- goto done; \
+ goto full; \
}
// Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE
#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode))
-/* Returns the length of the trace on success,
- * 0 if it failed to produce a worthwhile trace,
- * and -1 on an error.
+/* Returns 1 on success (added to trace), 0 on trace end.
*/
-static int
-translate_bytecode_to_trace(
- _PyInterpreterFrame *frame,
+int
+_PyJIT_translate_single_bytecode_to_trace(
PyThreadState *tstate,
- _PyUOpInstruction *trace,
- int buffer_size,
- _PyBloomFilter *dependencies, bool progress_needed)
+ _Py_CODEUNIT *this_instr,
+ _Py_CODEUNIT *next_instr,
+ PyCodeObject *code,
+ PyFunctionObject *func,
+ int oparg)
{
- bool first = true;
- PyCodeObject *code = tstate->interp->jit_tracer_initial_code;;
- PyFunctionObject *func = tstate->interp->jit_tracer_initial_func;
- PyCodeObject *initial_code = code;
- _Py_BloomFilter_Add(dependencies, initial_code);
- _Py_CODEUNIT *target_instr = tstate->interp->jit_tracer_initial_instr;
- _Py_CODEUNIT *initial_instr = tstate->interp->jit_tracer_code_buffer;
- int trace_length = 0;
- // Leave space for possible trailing _EXIT_TRACE
- int max_length = buffer_size-2;
+ if (Py_IsNone((PyObject *)func)) {
+ func = NULL;
+ }
+ bool progress_needed = (tstate->interp->jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0;;
+ _PyBloomFilter *dependencies = &tstate->interp->jit_tracer_dependencies;
+ _Py_BloomFilter_Add(dependencies, code);
+ _Py_CODEUNIT *target_instr = this_instr;
+ int trace_length = tstate->interp->jit_tracer_code_curr_size;
+ _PyUOpInstruction *trace = tstate->interp->jit_tracer_code_buffer;
+ int max_length = tstate->interp->jit_tracer_code_max_size;
#ifdef Py_DEBUG
char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
}
#endif
- DPRINTF(2,
- "Optimizing %s (%s:%d) at byte offset %d\n",
- PyUnicode_AsUTF8(code->co_qualname),
- PyUnicode_AsUTF8(code->co_filename),
- code->co_firstlineno,
- 2 * INSTR_IP(target_instr, code));
- ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)target_instr, INSTR_IP(target_instr, code));
- ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0);
uint32_t target = 0;
- bool preceded_by_for_iter = false;
-
- for (int x = 0; x < tstate->interp->jit_tracer_code_curr_size;) {
- target = INSTR_IP(target_instr, code);
- // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT
- max_length-=2;
- uint32_t opcode = initial_instr[x].op.code;
- uint32_t oparg = initial_instr[x].op.arg;
-
- DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg);
-
- if (opcode == EXTENDED_ARG) {
- x++;
- opcode = initial_instr[x].op.code;
- oparg = (oparg << 8) | initial_instr[x].op.arg;
- if (opcode == EXTENDED_ARG) {
- x--;
- goto done;
- }
- }
- assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG);
- if (opcode != NOP) {
- if (opcode != TIER1_GUARD_IP) {
- RESERVE_RAW(1, "_CHECK_VALIDITY");
- ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target);
- }
- if (!OPCODE_HAS_NO_SAVE_IP(opcode)) {
- RESERVE_RAW(2, "_SET_IP");
- ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target);
- }
- }
- /* Special case the first instruction,
- * so that we can guarantee forward progress */
- if (first && progress_needed) {
- assert(first);
- if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) {
- opcode = _PyOpcode_Deopt[opcode];
- }
- assert(!OPCODE_HAS_EXIT(opcode));
- assert(!OPCODE_HAS_DEOPT(opcode));
- }
+ target = INSTR_IP(target_instr, code);
+ // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT
+ max_length-=2;
+ uint32_t opcode = this_instr->op.code;
+ if ((uint16_t)oparg != (uint64_t)oparg) {
+ goto full;
+ }
- if (OPCODE_HAS_EXIT(opcode)) {
- // Make space for side exit and final _EXIT_TRACE:
- RESERVE_RAW(2, "_EXIT_TRACE");
- max_length--;
- }
- if (OPCODE_HAS_ERROR(opcode)) {
- // Make space for error stub and final _EXIT_TRACE:
- RESERVE_RAW(2, "_ERROR_POP_N");
- max_length--;
- }
- if (opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) {
- preceded_by_for_iter = true;
- // assert((initial_instr + x + 1 + _PyOpcode_Caches[FOR_ITER])->op.code == TIER1_GUARD_IP);
- // assert((initial_instr + x + 6 + _PyOpcode_Caches[FOR_ITER])->op.code == TIER1_SET_IP);
- // PyObject *temp = read_obj(&(initial_instr + x + 2 + _PyOpcode_Caches[FOR_ITER])->cache);
- // if (temp != NULL) {
- // code = ((PyFunctionObject*)temp)->func_code;
- // }
- // target_instr = (_Py_CODEUNIT *)(read_obj(&(initial_instr + x + 7 + _PyOpcode_Caches[FOR_ITER])->cache));
- // target = INSTR_IP(target_instr, code);
- //
- // (initial_instr + x + 1 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP;
- // (initial_instr + x + 2 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP;
- // (initial_instr + x + 3 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP;
- // (initial_instr + x + 4 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP;
- // (initial_instr + x + 5 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP;
- //
- // (initial_instr + x + 6 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP;
- // (initial_instr + x + 7 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP;
- // (initial_instr + x + 8 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP;
- // (initial_instr + x + 9 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP;
- // (initial_instr + x + 10 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP;
+ DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg);
+
+ if (opcode == EXTENDED_ARG) {
+ return 1;
+ }
+ if (opcode == ENTER_EXECUTOR) {
+ goto full;
+ }
+ assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG);
+ if (opcode == NOP) {
+ return 1;
+ }
+
+ RESERVE_RAW(1, "_CHECK_VALIDITY");
+ ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target);
+
+ if (!OPCODE_HAS_NO_SAVE_IP(opcode)) {
+ RESERVE_RAW(2, "_SET_IP");
+ ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target);
+ }
+
+ bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] &&
+ !(opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) &&
+ !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT);
+
+ if (needs_guard_ip) {
+ RESERVE_RAW(1, "_GUARD_IP");
+ }
+
+ /* Special case the first instruction,
+ * so that we can guarantee forward progress */
+ if (progress_needed && tstate->interp->jit_tracer_initial_instr == this_instr && tstate->interp->jit_tracer_code_curr_size == 0) {
+ if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) {
+ opcode = _PyOpcode_Deopt[opcode];
}
- switch (opcode) {
- case JUMP_BACKWARD_JIT:
- case JUMP_BACKWARD:
- ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target);
- _Py_FALLTHROUGH;
- case JUMP_BACKWARD_NO_INTERRUPT:
- ADD_TO_TRACE(_JUMP_BACKWARD_NO_INTERRUPT, oparg, 0, target);
- break;
- case TIER1_GUARD_IP:
- func = (PyFunctionObject *)read_obj(&(initial_instr + x + 1)->cache);
- assert(func == NULL || PyFunction_Check(func));
- if (func != NULL) {
- code = (PyCodeObject *)func->func_code;
- }
- assert((initial_instr + x + 5)->op.code == TIER1_SET_IP);
- target_instr = (_Py_CODEUNIT*)read_obj(&(initial_instr + x + 6)->cache);
- if (preceded_by_for_iter) {;
- preceded_by_for_iter = false;
- }
- else {
- ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)target_instr, 0)
- }
- break;
+ assert(!OPCODE_HAS_EXIT(opcode));
+ assert(!OPCODE_HAS_DEOPT(opcode));
+ }
- case TIER1_SET_IP:
- target_instr = (_Py_CODEUNIT*)read_obj(&(initial_instr + x + 1)->cache);
- break;
+ // Loop back to the start
+ if (tstate->interp->jit_tracer_initial_instr == this_instr && tstate->interp->jit_tracer_code_curr_size > 2) {
+ ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0);
+ goto done;
+ }
- case RESUME:
- /* Use a special tier 2 version of RESUME_CHECK to allow traces to
- * start with RESUME_CHECK */
- ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target);
- break;
+ if (OPCODE_HAS_EXIT(opcode)) {
+ // Make space for side exit and final _EXIT_TRACE:
+ RESERVE_RAW(2, "_EXIT_TRACE");
+ max_length--;
+ }
+ if (OPCODE_HAS_ERROR(opcode)) {
+ // Make space for error stub and final _EXIT_TRACE:
+ RESERVE_RAW(2, "_ERROR_POP_N");
+ max_length--;
+ }
- default:
- {
- const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode];
- if (expansion->nuops > 0) {
- // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE)
- int nuops = expansion->nuops;
- RESERVE(nuops + 1); /* One extra for exit */
- uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM
- for (int i = 0; i < nuops; i++) {
- oparg = orig_oparg;
- uint32_t uop = expansion->uops[i].uop;
- uint64_t operand = 0;
- // Add one to account for the actual opcode/oparg pair:
- int offset = expansion->uops[i].offset + 1;
- switch (expansion->uops[i].size) {
- case OPARG_SIMPLE:
- assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD);
- break;
- case OPARG_CACHE_1:
- operand = read_u16(&(initial_instr + x)[offset].cache);
- break;
- case OPARG_CACHE_2:
- operand = read_u32(&(initial_instr + x)[offset].cache);
- break;
- case OPARG_CACHE_4:
- operand = read_u64(&(initial_instr + x)[offset].cache);
- break;
- case OPARG_TOP: // First half of super-instr
- oparg = orig_oparg >> 4;
- break;
- case OPARG_BOTTOM: // Second half of super-instr
- oparg = orig_oparg & 0xF;
- break;
- case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset
- oparg = offset;
- assert(uop == _SAVE_RETURN_OFFSET);
- break;
- case OPARG_REPLACED:
- uop = _PyUOp_Replacements[uop];
- assert(uop != 0);
-
- uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255);
- if (uop == _TIER2_RESUME_CHECK) {
- target = next_inst;
- }
+ switch (opcode) {
+ case JUMP_BACKWARD_JIT:
+ case JUMP_BACKWARD:
+ ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target);
+ _Py_FALLTHROUGH;
+ case JUMP_BACKWARD_NO_INTERRUPT:
+ break;
+
+ case RESUME:
+ /* Use a special tier 2 version of RESUME_CHECK to allow traces to
+ * start with RESUME_CHECK */
+ ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target);
+ break;
+
+ default:
+ {
+ const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode];
+ if (expansion->nuops > 0) {
+ // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE)
+ int nuops = expansion->nuops;
+ RESERVE(nuops + 1); /* One extra for exit */
+ uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM
+ for (int i = 0; i < nuops; i++) {
+ oparg = orig_oparg;
+ uint32_t uop = expansion->uops[i].uop;
+ uint64_t operand = 0;
+ // Add one to account for the actual opcode/oparg pair:
+ int offset = expansion->uops[i].offset + 1;
+ switch (expansion->uops[i].size) {
+ case OPARG_SIMPLE:
+ assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD);
+ break;
+ case OPARG_CACHE_1:
+ operand = read_u16(&this_instr[offset].cache);
+ break;
+ case OPARG_CACHE_2:
+ operand = read_u32(&this_instr[offset].cache);
+ break;
+ case OPARG_CACHE_4:
+ operand = read_u64(&this_instr[offset].cache);
+ break;
+ case OPARG_TOP: // First half of super-instr
+ oparg = orig_oparg >> 4;
+ break;
+ case OPARG_BOTTOM: // Second half of super-instr
+ oparg = orig_oparg & 0xF;
+ break;
+ case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset
+ oparg = offset;
+ assert(uop == _SAVE_RETURN_OFFSET);
+ break;
+ case OPARG_REPLACED:
+ uop = _PyUOp_Replacements[uop];
+ assert(uop != 0);
+
+ uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255);
+ if (uop == _TIER2_RESUME_CHECK) {
+ target = next_inst;
+ }
#ifdef Py_DEBUG
- else {
- uint32_t jump_target = next_inst + oparg;
- assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR);
- assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER);
- }
+ else {
+ uint32_t jump_target = next_inst + oparg;
+ assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR);
+ assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER);
+ }
#endif
- break;
- case OPERAND1_1:
- assert(trace[trace_length-1].opcode == uop);
- operand = read_u16(&(initial_instr + x)[offset].cache);
- trace[trace_length-1].operand1 = operand;
- continue;
- case OPERAND1_2:
- assert(trace[trace_length-1].opcode == uop);
- operand = read_u32(&(initial_instr + x)[offset].cache);
- trace[trace_length-1].operand1 = operand;
- continue;
- case OPERAND1_4:
- assert(trace[trace_length-1].opcode == uop);
- operand = read_u64(&(initial_instr + x)[offset].cache);
- trace[trace_length-1].operand1 = operand;
- continue;
- default:
- fprintf(stderr,
- "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n",
- opcode, oparg, nuops, i,
- expansion->uops[i].size,
- expansion->uops[i].offset);
- Py_FatalError("garbled expansion");
+ break;
+ case OPERAND1_1:
+ assert(trace[trace_length-1].opcode == uop);
+ operand = read_u16(&this_instr[offset].cache);
+ trace[trace_length-1].operand1 = operand;
+ continue;
+ case OPERAND1_2:
+ assert(trace[trace_length-1].opcode == uop);
+ operand = read_u32(&this_instr[offset].cache);
+ trace[trace_length-1].operand1 = operand;
+ continue;
+ case OPERAND1_4:
+ assert(trace[trace_length-1].opcode == uop);
+ operand = read_u64(&this_instr[offset].cache);
+ trace[trace_length-1].operand1 = operand;
+ continue;
+ default:
+ fprintf(stderr,
+ "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n",
+ opcode, oparg, nuops, i,
+ expansion->uops[i].size,
+ expansion->uops[i].offset);
+ Py_FatalError("garbled expansion");
+ }
+ if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) {
+ if (func != NULL) {
+ operand = (uintptr_t)func;
}
-
- if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) {
- assert(i + 1 == nuops);
- _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];
- assert(next_instr->op.code == STORE_FAST);
- operand = next_instr->op.arg;
- // Skip the STORE_FAST:
- x++;
+ else if (code != NULL) {
+ operand = (uintptr_t)code | 1;
+ }
+ else {
+ operand = 0;
}
-
- // All other instructions
- ADD_TO_TRACE(uop, oparg, operand, target);
}
- break;
+ // All other instructions
+ ADD_TO_TRACE(uop, oparg, operand, target);
}
- DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]);
- OPT_UNSUPPORTED_OPCODE(opcode);
- goto done; // Break out of loop
- } // End default
-
- } // End switch (opcode)
-
- x++;
- // Add cache size for opcode
- x += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];
-
- if (opcode == CALL_LIST_APPEND) {
- assert(initial_instr[x].op.code == POP_TOP);
- x++;
- }
+ break;
+ }
+ DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]);
+ OPT_UNSUPPORTED_OPCODE(opcode);
+ } // End default
- if (x > tstate->interp->jit_tracer_code_curr_size) {
- goto done;
- }
- // Jump here after _PUSH_FRAME or likely branches.
- first = false;
- } // End for (;;)
+ } // End switch (opcode)
-done:
- // Looped back to top.
- if (!first && tstate->interp->jit_completed_loop && target_instr == tstate->interp->jit_tracer_initial_instr) {
- ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0);
+ if (needs_guard_ip) {
+ ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)next_instr, 0);
}
- // Skip short traces where we can't even translate a single instruction:
- if (first) {
- OPT_STAT_INC(trace_too_short);
- DPRINTF(2,
- "No trace for %s (%s:%d) at byte offset %d (no progress)\n",
- PyUnicode_AsUTF8(code->co_qualname),
- PyUnicode_AsUTF8(code->co_filename),
- code->co_firstlineno,
- 2 * INSTR_IP(initial_instr, code));
- return 0;
- }
- if (!is_terminator(&trace[trace_length-1])) {
- /* Allow space for _EXIT_TRACE */
- max_length += 2;
+ tstate->interp->jit_tracer_code_curr_size = trace_length;
+ tstate->interp->jit_tracer_code_max_size = max_length;
+ return 1;
+done:
+ tstate->interp->jit_tracer_code_curr_size = trace_length;
+ tstate->interp->jit_tracer_code_max_size = max_length;
+ return 0;
+full:
+ if (!is_terminator(&tstate->interp->jit_tracer_code_buffer[trace_length-1])) {
+ // Undo the last few instructions.
+ trace_length = tstate->interp->jit_tracer_code_curr_size;
ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target);
}
- DPRINTF(1,
- "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n",
- PyUnicode_AsUTF8(code->co_qualname),
- PyUnicode_AsUTF8(code->co_filename),
- code->co_firstlineno,
- 2 * INSTR_IP(tstate->interp->jit_tracer_initial_instr, (PyCodeObject *)tstate->interp->jit_tracer_initial_func->func_code),
- trace_length);
- OPT_HIST(trace_length, trace_length_hist);
- return trace_length;
+ tstate->interp->jit_tracer_code_curr_size = trace_length;
+ tstate->interp->jit_tracer_code_max_size = max_length;
+ return 0;
+}
+
+void
+_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth)
+{
+ PyCodeObject *code = _PyFrame_GetCode(frame);
+#ifdef Py_DEBUG
+ char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
+ int lltrace = 3;
+ if (python_lltrace != NULL && *python_lltrace >= '0') {
+ lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that
+ }
+ DPRINTF(2,
+ "Optimizing %s (%s:%d) at byte offset %d\n",
+ PyUnicode_AsUTF8(code->co_qualname),
+ PyUnicode_AsUTF8(code->co_filename),
+ code->co_firstlineno,
+ 2 * INSTR_IP(next_instr, code));
+#endif
+ add_to_trace(tstate->interp->jit_tracer_code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)next_instr, INSTR_IP(next_instr, code));
+ add_to_trace(tstate->interp->jit_tracer_code_buffer, 1, _MAKE_WARM, 0, 0, 0);
+ tstate->interp->jit_tracer_code_curr_size = 2;
+ tstate->interp->jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH;
+ tstate->interp->jit_tracer_initial_instr = next_instr;
+ tstate->interp->jit_tracer_initial_code = code;
+ tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame);
+ tstate->interp->jit_tracer_seen_initial_before = 0;
+ tstate->interp->jit_completed_loop = false;
+ tstate->interp->jit_tracer_initial_stack_depth = curr_stackdepth;
+ tstate->interp->jit_tracer_initial_chain_depth = chain_depth;
}
#undef RESERVE
return 0;
}
}
- _PyUOpInstruction *buffer = interp->jit_uop_buffer;
+ _PyUOpInstruction *buffer = interp->jit_tracer_code_buffer;
OPT_STAT_INC(attempts);
char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE");
bool is_noopt = true;
is_noopt = false;
}
int curr_stackentries = tstate->interp->jit_tracer_initial_stack_depth;
-// #ifdef Py_DEBUG
-// for (int x = 0; x < tstate->interp->jit_tracer_code_curr_size;) {
-// int opcode = tstate->interp->jit_tracer_code_buffer[x].op.code;
-// int oparg = tstate->interp->jit_tracer_code_buffer[x].op.arg;
-// fprintf(stdout, "%s(%d)\n", _PyOpcode_OpName[opcode], oparg);
-// x += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];
-// }
-// #endif
- int length = translate_bytecode_to_trace(frame, tstate, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed);
- if (length <= 0) {
- // Error or nothing translated
- return length;
- }
+ int length = interp->jit_tracer_code_curr_size;
+ assert(length > 0);
assert(length < UOP_MAX_TRACE_LENGTH);
OPT_STAT_INC(traces_created);
if (!is_noopt) {
interp->jit_uop_buffer = NULL;
}
if (interp->jit_tracer_code_buffer != NULL) {
- _PyObject_VirtualFree(interp->jit_tracer_code_buffer, TRACER_BUFFER_SIZE);
+ _PyObject_VirtualFree(interp->jit_tracer_code_buffer, UOP_BUFFER_SIZE);
interp->jit_tracer_code_buffer = NULL;
}
#endif
projects / thirdparty / Python / cpython.git / commitdiff
