Include/internal/pycore_opcode_metadata.h generated
Include/internal/pycore_*_generated.h generated
Include/internal/pycore_uop_ids.h generated
+Include/internal/pycore_uop_metadata.h generated
Include/opcode.h generated
Include/opcode_ids.h generated
Include/token.h generated
typedef struct {
uint16_t opcode;
uint16_t oparg;
- uint32_t target;
+ union {
+ uint32_t target;
+ uint32_t exit_index;
+ };
uint64_t operand; // A cache entry
} _PyUOpInstruction;
+typedef struct _exit_data {
+ uint32_t target;
+ int16_t temperature;
+ const struct _PyExecutorObject *executor;
+} _PyExitData;
+
typedef struct _PyExecutorObject {
PyObject_VAR_HEAD
+ const _PyUOpInstruction *trace;
_PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */
- void *jit_code;
+ uint32_t exit_count;
+ uint32_t code_size;
size_t jit_size;
- _PyUOpInstruction trace[1];
+ void *jit_code;
+ _PyExitData exits[1];
} _PyExecutorObject;
typedef struct _PyOptimizerObject _PyOptimizerObject;
/* These thresholds are treated as signed so do not exceed INT16_MAX
* Use INT16_MAX to indicate that the optimizer should never be called */
uint16_t resume_threshold;
+ uint16_t side_threshold;
uint16_t backedge_threshold;
/* Data needed by the optimizer goes here, but is opaque to the VM */
} _PyOptimizerObject;
_PyOptimizerObject *_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject* optimizer);
-PyAPI_FUNC(void) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer);
+PyAPI_FUNC(int) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer);
PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void);
PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset);
int
-_PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer);
+_PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr);
-void _Py_ExecutorInit(_PyExecutorObject *, _PyBloomFilter *);
+void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *);
void _Py_ExecutorClear(_PyExecutorObject *);
void _Py_BloomFilter_Init(_PyBloomFilter *);
void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj);
/* The thread's exception stack entry. (Always the last entry.) */
_PyErr_StackItem exc_state;
+ PyObject *previous_executor;
+
};
#ifdef Py_DEBUG
struct callable_cache callable_cache;
_PyOptimizerObject *optimizer;
_PyExecutorObject *executor_list_head;
- /* These values are shifted and offset to speed up check in JUMP_BACKWARD */
+
+ /* These two values are shifted and offset to speed up check in JUMP_BACKWARD */
uint32_t optimizer_resume_threshold;
uint32_t optimizer_backedge_threshold;
+ uint16_t optimizer_side_threshold;
+
uint32_t next_func_version;
_rare_events rare_events;
PyDict_WatchCallback builtins_dict_watcher;
typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate);
-int _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length);
+int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length);
void _PyJIT_Free(_PyExecutorObject *executor);
#endif // _Py_JIT
#define HAS_DEOPT_FLAG (128)
#define HAS_ERROR_FLAG (256)
#define HAS_ESCAPES_FLAG (512)
-#define HAS_PURE_FLAG (1024)
-#define HAS_PASSTHROUGH_FLAG (2048)
+#define HAS_EXIT_FLAG (1024)
+#define HAS_PURE_FLAG (2048)
+#define HAS_PASSTHROUGH_FLAG (4096)
#define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG))
#define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG))
#define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG))
#define OPCODE_HAS_DEOPT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_DEOPT_FLAG))
#define OPCODE_HAS_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_FLAG))
#define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG))
+#define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG))
#define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG))
#define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG))
[BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG },
- [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
- [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG },
- [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
- [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG },
- [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
+ [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
+ [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
+ [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
[LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
- [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
- [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
- [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
- [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
+ [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
+ [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
+ [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
- [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
- [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
+ [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
- [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
- [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
+ [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG },
[LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG },
[SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
- [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
+ [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
+ [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG },
[STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
[STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG },
[TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG },
- [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG },
- [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG },
- [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG },
- [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG },
- [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG },
+ [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
#define _CONTAINS_OP CONTAINS_OP
#define _CHECK_EG_MATCH CHECK_EG_MATCH
#define _CHECK_EXC_MATCH CHECK_EXC_MATCH
-#define _JUMP_BACKWARD JUMP_BACKWARD
#define _POP_JUMP_IF_FALSE 339
#define _POP_JUMP_IF_TRUE 340
#define _IS_NONE 341
#define _CHECK_GLOBALS 384
#define _CHECK_BUILTINS 385
#define _INTERNAL_INCREMENT_OPT_COUNTER 386
-#define _CHECK_VALIDITY_AND_SET_IP 387
-#define MAX_UOP_ID 387
+#define _COLD_EXIT 387
+#define _START_EXECUTOR 388
+#define _FATAL_ERROR 389
+#define _CHECK_VALIDITY_AND_SET_IP 390
+#define MAX_UOP_ID 390
#ifdef __cplusplus
}
[_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_UNARY_NOT] = HAS_PURE_FLAG,
[_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
- [_TO_BOOL_INT] = HAS_DEOPT_FLAG,
- [_TO_BOOL_LIST] = HAS_DEOPT_FLAG,
- [_TO_BOOL_NONE] = HAS_DEOPT_FLAG,
- [_TO_BOOL_STR] = HAS_DEOPT_FLAG,
- [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG,
+ [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_TO_BOOL_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
+ [_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
+ [_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
+ [_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
+ [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
[_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
[_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
- [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG,
[_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG,
[_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG,
- [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG,
[_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
[_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG,
[_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG,
[_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG,
- [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG,
- [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG,
- [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG,
- [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG,
+ [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
+ [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
+ [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
+ [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG,
[_SET_IP] = 0,
[_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG,
- [_EXIT_TRACE] = HAS_DEOPT_FLAG,
+ [_EXIT_TRACE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_CHECK_VALIDITY] = HAS_DEOPT_FLAG,
[_LOAD_CONST_INLINE] = HAS_PURE_FLAG,
[_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG,
[_CHECK_GLOBALS] = HAS_DEOPT_FLAG,
[_CHECK_BUILTINS] = HAS_DEOPT_FLAG,
[_INTERNAL_INCREMENT_OPT_COUNTER] = 0,
+ [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_START_EXECUTOR] = 0,
+ [_FATAL_ERROR] = HAS_ESCAPES_FLAG,
[_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG,
};
[_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE",
[_CHECK_VALIDITY] = "_CHECK_VALIDITY",
[_CHECK_VALIDITY_AND_SET_IP] = "_CHECK_VALIDITY_AND_SET_IP",
+ [_COLD_EXIT] = "_COLD_EXIT",
[_COMPARE_OP] = "_COMPARE_OP",
[_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT",
[_COMPARE_OP_INT] = "_COMPARE_OP_INT",
[_END_SEND] = "_END_SEND",
[_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK",
[_EXIT_TRACE] = "_EXIT_TRACE",
+ [_FATAL_ERROR] = "_FATAL_ERROR",
[_FORMAT_SIMPLE] = "_FORMAT_SIMPLE",
[_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC",
[_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO",
[_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE",
[_SET_IP] = "_SET_IP",
[_SET_UPDATE] = "_SET_UPDATE",
+ [_START_EXECUTOR] = "_START_EXECUTOR",
[_STORE_ATTR] = "_STORE_ATTR",
[_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE",
[_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT",
# on the *very next* allocation:
gc.collect()
gc.set_threshold(1, 0, 0)
+ sys._clear_internal_caches()
# Okay, so here's the nightmare scenario:
# - We're tracing the resumption of a generator, which creates a new
# frame object.
self.run_cases_test(input, output)
+ def test_deopt_and_exit(self):
+ input = """
+ pure op(OP, (arg1 -- out)) {
+ DEOPT_IF(1);
+ EXIT_IF(1);
+ }
+ """
+ output = ""
+ with self.assertRaises(Exception):
+ self.run_cases_test(input, output)
+
class TestGeneratedAbstractCases(unittest.TestCase):
def setUp(self) -> None:
super().setUp()
if (opt == Py_None) {
opt = NULL;
}
- PyUnstable_SetOptimizer((_PyOptimizerObject*)opt);
+ if (PyUnstable_SetOptimizer((_PyOptimizerObject*)opt) < 0) {
+ return NULL;
+ }
Py_RETURN_NONE;
}
macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL;
inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) {
- DEOPT_IF(!PyBool_Check(value));
+ EXIT_IF(!PyBool_Check(value));
STAT_INC(TO_BOOL, hit);
}
inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) {
- DEOPT_IF(!PyLong_CheckExact(value));
+ EXIT_IF(!PyLong_CheckExact(value));
STAT_INC(TO_BOOL, hit);
if (_PyLong_IsZero((PyLongObject *)value)) {
assert(_Py_IsImmortal(value));
}
inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) {
- DEOPT_IF(!PyList_CheckExact(value));
+ EXIT_IF(!PyList_CheckExact(value));
STAT_INC(TO_BOOL, hit);
res = Py_SIZE(value) ? Py_True : Py_False;
DECREF_INPUTS();
inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) {
// This one is a bit weird, because we expect *some* failures:
- DEOPT_IF(!Py_IsNone(value));
+ EXIT_IF(!Py_IsNone(value));
STAT_INC(TO_BOOL, hit);
res = Py_False;
}
inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) {
- DEOPT_IF(!PyUnicode_CheckExact(value));
+ EXIT_IF(!PyUnicode_CheckExact(value));
STAT_INC(TO_BOOL, hit);
if (value == &_Py_STR(empty)) {
assert(_Py_IsImmortal(value));
inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) {
// This one is a bit weird, because we expect *some* failures:
assert(version);
- DEOPT_IF(Py_TYPE(value)->tp_version_tag != version);
+ EXIT_IF(Py_TYPE(value)->tp_version_tag != version);
STAT_INC(TO_BOOL, hit);
DECREF_INPUTS();
res = Py_True;
};
op(_GUARD_BOTH_INT, (left, right -- left, right)) {
- DEOPT_IF(!PyLong_CheckExact(left));
- DEOPT_IF(!PyLong_CheckExact(right));
+ EXIT_IF(!PyLong_CheckExact(left));
+ EXIT_IF(!PyLong_CheckExact(right));
}
pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) {
_GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT;
op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) {
- DEOPT_IF(!PyFloat_CheckExact(left));
- DEOPT_IF(!PyFloat_CheckExact(right));
+ EXIT_IF(!PyFloat_CheckExact(left));
+ EXIT_IF(!PyFloat_CheckExact(right));
}
pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) {
_GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT;
op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) {
- DEOPT_IF(!PyUnicode_CheckExact(left));
- DEOPT_IF(!PyUnicode_CheckExact(right));
+ EXIT_IF(!PyUnicode_CheckExact(left));
+ EXIT_IF(!PyUnicode_CheckExact(right));
}
pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) {
op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) {
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
- DEOPT_IF(tp->tp_version_tag != type_version);
+ EXIT_IF(tp->tp_version_tag != type_version);
}
op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) {
}
inst(JUMP_BACKWARD, (unused/1 --)) {
+ TIER_ONE_ONLY
CHECK_EVAL_BREAKER();
assert(oparg <= INSTR_OFFSET());
JUMPBY(-oparg);
oparg >>= 8;
start--;
}
- int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer);
+ _PyExecutorObject *executor;
+ int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor);
ERROR_IF(optimized < 0, error);
if (optimized) {
- // Rewind and enter the executor:
- assert(start->op.code == ENTER_EXECUTOR);
- next_instr = start;
- this_instr[1].cache &= OPTIMIZER_BITS_MASK;
+ assert(tstate->previous_executor == NULL);
+ tstate->previous_executor = Py_None;
+ GOTO_TIER_TWO(executor);
}
else {
int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK;
inst(ENTER_EXECUTOR, (--)) {
TIER_ONE_ONLY
CHECK_EVAL_BREAKER();
-
PyCodeObject *code = _PyFrame_GetCode(frame);
- current_executor = code->co_executors->executors[oparg & 255];
- assert(current_executor->vm_data.index == INSTR_OFFSET() - 1);
- assert(current_executor->vm_data.code == code);
- assert(current_executor->vm_data.valid);
- Py_INCREF(current_executor);
- GOTO_TIER_TWO();
+ _PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
+ assert(executor->vm_data.index == INSTR_OFFSET() - 1);
+ assert(executor->vm_data.code == code);
+ assert(executor->vm_data.valid);
+ assert(tstate->previous_executor == NULL);
+ tstate->previous_executor = Py_None;
+ Py_INCREF(executor);
+ GOTO_TIER_TWO(executor);
}
replaced op(_POP_JUMP_IF_FALSE, (cond -- )) {
inst(CACHE, (--)) {
TIER_ONE_ONLY
assert(0 && "Executing a cache.");
- Py_UNREACHABLE();
+ Py_FatalError("Executing a cache.");
}
inst(RESERVED, (--)) {
TIER_ONE_ONLY
assert(0 && "Executing RESERVED instruction.");
- Py_UNREACHABLE();
+ Py_FatalError("Executing RESERVED instruction.");
}
///////// Tier-2 only opcodes /////////
op (_GUARD_IS_TRUE_POP, (flag -- )) {
SYNC_SP();
- DEOPT_IF(!Py_IsTrue(flag));
+ EXIT_IF(!Py_IsTrue(flag));
assert(Py_IsTrue(flag));
}
op (_GUARD_IS_FALSE_POP, (flag -- )) {
SYNC_SP();
- DEOPT_IF(!Py_IsFalse(flag));
+ EXIT_IF(!Py_IsFalse(flag));
assert(Py_IsFalse(flag));
}
SYNC_SP();
if (!Py_IsNone(val)) {
Py_DECREF(val);
- DEOPT_IF(1);
+ EXIT_IF(1);
}
}
op (_GUARD_IS_NOT_NONE_POP, (val -- )) {
SYNC_SP();
- DEOPT_IF(Py_IsNone(val));
+ EXIT_IF(Py_IsNone(val));
Py_DECREF(val);
}
op(_JUMP_TO_TOP, (--)) {
- next_uop = current_executor->trace;
+#ifndef _Py_JIT
+ next_uop = ¤t_executor->trace[1];
+#endif
CHECK_EVAL_BREAKER();
}
op(_EXIT_TRACE, (--)) {
TIER_TWO_ONLY
- DEOPT_IF(1);
+ EXIT_IF(1);
}
op(_CHECK_VALIDITY, (--)) {
exe->count++;
}
+ /* Only used for handling cold side exits, should never appear in
+ * a normal trace or as part of an instruction.
+ */
+ op(_COLD_EXIT, (--)) {
+ TIER_TWO_ONLY
+ _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor;
+ _PyExitData *exit = &previous->exits[oparg];
+ exit->temperature++;
+ PyCodeObject *code = _PyFrame_GetCode(frame);
+ _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target;
+ if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) {
+ GOTO_TIER_ONE(target);
+ }
+ _PyExecutorObject *executor;
+ if (target->op.code == ENTER_EXECUTOR) {
+ executor = code->co_executors->executors[target->op.arg];
+ Py_INCREF(executor);
+ } else {
+ int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor);
+ if (optimized <= 0) {
+ int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold;
+ exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp;
+ if (optimized < 0) {
+ Py_DECREF(previous);
+ tstate->previous_executor = Py_None;
+ ERROR_IF(1, error);
+ }
+ GOTO_TIER_ONE(target);
+ }
+ }
+ /* We need two references. One to store in exit->executor and
+ * one to keep the executor alive when executing. */
+ Py_INCREF(executor);
+ exit->executor = executor;
+ GOTO_TIER_TWO(executor);
+ }
+
+ op(_START_EXECUTOR, (executor/4 --)) {
+ TIER_TWO_ONLY
+ Py_DECREF(tstate->previous_executor);
+ tstate->previous_executor = NULL;
+#ifndef _Py_JIT
+ current_executor = (_PyExecutorObject*)executor;
+#endif
+ }
+
+ op(_FATAL_ERROR, (--)) {
+ TIER_TWO_ONLY
+ assert(0);
+ Py_FatalError("Fatal error uop executed.");
+ }
+
op(_CHECK_VALIDITY_AND_SET_IP, (instr_ptr/4 --)) {
TIER_TWO_ONLY
DEOPT_IF(!current_executor->vm_data.valid);
#include "pycore_moduleobject.h" // PyModuleObject
#include "pycore_object.h" // _PyObject_GC_TRACK()
#include "pycore_opcode_metadata.h" // EXTRA_CASES
+#include "pycore_optimizer.h" // _PyUOpExecutor_Type
#include "pycore_opcode_utils.h" // MAKE_FUNCTION_*
#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
goto resume_with_error;
}
- /* State shared between Tier 1 and Tier 2 interpreter */
- _PyExecutorObject *current_executor = NULL;
-
/* Local "register" variables.
* These are cached values from the frame and code object. */
-
_Py_CODEUNIT *next_instr;
PyObject **stack_pointer;
+#ifndef _Py_JIT
+ /* Tier 2 interpreter state */
+ _PyExecutorObject *current_executor = NULL;
+ const _PyUOpInstruction *next_uop = NULL;
+#endif
start_frame:
if (_Py_EnterRecursivePy(tstate)) {
enter_tier_two:
#ifdef _Py_JIT
-
- ; // ;)
- jit_func jitted = current_executor->jit_code;
- next_instr = jitted(frame, stack_pointer, tstate);
- frame = tstate->current_frame;
- Py_DECREF(current_executor);
- if (next_instr == NULL) {
- goto resume_with_error;
- }
- stack_pointer = _PyFrame_GetStackPointer(frame);
- DISPATCH();
-
+ assert(0);
#else
#undef LOAD_IP
#endif
OPT_STAT_INC(traces_executed);
- _PyUOpInstruction *next_uop = current_executor->trace;
uint16_t uopcode;
#ifdef Py_STATS
uint64_t trace_uop_execution_counter = 0;
#endif
+ assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT);
for (;;) {
uopcode = next_uop->opcode;
DPRINTF(3,
frame->return_offset = 0; // Don't leave this random
_PyFrame_SetStackPointer(frame, stack_pointer);
Py_DECREF(current_executor);
+ tstate->previous_executor = NULL;
goto resume_with_error;
// Jump here from DEOPT_IF()
deoptimize:
next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame));
- DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n",
+ DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d -> %s]\n",
uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target,
- (int)(next_uop - current_executor->trace - 1),
- _PyOpcode_OpName[frame->instr_ptr->op.code]);
+ _PyOpcode_OpName[next_instr->op.code]);
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
UOP_STAT_INC(uopcode, miss);
Py_DECREF(current_executor);
+ tstate->previous_executor = NULL;
DISPATCH();
+// Jump here from EXIT_IF()
+side_exit:
+ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
+ UOP_STAT_INC(uopcode, miss);
+ uint32_t exit_index = next_uop[-1].exit_index;
+ assert(exit_index < current_executor->exit_count);
+ _PyExitData *exit = ¤t_executor->exits[exit_index];
+ DPRINTF(2, "SIDE EXIT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", exit %u, temp %d, target %d -> %s]\n",
+ uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit_index, exit->temperature,
+ exit->target, _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]);
+ Py_INCREF(exit->executor);
+ tstate->previous_executor = (PyObject *)current_executor;
+ GOTO_TIER_TWO(exit->executor);
+
#endif // _Py_JIT
}
+
#if defined(__GNUC__)
# pragma GCC diagnostic pop
#elif defined(_MSC_VER) /* MS_WINDOWS */
/* Tier-switching macros. */
-#define GOTO_TIER_TWO() goto enter_tier_two;
+#ifdef _Py_JIT
+#define GOTO_TIER_TWO(EXECUTOR) \
+do { \
+ jit_func jitted = (EXECUTOR)->jit_code; \
+ next_instr = jitted(frame, stack_pointer, tstate); \
+ Py_DECREF(tstate->previous_executor); \
+ tstate->previous_executor = NULL; \
+ frame = tstate->current_frame; \
+ if (next_instr == NULL) { \
+ goto resume_with_error; \
+ } \
+ stack_pointer = _PyFrame_GetStackPointer(frame); \
+ DISPATCH(); \
+} while (0)
+#else
+#define GOTO_TIER_TWO(EXECUTOR) \
+do { \
+ next_uop = (EXECUTOR)->trace; \
+ assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); \
+ goto enter_tier_two; \
+} while (0)
+#endif
+
+#define GOTO_TIER_ONE(TARGET) \
+do { \
+ Py_DECREF(tstate->previous_executor); \
+ tstate->previous_executor = NULL; \
+ next_instr = target; \
+ DISPATCH(); \
+} while (0)
#define CURRENT_OPARG() (next_uop[-1].oparg)
case _TO_BOOL_BOOL: {
PyObject *value;
value = stack_pointer[-1];
- if (!PyBool_Check(value)) goto deoptimize;
+ if (!PyBool_Check(value)) goto side_exit;
STAT_INC(TO_BOOL, hit);
break;
}
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
- if (!PyLong_CheckExact(value)) goto deoptimize;
+ if (!PyLong_CheckExact(value)) goto side_exit;
STAT_INC(TO_BOOL, hit);
if (_PyLong_IsZero((PyLongObject *)value)) {
assert(_Py_IsImmortal(value));
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
- if (!PyList_CheckExact(value)) goto deoptimize;
+ if (!PyList_CheckExact(value)) goto side_exit;
STAT_INC(TO_BOOL, hit);
res = Py_SIZE(value) ? Py_True : Py_False;
Py_DECREF(value);
PyObject *res;
value = stack_pointer[-1];
// This one is a bit weird, because we expect *some* failures:
- if (!Py_IsNone(value)) goto deoptimize;
+ if (!Py_IsNone(value)) goto side_exit;
STAT_INC(TO_BOOL, hit);
res = Py_False;
stack_pointer[-1] = res;
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
- if (!PyUnicode_CheckExact(value)) goto deoptimize;
+ if (!PyUnicode_CheckExact(value)) goto side_exit;
STAT_INC(TO_BOOL, hit);
if (value == &_Py_STR(empty)) {
assert(_Py_IsImmortal(value));
uint32_t version = (uint32_t)CURRENT_OPERAND();
// This one is a bit weird, because we expect *some* failures:
assert(version);
- if (Py_TYPE(value)->tp_version_tag != version) goto deoptimize;
+ if (Py_TYPE(value)->tp_version_tag != version) goto side_exit;
STAT_INC(TO_BOOL, hit);
Py_DECREF(value);
res = Py_True;
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!PyLong_CheckExact(left)) goto deoptimize;
- if (!PyLong_CheckExact(right)) goto deoptimize;
+ if (!PyLong_CheckExact(left)) goto side_exit;
+ if (!PyLong_CheckExact(right)) goto side_exit;
break;
}
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!PyFloat_CheckExact(left)) goto deoptimize;
- if (!PyFloat_CheckExact(right)) goto deoptimize;
+ if (!PyFloat_CheckExact(left)) goto side_exit;
+ if (!PyFloat_CheckExact(right)) goto side_exit;
break;
}
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!PyUnicode_CheckExact(left)) goto deoptimize;
- if (!PyUnicode_CheckExact(right)) goto deoptimize;
+ if (!PyUnicode_CheckExact(left)) goto side_exit;
+ if (!PyUnicode_CheckExact(right)) goto side_exit;
break;
}
uint32_t type_version = (uint32_t)CURRENT_OPERAND();
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
- if (tp->tp_version_tag != type_version) goto deoptimize;
+ if (tp->tp_version_tag != type_version) goto side_exit;
break;
}
break;
}
- /* _JUMP_BACKWARD is not a viable micro-op for tier 2 */
-
/* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */
/* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */
PyObject *flag;
flag = stack_pointer[-1];
stack_pointer += -1;
- if (!Py_IsTrue(flag)) goto deoptimize;
+ if (!Py_IsTrue(flag)) goto side_exit;
assert(Py_IsTrue(flag));
break;
}
PyObject *flag;
flag = stack_pointer[-1];
stack_pointer += -1;
- if (!Py_IsFalse(flag)) goto deoptimize;
+ if (!Py_IsFalse(flag)) goto side_exit;
assert(Py_IsFalse(flag));
break;
}
stack_pointer += -1;
if (!Py_IsNone(val)) {
Py_DECREF(val);
- if (1) goto deoptimize;
+ if (1) goto side_exit;
}
break;
}
PyObject *val;
val = stack_pointer[-1];
stack_pointer += -1;
- if (Py_IsNone(val)) goto deoptimize;
+ if (Py_IsNone(val)) goto side_exit;
Py_DECREF(val);
break;
}
case _JUMP_TO_TOP: {
- next_uop = current_executor->trace;
+ #ifndef _Py_JIT
+ next_uop = ¤t_executor->trace[1];
+ #endif
CHECK_EVAL_BREAKER();
break;
}
case _EXIT_TRACE: {
TIER_TWO_ONLY
- if (1) goto deoptimize;
+ if (1) goto side_exit;
break;
}
break;
}
+ case _COLD_EXIT: {
+ oparg = CURRENT_OPARG();
+ TIER_TWO_ONLY
+ _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor;
+ _PyExitData *exit = &previous->exits[oparg];
+ exit->temperature++;
+ PyCodeObject *code = _PyFrame_GetCode(frame);
+ _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target;
+ if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) {
+ GOTO_TIER_ONE(target);
+ }
+ _PyExecutorObject *executor;
+ if (target->op.code == ENTER_EXECUTOR) {
+ executor = code->co_executors->executors[target->op.arg];
+ Py_INCREF(executor);
+ } else {
+ int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor);
+ if (optimized <= 0) {
+ int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold;
+ exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp;
+ if (optimized < 0) {
+ Py_DECREF(previous);
+ tstate->previous_executor = Py_None;
+ if (1) goto error_tier_two;
+ }
+ GOTO_TIER_ONE(target);
+ }
+ }
+ /* We need two references. One to store in exit->executor and
+ * one to keep the executor alive when executing. */
+ Py_INCREF(executor);
+ exit->executor = executor;
+ GOTO_TIER_TWO(executor);
+ break;
+ }
+
+ case _START_EXECUTOR: {
+ PyObject *executor = (PyObject *)CURRENT_OPERAND();
+ TIER_TWO_ONLY
+ Py_DECREF(tstate->previous_executor);
+ tstate->previous_executor = NULL;
+ #ifndef _Py_JIT
+ current_executor = (_PyExecutorObject*)executor;
+ #endif
+ break;
+ }
+
+ case _FATAL_ERROR: {
+ TIER_TWO_ONLY
+ assert(0);
+ Py_FatalError("Fatal error uop executed.");
+ break;
+ }
+
case _CHECK_VALIDITY_AND_SET_IP: {
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND();
TIER_TWO_ONLY
INSTRUCTION_STATS(CACHE);
TIER_ONE_ONLY
assert(0 && "Executing a cache.");
- Py_UNREACHABLE();
+ Py_FatalError("Executing a cache.");
+ DISPATCH();
}
TARGET(CALL) {
TIER_ONE_ONLY
CHECK_EVAL_BREAKER();
PyCodeObject *code = _PyFrame_GetCode(frame);
- current_executor = code->co_executors->executors[oparg & 255];
- assert(current_executor->vm_data.index == INSTR_OFFSET() - 1);
- assert(current_executor->vm_data.code == code);
- assert(current_executor->vm_data.valid);
- Py_INCREF(current_executor);
- GOTO_TIER_TWO();
+ _PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
+ assert(executor->vm_data.index == INSTR_OFFSET() - 1);
+ assert(executor->vm_data.code == code);
+ assert(executor->vm_data.valid);
+ assert(tstate->previous_executor == NULL);
+ tstate->previous_executor = Py_None;
+ Py_INCREF(executor);
+ GOTO_TIER_TWO(executor);
DISPATCH();
}
next_instr += 2;
INSTRUCTION_STATS(JUMP_BACKWARD);
/* Skip 1 cache entry */
+ TIER_ONE_ONLY
CHECK_EVAL_BREAKER();
assert(oparg <= INSTR_OFFSET());
JUMPBY(-oparg);
oparg >>= 8;
start--;
}
- int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer);
+ _PyExecutorObject *executor;
+ int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor);
if (optimized < 0) goto error;
if (optimized) {
- // Rewind and enter the executor:
- assert(start->op.code == ENTER_EXECUTOR);
- next_instr = start;
- this_instr[1].cache &= OPTIMIZER_BITS_MASK;
+ assert(tstate->previous_executor == NULL);
+ tstate->previous_executor = Py_None;
+ GOTO_TIER_TWO(executor);
}
else {
int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK;
INSTRUCTION_STATS(RESERVED);
TIER_ONE_ONLY
assert(0 && "Executing RESERVED instruction.");
- Py_UNREACHABLE();
+ Py_FatalError("Executing RESERVED instruction.");
+ DISPATCH();
}
TARGET(RESUME) {
// Compiles executor in-place. Don't forget to call _PyJIT_Free later!
int
-_PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length)
+_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length)
{
// Loop once to find the total compiled size:
size_t code_size = 0;
size_t data_size = 0;
for (size_t i = 0; i < length; i++) {
- _PyUOpInstruction *instruction = &trace[i];
+ _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i];
const StencilGroup *group = &stencil_groups[instruction->opcode];
code_size += group->code.body_size;
data_size += group->data.body_size;
// Loop again to emit the code:
char *code = memory;
char *data = memory + code_size;
+ char *top = code;
+ if (trace[0].opcode == _START_EXECUTOR) {
+ // Don't want to execute this more than once:
+ top += stencil_groups[_START_EXECUTOR].code.body_size;
+ }
for (size_t i = 0; i < length; i++) {
- _PyUOpInstruction *instruction = &trace[i];
+ _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i];
const StencilGroup *group = &stencil_groups[instruction->opcode];
// Think of patches as a dictionary mapping HoleValue to uint64_t:
uint64_t patches[] = GET_PATCHES();
patches[HoleValue_OPARG] = instruction->oparg;
patches[HoleValue_OPERAND] = instruction->operand;
patches[HoleValue_TARGET] = instruction->target;
- patches[HoleValue_TOP] = (uint64_t)memory;
+ patches[HoleValue_TOP] = (uint64_t)top;
patches[HoleValue_ZERO] = 0;
emit(group, patches);
code += group->code.body_size;
#include "opcode.h"
#include "pycore_interp.h"
#include "pycore_bitutils.h" // _Py_popcount32()
+#include "pycore_object.h" // _PyObject_GC_UNTRACK()
#include "pycore_opcode_metadata.h" // _PyOpcode_OpName[]
#include "pycore_opcode_utils.h" // MAX_REAL_OPCODE
#include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize()
.optimize = never_optimize,
.resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD,
.backedge_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD,
+ .side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD,
};
static uint32_t
-shift_and_offset_threshold(uint16_t threshold)
+shift_and_offset_threshold(uint32_t threshold)
{
return (threshold << OPTIMIZER_BITS_IN_COUNTER) + (1 << 15);
}
PyUnstable_GetOptimizer(void)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
- if (interp->optimizer == &_PyOptimizer_Default) {
- return NULL;
- }
assert(interp->optimizer_backedge_threshold ==
shift_and_offset_threshold(interp->optimizer->backedge_threshold));
assert(interp->optimizer_resume_threshold ==
shift_and_offset_threshold(interp->optimizer->resume_threshold));
+ if (interp->optimizer == &_PyOptimizer_Default) {
+ return NULL;
+ }
Py_INCREF(interp->optimizer);
return interp->optimizer;
}
+static _PyExecutorObject *
+make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies);
+
+static int
+init_cold_exit_executor(_PyExecutorObject *executor, int oparg);
+
+static int cold_exits_initialized = 0;
+static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 };
+
+static const _PyBloomFilter EMPTY_FILTER = { 0 };
+
_PyOptimizerObject *
_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer)
{
if (optimizer == NULL) {
optimizer = &_PyOptimizer_Default;
}
+ else if (cold_exits_initialized == 0) {
+ cold_exits_initialized = 1;
+ for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) {
+ if (init_cold_exit_executor(&COLD_EXITS[i], i)) {
+ return NULL;
+ }
+ }
+ }
_PyOptimizerObject *old = interp->optimizer;
+ if (old == NULL) {
+ old = &_PyOptimizer_Default;
+ }
Py_INCREF(optimizer);
interp->optimizer = optimizer;
interp->optimizer_backedge_threshold = shift_and_offset_threshold(optimizer->backedge_threshold);
interp->optimizer_resume_threshold = shift_and_offset_threshold(optimizer->resume_threshold);
+ interp->optimizer_side_threshold = optimizer->side_threshold;
+ if (optimizer == &_PyOptimizer_Default) {
+ assert(interp->optimizer_backedge_threshold > (1 << 16));
+ assert(interp->optimizer_resume_threshold > (1 << 16));
+ }
return old;
}
-void
+int
PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyOptimizerObject *old = _Py_SetOptimizer(interp, optimizer);
- Py_DECREF(old);
+ Py_XDECREF(old);
+ return old == NULL ? -1 : 0;
}
+/* Returns 1 if optimized, 0 if not optimized, and -1 for an error.
+ * If optimized, *executor_ptr contains a new reference to the executor
+ */
int
-_PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer)
+_PyOptimizer_Optimize(
+ _PyInterpreterFrame *frame, _Py_CODEUNIT *start,
+ PyObject **stack_pointer, _PyExecutorObject **executor_ptr)
{
PyCodeObject *code = (PyCodeObject *)frame->f_executable;
assert(PyCode_Check(code));
return 0;
}
_PyOptimizerObject *opt = interp->optimizer;
- _PyExecutorObject *executor = NULL;
- int err = opt->optimize(opt, frame, start, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame)));
+ int err = opt->optimize(opt, frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame)));
if (err <= 0) {
- assert(executor == NULL);
return err;
}
+ assert(*executor_ptr != NULL);
int index = get_index_for_executor(code, start);
if (index < 0) {
/* Out of memory. Don't raise and assume that the
* If an optimizer has already produced an executor,
* it might get confused by the executor disappearing,
* but there is not much we can do about that here. */
- Py_DECREF(executor);
+ Py_DECREF(*executor_ptr);
return 0;
}
- insert_executor(code, start, index, executor);
- Py_DECREF(executor);
+ insert_executor(code, start, index, *executor_ptr);
+ assert((*executor_ptr)->vm_data.valid);
return 1;
}
static void
uop_dealloc(_PyExecutorObject *self) {
+ _PyObject_GC_UNTRACK(self);
_Py_ExecutorClear(self);
#ifdef _Py_JIT
_PyJIT_Free(self);
#endif
- PyObject_Free(self);
+ PyObject_GC_Del(self);
}
const char *
static Py_ssize_t
uop_len(_PyExecutorObject *self)
{
- return Py_SIZE(self);
+ return self->code_size;
}
static PyObject *
.sq_item = (ssizeargfunc)uop_item,
};
+static int
+executor_clear(PyObject *o)
+{
+ _Py_ExecutorClear((_PyExecutorObject *)o);
+ return 0;
+}
+
+static int
+executor_traverse(PyObject *o, visitproc visit, void *arg)
+{
+ _PyExecutorObject *executor = (_PyExecutorObject *)o;
+ for (uint32_t i = 0; i < executor->exit_count; i++) {
+ Py_VISIT(executor->exits[i].executor);
+ }
+ return 0;
+}
+
PyTypeObject _PyUOpExecutor_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
.tp_name = "uop_executor",
- .tp_basicsize = offsetof(_PyExecutorObject, trace),
- .tp_itemsize = sizeof(_PyUOpInstruction),
- .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION,
+ .tp_basicsize = offsetof(_PyExecutorObject, exits),
+ .tp_itemsize = 1,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC,
.tp_dealloc = (destructor)uop_dealloc,
.tp_as_sequence = &uop_as_sequence,
.tp_methods = executor_methods,
+ .tp_traverse = executor_traverse,
+ .tp_clear = executor_clear,
};
/* TO DO -- Generate these tables */
[POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP,
};
+
#define CONFIDENCE_RANGE 1000
#define CONFIDENCE_CUTOFF 333
* NOPs are excluded from the count.
*/
static int
-compute_used(_PyUOpInstruction *buffer, uint32_t *used)
+compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr)
{
int count = 0;
+ int exit_count = 0;
SET_BIT(used, 0);
for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) {
if (!BIT_IS_SET(used, i)) {
}
count++;
int opcode = buffer[i].opcode;
+ if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) {
+ exit_count++;
+ }
if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) {
continue;
}
UNSET_BIT(used, i);
}
}
+ *exit_count_ptr = exit_count;
return count;
}
+/* Executor side exits */
+
+static _PyExecutorObject *
+allocate_executor(int exit_count, int length)
+{
+ int size = exit_count*sizeof(_PyExitData) + length*sizeof(_PyUOpInstruction);
+ _PyExecutorObject *res = PyObject_GC_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, size);
+ if (res == NULL) {
+ return NULL;
+ }
+ res->trace = (_PyUOpInstruction *)(res->exits + exit_count);
+ res->code_size = length;
+ res->exit_count = exit_count;
+ return res;
+}
+
/* Makes an executor from a buffer of uops.
* Account for the buffer having gaps and NOPs by computing a "used"
* bit vector and only copying the used uops. Here "used" means reachable
* and not a NOP.
*/
static _PyExecutorObject *
-make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies)
+make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies)
{
uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 };
- int length = compute_used(buffer, used);
- _PyExecutorObject *executor = PyObject_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, length);
+ int exit_count;
+ int length = compute_used(buffer, used, &exit_count);
+ _PyExecutorObject *executor = allocate_executor(exit_count, length+1);
if (executor == NULL) {
return NULL;
}
- int dest = length - 1;
+ /* Initialize exits */
+ for (int i = 0; i < exit_count; i++) {
+ executor->exits[i].executor = &COLD_EXITS[i];
+ executor->exits[i].temperature = 0;
+ }
+ int next_exit = exit_count-1;
+ _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length];
/* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */
for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) {
if (!BIT_IS_SET(used, i)) {
continue;
}
- executor->trace[dest] = buffer[i];
+ *dest = buffer[i];
int opcode = buffer[i].opcode;
if (opcode == _POP_JUMP_IF_FALSE ||
opcode == _POP_JUMP_IF_TRUE)
{
/* The oparg of the target will already have been set to its new offset */
- int oparg = executor->trace[dest].oparg;
- executor->trace[dest].oparg = buffer[oparg].oparg;
+ int oparg = dest->oparg;
+ dest->oparg = buffer[oparg].oparg;
+ }
+ if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) {
+ executor->exits[next_exit].target = buffer[i].target;
+ dest->exit_index = next_exit;
+ next_exit--;
}
/* Set the oparg to be the destination offset,
* so that we can set the oparg of earlier jumps correctly. */
- buffer[i].oparg = dest;
+ buffer[i].oparg = (uint16_t)(dest - executor->trace);
dest--;
}
- assert(dest == -1);
+ assert(next_exit == -1);
+ assert(dest == executor->trace);
+ dest->opcode = _START_EXECUTOR;
+ dest->operand = (uintptr_t)executor;
_Py_ExecutorInit(executor, dependencies);
#ifdef Py_DEBUG
char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
#ifdef _Py_JIT
executor->jit_code = NULL;
executor->jit_size = 0;
- if (_PyJIT_Compile(executor, executor->trace, Py_SIZE(executor))) {
+ if (_PyJIT_Compile(executor, executor->trace, length+1)) {
Py_DECREF(executor);
return NULL;
}
#endif
+ _PyObject_GC_TRACK(executor);
return executor;
}
+static int
+init_cold_exit_executor(_PyExecutorObject *executor, int oparg)
+{
+ _Py_SetImmortal(executor);
+ Py_SET_TYPE(executor, &_PyUOpExecutor_Type);
+ executor->trace = (_PyUOpInstruction *)executor->exits;
+ executor->code_size = 1;
+ executor->exit_count = 0;
+ _PyUOpInstruction *inst = (_PyUOpInstruction *)&executor->trace[0];
+ inst->opcode = _COLD_EXIT;
+ inst->oparg = oparg;
+ executor->vm_data.valid = true;
+ for (int i = 0; i < BLOOM_FILTER_WORDS; i++) {
+ assert(executor->vm_data.bloom.bits[i] == 0);
+ }
+#ifdef _Py_JIT
+ executor->jit_code = NULL;
+ executor->jit_size = 0;
+ if (_PyJIT_Compile(executor, executor->trace, 1)) {
+ return -1;
+ }
+#endif
+ return 0;
+}
+
static int
uop_optimize(
_PyOptimizerObject *self,
opt->resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD;
// Need a few iterations to settle specializations,
// and to ammortize the cost of optimization.
+ opt->side_threshold = 16;
opt->backedge_threshold = 16;
return (PyObject *)opt;
}
static void
counter_dealloc(_PyExecutorObject *self) {
- PyObject *opt = (PyObject *)self->trace[0].operand;
+ /* The optimizer is the operand of the second uop. */
+ PyObject *opt = (PyObject *)self->trace[1].operand;
Py_DECREF(opt);
uop_dealloc(self);
}
PyTypeObject _PyCounterExecutor_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
.tp_name = "counting_executor",
- .tp_basicsize = offsetof(_PyExecutorObject, trace),
- .tp_itemsize = sizeof(_PyUOpInstruction),
- .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION,
+ .tp_basicsize = offsetof(_PyExecutorObject, exits),
+ .tp_itemsize = 1,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC,
.tp_dealloc = (destructor)counter_dealloc,
.tp_methods = executor_methods,
+ .tp_traverse = executor_traverse,
+ .tp_clear = executor_clear,
};
static int
{ .opcode = _INTERNAL_INCREMENT_OPT_COUNTER },
{ .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) }
};
- _PyBloomFilter empty;
- _Py_BloomFilter_Init(&empty);
- _PyExecutorObject *executor = make_executor_from_uops(buffer, &empty);
+ _PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY_FILTER);
if (executor == NULL) {
return -1;
}
}
opt->base.optimize = counter_optimize;
opt->base.resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD;
+ opt->base.side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD;
opt->base.backedge_threshold = 0;
opt->count = 0;
return (PyObject *)opt;
static void
unlink_executor(_PyExecutorObject *executor)
{
- if (!executor->vm_data.valid) {
- return;
- }
_PyExecutorLinkListNode *links = &executor->vm_data.links;
_PyExecutorObject *next = links->next;
_PyExecutorObject *prev = links->previous;
/* This must be called by optimizers before using the executor */
void
-_Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set)
+_Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_set)
{
executor->vm_data.valid = true;
for (int i = 0; i < BLOOM_FILTER_WORDS; i++) {
void
_Py_ExecutorClear(_PyExecutorObject *executor)
{
+ if (!executor->vm_data.valid) {
+ return;
+ }
unlink_executor(executor);
PyCodeObject *code = executor->vm_data.code;
if (code == NULL) {
return;
}
+ for (uint32_t i = 0; i < executor->exit_count; i++) {
+ Py_DECREF(executor->exits[i].executor);
+ executor->exits[i].executor = &COLD_EXITS[i];
+ executor->exits[i].temperature = INT16_MIN;
+ }
_Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index];
assert(instruction->op.code == ENTER_EXECUTOR);
int index = instruction->op.arg;
if (opt == NULL) {
return _PyStatus_ERR("can't initialize optimizer");
}
- PyUnstable_SetOptimizer((_PyOptimizerObject *)opt);
+ if (PyUnstable_SetOptimizer((_PyOptimizerObject *)opt)) {
+ return _PyStatus_ERR("can't initialize optimizer");
+ }
Py_DECREF(opt);
}
}
}
_PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL);
+ assert(old != NULL);
Py_DECREF(old);
/* It is possible that any of the objects below have a finalizer
tstate->datastack_top = NULL;
tstate->datastack_limit = NULL;
tstate->what_event = -1;
+ tstate->previous_executor = NULL;
#ifdef Py_GIL_DISABLED
// Initialize biased reference counting inter-thread queue
--- /dev/null
+# The tier 2 execution engine
+
+## General idea
+
+When execution in tier 1 becomes "hot", that is the counter for that point in
+the code reaches some threshold, we create an executor and execute that
+instead of the tier 1 bytecode.
+
+Since each executor must exit, we also track the "hotness" of those
+exits and attach new executors to those exits.
+
+As the program executes, and the hot parts of the program get optimized,
+a graph of executors forms.
+
+## Superblocks and Executors
+
+Once a point in the code has become hot enough, we want to optimize it.
+Starting from that point we project the likely path of execution,
+using information gathered by tier 1 to guide that projection to
+form a "superblock", a mostly linear sequence of micro-ops.
+Although mostly linear, it may include a single loop.
+
+We then optimize this superblock to form an optimized superblock,
+which is equivalent but more efficient.
+
+A superblock is a representation of the code we want to execute,
+but it is not in executable form.
+The executable form is known as an executor.
+
+Executors are semantically equivalent to the superblock they are
+created from, but are in a form that can be efficiently executable.
+
+There are two execution engines for executors, and two types of executors:
+* The hardware which runs machine code executors created by the JIT compiler.
+* The tier 2 interpreter runs bytecode executors.
+
+It would be very wasteful to support both a tier 2 interpreter and
+JIT compiler in the same process.
+For now, we will make the choice of engine a configuration option,
+but we could make it a command line option in the future if that would prove useful.
+
+
+### Tier 2 Interpreter
+
+For platforms without a JIT and for testing, we need an interpreter
+for executors. It is similar in design to the tier 1 interpreter, but has a
+different instruction set, and does not adapt.
+
+### JIT compiler
+
+The JIT compiler converts superblocks into machine code executors.
+These have identical behavior to interpreted executors, except that
+they consume more memory for the generated machine code and are a lot faster.
+
+## Transfering control
+
+There are three types of control transfer that we need to consider:
+* Tier 1 to tier 2
+* Tier 2 to tier 1
+* One executor to another within tier 2
+
+Since we expect the graph of executors to span most of the hot
+part of the program, transfers from one executor to another should
+be the most common.
+Therefore, we want to make those transfers fast.
+
+### Tier 2 to tier 2
+
+#### Cold exits
+
+All side exits start cold and most stay cold, but a few become
+hot. We want to keep the memory consumption small for the many
+cold exits, but those that become hot need to be fast.
+However we cannot know in advance, which will be which.
+
+So that tier 2 to tier 2 transfers are fast for hot exits,
+exits must be implemented as executors. In order to patch
+executor exits when they get hot, a pointer to the current
+executor must be passed to the exit executor.
+
+#### Handling reference counts
+
+There must be an implicit reference to the currently executing
+executor, otherwise it might be freed.
+Consequently, we must increment the reference count of an
+executor just before executing it, and decrement it just after
+executing it.
+
+We want to minimize the amount of data that is passed from
+one executor to the next. In the JIT, this reduces the number
+of arguments in the tailcall, freeing up registers for other uses.
+It is less important in the interpreter, but following the same
+design as the JIT simplifies debugging and is good for performance.
+
+Provided that we incref the new executor before executing it, we
+can jump directly to the code of the executor, without needing
+to pass a reference to that executor object.
+However, we do need a reference to the previous executor,
+so that it can be decref'd and for handling of cold exits.
+To avoid messing up the JIT's register allocation, we pass a
+reference to the previous executor in the thread state's
+`previous_executor` field.
+
+#### The interpreter
+
+The tier 2 interpreter has a variable `current_executor` which
+points to the currently live executor. When transfering from executor
+`A` to executor `B` we do the following:
+(Initially `current_executor` points to `A`, and the refcount of
+`A` is elevated by one)
+
+1. Set the instruction pointer to start at the beginning of `B`
+2. Increment the reference count of `B`
+3. Start executing `B`
+
+We also make the first instruction in `B` do the following:
+1. Set `current_executor` to point to `B`
+2. Decrement the reference count of `A` (`A` is referenced by `tstate->previous_executor`)
+
+The net effect of the above is to safely decrement the refcount of `A`,
+increment the refcount of `B` and set `current_executor` to point to `B`.
+
+#### In the JIT
+
+Transfering control from one executor to another is done via tailcalls.
+
+The compiled executor should do the same, except that there is no local
+variable `current_executor`.
+
+### Tier 1 to tier 2
+
+Since the executor doesn't know if the previous code was tier 1 or tier 2,
+we need to make a transfer from tier 1 to tier 2 look like a tier 2 to tier 2
+transfer to the executor.
+
+We can then perform a tier 1 to tier 2 transfer by setting `current_executor`
+to `None`, and then performing a tier 2 to tier 2 transfer as above.
+
+### Tier 2 to tier 1
+
+Each micro-op that might exit to tier 1 contains a `target` value,
+which is the offset of the tier 1 instruction to exit to in the
+current code object.
+
+## Counters
+
+TO DO.
+The implementation will change soon, so there is no point in
+documenting it until then.
+
break;
}
- /* _JUMP_BACKWARD is not a viable micro-op for tier 2 */
-
/* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */
/* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */
break;
}
+ case _COLD_EXIT: {
+ break;
+ }
+
+ case _START_EXECUTOR: {
+ break;
+ }
+
+ case _FATAL_ERROR: {
+ break;
+ }
+
case _CHECK_VALIDITY_AND_SET_IP: {
break;
}
_abs('Objects/stringlib/unicode_format.h'): (10_000, 400),
_abs('Objects/typeobject.c'): (35_000, 200),
_abs('Python/compile.c'): (20_000, 500),
+ _abs('Python/optimizer.c'): (100_000, 5_000),
_abs('Python/parking_lot.c'): (40_000, 1000),
_abs('Python/pylifecycle.c'): (500_000, 5000),
_abs('Python/pystate.c'): (500_000, 5000),
Python/optimizer.c - _PyUOpExecutor_Type -
Python/optimizer.c - _PyUOpOptimizer_Type -
Python/optimizer.c - _PyOptimizer_Default -
+Python/optimizer.c - _ColdExit_Type -
+Python/optimizer.c - COLD_EXITS -
+Python/optimizer.c - Py_FatalErrorExecutor -
+Python/optimizer.c - EMPTY_FILTER -
+Python/optimizer.c - cold_exits_initialized -
##-----------------------
## test code
uses_co_names: bool
uses_locals: bool
has_free: bool
+ side_exit: bool
pure: bool
passthrough: bool
uses_co_names=any(p.uses_co_names for p in properties),
uses_locals=any(p.uses_locals for p in properties),
has_free=any(p.has_free for p in properties),
+ side_exit=any(p.side_exit for p in properties),
pure=all(p.pure for p in properties),
passthrough=all(p.passthrough for p in properties),
)
uses_co_names=False,
uses_locals=False,
has_free=False,
+ side_exit=False,
pure=False,
passthrough=False,
)
def convert_stack_item(item: parser.StackEffect) -> StackItem:
- return StackItem(
- item.name, item.type, item.cond, (item.size or "1")
- )
+ return StackItem(item.name, item.type, item.cond, (item.size or "1"))
def analyze_stack(op: parser.InstDef) -> StackEffect:
or variable_used(op, "PyCell_GET")
or variable_used(op, "PyCell_SET")
)
+ deopts_if = variable_used(op, "DEOPT_IF")
+ exits_if = variable_used(op, "EXIT_IF")
+ if deopts_if and exits_if:
+ tkn = op.tokens[0]
+ raise lexer.make_syntax_error(
+ "Op cannot contain both EXIT_IF and DEOPT_IF",
+ tkn.filename,
+ tkn.line,
+ tkn.column,
+ op.name,
+ )
infallible = is_infallible(op)
- deopts = variable_used(op, "DEOPT_IF")
passthrough = stack_effect_only_peeks(op) and infallible
return Properties(
escapes=makes_escaping_api_call(op),
infallible=infallible,
- deopts=deopts,
+ deopts=deopts_if or exits_if,
+ side_exit=exits_if,
oparg=variable_used(op, "oparg"),
jumps=variable_used(op, "JUMPBY"),
eval_breaker=variable_used(op, "CHECK_EVAL_BREAKER"),
REPLACEMENT_FUNCTIONS = {
+ "EXIT_IF": replace_deopt,
"DEOPT_IF": replace_deopt,
"ERROR_IF": replace_error,
"DECREF_INPUTS": replace_decrefs,
flags.append("HAS_EVAL_BREAK_FLAG")
if p.deopts:
flags.append("HAS_DEOPT_FLAG")
+ if p.side_exit:
+ flags.append("HAS_EXIT_FLAG")
if not p.infallible:
flags.append("HAS_ERROR_FLAG")
if p.escapes:
"DEOPT",
"ERROR",
"ESCAPES",
+ "EXIT",
"PURE",
"PASSTHROUGH",
]
out.emit(") goto deoptimize;\n")
+def tier2_replace_exit_if(
+ out: CWriter,
+ tkn: Token,
+ tkn_iter: Iterator[Token],
+ uop: Uop,
+ unused: Stack,
+ inst: Instruction | None,
+) -> None:
+ out.emit_at("if ", tkn)
+ out.emit(next(tkn_iter))
+ emit_to(out, tkn_iter, "RPAREN")
+ next(tkn_iter) # Semi colon
+ out.emit(") goto side_exit;\n")
+
+
TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy()
TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error
TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt
+TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if
def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None:
goto LABEL ## _tier_two; \
} while (0)
+#undef GOTO_TIER_TWO
+#define GOTO_TIER_TWO(EXECUTOR) \
+do { \
+ __attribute__((musttail)) \
+ return ((jit_func)((EXECUTOR)->jit_code))(frame, stack_pointer, tstate); \
+} while (0)
+
+#undef GOTO_TIER_ONE
+#define GOTO_TIER_ONE(TARGET) \
+do { \
+ _PyFrame_SetStackPointer(frame, stack_pointer); \
+ return TARGET; \
+} while (0)
+
#undef LOAD_IP
#define LOAD_IP(UNUSED) \
do { \
PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR)
int oparg;
int opcode = _JIT_OPCODE;
- _PyUOpInstruction *next_uop;
// Other stuff we need handy:
PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND)
pop_1_error_tier_two:
STACK_SHRINK(1);
error_tier_two:
- _PyFrame_SetStackPointer(frame, stack_pointer);
- return NULL;
+ tstate->previous_executor = (PyObject *)current_executor;
+ GOTO_TIER_ONE(NULL);
deoptimize:
- _PyFrame_SetStackPointer(frame, stack_pointer);
- return _PyCode_CODE(_PyFrame_GetCode(frame)) + _target;
+ tstate->previous_executor = (PyObject *)current_executor;
+ GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target);
+side_exit:
+ {
+ _PyExitData *exit = ¤t_executor->exits[_target];
+ Py_INCREF(exit->executor);
+ tstate->previous_executor = (PyObject *)current_executor;
+ GOTO_TIER_TWO(exit->executor);
+ }
}
projects / thirdparty / Python / cpython.git / commitdiff
