GH-136410: Faster side exits by using a cold exit stub (GH-136411)

diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index be582122118e44edec5af41ca3a7f2c9b90511bb..e13b2b373c47b1bb06dc907201d656e66cc5a4df 100644 (file)
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -198,6 +198,9 @@ struct _ts {
 
     PyObject *current_executor;
 
+    /* Internal to the JIT */
+    struct _PyExitData *jit_exit;
+
     uint64_t dict_global_version;
 
     /* Used to store/retrieve `threading.local` keys/values for this thread */

diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h
index 542a75617b4d3cf925ced6348332b5f39a6047c7..758bf5447ee32a5c414bd2d35b811711b5c7ca7c 100644 (file)
--- a/Include/internal/pycore_interp_structs.h
+++ b/Include/internal/pycore_interp_structs.h
@@ -943,6 +943,7 @@ struct _is {
     bool jit;
     struct _PyExecutorObject *executor_list_head;
     struct _PyExecutorObject *executor_deletion_list_head;
+    struct _PyExecutorObject *cold_executor;
     int executor_deletion_list_remaining_capacity;
     size_t trace_run_counter;
     _rare_events rare_events;

diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
index 8b7f12bf03d6242fc5f505031a6c088e2b4fe77d..1571e19a35032ede8a2f37e188a7c15f2ebc7f91 100644 (file)
--- a/Include/internal/pycore_optimizer.h
+++ b/Include/internal/pycore_optimizer.h
@@ -67,8 +67,9 @@ typedef struct {
 #endif
 } _PyUOpInstruction;
 
-typedef struct {
+typedef struct _PyExitData {
     uint32_t target;
+    uint16_t index;
     _Py_BackoffCounter temperature;
     struct _PyExecutorObject *executor;
 } _PyExitData;
@@ -354,6 +355,16 @@ PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored);
 
 PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth);
 
+static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit)
+{
+    _PyExitData *exit0 = exit - exit->index;
+    return (_PyExecutorObject *)(((char *)exit0) - offsetof(_PyExecutorObject, exits));
+}
+
+extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void);
+
+PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit);
+
 static inline int is_terminator(const _PyUOpInstruction *uop)
 {
     int opcode = uop->opcode;
@@ -363,6 +374,8 @@ static inline int is_terminator(const _PyUOpInstruction *uop)
     );
 }
 
+extern void _PyExecutor_Free(_PyExecutorObject *self);
+
 PyAPI_FUNC(int) _PyDumpExecutors(FILE *out);
 #ifdef _Py_TIER2
 extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp);

diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h
index a9432401525ebbd262545fef20fec06fc2b335da..684969a23c4d22dffc729231afc811eb231203e1 100644 (file)
--- a/Include/internal/pycore_uop_ids.h
+++ b/Include/internal/pycore_uop_ids.h
@@ -81,99 +81,100 @@ extern "C" {
 #define _CHECK_STACK_SPACE 357
 #define _CHECK_STACK_SPACE_OPERAND 358
 #define _CHECK_VALIDITY 359
-#define _COMPARE_OP 360
-#define _COMPARE_OP_FLOAT 361
-#define _COMPARE_OP_INT 362
-#define _COMPARE_OP_STR 363
-#define _CONTAINS_OP 364
-#define _CONTAINS_OP_DICT 365
-#define _CONTAINS_OP_SET 366
+#define _COLD_EXIT 360
+#define _COMPARE_OP 361
+#define _COMPARE_OP_FLOAT 362
+#define _COMPARE_OP_INT 363
+#define _COMPARE_OP_STR 364
+#define _CONTAINS_OP 365
+#define _CONTAINS_OP_DICT 366
+#define _CONTAINS_OP_SET 367
 #define _CONVERT_VALUE CONVERT_VALUE
-#define _COPY 367
-#define _COPY_1 368
-#define _COPY_2 369
-#define _COPY_3 370
+#define _COPY 368
+#define _COPY_1 369
+#define _COPY_2 370
+#define _COPY_3 371
 #define _COPY_FREE_VARS COPY_FREE_VARS
-#define _CREATE_INIT_FRAME 371
+#define _CREATE_INIT_FRAME 372
 #define _DELETE_ATTR DELETE_ATTR
 #define _DELETE_DEREF DELETE_DEREF
 #define _DELETE_FAST DELETE_FAST
 #define _DELETE_GLOBAL DELETE_GLOBAL
 #define _DELETE_NAME DELETE_NAME
 #define _DELETE_SUBSCR DELETE_SUBSCR
-#define _DEOPT 372
+#define _DEOPT 373
 #define _DICT_MERGE DICT_MERGE
 #define _DICT_UPDATE DICT_UPDATE
-#define _DO_CALL 373
-#define _DO_CALL_FUNCTION_EX 374
-#define _DO_CALL_KW 375
+#define _DO_CALL 374
+#define _DO_CALL_FUNCTION_EX 375
+#define _DO_CALL_KW 376
 #define _END_FOR END_FOR
 #define _END_SEND END_SEND
-#define _ERROR_POP_N 376
+#define _ERROR_POP_N 377
 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK
-#define _EXPAND_METHOD 377
-#define _EXPAND_METHOD_KW 378
-#define _FATAL_ERROR 379
+#define _EXPAND_METHOD 378
+#define _EXPAND_METHOD_KW 379
+#define _FATAL_ERROR 380
 #define _FORMAT_SIMPLE FORMAT_SIMPLE
 #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC
-#define _FOR_ITER 380
-#define _FOR_ITER_GEN_FRAME 381
-#define _FOR_ITER_TIER_TWO 382
+#define _FOR_ITER 381
+#define _FOR_ITER_GEN_FRAME 382
+#define _FOR_ITER_TIER_TWO 383
 #define _GET_AITER GET_AITER
 #define _GET_ANEXT GET_ANEXT
 #define _GET_AWAITABLE GET_AWAITABLE
 #define _GET_ITER GET_ITER
 #define _GET_LEN GET_LEN
 #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER
-#define _GUARD_BINARY_OP_EXTEND 383
-#define _GUARD_CALLABLE_ISINSTANCE 384
-#define _GUARD_CALLABLE_LEN 385
-#define _GUARD_CALLABLE_LIST_APPEND 386
-#define _GUARD_CALLABLE_STR_1 387
-#define _GUARD_CALLABLE_TUPLE_1 388
-#define _GUARD_CALLABLE_TYPE_1 389
-#define _GUARD_DORV_NO_DICT 390
-#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 391
-#define _GUARD_GLOBALS_VERSION 392
-#define _GUARD_IS_FALSE_POP 393
-#define _GUARD_IS_NONE_POP 394
-#define _GUARD_IS_NOT_NONE_POP 395
-#define _GUARD_IS_TRUE_POP 396
-#define _GUARD_KEYS_VERSION 397
-#define _GUARD_NOS_DICT 398
-#define _GUARD_NOS_FLOAT 399
-#define _GUARD_NOS_INT 400
-#define _GUARD_NOS_LIST 401
-#define _GUARD_NOS_NOT_NULL 402
-#define _GUARD_NOS_NULL 403
-#define _GUARD_NOS_OVERFLOWED 404
-#define _GUARD_NOS_TUPLE 405
-#define _GUARD_NOS_UNICODE 406
-#define _GUARD_NOT_EXHAUSTED_LIST 407
-#define _GUARD_NOT_EXHAUSTED_RANGE 408
-#define _GUARD_NOT_EXHAUSTED_TUPLE 409
-#define _GUARD_THIRD_NULL 410
-#define _GUARD_TOS_ANY_SET 411
-#define _GUARD_TOS_DICT 412
-#define _GUARD_TOS_FLOAT 413
-#define _GUARD_TOS_INT 414
-#define _GUARD_TOS_LIST 415
-#define _GUARD_TOS_OVERFLOWED 416
-#define _GUARD_TOS_SLICE 417
-#define _GUARD_TOS_TUPLE 418
-#define _GUARD_TOS_UNICODE 419
-#define _GUARD_TYPE_VERSION 420
-#define _GUARD_TYPE_VERSION_AND_LOCK 421
+#define _GUARD_BINARY_OP_EXTEND 384
+#define _GUARD_CALLABLE_ISINSTANCE 385
+#define _GUARD_CALLABLE_LEN 386
+#define _GUARD_CALLABLE_LIST_APPEND 387
+#define _GUARD_CALLABLE_STR_1 388
+#define _GUARD_CALLABLE_TUPLE_1 389
+#define _GUARD_CALLABLE_TYPE_1 390
+#define _GUARD_DORV_NO_DICT 391
+#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 392
+#define _GUARD_GLOBALS_VERSION 393
+#define _GUARD_IS_FALSE_POP 394
+#define _GUARD_IS_NONE_POP 395
+#define _GUARD_IS_NOT_NONE_POP 396
+#define _GUARD_IS_TRUE_POP 397
+#define _GUARD_KEYS_VERSION 398
+#define _GUARD_NOS_DICT 399
+#define _GUARD_NOS_FLOAT 400
+#define _GUARD_NOS_INT 401
+#define _GUARD_NOS_LIST 402
+#define _GUARD_NOS_NOT_NULL 403
+#define _GUARD_NOS_NULL 404
+#define _GUARD_NOS_OVERFLOWED 405
+#define _GUARD_NOS_TUPLE 406
+#define _GUARD_NOS_UNICODE 407
+#define _GUARD_NOT_EXHAUSTED_LIST 408
+#define _GUARD_NOT_EXHAUSTED_RANGE 409
+#define _GUARD_NOT_EXHAUSTED_TUPLE 410
+#define _GUARD_THIRD_NULL 411
+#define _GUARD_TOS_ANY_SET 412
+#define _GUARD_TOS_DICT 413
+#define _GUARD_TOS_FLOAT 414
+#define _GUARD_TOS_INT 415
+#define _GUARD_TOS_LIST 416
+#define _GUARD_TOS_OVERFLOWED 417
+#define _GUARD_TOS_SLICE 418
+#define _GUARD_TOS_TUPLE 419
+#define _GUARD_TOS_UNICODE 420
+#define _GUARD_TYPE_VERSION 421
+#define _GUARD_TYPE_VERSION_AND_LOCK 422
 #define _IMPORT_FROM IMPORT_FROM
 #define _IMPORT_NAME IMPORT_NAME
-#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 422
-#define _INIT_CALL_PY_EXACT_ARGS 423
-#define _INIT_CALL_PY_EXACT_ARGS_0 424
-#define _INIT_CALL_PY_EXACT_ARGS_1 425
-#define _INIT_CALL_PY_EXACT_ARGS_2 426
-#define _INIT_CALL_PY_EXACT_ARGS_3 427
-#define _INIT_CALL_PY_EXACT_ARGS_4 428
-#define _INSERT_NULL 429
+#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 423
+#define _INIT_CALL_PY_EXACT_ARGS 424
+#define _INIT_CALL_PY_EXACT_ARGS_0 425
+#define _INIT_CALL_PY_EXACT_ARGS_1 426
+#define _INIT_CALL_PY_EXACT_ARGS_2 427
+#define _INIT_CALL_PY_EXACT_ARGS_3 428
+#define _INIT_CALL_PY_EXACT_ARGS_4 429
+#define _INSERT_NULL 430
 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER
 #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION
 #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD
@@ -183,177 +184,177 @@ extern "C" {
 #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE
 #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE
 #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE
-#define _IS_NONE 430
+#define _IS_NONE 431
 #define _IS_OP IS_OP
-#define _ITER_CHECK_LIST 431
-#define _ITER_CHECK_RANGE 432
-#define _ITER_CHECK_TUPLE 433
-#define _ITER_JUMP_LIST 434
-#define _ITER_JUMP_RANGE 435
-#define _ITER_JUMP_TUPLE 436
-#define _ITER_NEXT_LIST 437
-#define _ITER_NEXT_LIST_TIER_TWO 438
-#define _ITER_NEXT_RANGE 439
-#define _ITER_NEXT_TUPLE 440
-#define _JUMP_TO_TOP 441
+#define _ITER_CHECK_LIST 432
+#define _ITER_CHECK_RANGE 433
+#define _ITER_CHECK_TUPLE 434
+#define _ITER_JUMP_LIST 435
+#define _ITER_JUMP_RANGE 436
+#define _ITER_JUMP_TUPLE 437
+#define _ITER_NEXT_LIST 438
+#define _ITER_NEXT_LIST_TIER_TWO 439
+#define _ITER_NEXT_RANGE 440
+#define _ITER_NEXT_TUPLE 441
+#define _JUMP_TO_TOP 442
 #define _LIST_APPEND LIST_APPEND
 #define _LIST_EXTEND LIST_EXTEND
-#define _LOAD_ATTR 442
-#define _LOAD_ATTR_CLASS 443
+#define _LOAD_ATTR 443
+#define _LOAD_ATTR_CLASS 444
 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
-#define _LOAD_ATTR_INSTANCE_VALUE 444
-#define _LOAD_ATTR_METHOD_LAZY_DICT 445
-#define _LOAD_ATTR_METHOD_NO_DICT 446
-#define _LOAD_ATTR_METHOD_WITH_VALUES 447
-#define _LOAD_ATTR_MODULE 448
-#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 449
-#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 450
-#define _LOAD_ATTR_PROPERTY_FRAME 451
-#define _LOAD_ATTR_SLOT 452
-#define _LOAD_ATTR_WITH_HINT 453
+#define _LOAD_ATTR_INSTANCE_VALUE 445
+#define _LOAD_ATTR_METHOD_LAZY_DICT 446
+#define _LOAD_ATTR_METHOD_NO_DICT 447
+#define _LOAD_ATTR_METHOD_WITH_VALUES 448
+#define _LOAD_ATTR_MODULE 449
+#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 450
+#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 451
+#define _LOAD_ATTR_PROPERTY_FRAME 452
+#define _LOAD_ATTR_SLOT 453
+#define _LOAD_ATTR_WITH_HINT 454
 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
-#define _LOAD_BYTECODE 454
+#define _LOAD_BYTECODE 455
 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT
 #define _LOAD_CONST LOAD_CONST
-#define _LOAD_CONST_INLINE 455
-#define _LOAD_CONST_INLINE_BORROW 456
-#define _LOAD_CONST_UNDER_INLINE 457
-#define _LOAD_CONST_UNDER_INLINE_BORROW 458
+#define _LOAD_CONST_INLINE 456
+#define _LOAD_CONST_INLINE_BORROW 457
+#define _LOAD_CONST_UNDER_INLINE 458
+#define _LOAD_CONST_UNDER_INLINE_BORROW 459
 #define _LOAD_DEREF LOAD_DEREF
-#define _LOAD_FAST 459
-#define _LOAD_FAST_0 460
-#define _LOAD_FAST_1 461
-#define _LOAD_FAST_2 462
-#define _LOAD_FAST_3 463
-#define _LOAD_FAST_4 464
-#define _LOAD_FAST_5 465
-#define _LOAD_FAST_6 466
-#define _LOAD_FAST_7 467
+#define _LOAD_FAST 460
+#define _LOAD_FAST_0 461
+#define _LOAD_FAST_1 462
+#define _LOAD_FAST_2 463
+#define _LOAD_FAST_3 464
+#define _LOAD_FAST_4 465
+#define _LOAD_FAST_5 466
+#define _LOAD_FAST_6 467
+#define _LOAD_FAST_7 468
 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
-#define _LOAD_FAST_BORROW 468
-#define _LOAD_FAST_BORROW_0 469
-#define _LOAD_FAST_BORROW_1 470
-#define _LOAD_FAST_BORROW_2 471
-#define _LOAD_FAST_BORROW_3 472
-#define _LOAD_FAST_BORROW_4 473
-#define _LOAD_FAST_BORROW_5 474
-#define _LOAD_FAST_BORROW_6 475
-#define _LOAD_FAST_BORROW_7 476
+#define _LOAD_FAST_BORROW 469
+#define _LOAD_FAST_BORROW_0 470
+#define _LOAD_FAST_BORROW_1 471
+#define _LOAD_FAST_BORROW_2 472
+#define _LOAD_FAST_BORROW_3 473
+#define _LOAD_FAST_BORROW_4 474
+#define _LOAD_FAST_BORROW_5 475
+#define _LOAD_FAST_BORROW_6 476
+#define _LOAD_FAST_BORROW_7 477
 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW
 #define _LOAD_FAST_CHECK LOAD_FAST_CHECK
 #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
 #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
 #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
-#define _LOAD_GLOBAL 477
-#define _LOAD_GLOBAL_BUILTINS 478
-#define _LOAD_GLOBAL_MODULE 479
+#define _LOAD_GLOBAL 478
+#define _LOAD_GLOBAL_BUILTINS 479
+#define _LOAD_GLOBAL_MODULE 480
 #define _LOAD_LOCALS LOAD_LOCALS
 #define _LOAD_NAME LOAD_NAME
-#define _LOAD_SMALL_INT 480
-#define _LOAD_SMALL_INT_0 481
-#define _LOAD_SMALL_INT_1 482
-#define _LOAD_SMALL_INT_2 483
-#define _LOAD_SMALL_INT_3 484
-#define _LOAD_SPECIAL 485
+#define _LOAD_SMALL_INT 481
+#define _LOAD_SMALL_INT_0 482
+#define _LOAD_SMALL_INT_1 483
+#define _LOAD_SMALL_INT_2 484
+#define _LOAD_SMALL_INT_3 485
+#define _LOAD_SPECIAL 486
 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR
 #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD
-#define _MAKE_CALLARGS_A_TUPLE 486
+#define _MAKE_CALLARGS_A_TUPLE 487
 #define _MAKE_CELL MAKE_CELL
 #define _MAKE_FUNCTION MAKE_FUNCTION
-#define _MAKE_WARM 487
+#define _MAKE_WARM 488
 #define _MAP_ADD MAP_ADD
 #define _MATCH_CLASS MATCH_CLASS
 #define _MATCH_KEYS MATCH_KEYS
 #define _MATCH_MAPPING MATCH_MAPPING
 #define _MATCH_SEQUENCE MATCH_SEQUENCE
-#define _MAYBE_EXPAND_METHOD 488
-#define _MAYBE_EXPAND_METHOD_KW 489
-#define _MONITOR_CALL 490
-#define _MONITOR_CALL_KW 491
-#define _MONITOR_JUMP_BACKWARD 492
-#define _MONITOR_RESUME 493
+#define _MAYBE_EXPAND_METHOD 489
+#define _MAYBE_EXPAND_METHOD_KW 490
+#define _MONITOR_CALL 491
+#define _MONITOR_CALL_KW 492
+#define _MONITOR_JUMP_BACKWARD 493
+#define _MONITOR_RESUME 494
 #define _NOP NOP
-#define _POP_CALL 494
-#define _POP_CALL_LOAD_CONST_INLINE_BORROW 495
-#define _POP_CALL_ONE 496
-#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 497
-#define _POP_CALL_TWO 498
-#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 499
+#define _POP_CALL 495
+#define _POP_CALL_LOAD_CONST_INLINE_BORROW 496
+#define _POP_CALL_ONE 497
+#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 498
+#define _POP_CALL_TWO 499
+#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 500
 #define _POP_EXCEPT POP_EXCEPT
 #define _POP_ITER POP_ITER
-#define _POP_JUMP_IF_FALSE 500
-#define _POP_JUMP_IF_TRUE 501
+#define _POP_JUMP_IF_FALSE 501
+#define _POP_JUMP_IF_TRUE 502
 #define _POP_TOP POP_TOP
-#define _POP_TOP_FLOAT 502
-#define _POP_TOP_INT 503
-#define _POP_TOP_LOAD_CONST_INLINE 504
-#define _POP_TOP_LOAD_CONST_INLINE_BORROW 505
-#define _POP_TOP_NOP 506
-#define _POP_TOP_UNICODE 507
-#define _POP_TWO 508
-#define _POP_TWO_LOAD_CONST_INLINE_BORROW 509
+#define _POP_TOP_FLOAT 503
+#define _POP_TOP_INT 504
+#define _POP_TOP_LOAD_CONST_INLINE 505
+#define _POP_TOP_LOAD_CONST_INLINE_BORROW 506
+#define _POP_TOP_NOP 507
+#define _POP_TOP_UNICODE 508
+#define _POP_TWO 509
+#define _POP_TWO_LOAD_CONST_INLINE_BORROW 510
 #define _PUSH_EXC_INFO PUSH_EXC_INFO
-#define _PUSH_FRAME 510
+#define _PUSH_FRAME 511
 #define _PUSH_NULL PUSH_NULL
-#define _PUSH_NULL_CONDITIONAL 511
-#define _PY_FRAME_GENERAL 512
-#define _PY_FRAME_KW 513
-#define _QUICKEN_RESUME 514
-#define _REPLACE_WITH_TRUE 515
+#define _PUSH_NULL_CONDITIONAL 512
+#define _PY_FRAME_GENERAL 513
+#define _PY_FRAME_KW 514
+#define _QUICKEN_RESUME 515
+#define _REPLACE_WITH_TRUE 516
 #define _RESUME_CHECK RESUME_CHECK
 #define _RETURN_GENERATOR RETURN_GENERATOR
 #define _RETURN_VALUE RETURN_VALUE
-#define _SAVE_RETURN_OFFSET 516
-#define _SEND 517
-#define _SEND_GEN_FRAME 518
+#define _SAVE_RETURN_OFFSET 517
+#define _SEND 518
+#define _SEND_GEN_FRAME 519
 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
 #define _SET_ADD SET_ADD
 #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
 #define _SET_UPDATE SET_UPDATE
-#define _START_EXECUTOR 519
-#define _STORE_ATTR 520
-#define _STORE_ATTR_INSTANCE_VALUE 521
-#define _STORE_ATTR_SLOT 522
-#define _STORE_ATTR_WITH_HINT 523
+#define _START_EXECUTOR 520
+#define _STORE_ATTR 521
+#define _STORE_ATTR_INSTANCE_VALUE 522
+#define _STORE_ATTR_SLOT 523
+#define _STORE_ATTR_WITH_HINT 524
 #define _STORE_DEREF STORE_DEREF
-#define _STORE_FAST 524
-#define _STORE_FAST_0 525
-#define _STORE_FAST_1 526
-#define _STORE_FAST_2 527
-#define _STORE_FAST_3 528
-#define _STORE_FAST_4 529
-#define _STORE_FAST_5 530
-#define _STORE_FAST_6 531
-#define _STORE_FAST_7 532
+#define _STORE_FAST 525
+#define _STORE_FAST_0 526
+#define _STORE_FAST_1 527
+#define _STORE_FAST_2 528
+#define _STORE_FAST_3 529
+#define _STORE_FAST_4 530
+#define _STORE_FAST_5 531
+#define _STORE_FAST_6 532
+#define _STORE_FAST_7 533
 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
 #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
 #define _STORE_GLOBAL STORE_GLOBAL
 #define _STORE_NAME STORE_NAME
-#define _STORE_SLICE 533
-#define _STORE_SUBSCR 534
-#define _STORE_SUBSCR_DICT 535
-#define _STORE_SUBSCR_LIST_INT 536
-#define _SWAP 537
-#define _SWAP_2 538
-#define _SWAP_3 539
-#define _TIER2_RESUME_CHECK 540
-#define _TO_BOOL 541
+#define _STORE_SLICE 534
+#define _STORE_SUBSCR 535
+#define _STORE_SUBSCR_DICT 536
+#define _STORE_SUBSCR_LIST_INT 537
+#define _SWAP 538
+#define _SWAP_2 539
+#define _SWAP_3 540
+#define _TIER2_RESUME_CHECK 541
+#define _TO_BOOL 542
 #define _TO_BOOL_BOOL TO_BOOL_BOOL
 #define _TO_BOOL_INT TO_BOOL_INT
-#define _TO_BOOL_LIST 542
+#define _TO_BOOL_LIST 543
 #define _TO_BOOL_NONE TO_BOOL_NONE
-#define _TO_BOOL_STR 543
+#define _TO_BOOL_STR 544
 #define _UNARY_INVERT UNARY_INVERT
 #define _UNARY_NEGATIVE UNARY_NEGATIVE
 #define _UNARY_NOT UNARY_NOT
 #define _UNPACK_EX UNPACK_EX
-#define _UNPACK_SEQUENCE 544
-#define _UNPACK_SEQUENCE_LIST 545
-#define _UNPACK_SEQUENCE_TUPLE 546
-#define _UNPACK_SEQUENCE_TWO_TUPLE 547
+#define _UNPACK_SEQUENCE 545
+#define _UNPACK_SEQUENCE_LIST 546
+#define _UNPACK_SEQUENCE_TUPLE 547
+#define _UNPACK_SEQUENCE_TWO_TUPLE 548
 #define _WITH_EXCEPT_START WITH_EXCEPT_START
 #define _YIELD_VALUE YIELD_VALUE
-#define MAX_UOP_ID 547
+#define MAX_UOP_ID 548
 
 #ifdef __cplusplus
 }

diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h
index ff7e800aa9bb1a899d645393bb6680d10902459c..3348ce0f204f5aa500ddc6bb7cfcecf29e264cb9 100644 (file)
--- a/Include/internal/pycore_uop_metadata.h
+++ b/Include/internal/pycore_uop_metadata.h
@@ -330,12 +330,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_LOAD_CONST_UNDER_INLINE] = 0,
     [_LOAD_CONST_UNDER_INLINE_BORROW] = 0,
     [_CHECK_FUNCTION] = HAS_DEOPT_FLAG,
-    [_START_EXECUTOR] = 0,
+    [_START_EXECUTOR] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
     [_MAKE_WARM] = 0,
     [_FATAL_ERROR] = 0,
     [_DEOPT] = 0,
     [_ERROR_POP_N] = HAS_ARG_FLAG,
     [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG,
+    [_COLD_EXIT] = HAS_ESCAPES_FLAG,
 };
 
 const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = {
@@ -419,6 +420,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
     [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE",
     [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND",
     [_CHECK_VALIDITY] = "_CHECK_VALIDITY",
+    [_COLD_EXIT] = "_COLD_EXIT",
     [_COMPARE_OP] = "_COMPARE_OP",
     [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT",
     [_COMPARE_OP_INT] = "_COMPARE_OP_INT",
@@ -1301,6 +1303,8 @@ int _PyUop_num_popped(int opcode, int oparg)
             return 0;
         case _TIER2_RESUME_CHECK:
             return 0;
+        case _COLD_EXIT:
+            return 0;
         default:
             return -1;
     }

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index d9abc4c53d1f507c186f96f407454a49da428619..cdc0a785c6d72e73e9d51a7e5feb32427e946f7b 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -2964,6 +2964,8 @@ dummy_func(
                 else {
                     this_instr[1].counter = initial_jump_backoff_counter();
                     assert(tstate->current_executor == NULL);
+                    assert(executor != tstate->interp->cold_executor);
+                    tstate->jit_exit = NULL;
                     GOTO_TIER_TWO(executor);
                 }
             }
@@ -3028,6 +3030,8 @@ dummy_func(
                 }
                 DISPATCH_GOTO();
             }
+            assert(executor != tstate->interp->cold_executor);
+            tstate->jit_exit = NULL;
             GOTO_TIER_TWO(executor);
             #else
             Py_FatalError("ENTER_EXECUTOR is not supported in this build");
@@ -5238,9 +5242,8 @@ dummy_func(
 
         tier2 op(_EXIT_TRACE, (exit_p/4 --)) {
             _PyExitData *exit = (_PyExitData *)exit_p;
-            PyCodeObject *code = _PyFrame_GetCode(frame);
-            _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
         #if defined(Py_DEBUG) && !defined(_Py_JIT)
+            _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
             OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
             if (frame->lltrace >= 2) {
                 printf("SIDE EXIT: [UOp ");
@@ -5251,32 +5254,7 @@ dummy_func(
                     _PyOpcode_OpName[target->op.code]);
             }
         #endif
-            if (exit->executor && !exit->executor->vm_data.valid) {
-                exit->temperature = initial_temperature_backoff_counter();
-                Py_CLEAR(exit->executor);
-            }
-            if (exit->executor == NULL) {
-                _Py_BackoffCounter temperature = exit->temperature;
-                if (!backoff_counter_triggers(temperature)) {
-                    exit->temperature = advance_backoff_counter(temperature);
-                    GOTO_TIER_ONE(target);
-                }
-                _PyExecutorObject *executor;
-                if (target->op.code == ENTER_EXECUTOR) {
-                    executor = code->co_executors->executors[target->op.arg];
-                    Py_INCREF(executor);
-                }
-                else {
-                    int chain_depth = current_executor->vm_data.chain_depth + 1;
-                    int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth);
-                    if (optimized <= 0) {
-                        exit->temperature = restart_backoff_counter(temperature);
-                        GOTO_TIER_ONE(optimized < 0 ? NULL : target);
-                    }
-                    exit->temperature = initial_temperature_backoff_counter();
-                }
-                exit->executor = executor;
-            }
+            tstate->jit_exit = exit;
             GOTO_TIER_TWO(exit->executor);
         }
 
@@ -5375,7 +5353,14 @@ dummy_func(
 #ifndef _Py_JIT
             current_executor = (_PyExecutorObject*)executor;
 #endif
-            assert(((_PyExecutorObject *)executor)->vm_data.valid);
+            assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor);
+            tstate->current_executor = (PyObject *)executor;
+            if (!current_executor->vm_data.valid) {
+                assert(tstate->jit_exit->executor == current_executor);
+                assert(tstate->current_executor == executor);
+                _PyExecutor_ClearExit(tstate->jit_exit);
+                DEOPT_IF(true);
+            }
         }
 
         tier2 op(_MAKE_WARM, (--)) {
@@ -5414,6 +5399,37 @@ dummy_func(
             assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version));
         }
 
+        tier2 op(_COLD_EXIT, ( -- )) {
+            _PyExitData *exit = tstate->jit_exit;
+            assert(exit != NULL);
+            _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
+            _Py_BackoffCounter temperature = exit->temperature;
+            if (!backoff_counter_triggers(temperature)) {
+                exit->temperature = advance_backoff_counter(temperature);
+                GOTO_TIER_ONE(target);
+            }
+            _PyExecutorObject *executor;
+            if (target->op.code == ENTER_EXECUTOR) {
+                PyCodeObject *code = _PyFrame_GetCode(frame);
+                executor = code->co_executors->executors[target->op.arg];
+                Py_INCREF(executor);
+            }
+            else {
+                _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
+                assert(tstate->current_executor == (PyObject *)previous_executor);
+                int chain_depth = previous_executor->vm_data.chain_depth + 1;
+                int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth);
+                if (optimized <= 0) {
+                    exit->temperature = restart_backoff_counter(temperature);
+                    GOTO_TIER_ONE(optimized < 0 ? NULL : target);
+                }
+                exit->temperature = initial_temperature_backoff_counter();
+            }
+            assert(tstate->jit_exit == exit);
+            exit->executor = executor;
+            GOTO_TIER_TWO(exit->executor);
+        }
+
         label(pop_2_error) {
             stack_pointer -= 2;
             assert(WITHIN_STACK_BOUNDS());

diff --git a/Python/ceval.c b/Python/ceval.c
index 9ccd42bdf0a55c94778e084513601350ba50fd10..e1045808af93d895972db790b874f6bfae149fe4 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1158,7 +1158,7 @@ enter_tier_two:
     uint64_t trace_uop_execution_counter = 0;
 #endif
 
-    assert(next_uop->opcode == _START_EXECUTOR);
+    assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT);
 tier2_dispatch:
     for (;;) {
         uopcode = next_uop->opcode;

diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index 187ec8fdd265840b7bd571afec66ff559d12f934..9ebcefbcb858503b0fc324d8c84e4fa99bef4960 100644 (file)
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -359,7 +359,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer)
 do {                                                   \
     OPT_STAT_INC(traces_executed);                     \
     _PyExecutorObject *_executor = (EXECUTOR);         \
-    tstate->current_executor = (PyObject *)_executor;  \
     jit_func jitted = _executor->jit_code;             \
     /* Keep the shim frame alive via the executor: */  \
     Py_INCREF(_executor);                              \
@@ -378,9 +377,8 @@ do {                                                   \
 do { \
     OPT_STAT_INC(traces_executed); \
     _PyExecutorObject *_executor = (EXECUTOR); \
-    tstate->current_executor = (PyObject *)_executor; \
     next_uop = _executor->trace; \
-    assert(next_uop->opcode == _START_EXECUTOR); \
+    assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); \
     goto enter_tier_two; \
 } while (0)
 #endif
@@ -390,7 +388,6 @@ do { \
     {                                                                 \
         tstate->current_executor = NULL;                              \
         next_instr = (TARGET);                                        \
-        assert(tstate->current_executor == NULL);                     \
         OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \
         _PyFrame_SetStackPointer(frame, stack_pointer);               \
         stack_pointer = _PyFrame_GetStackPointer(frame);              \

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index e152865e4ec9e8578e9c776dcbe89a7814ab147a..c10797e3d8b1b2733e25f208db2817a048843746 100644 (file)
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -7113,9 +7113,8 @@
         case _EXIT_TRACE: {
             PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
             _PyExitData *exit = (_PyExitData *)exit_p;
-            PyCodeObject *code = _PyFrame_GetCode(frame);
-            _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
+            _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
             OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
             if (frame->lltrace >= 2) {
                 _PyFrame_SetStackPointer(frame, stack_pointer);
@@ -7128,36 +7127,7 @@
                 stack_pointer = _PyFrame_GetStackPointer(frame);
             }
             #endif
-            if (exit->executor && !exit->executor->vm_data.valid) {
-                exit->temperature = initial_temperature_backoff_counter();
-                _PyFrame_SetStackPointer(frame, stack_pointer);
-                Py_CLEAR(exit->executor);
-                stack_pointer = _PyFrame_GetStackPointer(frame);
-            }
-            if (exit->executor == NULL) {
-                _Py_BackoffCounter temperature = exit->temperature;
-                if (!backoff_counter_triggers(temperature)) {
-                    exit->temperature = advance_backoff_counter(temperature);
-                    GOTO_TIER_ONE(target);
-                }
-                _PyExecutorObject *executor;
-                if (target->op.code == ENTER_EXECUTOR) {
-                    executor = code->co_executors->executors[target->op.arg];
-                    Py_INCREF(executor);
-                }
-                else {
-                    int chain_depth = current_executor->vm_data.chain_depth + 1;
-                    _PyFrame_SetStackPointer(frame, stack_pointer);
-                    int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth);
-                    stack_pointer = _PyFrame_GetStackPointer(frame);
-                    if (optimized <= 0) {
-                        exit->temperature = restart_backoff_counter(temperature);
-                        GOTO_TIER_ONE(optimized < 0 ? NULL : target);
-                    }
-                    exit->temperature = initial_temperature_backoff_counter();
-                }
-                exit->executor = executor;
-            }
+            tstate->jit_exit = exit;
             GOTO_TIER_TWO(exit->executor);
             break;
         }
@@ -7438,7 +7408,19 @@
             #ifndef _Py_JIT
             current_executor = (_PyExecutorObject*)executor;
             #endif
-            assert(((_PyExecutorObject *)executor)->vm_data.valid);
+            assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor);
+            tstate->current_executor = (PyObject *)executor;
+            if (!current_executor->vm_data.valid) {
+                assert(tstate->jit_exit->executor == current_executor);
+                assert(tstate->current_executor == executor);
+                _PyFrame_SetStackPointer(frame, stack_pointer);
+                _PyExecutor_ClearExit(tstate->jit_exit);
+                stack_pointer = _PyFrame_GetStackPointer(frame);
+                if (true) {
+                    UOP_STAT_INC(uopcode, miss);
+                    JUMP_TO_JUMP_TARGET();
+                }
+            }
             break;
         }
 
@@ -7487,4 +7469,40 @@
             break;
         }
 
+        case _COLD_EXIT: {
+            _PyExitData *exit = tstate->jit_exit;
+            assert(exit != NULL);
+            _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
+            _Py_BackoffCounter temperature = exit->temperature;
+            if (!backoff_counter_triggers(temperature)) {
+                exit->temperature = advance_backoff_counter(temperature);
+                GOTO_TIER_ONE(target);
+            }
+            _PyExecutorObject *executor;
+            if (target->op.code == ENTER_EXECUTOR) {
+                PyCodeObject *code = _PyFrame_GetCode(frame);
+                executor = code->co_executors->executors[target->op.arg];
+                Py_INCREF(executor);
+            }
+            else {
+                _PyFrame_SetStackPointer(frame, stack_pointer);
+                _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
+                stack_pointer = _PyFrame_GetStackPointer(frame);
+                assert(tstate->current_executor == (PyObject *)previous_executor);
+                int chain_depth = previous_executor->vm_data.chain_depth + 1;
+                _PyFrame_SetStackPointer(frame, stack_pointer);
+                int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth);
+                stack_pointer = _PyFrame_GetStackPointer(frame);
+                if (optimized <= 0) {
+                    exit->temperature = restart_backoff_counter(temperature);
+                    GOTO_TIER_ONE(optimized < 0 ? NULL : target);
+                }
+                exit->temperature = initial_temperature_backoff_counter();
+            }
+            assert(tstate->jit_exit == exit);
+            exit->executor = executor;
+            GOTO_TIER_TWO(exit->executor);
+            break;
+        }
+
 #undef TIER_TWO

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index aa1eb373b7ba4b81e1c9eb03b284523528d686b0..b83ab941ed2b70e2e2a1f09e42a8adbb0f8bdf43 100644 (file)
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -5595,6 +5595,8 @@
                 }
                 DISPATCH_GOTO();
             }
+            assert(executor != tstate->interp->cold_executor);
+            tstate->jit_exit = NULL;
             GOTO_TIER_TWO(executor);
             #else
             Py_FatalError("ENTER_EXECUTOR is not supported in this build");
@@ -7793,6 +7795,8 @@
                         this_instr[1].counter = initial_jump_backoff_counter();
                         stack_pointer = _PyFrame_GetStackPointer(frame);
                         assert(tstate->current_executor == NULL);
+                        assert(executor != tstate->interp->cold_executor);
+                        tstate->jit_exit = NULL;
                         GOTO_TIER_TWO(executor);
                     }
                 }

diff --git a/Python/jit.c b/Python/jit.c
index e4927d9ff040718f4696f8c4a3a1e0bf3edba5d8..bd6a5e17a4164f0f27234b5d6c4d620df7136b01 100644 (file)
--- a/Python/jit.c
+++ b/Python/jit.c
@@ -546,7 +546,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz
     group->emit(code, data, executor, NULL, &state);
     code += group->code_size;
     data += group->data_size;
-    assert(trace[0].opcode == _START_EXECUTOR);
+    assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT);
     for (size_t i = 0; i < length; i++) {
         const _PyUOpInstruction *instruction = &trace[i];
         group = &stencil_groups[instruction->opcode];

diff --git a/Python/optimizer.c b/Python/optimizer.c
index 8d01d605ef4a2ad319ce60e95fc9be26bcd0ed8a..6a9b6824a531ac7539e63450de05be5ed6ea0c48 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -205,8 +205,8 @@ static int executor_clear(PyObject *executor);
 static void unlink_executor(_PyExecutorObject *executor);
 
 
-static void
-free_executor(_PyExecutorObject *self)
+void
+_PyExecutor_Free(_PyExecutorObject *self)
 {
 #ifdef _Py_JIT
     _PyJIT_Free(self);
@@ -242,7 +242,7 @@ _Py_ClearExecutorDeletionList(PyInterpreterState *interp)
         }
         else {
             *prev_to_next_ptr = exec->vm_data.links.next;
-            free_executor(exec);
+            _PyExecutor_Free(exec);
         }
         exec = *prev_to_next_ptr;
     }
@@ -1129,7 +1129,7 @@ sanity_check(_PyExecutorObject *executor)
     }
     bool ended = false;
     uint32_t i = 0;
-    CHECK(executor->trace[0].opcode == _START_EXECUTOR);
+    CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT);
     for (; i < executor->code_size; i++) {
         const _PyUOpInstruction *inst = &executor->trace[i];
         uint16_t opcode = inst->opcode;
@@ -1182,9 +1182,11 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil
     }
 
     /* Initialize exits */
+    _PyExecutorObject *cold = _PyExecutor_GetColdExecutor();
     for (int i = 0; i < exit_count; i++) {
-        executor->exits[i].executor = NULL;
+        executor->exits[i].index = i;
         executor->exits[i].temperature = initial_temperature_backoff_counter();
+        executor->exits[i].executor = cold;
     }
     int next_exit = exit_count-1;
     _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length];
@@ -1462,6 +1464,46 @@ _Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_s
     link_executor(executor);
 }
 
+_PyExecutorObject *
+_PyExecutor_GetColdExecutor(void)
+{
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    if (interp->cold_executor != NULL) {
+        return interp->cold_executor;
+    }
+    _PyExecutorObject *cold = allocate_executor(0, 1);
+    if (cold == NULL) {
+        Py_FatalError("Cannot allocate core JIT code");
+    }
+    ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_EXIT;
+#ifdef _Py_JIT
+    cold->jit_code = NULL;
+    cold->jit_side_entry = NULL;
+    cold->jit_size = 0;
+    // This is initialized to true so we can prevent the executor
+    // from being immediately detected as cold and invalidated.
+    cold->vm_data.warm = true;
+    if (_PyJIT_Compile(cold, cold->trace, 1)) {
+        Py_DECREF(cold);
+        Py_FatalError("Cannot allocate core JIT code");
+    }
+#endif
+    _Py_SetImmortal((PyObject *)cold);
+    interp->cold_executor = cold;
+    return cold;
+}
+
+void
+_PyExecutor_ClearExit(_PyExitData *exit)
+{
+    if (exit == NULL) {
+        return;
+    }
+    _PyExecutorObject *old = exit->executor;
+    exit->executor = _PyExecutor_GetColdExecutor();
+    Py_DECREF(old);
+}
+
 /* Detaches the executor from the code object (if any) that
  * holds a reference to it */
 void
@@ -1492,14 +1534,18 @@ executor_clear(PyObject *op)
     assert(executor->vm_data.valid == 1);
     unlink_executor(executor);
     executor->vm_data.valid = 0;
+
     /* It is possible for an executor to form a reference
      * cycle with itself, so decref'ing a side exit could
      * free the executor unless we hold a strong reference to it
      */
+    _PyExecutorObject *cold = _PyExecutor_GetColdExecutor();
     Py_INCREF(executor);
     for (uint32_t i = 0; i < executor->exit_count; i++) {
         executor->exits[i].temperature = initial_unreachable_backoff_counter();
-        Py_CLEAR(executor->exits[i].executor);
+        _PyExecutorObject *e = executor->exits[i].executor;
+        executor->exits[i].executor = cold;
+        Py_DECREF(e);
     }
     _Py_ExecutorDetach(executor);
     Py_DECREF(executor);
@@ -1741,4 +1787,11 @@ _PyDumpExecutors(FILE *out)
     return -1;
 }
 
+void
+_PyExecutor_Free(struct _PyExecutorObject *self)
+{
+    /* This should never be called */
+    Py_UNREACHABLE();
+}
+
 #endif /* _Py_TIER2 */

diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h
index 99206f01618d7956cc81f98ea87ca75a55e6fd39..6ec85a93f46b3e3d517ee47a54c21dbc5f919470 100644 (file)
--- a/Python/optimizer_cases.c.h
+++ b/Python/optimizer_cases.c.h
@@ -3206,3 +3206,7 @@
             break;
         }
 
+        case _COLD_EXIT: {
+            break;
+        }
+

diff --git a/Python/pystate.c b/Python/pystate.c
index 04ca6edb4aaa0ebc19fddc36bb8b33f549f1bef9..c77868bc717daf2f78ed5bd7d2e582ea7e982992 100644 (file)
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -815,7 +815,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
     /* Last garbage collection on this interpreter */
     _PyGC_CollectNoFail(tstate);
     _PyGC_Fini(interp);
-
+    struct _PyExecutorObject *cold = interp->cold_executor;
+    if (cold != NULL) {
+        interp->cold_executor = NULL;
+        assert(cold->vm_data.valid);
+        assert(cold->vm_data.warm);
+        _PyExecutor_Free(cold);
+    }
     /* We don't clear sysdict and builtins until the end of this function.
        Because clearing other attributes can execute arbitrary Python code
        which requires sysdict and builtins. */
@@ -1469,6 +1475,7 @@ init_threadstate(_PyThreadStateImpl *_tstate,
     tstate->datastack_limit = NULL;
     tstate->what_event = -1;
     tstate->current_executor = NULL;
+    tstate->jit_exit = NULL;
     tstate->dict_global_version = 0;
 
     _tstate->c_stack_soft_limit = UINTPTR_MAX;

diff --git a/Tools/jit/template.c b/Tools/jit/template.c
index d07f56e9ce6b42f4354d6193c9483897698fa1e5..7bbc44a08a0a989302ddd6a20f91062876739c66 100644 (file)
--- a/Tools/jit/template.c
+++ b/Tools/jit/template.c
@@ -51,7 +51,6 @@
 do {                                                                       \
     OPT_STAT_INC(traces_executed);                                         \
     _PyExecutorObject *_executor = (EXECUTOR);                             \
-    tstate->current_executor = (PyObject *)_executor;                      \
     jit_func_preserve_none jitted = _executor->jit_side_entry;             \
     __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \
 } while (0)