* Factor some instructions into micro-ops to isolate CHECK_EVAL_BREAKER for escape analysis
* Eliminate CHECK_EVAL_BREAKER macro
return 5;
case YIELD_VALUE:
return 1;
+ case _DO_CALL_FUNCTION_EX:
+ return 3 + (oparg & 1);
default:
return -1;
}
return 6;
case YIELD_VALUE:
return 1;
+ case _DO_CALL_FUNCTION_EX:
+ return 1;
default:
return -1;
}
[INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG },
[INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG },
+ [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
[INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
[RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[RESERVED] = { true, INSTR_FMT_IX, 0 },
- [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG },
[RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG },
[RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
[WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG },
+ [_DO_CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG },
[LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG },
[UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE",
[WITH_EXCEPT_START] = "WITH_EXCEPT_START",
[YIELD_VALUE] = "YIELD_VALUE",
+ [_DO_CALL_FUNCTION_EX] = "_DO_CALL_FUNCTION_EX",
};
#endif
extern const uint8_t _PyOpcode_Caches[256];
#ifdef NEED_OPCODE_METADATA
const uint8_t _PyOpcode_Caches[256] = {
- [JUMP_BACKWARD] = 1,
[TO_BOOL] = 3,
[BINARY_SUBSCR] = 1,
[STORE_SUBSCR] = 1,
[LOAD_ATTR] = 9,
[COMPARE_OP] = 1,
[CONTAINS_OP] = 1,
+ [JUMP_BACKWARD] = 1,
[POP_JUMP_IF_TRUE] = 1,
[POP_JUMP_IF_FALSE] = 1,
[POP_JUMP_IF_NONE] = 1,
[UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE,
[WITH_EXCEPT_START] = WITH_EXCEPT_START,
[YIELD_VALUE] = YIELD_VALUE,
+ [_DO_CALL_FUNCTION_EX] = _DO_CALL_FUNCTION_EX,
};
#endif // NEED_OPCODE_METADATA
#define EXTRA_CASES \
- case 116: \
case 117: \
case 118: \
case 119: \
#define _CALL_BUILTIN_FAST 315
#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 316
#define _CALL_BUILTIN_O 317
-#define _CALL_FUNCTION_EX CALL_FUNCTION_EX
#define _CALL_INTRINSIC_1 CALL_INTRINSIC_1
#define _CALL_INTRINSIC_2 CALL_INTRINSIC_2
#define _CALL_ISINSTANCE CALL_ISINSTANCE
-#define _CALL_KW CALL_KW
#define _CALL_LEN CALL_LEN
#define _CALL_LIST_APPEND CALL_LIST_APPEND
#define _CALL_METHOD_DESCRIPTOR_FAST 318
#define _CHECK_METHOD_VERSION 335
#define _CHECK_PEP_523 336
#define _CHECK_PERIODIC 337
-#define _CHECK_STACK_SPACE 338
-#define _CHECK_STACK_SPACE_OPERAND 339
-#define _CHECK_VALIDITY 340
-#define _CHECK_VALIDITY_AND_SET_IP 341
-#define _COMPARE_OP 342
-#define _COMPARE_OP_FLOAT 343
-#define _COMPARE_OP_INT 344
-#define _COMPARE_OP_STR 345
-#define _CONTAINS_OP 346
+#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 338
+#define _CHECK_STACK_SPACE 339
+#define _CHECK_STACK_SPACE_OPERAND 340
+#define _CHECK_VALIDITY 341
+#define _CHECK_VALIDITY_AND_SET_IP 342
+#define _COMPARE_OP 343
+#define _COMPARE_OP_FLOAT 344
+#define _COMPARE_OP_INT 345
+#define _COMPARE_OP_STR 346
+#define _CONTAINS_OP 347
#define _CONTAINS_OP_DICT CONTAINS_OP_DICT
#define _CONTAINS_OP_SET CONTAINS_OP_SET
#define _CONVERT_VALUE CONVERT_VALUE
#define _DELETE_GLOBAL DELETE_GLOBAL
#define _DELETE_NAME DELETE_NAME
#define _DELETE_SUBSCR DELETE_SUBSCR
-#define _DEOPT 347
+#define _DEOPT 348
#define _DICT_MERGE DICT_MERGE
#define _DICT_UPDATE DICT_UPDATE
-#define _DO_CALL 348
-#define _DYNAMIC_EXIT 349
+#define _DO_CALL 349
+#define _DO_CALL_KW 350
+#define _DYNAMIC_EXIT 351
#define _END_SEND END_SEND
-#define _ERROR_POP_N 350
+#define _ERROR_POP_N 352
#define _EXIT_INIT_CHECK EXIT_INIT_CHECK
-#define _EXPAND_METHOD 351
-#define _FATAL_ERROR 352
+#define _EXPAND_METHOD 353
+#define _FATAL_ERROR 354
#define _FORMAT_SIMPLE FORMAT_SIMPLE
#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC
-#define _FOR_ITER 353
-#define _FOR_ITER_GEN_FRAME 354
-#define _FOR_ITER_TIER_TWO 355
+#define _FOR_ITER 355
+#define _FOR_ITER_GEN_FRAME 356
+#define _FOR_ITER_TIER_TWO 357
#define _GET_AITER GET_AITER
#define _GET_ANEXT GET_ANEXT
#define _GET_AWAITABLE GET_AWAITABLE
#define _GET_ITER GET_ITER
#define _GET_LEN GET_LEN
#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER
-#define _GUARD_BOTH_FLOAT 356
-#define _GUARD_BOTH_INT 357
-#define _GUARD_BOTH_UNICODE 358
-#define _GUARD_BUILTINS_VERSION 359
-#define _GUARD_DORV_NO_DICT 360
-#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 361
-#define _GUARD_GLOBALS_VERSION 362
-#define _GUARD_IS_FALSE_POP 363
-#define _GUARD_IS_NONE_POP 364
-#define _GUARD_IS_NOT_NONE_POP 365
-#define _GUARD_IS_TRUE_POP 366
-#define _GUARD_KEYS_VERSION 367
-#define _GUARD_NOS_FLOAT 368
-#define _GUARD_NOS_INT 369
-#define _GUARD_NOT_EXHAUSTED_LIST 370
-#define _GUARD_NOT_EXHAUSTED_RANGE 371
-#define _GUARD_NOT_EXHAUSTED_TUPLE 372
-#define _GUARD_TOS_FLOAT 373
-#define _GUARD_TOS_INT 374
-#define _GUARD_TYPE_VERSION 375
+#define _GUARD_BOTH_FLOAT 358
+#define _GUARD_BOTH_INT 359
+#define _GUARD_BOTH_UNICODE 360
+#define _GUARD_BUILTINS_VERSION 361
+#define _GUARD_DORV_NO_DICT 362
+#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 363
+#define _GUARD_GLOBALS_VERSION 364
+#define _GUARD_IS_FALSE_POP 365
+#define _GUARD_IS_NONE_POP 366
+#define _GUARD_IS_NOT_NONE_POP 367
+#define _GUARD_IS_TRUE_POP 368
+#define _GUARD_KEYS_VERSION 369
+#define _GUARD_NOS_FLOAT 370
+#define _GUARD_NOS_INT 371
+#define _GUARD_NOT_EXHAUSTED_LIST 372
+#define _GUARD_NOT_EXHAUSTED_RANGE 373
+#define _GUARD_NOT_EXHAUSTED_TUPLE 374
+#define _GUARD_TOS_FLOAT 375
+#define _GUARD_TOS_INT 376
+#define _GUARD_TYPE_VERSION 377
#define _IMPORT_FROM IMPORT_FROM
#define _IMPORT_NAME IMPORT_NAME
-#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 376
-#define _INIT_CALL_PY_EXACT_ARGS 377
-#define _INIT_CALL_PY_EXACT_ARGS_0 378
-#define _INIT_CALL_PY_EXACT_ARGS_1 379
-#define _INIT_CALL_PY_EXACT_ARGS_2 380
-#define _INIT_CALL_PY_EXACT_ARGS_3 381
-#define _INIT_CALL_PY_EXACT_ARGS_4 382
+#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 378
+#define _INIT_CALL_PY_EXACT_ARGS 379
+#define _INIT_CALL_PY_EXACT_ARGS_0 380
+#define _INIT_CALL_PY_EXACT_ARGS_1 381
+#define _INIT_CALL_PY_EXACT_ARGS_2 382
+#define _INIT_CALL_PY_EXACT_ARGS_3 383
+#define _INIT_CALL_PY_EXACT_ARGS_4 384
#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX
#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW
#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER
#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION
-#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD
#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD
#define _INSTRUMENTED_LINE INSTRUMENTED_LINE
#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR
#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE
#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE
#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE
-#define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME
-#define _INTERNAL_INCREMENT_OPT_COUNTER 383
-#define _IS_NONE 384
+#define _INTERNAL_INCREMENT_OPT_COUNTER 385
+#define _IS_NONE 386
#define _IS_OP IS_OP
-#define _ITER_CHECK_LIST 385
-#define _ITER_CHECK_RANGE 386
-#define _ITER_CHECK_TUPLE 387
-#define _ITER_JUMP_LIST 388
-#define _ITER_JUMP_RANGE 389
-#define _ITER_JUMP_TUPLE 390
-#define _ITER_NEXT_LIST 391
-#define _ITER_NEXT_RANGE 392
-#define _ITER_NEXT_TUPLE 393
-#define _JUMP_TO_TOP 394
+#define _ITER_CHECK_LIST 387
+#define _ITER_CHECK_RANGE 388
+#define _ITER_CHECK_TUPLE 389
+#define _ITER_JUMP_LIST 390
+#define _ITER_JUMP_RANGE 391
+#define _ITER_JUMP_TUPLE 392
+#define _ITER_NEXT_LIST 393
+#define _ITER_NEXT_RANGE 394
+#define _ITER_NEXT_TUPLE 395
+#define _JUMP_TO_TOP 396
#define _LIST_APPEND LIST_APPEND
#define _LIST_EXTEND LIST_EXTEND
-#define _LOAD_ATTR 395
-#define _LOAD_ATTR_CLASS 396
-#define _LOAD_ATTR_CLASS_0 397
-#define _LOAD_ATTR_CLASS_1 398
+#define _LOAD_ATTR 397
+#define _LOAD_ATTR_CLASS 398
+#define _LOAD_ATTR_CLASS_0 399
+#define _LOAD_ATTR_CLASS_1 400
#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
-#define _LOAD_ATTR_INSTANCE_VALUE 399
-#define _LOAD_ATTR_INSTANCE_VALUE_0 400
-#define _LOAD_ATTR_INSTANCE_VALUE_1 401
-#define _LOAD_ATTR_METHOD_LAZY_DICT 402
-#define _LOAD_ATTR_METHOD_NO_DICT 403
-#define _LOAD_ATTR_METHOD_WITH_VALUES 404
-#define _LOAD_ATTR_MODULE 405
-#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 406
-#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 407
-#define _LOAD_ATTR_PROPERTY_FRAME 408
-#define _LOAD_ATTR_SLOT 409
-#define _LOAD_ATTR_SLOT_0 410
-#define _LOAD_ATTR_SLOT_1 411
-#define _LOAD_ATTR_WITH_HINT 412
+#define _LOAD_ATTR_INSTANCE_VALUE 401
+#define _LOAD_ATTR_INSTANCE_VALUE_0 402
+#define _LOAD_ATTR_INSTANCE_VALUE_1 403
+#define _LOAD_ATTR_METHOD_LAZY_DICT 404
+#define _LOAD_ATTR_METHOD_NO_DICT 405
+#define _LOAD_ATTR_METHOD_WITH_VALUES 406
+#define _LOAD_ATTR_MODULE 407
+#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 408
+#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 409
+#define _LOAD_ATTR_PROPERTY_FRAME 410
+#define _LOAD_ATTR_SLOT 411
+#define _LOAD_ATTR_SLOT_0 412
+#define _LOAD_ATTR_SLOT_1 413
+#define _LOAD_ATTR_WITH_HINT 414
#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
#define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT
#define _LOAD_CONST LOAD_CONST
-#define _LOAD_CONST_INLINE 413
-#define _LOAD_CONST_INLINE_BORROW 414
-#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 415
-#define _LOAD_CONST_INLINE_WITH_NULL 416
+#define _LOAD_CONST_INLINE 415
+#define _LOAD_CONST_INLINE_BORROW 416
+#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 417
+#define _LOAD_CONST_INLINE_WITH_NULL 418
#define _LOAD_DEREF LOAD_DEREF
-#define _LOAD_FAST 417
-#define _LOAD_FAST_0 418
-#define _LOAD_FAST_1 419
-#define _LOAD_FAST_2 420
-#define _LOAD_FAST_3 421
-#define _LOAD_FAST_4 422
-#define _LOAD_FAST_5 423
-#define _LOAD_FAST_6 424
-#define _LOAD_FAST_7 425
+#define _LOAD_FAST 419
+#define _LOAD_FAST_0 420
+#define _LOAD_FAST_1 421
+#define _LOAD_FAST_2 422
+#define _LOAD_FAST_3 423
+#define _LOAD_FAST_4 424
+#define _LOAD_FAST_5 425
+#define _LOAD_FAST_6 426
+#define _LOAD_FAST_7 427
#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
#define _LOAD_FAST_CHECK LOAD_FAST_CHECK
#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
-#define _LOAD_GLOBAL 426
-#define _LOAD_GLOBAL_BUILTINS 427
-#define _LOAD_GLOBAL_MODULE 428
+#define _LOAD_GLOBAL 428
+#define _LOAD_GLOBAL_BUILTINS 429
+#define _LOAD_GLOBAL_MODULE 430
#define _LOAD_LOCALS LOAD_LOCALS
#define _LOAD_NAME LOAD_NAME
#define _LOAD_SPECIAL LOAD_SPECIAL
#define _MATCH_KEYS MATCH_KEYS
#define _MATCH_MAPPING MATCH_MAPPING
#define _MATCH_SEQUENCE MATCH_SEQUENCE
-#define _MAYBE_EXPAND_METHOD 429
-#define _MONITOR_CALL 430
+#define _MAYBE_EXPAND_METHOD 431
+#define _MONITOR_CALL 432
+#define _MONITOR_JUMP_BACKWARD 433
+#define _MONITOR_RESUME 434
#define _NOP NOP
#define _POP_EXCEPT POP_EXCEPT
-#define _POP_JUMP_IF_FALSE 431
-#define _POP_JUMP_IF_TRUE 432
+#define _POP_JUMP_IF_FALSE 435
+#define _POP_JUMP_IF_TRUE 436
#define _POP_TOP POP_TOP
-#define _POP_TOP_LOAD_CONST_INLINE_BORROW 433
+#define _POP_TOP_LOAD_CONST_INLINE_BORROW 437
#define _PUSH_EXC_INFO PUSH_EXC_INFO
-#define _PUSH_FRAME 434
+#define _PUSH_FRAME 438
#define _PUSH_NULL PUSH_NULL
-#define _PY_FRAME_GENERAL 435
-#define _REPLACE_WITH_TRUE 436
+#define _PY_FRAME_GENERAL 439
+#define _QUICKEN_RESUME 440
+#define _REPLACE_WITH_TRUE 441
#define _RESUME_CHECK RESUME_CHECK
#define _RETURN_GENERATOR RETURN_GENERATOR
#define _RETURN_VALUE RETURN_VALUE
-#define _SAVE_RETURN_OFFSET 437
-#define _SEND 438
-#define _SEND_GEN_FRAME 439
+#define _SAVE_RETURN_OFFSET 442
+#define _SEND 443
+#define _SEND_GEN_FRAME 444
#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
#define _SET_ADD SET_ADD
#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
#define _SET_UPDATE SET_UPDATE
-#define _START_EXECUTOR 440
-#define _STORE_ATTR 441
-#define _STORE_ATTR_INSTANCE_VALUE 442
-#define _STORE_ATTR_SLOT 443
-#define _STORE_ATTR_WITH_HINT 444
+#define _START_EXECUTOR 445
+#define _STORE_ATTR 446
+#define _STORE_ATTR_INSTANCE_VALUE 447
+#define _STORE_ATTR_SLOT 448
+#define _STORE_ATTR_WITH_HINT 449
#define _STORE_DEREF STORE_DEREF
-#define _STORE_FAST 445
-#define _STORE_FAST_0 446
-#define _STORE_FAST_1 447
-#define _STORE_FAST_2 448
-#define _STORE_FAST_3 449
-#define _STORE_FAST_4 450
-#define _STORE_FAST_5 451
-#define _STORE_FAST_6 452
-#define _STORE_FAST_7 453
+#define _STORE_FAST 450
+#define _STORE_FAST_0 451
+#define _STORE_FAST_1 452
+#define _STORE_FAST_2 453
+#define _STORE_FAST_3 454
+#define _STORE_FAST_4 455
+#define _STORE_FAST_5 456
+#define _STORE_FAST_6 457
+#define _STORE_FAST_7 458
#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
#define _STORE_GLOBAL STORE_GLOBAL
#define _STORE_NAME STORE_NAME
#define _STORE_SLICE STORE_SLICE
-#define _STORE_SUBSCR 454
+#define _STORE_SUBSCR 459
#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT
#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT
#define _SWAP SWAP
-#define _TIER2_RESUME_CHECK 455
-#define _TO_BOOL 456
+#define _TIER2_RESUME_CHECK 460
+#define _TO_BOOL 461
#define _TO_BOOL_BOOL TO_BOOL_BOOL
#define _TO_BOOL_INT TO_BOOL_INT
#define _TO_BOOL_LIST TO_BOOL_LIST
#define _UNARY_NEGATIVE UNARY_NEGATIVE
#define _UNARY_NOT UNARY_NOT
#define _UNPACK_EX UNPACK_EX
-#define _UNPACK_SEQUENCE 457
+#define _UNPACK_SEQUENCE 462
#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST
#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE
#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE
#define _WITH_EXCEPT_START WITH_EXCEPT_START
#define _YIELD_VALUE YIELD_VALUE
-#define MAX_UOP_ID 457
+#define __DO_CALL_FUNCTION_EX _DO_CALL_FUNCTION_EX
+#define MAX_UOP_ID 462
#ifdef __cplusplus
}
#ifdef NEED_OPCODE_METADATA
const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_NOP] = HAS_PURE_FLAG,
+ [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_RESUME_CHECK] = HAS_DEOPT_FLAG,
[_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG,
[_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG,
[_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG,
[_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG,
- [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG,
[_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG,
[_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG,
[_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION",
[_CHECK_PEP_523] = "_CHECK_PEP_523",
[_CHECK_PERIODIC] = "_CHECK_PERIODIC",
+ [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM",
[_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE",
[_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND",
[_CHECK_VALIDITY] = "_CHECK_VALIDITY",
switch(opcode) {
case _NOP:
return 0;
+ case _CHECK_PERIODIC:
+ return 0;
+ case _CHECK_PERIODIC_IF_NOT_YIELD_FROM:
+ return 0;
case _RESUME_CHECK:
return 0;
case _LOAD_FAST_CHECK:
return 1;
case _MAYBE_EXPAND_METHOD:
return 2 + oparg;
- case _CHECK_PERIODIC:
- return 0;
case _PY_FRAME_GENERAL:
return 2 + oparg;
case _CHECK_FUNCTION_VERSION:
#define UNPACK_EX 113
#define UNPACK_SEQUENCE 114
#define YIELD_VALUE 115
+#define _DO_CALL_FUNCTION_EX 116
#define RESUME 149
#define BINARY_OP_ADD_FLOAT 150
#define BINARY_OP_ADD_INT 151
#define UNPACK_SEQUENCE_LIST 220
#define UNPACK_SEQUENCE_TUPLE 221
#define UNPACK_SEQUENCE_TWO_TUPLE 222
-#define INSTRUMENTED_RESUME 236
-#define INSTRUMENTED_END_FOR 237
-#define INSTRUMENTED_END_SEND 238
-#define INSTRUMENTED_LOAD_SUPER_ATTR 239
-#define INSTRUMENTED_FOR_ITER 240
-#define INSTRUMENTED_CALL_KW 241
-#define INSTRUMENTED_CALL_FUNCTION_EX 242
-#define INSTRUMENTED_INSTRUCTION 243
-#define INSTRUMENTED_JUMP_FORWARD 244
-#define INSTRUMENTED_JUMP_BACKWARD 245
-#define INSTRUMENTED_POP_JUMP_IF_TRUE 246
-#define INSTRUMENTED_POP_JUMP_IF_FALSE 247
-#define INSTRUMENTED_POP_JUMP_IF_NONE 248
-#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 249
-#define INSTRUMENTED_RETURN_VALUE 250
-#define INSTRUMENTED_RETURN_CONST 251
-#define INSTRUMENTED_YIELD_VALUE 252
-#define INSTRUMENTED_CALL 253
+#define INSTRUMENTED_END_FOR 236
+#define INSTRUMENTED_END_SEND 237
+#define INSTRUMENTED_LOAD_SUPER_ATTR 238
+#define INSTRUMENTED_FOR_ITER 239
+#define INSTRUMENTED_CALL_KW 240
+#define INSTRUMENTED_CALL_FUNCTION_EX 241
+#define INSTRUMENTED_INSTRUCTION 242
+#define INSTRUMENTED_JUMP_FORWARD 243
+#define INSTRUMENTED_POP_JUMP_IF_TRUE 244
+#define INSTRUMENTED_POP_JUMP_IF_FALSE 245
+#define INSTRUMENTED_POP_JUMP_IF_NONE 246
+#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 247
+#define INSTRUMENTED_RESUME 248
+#define INSTRUMENTED_RETURN_VALUE 249
+#define INSTRUMENTED_RETURN_CONST 250
+#define INSTRUMENTED_YIELD_VALUE 251
+#define INSTRUMENTED_CALL 252
+#define INSTRUMENTED_JUMP_BACKWARD 253
#define INSTRUMENTED_LINE 254
#define ENTER_EXECUTOR 255
#define JUMP 256
'UNPACK_EX': 113,
'UNPACK_SEQUENCE': 114,
'YIELD_VALUE': 115,
- 'INSTRUMENTED_RESUME': 236,
- 'INSTRUMENTED_END_FOR': 237,
- 'INSTRUMENTED_END_SEND': 238,
- 'INSTRUMENTED_LOAD_SUPER_ATTR': 239,
- 'INSTRUMENTED_FOR_ITER': 240,
- 'INSTRUMENTED_CALL_KW': 241,
- 'INSTRUMENTED_CALL_FUNCTION_EX': 242,
- 'INSTRUMENTED_INSTRUCTION': 243,
- 'INSTRUMENTED_JUMP_FORWARD': 244,
- 'INSTRUMENTED_JUMP_BACKWARD': 245,
- 'INSTRUMENTED_POP_JUMP_IF_TRUE': 246,
- 'INSTRUMENTED_POP_JUMP_IF_FALSE': 247,
- 'INSTRUMENTED_POP_JUMP_IF_NONE': 248,
- 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 249,
- 'INSTRUMENTED_RETURN_VALUE': 250,
- 'INSTRUMENTED_RETURN_CONST': 251,
- 'INSTRUMENTED_YIELD_VALUE': 252,
- 'INSTRUMENTED_CALL': 253,
+ '_DO_CALL_FUNCTION_EX': 116,
+ 'INSTRUMENTED_END_FOR': 236,
+ 'INSTRUMENTED_END_SEND': 237,
+ 'INSTRUMENTED_LOAD_SUPER_ATTR': 238,
+ 'INSTRUMENTED_FOR_ITER': 239,
+ 'INSTRUMENTED_CALL_KW': 240,
+ 'INSTRUMENTED_CALL_FUNCTION_EX': 241,
+ 'INSTRUMENTED_INSTRUCTION': 242,
+ 'INSTRUMENTED_JUMP_FORWARD': 243,
+ 'INSTRUMENTED_POP_JUMP_IF_TRUE': 244,
+ 'INSTRUMENTED_POP_JUMP_IF_FALSE': 245,
+ 'INSTRUMENTED_POP_JUMP_IF_NONE': 246,
+ 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 247,
+ 'INSTRUMENTED_RESUME': 248,
+ 'INSTRUMENTED_RETURN_VALUE': 249,
+ 'INSTRUMENTED_RETURN_CONST': 250,
+ 'INSTRUMENTED_YIELD_VALUE': 251,
+ 'INSTRUMENTED_CALL': 252,
+ 'INSTRUMENTED_JUMP_BACKWARD': 253,
'JUMP': 256,
'JUMP_NO_INTERRUPT': 257,
'LOAD_CLOSURE': 258,
"""
self.run_cases_test(input, output)
- def test_predictions_and_eval_breaker(self):
+ def test_predictions(self):
input = """
inst(OP1, (arg -- rest)) {
}
inst(OP3, (arg -- res)) {
DEOPT_IF(xxx);
res = Py_None;
- CHECK_EVAL_BREAKER();
}
family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 };
"""
DEOPT_IF(xxx, OP1);
res = Py_None;
stack_pointer[-1] = res;
- CHECK_EVAL_BREAKER();
DISPATCH();
}
"""
#include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_backoff.h"
#include "pycore_cell.h" // PyCell_GetRef()
+#include "pycore_ceval.h"
#include "pycore_code.h"
#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS
#include "pycore_function.h"
RESUME_CHECK,
};
- tier1 inst(RESUME, (--)) {
- assert(frame == tstate->current_frame);
+ op(_CHECK_PERIODIC, (--)) {
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ ERROR_IF(err != 0, error);
+ }
+ }
+
+ op(_CHECK_PERIODIC_IF_NOT_YIELD_FROM, (--)) {
+ if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ ERROR_IF(err != 0, error);
+ }
+ }
+ }
+
+ op(_QUICKEN_RESUME, (--)) {
+ #if ENABLE_SPECIALIZATION
+ if (tstate->tracing == 0 && this_instr->op.code == RESUME) {
+ FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
+ }
+ #endif /* ENABLE_SPECIALIZATION */
+ }
+
+ tier1 op(_MAYBE_INSTRUMENT, (--)) {
if (tstate->tracing == 0) {
- uintptr_t global_version =
- _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
- ~_PY_EVAL_EVENTS_MASK;
- PyCodeObject* code = _PyFrame_GetCode(frame);
- uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version);
- assert((code_version & 255) == 0);
+ uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;
+ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
if (code_version != global_version) {
int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
- ERROR_IF(err, error);
+ if (err) {
+ ERROR_NO_POP();
+ }
next_instr = this_instr;
DISPATCH();
}
- assert(this_instr->op.code == RESUME ||
- this_instr->op.code == RESUME_CHECK ||
- this_instr->op.code == INSTRUMENTED_RESUME ||
- this_instr->op.code == ENTER_EXECUTOR);
- if (this_instr->op.code == RESUME) {
- #if ENABLE_SPECIALIZATION
- FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
- #endif /* ENABLE_SPECIALIZATION */
- }
- }
- if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
- CHECK_EVAL_BREAKER();
}
}
+ macro(RESUME) =
+ _MAYBE_INSTRUMENT +
+ _QUICKEN_RESUME +
+ _CHECK_PERIODIC_IF_NOT_YIELD_FROM;
+
inst(RESUME_CHECK, (--)) {
#if defined(__EMSCRIPTEN__)
DEOPT_IF(_Py_emscripten_signal_clock == 0);
DEOPT_IF(eval_breaker != version);
}
- inst(INSTRUMENTED_RESUME, (--)) {
- uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;
- uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
- if (code_version != global_version && tstate->tracing == 0) {
- int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
- if (err) {
- ERROR_NO_POP();
- }
- next_instr = this_instr;
- }
- else {
- if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
- CHECK_EVAL_BREAKER();
- }
- _PyFrame_SetStackPointer(frame, stack_pointer);
- int err = _Py_call_instrumentation(
- tstate, oparg > 0, frame, this_instr);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- ERROR_IF(err, error);
- if (frame->instr_ptr != this_instr) {
- /* Instrumentation has jumped */
- next_instr = frame->instr_ptr;
- DISPATCH();
- }
+ op(_MONITOR_RESUME, (--)) {
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ int err = _Py_call_instrumentation(
+ tstate, oparg > 0, frame, this_instr);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ ERROR_IF(err, error);
+ if (frame->instr_ptr != this_instr) {
+ /* Instrumentation has jumped */
+ next_instr = frame->instr_ptr;
}
}
+ macro(INSTRUMENTED_RESUME) =
+ _MAYBE_INSTRUMENT +
+ _CHECK_PERIODIC_IF_NOT_YIELD_FROM +
+ _MONITOR_RESUME;
+
pseudo(LOAD_CLOSURE, (-- unused)) = {
LOAD_FAST,
};
JUMPBY(oparg);
}
- tier1 inst(JUMP_BACKWARD, (unused/1 --)) {
- CHECK_EVAL_BREAKER();
+ tier1 op(_JUMP_BACKWARD, (the_counter/1 --)) {
assert(oparg <= INSTR_OFFSET());
JUMPBY(-oparg);
#ifdef _Py_TIER2
#endif /* _Py_TIER2 */
}
+ macro(JUMP_BACKWARD) =
+ _CHECK_PERIODIC +
+ _JUMP_BACKWARD;
+
pseudo(JUMP, (--)) = {
JUMP_FORWARD,
JUMP_BACKWARD,
res = PyStackRef_FromPyObjectSteal(res_o);
}
- op(_CHECK_PERIODIC, (--)) {
- CHECK_EVAL_BREAKER();
- }
-
op(_MONITOR_CALL, (func, maybe_self, args[oparg] -- func, maybe_self, args[oparg])) {
int is_meth = !PyStackRef_IsNull(maybe_self);
PyObject *function = PyStackRef_AsPyObjectBorrow(func);
GO_TO_INSTRUCTION(CALL_KW);
}
- inst(CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) {
+ op(_DO_CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) {
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames);
}
ERROR_IF(res_o == NULL, error);
res = PyStackRef_FromPyObjectSteal(res_o);
- CHECK_EVAL_BREAKER();
}
+ macro(CALL_KW) =
+ _DO_CALL_KW +
+ _CHECK_PERIODIC;
+
inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) {
GO_TO_INSTRUCTION(CALL_FUNCTION_EX);
}
- inst(CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) {
+ inst(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) {
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
DECREF_INPUTS();
assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL);
ERROR_IF(PyStackRef_IsNull(result), error);
- CHECK_EVAL_BREAKER();
}
+ macro(CALL_FUNCTION_EX) =
+ _DO_CALL_FUNCTION_EX +
+ _CHECK_PERIODIC;
+
+
inst(MAKE_FUNCTION, (codeobj_st -- func)) {
PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st);
INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP);
}
- inst(INSTRUMENTED_JUMP_BACKWARD, (unused/1 -- )) {
- CHECK_EVAL_BREAKER();
+ op(_MONITOR_JUMP_BACKWARD, (-- )) {
INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP);
}
+ macro(INSTRUMENTED_JUMP_BACKWARD) =
+ unused/1 +
+ _CHECK_PERIODIC +
+ _MONITOR_JUMP_BACKWARD;
+
inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) {
_PyStackRef cond = POP();
assert(PyStackRef_BoolCheck(cond));
// Use this instead of 'goto error' so Tier 2 can go to a different label
#define GOTO_ERROR(LABEL) goto LABEL
-#define CHECK_EVAL_BREAKER() \
- _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \
- QSBR_QUIESCENT_STATE(tstate); \
- if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { \
- if (_Py_HandlePending(tstate) != 0) { \
- GOTO_ERROR(error); \
- } \
- }
-
-
/* Tuple access macros */
#ifndef Py_DEBUG
break;
}
+ case _CHECK_PERIODIC: {
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) JUMP_TO_ERROR();
+ }
+ break;
+ }
+
+ case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: {
+ oparg = CURRENT_OPARG();
+ if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) JUMP_TO_ERROR();
+ }
+ }
+ break;
+ }
+
+ /* _QUICKEN_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
+
case _RESUME_CHECK: {
#if defined(__EMSCRIPTEN__)
if (_Py_emscripten_signal_clock == 0) {
break;
}
- /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 because it is instrumented */
+ /* _MONITOR_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _LOAD_FAST_CHECK: {
_PyStackRef value;
/* _DO_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
- case _CHECK_PERIODIC: {
- CHECK_EVAL_BREAKER();
- break;
- }
-
/* _MONITOR_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _PY_FRAME_GENERAL: {
/* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 because it is instrumented */
- /* _CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
+ /* _DO_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
/* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */
- /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
+ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _MAKE_FUNCTION: {
_PyStackRef codeobj_st;
/* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */
- /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 because it is instrumented */
+ /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
/* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
_PyStackRef callargs_st;
_PyStackRef kwargs_st = PyStackRef_NULL;
_PyStackRef result;
+ // __DO_CALL_FUNCTION_EX
if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; }
callargs_st = stack_pointer[-1 - (oparg & 1)];
func_st = stack_pointer[-3 - (oparg & 1)];
- PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
- PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
- PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
- // DICT_MERGE is called before this opcode if there are kwargs.
- // It converts all dict subtypes in kwargs into regular dicts.
- assert(kwargs == NULL || PyDict_CheckExact(kwargs));
- if (!PyTuple_CheckExact(callargs)) {
- int err = check_args_iterable(tstate, func, callargs);
- if (err < 0) {
- goto error;
- }
- PyObject *tuple = PySequence_Tuple(callargs);
- if (tuple == NULL) {
- goto error;
- }
- PyStackRef_CLOSE(callargs_st);
- callargs_st = PyStackRef_FromPyObjectSteal(tuple);
- callargs = tuple;
- }
- assert(PyTuple_CheckExact(callargs));
- EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func);
- if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) {
- PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ?
- PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING;
- int err = _Py_call_instrumentation_2args(
- tstate, PY_MONITORING_EVENT_CALL,
- frame, this_instr, func, arg);
- if (err) goto error;
- result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
- if (!PyFunction_Check(func) && !PyMethod_Check(func)) {
- if (PyStackRef_IsNull(result)) {
- _Py_call_instrumentation_exc2(
- tstate, PY_MONITORING_EVENT_C_RAISE,
- frame, this_instr, func, arg);
+ {
+ PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
+ PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
+ PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
+ // DICT_MERGE is called before this opcode if there are kwargs.
+ // It converts all dict subtypes in kwargs into regular dicts.
+ assert(kwargs == NULL || PyDict_CheckExact(kwargs));
+ if (!PyTuple_CheckExact(callargs)) {
+ int err = check_args_iterable(tstate, func, callargs);
+ if (err < 0) {
+ goto error;
}
- else {
- int err = _Py_call_instrumentation_2args(
- tstate, PY_MONITORING_EVENT_C_RETURN,
- frame, this_instr, func, arg);
- if (err < 0) {
- PyStackRef_CLEAR(result);
+ PyObject *tuple = PySequence_Tuple(callargs);
+ if (tuple == NULL) {
+ goto error;
+ }
+ PyStackRef_CLOSE(callargs_st);
+ callargs_st = PyStackRef_FromPyObjectSteal(tuple);
+ callargs = tuple;
+ }
+ assert(PyTuple_CheckExact(callargs));
+ EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func);
+ if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) {
+ PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ?
+ PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING;
+ int err = _Py_call_instrumentation_2args(
+ tstate, PY_MONITORING_EVENT_CALL,
+ frame, this_instr, func, arg);
+ if (err) goto error;
+ result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
+ if (!PyFunction_Check(func) && !PyMethod_Check(func)) {
+ if (PyStackRef_IsNull(result)) {
+ _Py_call_instrumentation_exc2(
+ tstate, PY_MONITORING_EVENT_C_RAISE,
+ frame, this_instr, func, arg);
+ }
+ else {
+ int err = _Py_call_instrumentation_2args(
+ tstate, PY_MONITORING_EVENT_C_RETURN,
+ frame, this_instr, func, arg);
+ if (err < 0) {
+ PyStackRef_CLEAR(result);
+ }
+ }
+ }
+ }
+ else {
+ if (Py_TYPE(func) == &PyFunction_Type &&
+ tstate->interp->eval_frame == NULL &&
+ ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
+ assert(PyTuple_CheckExact(callargs));
+ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
+ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
+ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
+ (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals,
+ nargs, callargs, kwargs);
+ // Need to manually shrink the stack since we exit with DISPATCH_INLINED.
+ STACK_SHRINK(oparg + 3);
+ if (new_frame == NULL) {
+ goto error;
}
+ assert(next_instr - this_instr == 1);
+ frame->return_offset = 1;
+ DISPATCH_INLINED(new_frame);
}
+ result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
+ }
+ PyStackRef_CLOSE(func_st);
+ PyStackRef_CLOSE(callargs_st);
+ PyStackRef_XCLOSE(kwargs_st);
+ assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL);
+ if (PyStackRef_IsNull(result)) {
+ stack_pointer += -3 - (oparg & 1);
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
}
}
- else {
- if (Py_TYPE(func) == &PyFunction_Type &&
- tstate->interp->eval_frame == NULL &&
- ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
- assert(PyTuple_CheckExact(callargs));
- Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
- int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
- _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
- (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals,
- nargs, callargs, kwargs);
- // Need to manually shrink the stack since we exit with DISPATCH_INLINED.
- STACK_SHRINK(oparg + 3);
- if (new_frame == NULL) {
+ // _CHECK_PERIODIC
+ {
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-3 - (oparg & 1)] = result;
+ stack_pointer += -2 - (oparg & 1);
+ assert(WITHIN_STACK_BOUNDS());
goto error;
}
- assert(next_instr - this_instr == 1);
- frame->return_offset = 1;
- DISPATCH_INLINED(new_frame);
}
- result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
- }
- PyStackRef_CLOSE(func_st);
- PyStackRef_CLOSE(callargs_st);
- PyStackRef_XCLOSE(kwargs_st);
- assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL);
- if (PyStackRef_IsNull(result)) {
- stack_pointer += -3 - (oparg & 1);
- assert(WITHIN_STACK_BOUNDS());
- goto error;
}
stack_pointer[-3 - (oparg & 1)] = result;
stack_pointer += -2 - (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
_PyStackRef *args;
_PyStackRef kwnames;
_PyStackRef res;
+ // _DO_CALL_KW
kwnames = stack_pointer[-1];
args = &stack_pointer[-1 - oparg];
self_or_null = stack_pointer[-2 - oparg];
callable = stack_pointer[-3 - oparg];
- PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
- PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
- PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames);
- // oparg counts all of the args, but *not* self:
- int total_args = oparg;
- if (self_or_null_o != NULL) {
- args--;
- total_args++;
- }
- if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) {
- args--;
- total_args++;
- PyObject *self = ((PyMethodObject *)callable_o)->im_self;
- args[0] = PyStackRef_FromPyObjectNew(self);
- PyObject *method = ((PyMethodObject *)callable_o)->im_func;
- args[-1] = PyStackRef_FromPyObjectNew(method);
- PyStackRef_CLOSE(callable);
- callable_o = method;
- callable = args[-1];
- }
- int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o);
- // Check if the call can be inlined or not
- if (Py_TYPE(callable_o) == &PyFunction_Type &&
- tstate->interp->eval_frame == NULL &&
- ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall)
{
- int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
- _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
- tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, positional_args, kwnames_o
- );
- PyStackRef_CLOSE(kwnames);
- // Manipulate stack directly since we leave using DISPATCH_INLINED().
- STACK_SHRINK(oparg + 3);
- // The frame has stolen all the arguments from the stack,
- // so there is no need to clean them up.
- if (new_frame == NULL) {
- goto error;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
+ PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames);
+ // oparg counts all of the args, but *not* self:
+ int total_args = oparg;
+ if (self_or_null_o != NULL) {
+ args--;
+ total_args++;
}
- assert(next_instr - this_instr == 1);
- frame->return_offset = 1;
- DISPATCH_INLINED(new_frame);
- }
- /* Callable is not a normal Python function */
- STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
- if (CONVERSION_FAILED(args_o)) {
- PyStackRef_CLOSE(callable);
- PyStackRef_CLOSE(self_or_null);
- for (int _i = oparg; --_i >= 0;) {
- PyStackRef_CLOSE(args[_i]);
+ if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) {
+ args--;
+ total_args++;
+ PyObject *self = ((PyMethodObject *)callable_o)->im_self;
+ args[0] = PyStackRef_FromPyObjectNew(self);
+ PyObject *method = ((PyMethodObject *)callable_o)->im_func;
+ args[-1] = PyStackRef_FromPyObjectNew(method);
+ PyStackRef_CLOSE(callable);
+ callable_o = method;
+ callable = args[-1];
+ }
+ int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o);
+ // Check if the call can be inlined or not
+ if (Py_TYPE(callable_o) == &PyFunction_Type &&
+ tstate->interp->eval_frame == NULL &&
+ ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall)
+ {
+ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
+ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
+ tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
+ args, positional_args, kwnames_o
+ );
+ PyStackRef_CLOSE(kwnames);
+ // Manipulate stack directly since we leave using DISPATCH_INLINED().
+ STACK_SHRINK(oparg + 3);
+ // The frame has stolen all the arguments from the stack,
+ // so there is no need to clean them up.
+ if (new_frame == NULL) {
+ goto error;
+ }
+ assert(next_instr - this_instr == 1);
+ frame->return_offset = 1;
+ DISPATCH_INLINED(new_frame);
+ }
+ /* Callable is not a normal Python function */
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ PyStackRef_CLOSE(kwnames);
+ if (true) {
+ stack_pointer += -3 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
+ PyObject *res_o = PyObject_Vectorcall(
+ callable_o, args_o,
+ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
+ kwnames_o);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ if (opcode == INSTRUMENTED_CALL_KW) {
+ PyObject *arg = total_args == 0 ?
+ &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]);
+ if (res_o == NULL) {
+ _Py_call_instrumentation_exc2(
+ tstate, PY_MONITORING_EVENT_C_RAISE,
+ frame, this_instr, callable_o, arg);
+ }
+ else {
+ int err = _Py_call_instrumentation_2args(
+ tstate, PY_MONITORING_EVENT_C_RETURN,
+ frame, this_instr, callable_o, arg);
+ if (err < 0) {
+ Py_CLEAR(res_o);
+ }
+ }
}
PyStackRef_CLOSE(kwnames);
- if (true) {
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(callable);
+ for (int i = 0; i < total_args; i++) {
+ PyStackRef_CLOSE(args[i]);
+ }
+ if (res_o == NULL) {
stack_pointer += -3 - oparg;
assert(WITHIN_STACK_BOUNDS());
goto error;
}
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
- PyObject *res_o = PyObject_Vectorcall(
- callable_o, args_o,
- positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
- kwnames_o);
- STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
- if (opcode == INSTRUMENTED_CALL_KW) {
- PyObject *arg = total_args == 0 ?
- &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]);
- if (res_o == NULL) {
- _Py_call_instrumentation_exc2(
- tstate, PY_MONITORING_EVENT_C_RAISE,
- frame, this_instr, callable_o, arg);
- }
- else {
- int err = _Py_call_instrumentation_2args(
- tstate, PY_MONITORING_EVENT_C_RETURN,
- frame, this_instr, callable_o, arg);
- if (err < 0) {
- Py_CLEAR(res_o);
+ // _CHECK_PERIODIC
+ {
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-3 - oparg] = res;
+ stack_pointer += -2 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
}
}
}
- PyStackRef_CLOSE(kwnames);
- assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- PyStackRef_CLOSE(callable);
- for (int i = 0; i < total_args; i++) {
- PyStackRef_CLOSE(args[i]);
- }
- if (res_o == NULL) {
- stack_pointer += -3 - oparg;
- assert(WITHIN_STACK_BOUNDS());
- goto error;
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-3 - oparg] = res;
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) goto pop_2_error;
+ }
}
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) goto pop_2_error;
+ }
}
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
}
// _CHECK_PERIODIC
{
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) {
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ }
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
- CHECK_EVAL_BREAKER();
DISPATCH();
}
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD);
/* Skip 1 cache entry */
- CHECK_EVAL_BREAKER();
- INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP);
+ // _CHECK_PERIODIC
+ {
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) goto error;
+ }
+ }
+ // _MONITOR_JUMP_BACKWARD
+ {
+ INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP);
+ }
DISPATCH();
}
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RESUME);
- uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;
- uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
- if (code_version != global_version && tstate->tracing == 0) {
- int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
- if (err) {
- goto error;
+ // _MAYBE_INSTRUMENT
+ {
+ if (tstate->tracing == 0) {
+ uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;
+ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
+ if (code_version != global_version) {
+ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
+ if (err) {
+ goto error;
+ }
+ next_instr = this_instr;
+ DISPATCH();
+ }
}
- next_instr = this_instr;
}
- else {
+ // _CHECK_PERIODIC_IF_NOT_YIELD_FROM
+ {
if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
- CHECK_EVAL_BREAKER();
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) goto error;
+ }
}
+ }
+ // _MONITOR_RESUME
+ {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation(
tstate, oparg > 0, frame, this_instr);
if (frame->instr_ptr != this_instr) {
/* Instrumentation has jumped */
next_instr = frame->instr_ptr;
- DISPATCH();
}
}
DISPATCH();
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(JUMP_BACKWARD);
- /* Skip 1 cache entry */
- CHECK_EVAL_BREAKER();
- assert(oparg <= INSTR_OFFSET());
- JUMPBY(-oparg);
- #ifdef _Py_TIER2
- #if ENABLE_SPECIALIZATION
- _Py_BackoffCounter counter = this_instr[1].counter;
- if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) {
- _Py_CODEUNIT *start = this_instr;
- /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */
- while (oparg > 255) {
- oparg >>= 8;
- start--;
- }
- _PyExecutorObject *executor;
- int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0);
- if (optimized < 0) goto error;
- if (optimized) {
- assert(tstate->previous_executor == NULL);
- tstate->previous_executor = Py_None;
- GOTO_TIER_TWO(executor);
+ // _CHECK_PERIODIC
+ {
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) goto error;
+ }
+ }
+ // _JUMP_BACKWARD
+ {
+ uint16_t the_counter = read_u16(&this_instr[1].cache);
+ (void)the_counter;
+ assert(oparg <= INSTR_OFFSET());
+ JUMPBY(-oparg);
+ #ifdef _Py_TIER2
+ #if ENABLE_SPECIALIZATION
+ _Py_BackoffCounter counter = this_instr[1].counter;
+ if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) {
+ _Py_CODEUNIT *start = this_instr;
+ /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */
+ while (oparg > 255) {
+ oparg >>= 8;
+ start--;
+ }
+ _PyExecutorObject *executor;
+ int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0);
+ if (optimized < 0) goto error;
+ if (optimized) {
+ assert(tstate->previous_executor == NULL);
+ tstate->previous_executor = Py_None;
+ GOTO_TIER_TWO(executor);
+ }
+ else {
+ this_instr[1].counter = restart_backoff_counter(counter);
+ }
}
else {
- this_instr[1].counter = restart_backoff_counter(counter);
+ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
}
+ #endif /* ENABLE_SPECIALIZATION */
+ #endif /* _Py_TIER2 */
}
- else {
- ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
- }
- #endif /* ENABLE_SPECIALIZATION */
- #endif /* _Py_TIER2 */
DISPATCH();
}
PREDICTED(RESUME);
_Py_CODEUNIT *this_instr = next_instr - 1;
(void)this_instr;
- assert(frame == tstate->current_frame);
- if (tstate->tracing == 0) {
- uintptr_t global_version =
- _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
- ~_PY_EVAL_EVENTS_MASK;
- PyCodeObject* code = _PyFrame_GetCode(frame);
- uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version);
- assert((code_version & 255) == 0);
- if (code_version != global_version) {
- int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
- if (err) goto error;
- next_instr = this_instr;
- DISPATCH();
+ // _MAYBE_INSTRUMENT
+ {
+ if (tstate->tracing == 0) {
+ uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;
+ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
+ if (code_version != global_version) {
+ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
+ if (err) {
+ goto error;
+ }
+ next_instr = this_instr;
+ DISPATCH();
+ }
}
- assert(this_instr->op.code == RESUME ||
- this_instr->op.code == RESUME_CHECK ||
- this_instr->op.code == INSTRUMENTED_RESUME ||
- this_instr->op.code == ENTER_EXECUTOR);
- if (this_instr->op.code == RESUME) {
- #if ENABLE_SPECIALIZATION
+ }
+ // _QUICKEN_RESUME
+ {
+ #if ENABLE_SPECIALIZATION
+ if (tstate->tracing == 0 && this_instr->op.code == RESUME) {
FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
- #endif /* ENABLE_SPECIALIZATION */
}
+ #endif /* ENABLE_SPECIALIZATION */
}
- if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
- CHECK_EVAL_BREAKER();
+ // _CHECK_PERIODIC_IF_NOT_YIELD_FROM
+ {
+ if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
+ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
+ QSBR_QUIESCENT_STATE(tstate); \
+ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ int err = _Py_HandlePending(tstate);
+ if (err != 0) goto error;
+ }
+ }
}
DISPATCH();
}
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
+
+ TARGET(_DO_CALL_FUNCTION_EX) {
+ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+ (void)this_instr;
+ next_instr += 1;
+ INSTRUCTION_STATS(_DO_CALL_FUNCTION_EX);
+ _PyStackRef func_st;
+ _PyStackRef callargs_st;
+ _PyStackRef kwargs_st = PyStackRef_NULL;
+ _PyStackRef result;
+ if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; }
+ callargs_st = stack_pointer[-1 - (oparg & 1)];
+ func_st = stack_pointer[-3 - (oparg & 1)];
+ PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
+ PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
+ PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
+ // DICT_MERGE is called before this opcode if there are kwargs.
+ // It converts all dict subtypes in kwargs into regular dicts.
+ assert(kwargs == NULL || PyDict_CheckExact(kwargs));
+ if (!PyTuple_CheckExact(callargs)) {
+ int err = check_args_iterable(tstate, func, callargs);
+ if (err < 0) {
+ goto error;
+ }
+ PyObject *tuple = PySequence_Tuple(callargs);
+ if (tuple == NULL) {
+ goto error;
+ }
+ PyStackRef_CLOSE(callargs_st);
+ callargs_st = PyStackRef_FromPyObjectSteal(tuple);
+ callargs = tuple;
+ }
+ assert(PyTuple_CheckExact(callargs));
+ EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func);
+ if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) {
+ PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ?
+ PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING;
+ int err = _Py_call_instrumentation_2args(
+ tstate, PY_MONITORING_EVENT_CALL,
+ frame, this_instr, func, arg);
+ if (err) goto error;
+ result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
+ if (!PyFunction_Check(func) && !PyMethod_Check(func)) {
+ if (PyStackRef_IsNull(result)) {
+ _Py_call_instrumentation_exc2(
+ tstate, PY_MONITORING_EVENT_C_RAISE,
+ frame, this_instr, func, arg);
+ }
+ else {
+ int err = _Py_call_instrumentation_2args(
+ tstate, PY_MONITORING_EVENT_C_RETURN,
+ frame, this_instr, func, arg);
+ if (err < 0) {
+ PyStackRef_CLEAR(result);
+ }
+ }
+ }
+ }
+ else {
+ if (Py_TYPE(func) == &PyFunction_Type &&
+ tstate->interp->eval_frame == NULL &&
+ ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
+ assert(PyTuple_CheckExact(callargs));
+ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
+ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
+ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
+ (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals,
+ nargs, callargs, kwargs);
+ // Need to manually shrink the stack since we exit with DISPATCH_INLINED.
+ STACK_SHRINK(oparg + 3);
+ if (new_frame == NULL) {
+ goto error;
+ }
+ assert(next_instr - this_instr == 1);
+ frame->return_offset = 1;
+ DISPATCH_INLINED(new_frame);
+ }
+ result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
+ }
+ PyStackRef_CLOSE(func_st);
+ PyStackRef_CLOSE(callargs_st);
+ PyStackRef_XCLOSE(kwargs_st);
+ assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL);
+ if (PyStackRef_IsNull(result)) {
+ stack_pointer += -3 - (oparg & 1);
+ assert(WITHIN_STACK_BOUNDS());
+ goto error;
+ }
+ stack_pointer[-3 - (oparg & 1)] = result;
+ stack_pointer += -2 - (oparg & 1);
+ assert(WITHIN_STACK_BOUNDS());
+ DISPATCH();
+ }
#undef TIER_ONE
&&TARGET_UNPACK_EX,
&&TARGET_UNPACK_SEQUENCE,
&&TARGET_YIELD_VALUE,
- &&_unknown_opcode,
+ &&TARGET__DO_CALL_FUNCTION_EX,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
- &&TARGET_INSTRUMENTED_RESUME,
&&TARGET_INSTRUMENTED_END_FOR,
&&TARGET_INSTRUMENTED_END_SEND,
&&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR,
&&TARGET_INSTRUMENTED_CALL_FUNCTION_EX,
&&TARGET_INSTRUMENTED_INSTRUCTION,
&&TARGET_INSTRUMENTED_JUMP_FORWARD,
- &&TARGET_INSTRUMENTED_JUMP_BACKWARD,
&&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE,
&&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE,
&&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE,
&&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE,
+ &&TARGET_INSTRUMENTED_RESUME,
&&TARGET_INSTRUMENTED_RETURN_VALUE,
&&TARGET_INSTRUMENTED_RETURN_CONST,
&&TARGET_INSTRUMENTED_YIELD_VALUE,
&&TARGET_INSTRUMENTED_CALL,
+ &&TARGET_INSTRUMENTED_JUMP_BACKWARD,
&&TARGET_INSTRUMENTED_LINE,
&&TARGET_ENTER_EXECUTOR,
};
break;
}
+ case _CHECK_PERIODIC: {
+ break;
+ }
+
+ case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: {
+ break;
+ }
+
+ /* _QUICKEN_RESUME is not a viable micro-op for tier 2 */
+
case _RESUME_CHECK: {
break;
}
- /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */
+ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */
case _LOAD_FAST_CHECK: {
_Py_UopsSymbol *value;
/* _DO_CALL is not a viable micro-op for tier 2 */
- case _CHECK_PERIODIC: {
- break;
- }
-
/* _MONITOR_CALL is not a viable micro-op for tier 2 */
case _PY_FRAME_GENERAL: {
/* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */
- /* _CALL_KW is not a viable micro-op for tier 2 */
+ /* _DO_CALL_KW is not a viable micro-op for tier 2 */
/* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */
- /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */
+ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */
case _MAKE_FUNCTION: {
_Py_UopsSymbol *func;
/* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */
- /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */
+ /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 */
/* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */
oparg: bool
jumps: bool
eval_breaker: bool
- ends_with_eval_breaker: bool
needs_this: bool
always_exits: bool
stores_sp: bool
oparg=any(p.oparg for p in properties),
jumps=any(p.jumps for p in properties),
eval_breaker=any(p.eval_breaker for p in properties),
- ends_with_eval_breaker=any(p.ends_with_eval_breaker for p in properties),
needs_this=any(p.needs_this for p in properties),
always_exits=any(p.always_exits for p in properties),
stores_sp=any(p.stores_sp for p in properties),
oparg=False,
jumps=False,
eval_breaker=False,
- ends_with_eval_breaker=False,
needs_this=False,
always_exits=False,
stores_sp=False,
return "has unused cache entries"
if self.properties.error_with_pop and self.properties.error_without_pop:
return "has both popping and not-popping errors"
- if self.properties.eval_breaker:
- if self.properties.error_with_pop or self.properties.error_without_pop:
- return "has error handling and eval-breaker check"
- if self.properties.side_exit:
- return "exits and eval-breaker check"
- if self.properties.deopts:
- return "deopts and eval-breaker check"
return None
def is_viable(self) -> bool:
}
-def eval_breaker_at_end(op: parser.InstDef) -> bool:
- return op.tokens[-5].text == "CHECK_EVAL_BREAKER"
-
-
def always_exits(op: parser.InstDef) -> bool:
depth = 0
tkn_iter = iter(op.tokens)
side_exit=exits_if,
oparg=oparg_used(op),
jumps=variable_used(op, "JUMPBY"),
- eval_breaker=variable_used(op, "CHECK_EVAL_BREAKER"),
- ends_with_eval_breaker=eval_breaker_at_end(op),
+ eval_breaker="CHECK_PERIODIC" in op.name,
needs_this=variable_used(op, "this_instr"),
always_exits=always_exits(op),
stores_sp=variable_used(op, "SYNC_SP"),
"ERROR_IF": self.error_if,
"ERROR_NO_POP": self.error_no_pop,
"DECREF_INPUTS": self.decref_inputs,
- "CHECK_EVAL_BREAKER": self.check_eval_breaker,
"SYNC_SP": self.sync_sp,
"PyStackRef_FromPyObjectNew": self.py_stack_ref_from_py_object_new,
}
next(tkn_iter)
stack.flush(self.out)
- def check_eval_breaker(
- self,
- tkn: Token,
- tkn_iter: Iterator[Token],
- uop: Uop,
- stack: Stack,
- inst: Instruction | None,
- ) -> None:
- next(tkn_iter)
- next(tkn_iter)
- next(tkn_iter)
- if not uop.properties.ends_with_eval_breaker:
- self.out.emit_at("CHECK_EVAL_BREAKER();", tkn)
-
def py_stack_ref_from_py_object_new(
self,
tkn: Token,
out.start_line()
if not inst.parts[-1].properties.always_exits:
stack.flush(out)
- if inst.parts[-1].properties.ends_with_eval_breaker:
- out.emit("CHECK_EVAL_BREAKER();\n")
out.emit("DISPATCH();\n")
out.start_line()
out.emit("}")
out.start_line()
if not uop.properties.always_exits:
stack.flush(out)
- if uop.properties.ends_with_eval_breaker:
- out.emit("CHECK_EVAL_BREAKER();\n")
out.emit("break;\n")
out.start_line()
out.emit("}")
projects / thirdparty / Python / cpython.git / commitdiff
