return op->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS);
}
+static inline int
+PyLong_CheckCompact(const PyObject *op)
+{
+ return PyLong_CheckExact(op) && _PyLong_IsCompact((PyLongObject *)op);
+}
+
#define PyUnstable_Long_IsCompact _PyLong_IsCompact
static inline Py_ssize_t
// Export for 'math' shared extension
PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t);
-PyAPI_FUNC(PyObject*) _PyCompactLong_Add(PyLongObject *left, PyLongObject *right);
-PyAPI_FUNC(PyObject*) _PyCompactLong_Multiply(PyLongObject *left, PyLongObject *right);
-PyAPI_FUNC(PyObject*) _PyCompactLong_Subtract(PyLongObject *left, PyLongObject *right);
+PyAPI_FUNC(_PyStackRef) _PyCompactLong_Add(PyLongObject *left, PyLongObject *right);
+PyAPI_FUNC(_PyStackRef) _PyCompactLong_Multiply(PyLongObject *left, PyLongObject *right);
+PyAPI_FUNC(_PyStackRef) _PyCompactLong_Subtract(PyLongObject *left, PyLongObject *right);
// Export for 'binascii' shared extension.
PyAPI_DATA(unsigned char) _PyLong_DigitValue[256];
assert(PyLong_Check(b));
return (a->long_value.lv_tag | b->long_value.lv_tag) < (2 << NON_SIZE_BITS);
}
-
static inline bool
_PyLong_IsZero(const PyLongObject *op)
{
const struct opcode_metadata _PyOpcode_opcode_metadata[267] = {
[BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
- [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
+ [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG },
[BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
- [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
+ [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG },
[BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG },
[BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
- [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
+ [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG },
[BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BUILD_INTERPOLATION] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG },
- [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG },
[COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG },
[CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
JIT_SYM_KNOWN_VALUE_TAG = 7,
JIT_SYM_TUPLE_TAG = 8,
JIT_SYM_TRUTHINESS_TAG = 9,
+ JIT_SYM_COMPACT_INT = 10,
} JitSymType;
typedef struct _jit_opt_known_class {
uint16_t value;
} JitOptTruthiness;
+typedef struct {
+ uint8_t tag;
+} JitOptCompactInt;
+
typedef union _jit_opt_symbol {
uint8_t tag;
JitOptKnownClass cls;
JitOptKnownVersion version;
JitOptTuple tuple;
JitOptTruthiness truthiness;
+ JitOptCompactInt compact;
} JitOptSymbol;
extern JitOptRef _Py_uop_sym_new_not_null(JitOptContext *ctx);
extern JitOptRef _Py_uop_sym_new_type(
JitOptContext *ctx, PyTypeObject *typ);
+
extern JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val);
extern JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx);
extern bool _Py_uop_sym_has_type(JitOptRef sym);
extern JitOptRef _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef sym, int item);
extern int _Py_uop_sym_tuple_length(JitOptRef sym);
extern JitOptRef _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef value, bool truthy);
+extern bool _Py_uop_sym_is_compact_int(JitOptRef sym);
+extern JitOptRef _Py_uop_sym_new_compact_int(JitOptContext *ctx);
+extern void _Py_uop_sym_set_compact_int(JitOptContext *ctx, JitOptRef sym);
extern void _Py_uop_abstractcontext_init(JitOptContext *ctx);
extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx);
#define _GUARD_NOS_LIST 401
#define _GUARD_NOS_NOT_NULL 402
#define _GUARD_NOS_NULL 403
-#define _GUARD_NOS_TUPLE 404
-#define _GUARD_NOS_UNICODE 405
-#define _GUARD_NOT_EXHAUSTED_LIST 406
-#define _GUARD_NOT_EXHAUSTED_RANGE 407
-#define _GUARD_NOT_EXHAUSTED_TUPLE 408
-#define _GUARD_THIRD_NULL 409
-#define _GUARD_TOS_ANY_SET 410
-#define _GUARD_TOS_DICT 411
-#define _GUARD_TOS_FLOAT 412
-#define _GUARD_TOS_INT 413
-#define _GUARD_TOS_LIST 414
-#define _GUARD_TOS_SLICE 415
-#define _GUARD_TOS_TUPLE 416
-#define _GUARD_TOS_UNICODE 417
-#define _GUARD_TYPE_VERSION 418
-#define _GUARD_TYPE_VERSION_AND_LOCK 419
+#define _GUARD_NOS_OVERFLOWED 404
+#define _GUARD_NOS_TUPLE 405
+#define _GUARD_NOS_UNICODE 406
+#define _GUARD_NOT_EXHAUSTED_LIST 407
+#define _GUARD_NOT_EXHAUSTED_RANGE 408
+#define _GUARD_NOT_EXHAUSTED_TUPLE 409
+#define _GUARD_THIRD_NULL 410
+#define _GUARD_TOS_ANY_SET 411
+#define _GUARD_TOS_DICT 412
+#define _GUARD_TOS_FLOAT 413
+#define _GUARD_TOS_INT 414
+#define _GUARD_TOS_LIST 415
+#define _GUARD_TOS_OVERFLOWED 416
+#define _GUARD_TOS_SLICE 417
+#define _GUARD_TOS_TUPLE 418
+#define _GUARD_TOS_UNICODE 419
+#define _GUARD_TYPE_VERSION 420
+#define _GUARD_TYPE_VERSION_AND_LOCK 421
#define _IMPORT_FROM IMPORT_FROM
#define _IMPORT_NAME IMPORT_NAME
-#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 420
-#define _INIT_CALL_PY_EXACT_ARGS 421
-#define _INIT_CALL_PY_EXACT_ARGS_0 422
-#define _INIT_CALL_PY_EXACT_ARGS_1 423
-#define _INIT_CALL_PY_EXACT_ARGS_2 424
-#define _INIT_CALL_PY_EXACT_ARGS_3 425
-#define _INIT_CALL_PY_EXACT_ARGS_4 426
-#define _INSERT_NULL 427
+#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 422
+#define _INIT_CALL_PY_EXACT_ARGS 423
+#define _INIT_CALL_PY_EXACT_ARGS_0 424
+#define _INIT_CALL_PY_EXACT_ARGS_1 425
+#define _INIT_CALL_PY_EXACT_ARGS_2 426
+#define _INIT_CALL_PY_EXACT_ARGS_3 427
+#define _INIT_CALL_PY_EXACT_ARGS_4 428
+#define _INSERT_NULL 429
#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER
#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION
#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD
#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE
#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE
#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE
-#define _IS_NONE 428
+#define _IS_NONE 430
#define _IS_OP IS_OP
-#define _ITER_CHECK_LIST 429
-#define _ITER_CHECK_RANGE 430
-#define _ITER_CHECK_TUPLE 431
-#define _ITER_JUMP_LIST 432
-#define _ITER_JUMP_RANGE 433
-#define _ITER_JUMP_TUPLE 434
-#define _ITER_NEXT_LIST 435
-#define _ITER_NEXT_LIST_TIER_TWO 436
-#define _ITER_NEXT_RANGE 437
-#define _ITER_NEXT_TUPLE 438
-#define _JUMP_TO_TOP 439
+#define _ITER_CHECK_LIST 431
+#define _ITER_CHECK_RANGE 432
+#define _ITER_CHECK_TUPLE 433
+#define _ITER_JUMP_LIST 434
+#define _ITER_JUMP_RANGE 435
+#define _ITER_JUMP_TUPLE 436
+#define _ITER_NEXT_LIST 437
+#define _ITER_NEXT_LIST_TIER_TWO 438
+#define _ITER_NEXT_RANGE 439
+#define _ITER_NEXT_TUPLE 440
+#define _JUMP_TO_TOP 441
#define _LIST_APPEND LIST_APPEND
#define _LIST_EXTEND LIST_EXTEND
-#define _LOAD_ATTR 440
-#define _LOAD_ATTR_CLASS 441
+#define _LOAD_ATTR 442
+#define _LOAD_ATTR_CLASS 443
#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
-#define _LOAD_ATTR_INSTANCE_VALUE 442
-#define _LOAD_ATTR_METHOD_LAZY_DICT 443
-#define _LOAD_ATTR_METHOD_NO_DICT 444
-#define _LOAD_ATTR_METHOD_WITH_VALUES 445
-#define _LOAD_ATTR_MODULE 446
-#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 447
-#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 448
-#define _LOAD_ATTR_PROPERTY_FRAME 449
-#define _LOAD_ATTR_SLOT 450
-#define _LOAD_ATTR_WITH_HINT 451
+#define _LOAD_ATTR_INSTANCE_VALUE 444
+#define _LOAD_ATTR_METHOD_LAZY_DICT 445
+#define _LOAD_ATTR_METHOD_NO_DICT 446
+#define _LOAD_ATTR_METHOD_WITH_VALUES 447
+#define _LOAD_ATTR_MODULE 448
+#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 449
+#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 450
+#define _LOAD_ATTR_PROPERTY_FRAME 451
+#define _LOAD_ATTR_SLOT 452
+#define _LOAD_ATTR_WITH_HINT 453
#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
-#define _LOAD_BYTECODE 452
+#define _LOAD_BYTECODE 454
#define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT
#define _LOAD_CONST LOAD_CONST
-#define _LOAD_CONST_INLINE 453
-#define _LOAD_CONST_INLINE_BORROW 454
-#define _LOAD_CONST_UNDER_INLINE 455
-#define _LOAD_CONST_UNDER_INLINE_BORROW 456
+#define _LOAD_CONST_INLINE 455
+#define _LOAD_CONST_INLINE_BORROW 456
+#define _LOAD_CONST_UNDER_INLINE 457
+#define _LOAD_CONST_UNDER_INLINE_BORROW 458
#define _LOAD_DEREF LOAD_DEREF
-#define _LOAD_FAST 457
-#define _LOAD_FAST_0 458
-#define _LOAD_FAST_1 459
-#define _LOAD_FAST_2 460
-#define _LOAD_FAST_3 461
-#define _LOAD_FAST_4 462
-#define _LOAD_FAST_5 463
-#define _LOAD_FAST_6 464
-#define _LOAD_FAST_7 465
+#define _LOAD_FAST 459
+#define _LOAD_FAST_0 460
+#define _LOAD_FAST_1 461
+#define _LOAD_FAST_2 462
+#define _LOAD_FAST_3 463
+#define _LOAD_FAST_4 464
+#define _LOAD_FAST_5 465
+#define _LOAD_FAST_6 466
+#define _LOAD_FAST_7 467
#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
-#define _LOAD_FAST_BORROW 466
-#define _LOAD_FAST_BORROW_0 467
-#define _LOAD_FAST_BORROW_1 468
-#define _LOAD_FAST_BORROW_2 469
-#define _LOAD_FAST_BORROW_3 470
-#define _LOAD_FAST_BORROW_4 471
-#define _LOAD_FAST_BORROW_5 472
-#define _LOAD_FAST_BORROW_6 473
-#define _LOAD_FAST_BORROW_7 474
+#define _LOAD_FAST_BORROW 468
+#define _LOAD_FAST_BORROW_0 469
+#define _LOAD_FAST_BORROW_1 470
+#define _LOAD_FAST_BORROW_2 471
+#define _LOAD_FAST_BORROW_3 472
+#define _LOAD_FAST_BORROW_4 473
+#define _LOAD_FAST_BORROW_5 474
+#define _LOAD_FAST_BORROW_6 475
+#define _LOAD_FAST_BORROW_7 476
#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW
#define _LOAD_FAST_CHECK LOAD_FAST_CHECK
#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
-#define _LOAD_GLOBAL 475
-#define _LOAD_GLOBAL_BUILTINS 476
-#define _LOAD_GLOBAL_MODULE 477
+#define _LOAD_GLOBAL 477
+#define _LOAD_GLOBAL_BUILTINS 478
+#define _LOAD_GLOBAL_MODULE 479
#define _LOAD_LOCALS LOAD_LOCALS
#define _LOAD_NAME LOAD_NAME
-#define _LOAD_SMALL_INT 478
-#define _LOAD_SMALL_INT_0 479
-#define _LOAD_SMALL_INT_1 480
-#define _LOAD_SMALL_INT_2 481
-#define _LOAD_SMALL_INT_3 482
-#define _LOAD_SPECIAL 483
+#define _LOAD_SMALL_INT 480
+#define _LOAD_SMALL_INT_0 481
+#define _LOAD_SMALL_INT_1 482
+#define _LOAD_SMALL_INT_2 483
+#define _LOAD_SMALL_INT_3 484
+#define _LOAD_SPECIAL 485
#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR
#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD
-#define _MAKE_CALLARGS_A_TUPLE 484
+#define _MAKE_CALLARGS_A_TUPLE 486
#define _MAKE_CELL MAKE_CELL
#define _MAKE_FUNCTION MAKE_FUNCTION
-#define _MAKE_WARM 485
+#define _MAKE_WARM 487
#define _MAP_ADD MAP_ADD
#define _MATCH_CLASS MATCH_CLASS
#define _MATCH_KEYS MATCH_KEYS
#define _MATCH_MAPPING MATCH_MAPPING
#define _MATCH_SEQUENCE MATCH_SEQUENCE
-#define _MAYBE_EXPAND_METHOD 486
-#define _MAYBE_EXPAND_METHOD_KW 487
-#define _MONITOR_CALL 488
-#define _MONITOR_CALL_KW 489
-#define _MONITOR_JUMP_BACKWARD 490
-#define _MONITOR_RESUME 491
+#define _MAYBE_EXPAND_METHOD 488
+#define _MAYBE_EXPAND_METHOD_KW 489
+#define _MONITOR_CALL 490
+#define _MONITOR_CALL_KW 491
+#define _MONITOR_JUMP_BACKWARD 492
+#define _MONITOR_RESUME 493
#define _NOP NOP
-#define _POP_CALL 492
-#define _POP_CALL_LOAD_CONST_INLINE_BORROW 493
-#define _POP_CALL_ONE 494
-#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 495
-#define _POP_CALL_TWO 496
-#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 497
+#define _POP_CALL 494
+#define _POP_CALL_LOAD_CONST_INLINE_BORROW 495
+#define _POP_CALL_ONE 496
+#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 497
+#define _POP_CALL_TWO 498
+#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 499
#define _POP_EXCEPT POP_EXCEPT
#define _POP_ITER POP_ITER
-#define _POP_JUMP_IF_FALSE 498
-#define _POP_JUMP_IF_TRUE 499
+#define _POP_JUMP_IF_FALSE 500
+#define _POP_JUMP_IF_TRUE 501
#define _POP_TOP POP_TOP
-#define _POP_TOP_LOAD_CONST_INLINE 500
-#define _POP_TOP_LOAD_CONST_INLINE_BORROW 501
-#define _POP_TWO 502
-#define _POP_TWO_LOAD_CONST_INLINE_BORROW 503
+#define _POP_TOP_LOAD_CONST_INLINE 502
+#define _POP_TOP_LOAD_CONST_INLINE_BORROW 503
+#define _POP_TWO 504
+#define _POP_TWO_LOAD_CONST_INLINE_BORROW 505
#define _PUSH_EXC_INFO PUSH_EXC_INFO
-#define _PUSH_FRAME 504
+#define _PUSH_FRAME 506
#define _PUSH_NULL PUSH_NULL
-#define _PUSH_NULL_CONDITIONAL 505
-#define _PY_FRAME_GENERAL 506
-#define _PY_FRAME_KW 507
-#define _QUICKEN_RESUME 508
-#define _REPLACE_WITH_TRUE 509
+#define _PUSH_NULL_CONDITIONAL 507
+#define _PY_FRAME_GENERAL 508
+#define _PY_FRAME_KW 509
+#define _QUICKEN_RESUME 510
+#define _REPLACE_WITH_TRUE 511
#define _RESUME_CHECK RESUME_CHECK
#define _RETURN_GENERATOR RETURN_GENERATOR
#define _RETURN_VALUE RETURN_VALUE
-#define _SAVE_RETURN_OFFSET 510
-#define _SEND 511
-#define _SEND_GEN_FRAME 512
+#define _SAVE_RETURN_OFFSET 512
+#define _SEND 513
+#define _SEND_GEN_FRAME 514
#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
#define _SET_ADD SET_ADD
#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
#define _SET_UPDATE SET_UPDATE
-#define _START_EXECUTOR 513
-#define _STORE_ATTR 514
-#define _STORE_ATTR_INSTANCE_VALUE 515
-#define _STORE_ATTR_SLOT 516
-#define _STORE_ATTR_WITH_HINT 517
+#define _START_EXECUTOR 515
+#define _STORE_ATTR 516
+#define _STORE_ATTR_INSTANCE_VALUE 517
+#define _STORE_ATTR_SLOT 518
+#define _STORE_ATTR_WITH_HINT 519
#define _STORE_DEREF STORE_DEREF
-#define _STORE_FAST 518
-#define _STORE_FAST_0 519
-#define _STORE_FAST_1 520
-#define _STORE_FAST_2 521
-#define _STORE_FAST_3 522
-#define _STORE_FAST_4 523
-#define _STORE_FAST_5 524
-#define _STORE_FAST_6 525
-#define _STORE_FAST_7 526
+#define _STORE_FAST 520
+#define _STORE_FAST_0 521
+#define _STORE_FAST_1 522
+#define _STORE_FAST_2 523
+#define _STORE_FAST_3 524
+#define _STORE_FAST_4 525
+#define _STORE_FAST_5 526
+#define _STORE_FAST_6 527
+#define _STORE_FAST_7 528
#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
#define _STORE_GLOBAL STORE_GLOBAL
#define _STORE_NAME STORE_NAME
-#define _STORE_SLICE 527
-#define _STORE_SUBSCR 528
-#define _STORE_SUBSCR_DICT 529
-#define _STORE_SUBSCR_LIST_INT 530
-#define _SWAP 531
-#define _SWAP_2 532
-#define _SWAP_3 533
-#define _TIER2_RESUME_CHECK 534
-#define _TO_BOOL 535
+#define _STORE_SLICE 529
+#define _STORE_SUBSCR 530
+#define _STORE_SUBSCR_DICT 531
+#define _STORE_SUBSCR_LIST_INT 532
+#define _SWAP 533
+#define _SWAP_2 534
+#define _SWAP_3 535
+#define _TIER2_RESUME_CHECK 536
+#define _TO_BOOL 537
#define _TO_BOOL_BOOL TO_BOOL_BOOL
#define _TO_BOOL_INT TO_BOOL_INT
-#define _TO_BOOL_LIST 536
+#define _TO_BOOL_LIST 538
#define _TO_BOOL_NONE TO_BOOL_NONE
-#define _TO_BOOL_STR 537
+#define _TO_BOOL_STR 539
#define _UNARY_INVERT UNARY_INVERT
#define _UNARY_NEGATIVE UNARY_NEGATIVE
#define _UNARY_NOT UNARY_NOT
#define _UNPACK_EX UNPACK_EX
-#define _UNPACK_SEQUENCE 538
-#define _UNPACK_SEQUENCE_LIST 539
-#define _UNPACK_SEQUENCE_TUPLE 540
-#define _UNPACK_SEQUENCE_TWO_TUPLE 541
+#define _UNPACK_SEQUENCE 540
+#define _UNPACK_SEQUENCE_LIST 541
+#define _UNPACK_SEQUENCE_TUPLE 542
+#define _UNPACK_SEQUENCE_TWO_TUPLE 543
#define _WITH_EXCEPT_START WITH_EXCEPT_START
#define _YIELD_VALUE YIELD_VALUE
-#define MAX_UOP_ID 541
+#define MAX_UOP_ID 543
#ifdef __cplusplus
}
[_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_NOS_INT] = HAS_EXIT_FLAG,
[_GUARD_TOS_INT] = HAS_EXIT_FLAG,
- [_BINARY_OP_MULTIPLY_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG,
- [_BINARY_OP_ADD_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG,
- [_BINARY_OP_SUBTRACT_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG,
+ [_GUARD_NOS_OVERFLOWED] = HAS_EXIT_FLAG,
+ [_GUARD_TOS_OVERFLOWED] = HAS_EXIT_FLAG,
+ [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG,
+ [_BINARY_OP_ADD_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG,
+ [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG,
[_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG,
[_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG,
[_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
[_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
[_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_COMPARE_OP_FLOAT] = HAS_ARG_FLAG,
- [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
+ [_COMPARE_OP_INT] = HAS_ARG_FLAG,
[_COMPARE_OP_STR] = HAS_ARG_FLAG,
[_IS_OP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
[_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_NOS_LIST] = "_GUARD_NOS_LIST",
[_GUARD_NOS_NOT_NULL] = "_GUARD_NOS_NOT_NULL",
[_GUARD_NOS_NULL] = "_GUARD_NOS_NULL",
+ [_GUARD_NOS_OVERFLOWED] = "_GUARD_NOS_OVERFLOWED",
[_GUARD_NOS_TUPLE] = "_GUARD_NOS_TUPLE",
[_GUARD_NOS_UNICODE] = "_GUARD_NOS_UNICODE",
[_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST",
[_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT",
[_GUARD_TOS_INT] = "_GUARD_TOS_INT",
[_GUARD_TOS_LIST] = "_GUARD_TOS_LIST",
+ [_GUARD_TOS_OVERFLOWED] = "_GUARD_TOS_OVERFLOWED",
[_GUARD_TOS_SLICE] = "_GUARD_TOS_SLICE",
[_GUARD_TOS_TUPLE] = "_GUARD_TOS_TUPLE",
[_GUARD_TOS_UNICODE] = "_GUARD_TOS_UNICODE",
return 0;
case _GUARD_TOS_INT:
return 0;
+ case _GUARD_NOS_OVERFLOWED:
+ return 0;
+ case _GUARD_TOS_OVERFLOWED:
+ return 0;
case _BINARY_OP_MULTIPLY_INT:
return 2;
case _BINARY_OP_ADD_INT:
x = 0
for i in range(m):
for j in MyIter(n):
- x += 1000*i + j
+ x += j
return x
- x = testfunc(TIER2_THRESHOLD, TIER2_THRESHOLD)
+ x = testfunc(TIER2_THRESHOLD, 2)
- self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * TIER2_THRESHOLD * 1001)
+ self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * 2)
ex = get_first_executor(testfunc)
self.assertIsNotNone(ex)
--- /dev/null
+Changes specialization of ``BINARY_OP`` for ints to only specialize for
+"compact" ints. This streamlines the fast path at the cost of fewer
+specializations when very large integers are used.
#include "pycore_long.h" // _Py_SmallInts
#include "pycore_object.h" // _PyObject_Init()
#include "pycore_runtime.h" // _PY_NSMALLPOSINTS
+#include "pycore_stackref.h"
#include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin()
#include "pycore_unicodeobject.h" // _PyUnicode_Equal()
return (PyLongObject*)_PyLong_FromLarge(x);
}
+/* Create a new medium int object from a medium int.
+ * Do not raise. Return NULL if not medium or can't allocate. */
+static inline _PyStackRef
+medium_from_stwodigits(stwodigits x)
+{
+ if (IS_SMALL_INT(x)) {
+ return PyStackRef_FromPyObjectBorrow(get_small_int((sdigit)x));
+ }
+ assert(x != 0);
+ if(!is_medium_int(x)) {
+ return PyStackRef_NULL;
+ }
+ PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints);
+ if (v == NULL) {
+ v = PyObject_Malloc(sizeof(PyLongObject));
+ if (v == NULL) {
+ return PyStackRef_NULL;
+ }
+ _PyObject_Init((PyObject*)v, &PyLong_Type);
+ }
+ digit abs_x = x < 0 ? -x : x;
+ _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1);
+ v->long_value.ob_digit[0] = abs_x;
+ return PyStackRef_FromPyObjectStealMortal((PyObject *)v);
+}
+
+
/* If a freshly-allocated int is already shared, it must
be a small integer, so negating it must go to PyLong_FromLong */
Py_LOCAL_INLINE(void)
return z;
}
-PyObject *
+_PyStackRef
_PyCompactLong_Add(PyLongObject *a, PyLongObject *b)
{
assert(_PyLong_BothAreCompact(a, b));
- stwodigits z = medium_value(a) + medium_value(b);
- return (PyObject *)_PyLong_FromSTwoDigits(z);
+ stwodigits v = medium_value(a) + medium_value(b);
+ return medium_from_stwodigits(v);
}
static PyObject *
return z;
}
-PyObject *
+_PyStackRef
_PyCompactLong_Subtract(PyLongObject *a, PyLongObject *b)
{
assert(_PyLong_BothAreCompact(a, b));
- return (PyObject *)_PyLong_FromSTwoDigits(medium_value(a) - medium_value(b));
+ stwodigits v = medium_value(a) - medium_value(b);
+ return medium_from_stwodigits(v);
}
static PyObject *
return z;
}
-PyObject *
+/* This function returns NULL if the result is not compact,
+ * or if it fails to allocate, but never raises */
+_PyStackRef
_PyCompactLong_Multiply(PyLongObject *a, PyLongObject *b)
{
assert(_PyLong_BothAreCompact(a, b));
stwodigits v = medium_value(a) * medium_value(b);
- return (PyObject *)_PyLong_FromSTwoDigits(v);
+ return medium_from_stwodigits(v);
}
static PyObject *
op(_GUARD_NOS_INT, (left, unused -- left, unused)) {
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- EXIT_IF(!PyLong_CheckExact(left_o));
+ EXIT_IF(!PyLong_CheckCompact(left_o));
}
op(_GUARD_TOS_INT, (value -- value)) {
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- EXIT_IF(!PyLong_CheckExact(value_o));
+ EXIT_IF(!PyLong_CheckCompact(value_o));
+ }
+
+ op(_GUARD_NOS_OVERFLOWED, (left, unused -- left, unused)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ assert(Py_TYPE(left_o) == &PyLong_Type);
+ EXIT_IF(!_PyLong_IsCompact((PyLongObject *)left_o));
+ }
+
+ op(_GUARD_TOS_OVERFLOWED, (value -- value)) {
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ assert(Py_TYPE(value_o) == &PyLong_Type);
+ EXIT_IF(!_PyLong_IsCompact((PyLongObject *)value_o));
}
pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) {
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
- DEOPT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
+ res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
+ EXIT_IF(PyStackRef_IsNull(res));
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
INPUTS_DEAD();
- ERROR_IF(res_o == NULL);
- res = PyStackRef_FromPyObjectSteal(res_o);
}
pure op(_BINARY_OP_ADD_INT, (left, right -- res)) {
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
- DEOPT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
+ res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
+ EXIT_IF(PyStackRef_IsNull(res));
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
INPUTS_DEAD();
- ERROR_IF(res_o == NULL);
- res = PyStackRef_FromPyObjectSteal(res_o);
}
pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) {
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
- DEOPT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
+ res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
+ EXIT_IF(PyStackRef_IsNull(res));
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
INPUTS_DEAD();
- ERROR_IF(res_o == NULL);
- res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(BINARY_OP_MULTIPLY_INT) =
_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_MULTIPLY_INT;
+
macro(BINARY_OP_ADD_INT) =
_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT;
+
macro(BINARY_OP_SUBTRACT_INT) =
_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o));
- DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o));
+ assert(_PyLong_IsCompact((PyLongObject *)left_o));
+ assert(_PyLong_IsCompact((PyLongObject *)right_o));
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right_o) <= 1);
_PyStackRef left;
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- if (!PyLong_CheckExact(left_o)) {
+ if (!PyLong_CheckCompact(left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
_PyStackRef value;
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- if (!PyLong_CheckExact(value_o)) {
+ if (!PyLong_CheckCompact(value_o)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
+ case _GUARD_NOS_OVERFLOWED: {
+ _PyStackRef left;
+ left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ assert(Py_TYPE(left_o) == &PyLong_Type);
+ if (!_PyLong_IsCompact((PyLongObject *)left_o)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ break;
+ }
+
+ case _GUARD_TOS_OVERFLOWED: {
+ _PyStackRef value;
+ value = stack_pointer[-1];
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ assert(Py_TYPE(value_o) == &PyLong_Type);
+ if (!_PyLong_IsCompact((PyLongObject *)value_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
- if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
+ STAT_INC(BINARY_OP, hit);
+ res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- JUMP_TO_ERROR();
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
- if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
+ STAT_INC(BINARY_OP, hit);
+ res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- JUMP_TO_ERROR();
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
- if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
+ STAT_INC(BINARY_OP, hit);
+ res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- JUMP_TO_ERROR();
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- if (!_PyLong_IsCompact((PyLongObject *)left_o)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
- if (!_PyLong_IsCompact((PyLongObject *)right_o)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
+ assert(_PyLong_IsCompact((PyLongObject *)left_o));
+ assert(_PyLong_IsCompact((PyLongObject *)right_o));
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right_o) <= 1);
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- if (!PyLong_CheckExact(value_o)) {
+ if (!PyLong_CheckCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
{
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- if (!PyLong_CheckExact(left_o)) {
+ if (!PyLong_CheckCompact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
- if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
+ STAT_INC(BINARY_OP, hit);
+ res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
- STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- JUMP_TO_LABEL(pop_2_error);
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- if (!PyLong_CheckExact(value_o)) {
+ if (!PyLong_CheckCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
{
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- if (!PyLong_CheckExact(left_o)) {
+ if (!PyLong_CheckCompact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
- if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
+ STAT_INC(BINARY_OP, hit);
+ res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
- STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- JUMP_TO_LABEL(pop_2_error);
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- if (!PyLong_CheckExact(value_o)) {
+ if (!PyLong_CheckCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- if (!PyLong_CheckExact(value_o)) {
+ if (!PyLong_CheckCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- if (!PyLong_CheckExact(value_o)) {
+ if (!PyLong_CheckCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- if (!PyLong_CheckExact(value_o)) {
+ if (!PyLong_CheckCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
{
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- if (!PyLong_CheckExact(left_o)) {
+ if (!PyLong_CheckCompact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
- if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
+ STAT_INC(BINARY_OP, hit);
+ res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
+ if (PyStackRef_IsNull(res)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
- STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- JUMP_TO_LABEL(pop_2_error);
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- if (!PyLong_CheckExact(value_o)) {
+ if (!PyLong_CheckCompact(value_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
JUMP_TO_PREDICTED(COMPARE_OP);
{
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- if (!PyLong_CheckExact(left_o)) {
+ if (!PyLong_CheckCompact(left_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
JUMP_TO_PREDICTED(COMPARE_OP);
right = value;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- if (!_PyLong_IsCompact((PyLongObject *)left_o)) {
- UPDATE_MISS_STATS(COMPARE_OP);
- assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
- JUMP_TO_PREDICTED(COMPARE_OP);
- }
- if (!_PyLong_IsCompact((PyLongObject *)right_o)) {
- UPDATE_MISS_STATS(COMPARE_OP);
- assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
- JUMP_TO_PREDICTED(COMPARE_OP);
- }
+ assert(_PyLong_IsCompact((PyLongObject *)left_o));
+ assert(_PyLong_IsCompact((PyLongObject *)right_o));
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right_o) <= 1);
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- if (!PyLong_CheckExact(value_o)) {
+ if (!PyLong_CheckCompact(value_o)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE)
#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION)
#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST)
+#define sym_set_compact_int(SYM) _Py_uop_sym_set_compact_int(ctx, SYM)
#define sym_is_bottom _Py_uop_sym_is_bottom
#define sym_truthiness _Py_uop_sym_truthiness
#define frame_new _Py_uop_frame_new
#define sym_tuple_getitem _Py_uop_sym_tuple_getitem
#define sym_tuple_length _Py_uop_sym_tuple_length
#define sym_is_immortal _Py_uop_sym_is_immortal
+#define sym_is_compact_int _Py_uop_sym_is_compact_int
+#define sym_new_compact_int _Py_uop_sym_new_compact_int
#define sym_new_truthiness _Py_uop_sym_new_truthiness
static int
#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE)
#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION)
#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST)
+#define sym_set_compact_int(SYM) _Py_uop_sym_set_compact_int(ctx, SYM)
#define sym_is_bottom _Py_uop_sym_is_bottom
#define frame_new _Py_uop_frame_new
#define frame_pop _Py_uop_frame_pop
#define sym_tuple_getitem _Py_uop_sym_tuple_getitem
#define sym_tuple_length _Py_uop_sym_tuple_length
#define sym_is_immortal _Py_uop_sym_is_immortal
+#define sym_new_compact_int _Py_uop_sym_new_compact_int
+#define sym_is_compact_int _Py_uop_sym_is_compact_int
#define sym_new_truthiness _Py_uop_sym_new_truthiness
extern int
}
op(_GUARD_TOS_INT, (value -- value)) {
- if (sym_matches_type(value, &PyLong_Type)) {
+ if (sym_is_compact_int(value)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
- sym_set_type(value, &PyLong_Type);
+ else {
+ if (sym_get_type(value) == &PyLong_Type) {
+ REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0);
+ }
+ sym_set_compact_int(value);
+ }
}
op(_GUARD_NOS_INT, (left, unused -- left, unused)) {
- if (sym_matches_type(left, &PyLong_Type)) {
+ if (sym_is_compact_int(left)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
- sym_set_type(left, &PyLong_Type);
+ else {
+ if (sym_get_type(left) == &PyLong_Type) {
+ REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0);
+ }
+ sym_set_compact_int(left);
+ }
}
op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) {
}
op(_BINARY_OP_ADD_INT, (left, right -- res)) {
- res = sym_new_type(ctx, &PyLong_Type);
+ res = sym_new_compact_int(ctx);
}
op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) {
- res = sym_new_type(ctx, &PyLong_Type);
+ res = sym_new_compact_int(ctx);
}
op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) {
- res = sym_new_type(ctx, &PyLong_Type);
+ res = sym_new_compact_int(ctx);
}
op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) {
res = sym_new_truthiness(ctx, value, false);
}
+ op(_UNARY_NEGATIVE, (value -- res)) {
+ if (sym_is_compact_int(value)) {
+ res = sym_new_compact_int(ctx);
+ }
+ else {
+ res = sym_new_not_null(ctx);
+ }
+ }
+
op(_UNARY_INVERT, (value -- res)) {
if (sym_matches_type(value, &PyLong_Type)) {
res = sym_new_type(ctx, &PyLong_Type);
}
case _UNARY_NEGATIVE: {
+ JitOptRef value;
JitOptRef res;
- res = sym_new_not_null(ctx);
+ value = stack_pointer[-1];
+ if (sym_is_compact_int(value)) {
+ res = sym_new_compact_int(ctx);
+ }
+ else {
+ res = sym_new_not_null(ctx);
+ }
stack_pointer[-1] = res;
break;
}
case _GUARD_NOS_INT: {
JitOptRef left;
left = stack_pointer[-2];
- if (sym_matches_type(left, &PyLong_Type)) {
+ if (sym_is_compact_int(left)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
- sym_set_type(left, &PyLong_Type);
+ else {
+ if (sym_get_type(left) == &PyLong_Type) {
+ REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0);
+ }
+ sym_set_compact_int(left);
+ }
break;
}
case _GUARD_TOS_INT: {
JitOptRef value;
value = stack_pointer[-1];
- if (sym_matches_type(value, &PyLong_Type)) {
+ if (sym_is_compact_int(value)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
- sym_set_type(value, &PyLong_Type);
+ else {
+ if (sym_get_type(value) == &PyLong_Type) {
+ REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0);
+ }
+ sym_set_compact_int(value);
+ }
+ break;
+ }
+
+ case _GUARD_NOS_OVERFLOWED: {
+ break;
+ }
+
+ case _GUARD_TOS_OVERFLOWED: {
break;
}
case _BINARY_OP_MULTIPLY_INT: {
JitOptRef res;
- res = sym_new_type(ctx, &PyLong_Type);
+ res = sym_new_compact_int(ctx);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
case _BINARY_OP_ADD_INT: {
JitOptRef res;
- res = sym_new_type(ctx, &PyLong_Type);
+ res = sym_new_compact_int(ctx);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
case _BINARY_OP_SUBTRACT_INT: {
JitOptRef res;
- res = sym_new_type(ctx, &PyLong_Type);
+ res = sym_new_compact_int(ctx);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
| |
NULL |
| | <- Anything below this level is an object.
-| NON_NULL
-| | | <- Anything below this level has a known type version.
-| TYPE_VERSION |
-| | | <- Anything below this level has a known type.
-| KNOWN_CLASS |
-| | | | <- Anything below this level has a known truthiness.
-| | | TRUTHINESS
-| | | |
-| TUPLE | |
-| | | | <- Anything below this level is a known constant.
-| KNOWN_VALUE
+| NON_NULL-+
+| | | <- Anything below this level has a known type version.
+| TYPE_VERSION |
+| | | <- Anything below this level has a known type.
+| KNOWN_CLASS |
+| | | | | |
+| | | INT* | |
+| | | | | | <- Anything below this level has a known truthiness.
+| | | | | TRUTHINESS
+| | | | | |
+| TUPLE | | | |
+| | | | | | <- Anything below this level is a known constant.
+| KNOWN_VALUE--+
| | <- Anything below this level is unreachable.
BOTTOM
the same symbol, that would be a contradiction, and the symbol would be set to
BOTTOM (indicating that the code is unreachable).
+INT* is a limited range int, currently a "compact" int.
+
*/
#ifdef Py_DEBUG
sym_set_bottom(ctx, sym);
}
return;
+ case JIT_SYM_COMPACT_INT:
+ if (typ != &PyLong_Type) {
+ sym_set_bottom(ctx, sym);
+ }
+ return;
}
}
return false;
}
return true;
+ case JIT_SYM_COMPACT_INT:
+ if (version != PyLong_Type.tp_version_tag) {
+ sym_set_bottom(ctx, sym);
+ return false;
+ }
+ return true;
}
Py_UNREACHABLE();
}
// TODO: More types (GH-130415)!
make_const(sym, const_val);
return;
+ case JIT_SYM_COMPACT_INT:
+ if (PyLong_CheckCompact(const_val)) {
+ make_const(sym, const_val);
+ }
+ else {
+ sym_set_bottom(ctx, sym);
+ }
+ return;
}
}
return &PyTuple_Type;
case JIT_SYM_TRUTHINESS_TAG:
return &PyBool_Type;
+ case JIT_SYM_COMPACT_INT:
+ return &PyLong_Type;
+
}
Py_UNREACHABLE();
}
return PyTuple_Type.tp_version_tag;
case JIT_SYM_TRUTHINESS_TAG:
return PyBool_Type.tp_version_tag;
+ case JIT_SYM_COMPACT_INT:
+ return PyLong_Type.tp_version_tag;
}
Py_UNREACHABLE();
}
case JIT_SYM_BOTTOM_TAG:
case JIT_SYM_NON_NULL_TAG:
case JIT_SYM_UNKNOWN_TAG:
+ case JIT_SYM_COMPACT_INT:
return -1;
case JIT_SYM_KNOWN_CLASS_TAG:
/* TODO :
return false;
}
+bool
+_Py_uop_sym_is_compact_int(JitOptRef ref)
+{
+ JitOptSymbol *sym = PyJitRef_Unwrap(ref);
+ if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) {
+ return (bool)PyLong_CheckCompact(sym->value.value);
+ }
+ return sym->tag == JIT_SYM_COMPACT_INT;
+}
+
bool
_Py_uop_sym_is_immortal(JitOptRef ref)
{
return _Py_uop_symbol_is_immortal(sym);
}
+void
+_Py_uop_sym_set_compact_int(JitOptContext *ctx, JitOptRef ref)
+{
+ JitOptSymbol *sym = PyJitRef_Unwrap(ref);
+ JitSymType tag = sym->tag;
+ switch(tag) {
+ case JIT_SYM_NULL_TAG:
+ sym_set_bottom(ctx, sym);
+ return;
+ case JIT_SYM_KNOWN_CLASS_TAG:
+ if (sym->cls.type == &PyLong_Type) {
+ sym->tag = JIT_SYM_COMPACT_INT;
+ } else {
+ sym_set_bottom(ctx, sym);
+ }
+ return;
+ case JIT_SYM_TYPE_VERSION_TAG:
+ if (sym->version.version == PyLong_Type.tp_version_tag) {
+ sym->tag = JIT_SYM_COMPACT_INT;
+ }
+ else {
+ sym_set_bottom(ctx, sym);
+ }
+ return;
+ case JIT_SYM_KNOWN_VALUE_TAG:
+ if (!PyLong_CheckCompact(sym->value.value)) {
+ Py_CLEAR(sym->value.value);
+ sym_set_bottom(ctx, sym);
+ }
+ return;
+ case JIT_SYM_TUPLE_TAG:
+ case JIT_SYM_TRUTHINESS_TAG:
+ sym_set_bottom(ctx, sym);
+ return;
+ case JIT_SYM_BOTTOM_TAG:
+ case JIT_SYM_COMPACT_INT:
+ return;
+ case JIT_SYM_NON_NULL_TAG:
+ case JIT_SYM_UNKNOWN_TAG:
+ sym->tag = JIT_SYM_COMPACT_INT;
+ return;
+ }
+}
+
JitOptRef
_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy)
{
return PyJitRef_Wrap(res);
}
+JitOptRef
+_Py_uop_sym_new_compact_int(JitOptContext *ctx)
+{
+ JitOptSymbol *sym = sym_new(ctx);
+ if (sym == NULL) {
+ return out_of_space_ref(ctx);
+ }
+ sym->tag = JIT_SYM_COMPACT_INT;
+ return PyJitRef_Wrap(sym);
+}
+
// 0 on success, -1 on error.
_Py_UOpsAbstractFrame *
_Py_uop_frame_new(
_Py_uop_abstractcontext_init(ctx);
PyObject *val_42 = NULL;
PyObject *val_43 = NULL;
+ PyObject *val_big = NULL;
PyObject *tuple = NULL;
// Use a single 'sym' variable so copy-pasting tests is easier.
TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == Py_False, "truthiness is not False");
TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == true, "value is not constant");
TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == Py_True, "value is not True");
+
+
+ val_big = PyNumber_Lshift(_PyLong_GetOne(), PyLong_FromLong(66));
+ if (val_big == NULL) {
+ goto fail;
+ }
+
+ JitOptRef ref_42 = _Py_uop_sym_new_const(ctx, val_42);
+ JitOptRef ref_big = _Py_uop_sym_new_const(ctx, val_big);
+ JitOptRef ref_int = _Py_uop_sym_new_compact_int(ctx);
+ TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_42), "42 is not a compact int");
+ TEST_PREDICATE(!_Py_uop_sym_is_compact_int(ref_big), "(1 << 66) is a compact int");
+ TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_int), "compact int is not a compact int");
+ TEST_PREDICATE(_Py_uop_sym_matches_type(ref_int, &PyLong_Type), "compact int is not an int");
+
+ _Py_uop_sym_set_type(ctx, ref_int, &PyLong_Type); // Should have no effect
+ TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_int), "compact int is not a compact int after cast");
+ TEST_PREDICATE(_Py_uop_sym_matches_type(ref_int, &PyLong_Type), "compact int is not an int after cast");
+
+ _Py_uop_sym_set_type(ctx, ref_int, &PyFloat_Type); // Should make it bottom
+ TEST_PREDICATE(_Py_uop_sym_is_bottom(ref_int), "compact int cast to float isn't bottom");
+
+ ref_int = _Py_uop_sym_new_compact_int(ctx);
+ _Py_uop_sym_set_const(ctx, ref_int, val_43);
+ TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_int), "43 is not a compact int");
+ TEST_PREDICATE(_Py_uop_sym_matches_type(ref_int, &PyLong_Type), "43 is not an int");
+ TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref_int) == val_43, "43 isn't 43");
+
_Py_uop_abstractcontext_fini(ctx);
Py_DECREF(val_42);
Py_DECREF(val_43);
+ Py_DECREF(val_big);
Py_DECREF(tuple);
Py_RETURN_NONE;
_Py_uop_abstractcontext_fini(ctx);
Py_XDECREF(val_42);
Py_XDECREF(val_43);
+ Py_XDECREF(val_big);
Py_DECREF(tuple);
return NULL;
}
"PyStackRef_IsValid",
"PyStackRef_Wrap",
"PyStackRef_Unwrap",
+ "PyLong_CheckCompact",
)
projects / thirdparty / Python / cpython.git / commitdiff
