Specialize arithmetic only on compact ints. This also makes int operations non-escaping in the JIT and in tier 1.
| | | |
| | | |--51.67%--_PyEval_EvalFrameDefault
| | | | |
- | | | | |--11.52%--_PyLong_Add
+ | | | | |--11.52%--_PyCompactLong_Add
| | | | | |
| | | | | |--2.97%--_PyObject_Malloc
...
| | | |
| | | |--51.81%--_PyEval_EvalFrameDefault
| | | | |
- | | | | |--13.77%--_PyLong_Add
+ | | | | |--13.77%--_PyCompactLong_Add
| | | | | |
| | | | | |--3.26%--_PyObject_Malloc
// Export for 'math' shared extension
PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t);
-PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right);
-PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right);
-PyAPI_FUNC(PyObject*) _PyLong_Subtract(PyLongObject *left, PyLongObject *right);
+PyAPI_FUNC(PyObject*) _PyCompactLong_Add(PyLongObject *left, PyLongObject *right);
+PyAPI_FUNC(PyObject*) _PyCompactLong_Multiply(PyLongObject *left, PyLongObject *right);
+PyAPI_FUNC(PyObject*) _PyCompactLong_Subtract(PyLongObject *left, PyLongObject *right);
// Export for 'binascii' shared extension.
PyAPI_DATA(unsigned char) _PyLong_DigitValue[256];
const struct opcode_metadata _PyOpcode_opcode_metadata[267] = {
[BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
- [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
- [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG },
[BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
- [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BUILD_INTERPOLATION] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_NOS_INT] = HAS_EXIT_FLAG,
[_GUARD_TOS_INT] = HAS_EXIT_FLAG,
- [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG,
- [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG,
- [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG,
+ [_BINARY_OP_MULTIPLY_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG,
+ [_BINARY_OP_ADD_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG,
+ [_BINARY_OP_SUBTRACT_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG,
[_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG,
[_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG,
[_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
--- /dev/null
+Specialize integer operations only on compact integers. This is a CPython internal change.
}
PyObject *
-_PyLong_Add(PyLongObject *a, PyLongObject *b)
+_PyCompactLong_Add(PyLongObject *a, PyLongObject *b)
{
- return (PyObject*)long_add(a, b);
+ assert(_PyLong_BothAreCompact(a, b));
+ stwodigits z = medium_value(a) + medium_value(b);
+ return (PyObject *)_PyLong_FromSTwoDigits(z);
}
static PyObject *
}
PyObject *
-_PyLong_Subtract(PyLongObject *a, PyLongObject *b)
+_PyCompactLong_Subtract(PyLongObject *a, PyLongObject *b)
{
- return (PyObject*)long_sub(a, b);
+ assert(_PyLong_BothAreCompact(a, b));
+ return (PyObject *)_PyLong_FromSTwoDigits(medium_value(a) - medium_value(b));
}
static PyObject *
}
PyObject *
-_PyLong_Multiply(PyLongObject *a, PyLongObject *b)
+_PyCompactLong_Multiply(PyLongObject *a, PyLongObject *b)
{
- return (PyObject*)long_mul(a, b);
+ assert(_PyLong_BothAreCompact(a, b));
+ stwodigits v = medium_value(a) * medium_value(b);
+ return (PyObject *)_PyLong_FromSTwoDigits(v);
}
static PyObject *
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ DEOPT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
+ PyObject *res_o = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
INPUTS_DEAD();
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ DEOPT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
+ PyObject *res_o = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
INPUTS_DEAD();
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ DEOPT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o));
STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
+ PyObject *res_o = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
INPUTS_DEAD();
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ PyObject *res_o = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ PyObject *res_o = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ PyObject *res_o = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ PyObject *res_o = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ PyObject *res_o = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
assert(PyLong_CheckExact(right_o));
+ if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
STAT_INC(BINARY_OP, hit);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ PyObject *res_o = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
}
op(_BINARY_OP_ADD_INT, (left, right -- res)) {
- if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) {
- assert(PyLong_CheckExact(sym_get_const(ctx, left)));
- assert(PyLong_CheckExact(sym_get_const(ctx, right)));
- PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left),
- (PyLongObject *)sym_get_const(ctx, right));
- if (temp == NULL) {
- goto error;
- }
- res = sym_new_const(ctx, temp);
- Py_DECREF(temp);
- // TODO gh-115506:
- // replace opcode with constant propagated one and add tests!
- }
- else {
- res = sym_new_type(ctx, &PyLong_Type);
- }
+ res = sym_new_type(ctx, &PyLong_Type);
}
op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) {
- if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) {
- assert(PyLong_CheckExact(sym_get_const(ctx, left)));
- assert(PyLong_CheckExact(sym_get_const(ctx, right)));
- PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left),
- (PyLongObject *)sym_get_const(ctx, right));
- if (temp == NULL) {
- goto error;
- }
- res = sym_new_const(ctx, temp);
- Py_DECREF(temp);
- // TODO gh-115506:
- // replace opcode with constant propagated one and add tests!
- }
- else {
- res = sym_new_type(ctx, &PyLong_Type);
- }
+ res = sym_new_type(ctx, &PyLong_Type);
}
op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) {
- if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) {
- assert(PyLong_CheckExact(sym_get_const(ctx, left)));
- assert(PyLong_CheckExact(sym_get_const(ctx, right)));
- PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left),
- (PyLongObject *)sym_get_const(ctx, right));
- if (temp == NULL) {
- goto error;
- }
- res = sym_new_const(ctx, temp);
- Py_DECREF(temp);
- // TODO gh-115506:
- // replace opcode with constant propagated one and add tests!
- }
- else {
- res = sym_new_type(ctx, &PyLong_Type);
- }
+ res = sym_new_type(ctx, &PyLong_Type);
}
op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) {
}
case _BINARY_OP_MULTIPLY_INT: {
- JitOptSymbol *right;
- JitOptSymbol *left;
JitOptSymbol *res;
- right = stack_pointer[-1];
- left = stack_pointer[-2];
- if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) {
- assert(PyLong_CheckExact(sym_get_const(ctx, left)));
- assert(PyLong_CheckExact(sym_get_const(ctx, right)));
- PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left),
- (PyLongObject *)sym_get_const(ctx, right));
- if (temp == NULL) {
- goto error;
- }
- res = sym_new_const(ctx, temp);
- stack_pointer[-2] = res;
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- Py_DECREF(temp);
- }
- else {
- res = sym_new_type(ctx, &PyLong_Type);
- stack_pointer += -1;
- }
- stack_pointer[-1] = res;
+ res = sym_new_type(ctx, &PyLong_Type);
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
break;
}
case _BINARY_OP_ADD_INT: {
- JitOptSymbol *right;
- JitOptSymbol *left;
JitOptSymbol *res;
- right = stack_pointer[-1];
- left = stack_pointer[-2];
- if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) {
- assert(PyLong_CheckExact(sym_get_const(ctx, left)));
- assert(PyLong_CheckExact(sym_get_const(ctx, right)));
- PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left),
- (PyLongObject *)sym_get_const(ctx, right));
- if (temp == NULL) {
- goto error;
- }
- res = sym_new_const(ctx, temp);
- stack_pointer[-2] = res;
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- Py_DECREF(temp);
- }
- else {
- res = sym_new_type(ctx, &PyLong_Type);
- stack_pointer += -1;
- }
- stack_pointer[-1] = res;
+ res = sym_new_type(ctx, &PyLong_Type);
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
break;
}
case _BINARY_OP_SUBTRACT_INT: {
- JitOptSymbol *right;
- JitOptSymbol *left;
JitOptSymbol *res;
- right = stack_pointer[-1];
- left = stack_pointer[-2];
- if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) {
- assert(PyLong_CheckExact(sym_get_const(ctx, left)));
- assert(PyLong_CheckExact(sym_get_const(ctx, right)));
- PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left),
- (PyLongObject *)sym_get_const(ctx, right));
- if (temp == NULL) {
- goto error;
- }
- res = sym_new_const(ctx, temp);
- stack_pointer[-2] = res;
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- Py_DECREF(temp);
- }
- else {
- res = sym_new_type(ctx, &PyLong_Type);
- stack_pointer += -1;
- }
- stack_pointer[-1] = res;
+ res = sym_new_type(ctx, &PyLong_Type);
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
break;
}
"_PyLong_IsNegative",
"_PyLong_IsNonNegativeCompact",
"_PyLong_IsZero",
+ "_PyLong_BothAreCompact",
+ "_PyCompactLong_Add",
+ "_PyCompactLong_Multiply",
+ "_PyCompactLong_Subtract",
"_PyManagedDictPointer_IsValues",
"_PyObject_GC_IS_SHARED",
"_PyObject_GC_IS_TRACKED",