_LOAD_ATTR_INSTANCE_VALUE +
unused/5; // Skip over rest of cache
- op(_CHECK_ATTR_MODULE, (type_version/2, owner -- owner)) {
+ op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) {
DEOPT_IF(!PyModule_CheckExact(owner));
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict != NULL);
- DEOPT_IF(dict->ma_keys->dk_version != type_version);
+ DEOPT_IF(dict->ma_keys->dk_version != dict_version);
}
op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) {
case _CHECK_ATTR_MODULE: {
PyObject *owner;
owner = stack_pointer[-1];
- uint32_t type_version = (uint32_t)CURRENT_OPERAND();
+ uint32_t dict_version = (uint32_t)CURRENT_OPERAND();
if (!PyModule_CheckExact(owner)) goto deoptimize;
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict != NULL);
- if (dict->ma_keys->dk_version != type_version) goto deoptimize;
+ if (dict->ma_keys->dk_version != dict_version) goto deoptimize;
break;
}
// _CHECK_ATTR_MODULE
owner = stack_pointer[-1];
{
- uint32_t type_version = read_u32(&this_instr[2].cache);
+ uint32_t dict_version = read_u32(&this_instr[2].cache);
DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict != NULL);
- DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR);
+ DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR);
}
// _LOAD_ATTR_MODULE
{
return (sym->flags & (IS_NULL | NOT_NULL)) == IS_NULL;
}
+static inline bool
+sym_is_const(_Py_UOpsSymType *sym)
+{
+ return (sym->flags & TRUE_CONST) != 0;
+}
+
+static inline PyObject *
+sym_get_const(_Py_UOpsSymType *sym)
+{
+ assert(sym_is_const(sym));
+ assert(sym->const_val);
+ return sym->const_val;
+}
+
static inline void
sym_set_type(_Py_UOpsSymType *sym, PyTypeObject *tp)
{
return temp;
}
-static inline bool
-is_const(_Py_UOpsSymType *sym)
-{
- return sym->const_val != NULL;
-}
-
-static inline PyObject *
-get_const(_Py_UOpsSymType *sym)
-{
- return sym->const_val;
-}
-
static _Py_UOpsSymType*
sym_new_null(_Py_UOpsAbstractInterpContext *ctx)
{
return 0;
}
-static void
-global_to_const(_PyUOpInstruction *inst, PyObject *obj)
+static PyObject *
+convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj)
{
- assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS);
+ assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE);
assert(PyDict_CheckExact(obj));
PyDictObject *dict = (PyDictObject *)obj;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
assert(inst->operand <= UINT16_MAX);
+ if ((int)inst->operand >= dict->ma_keys->dk_nentries) {
+ return NULL;
+ }
PyObject *res = entries[inst->operand].me_value;
if (res == NULL) {
- return;
+ return NULL;
}
if (_Py_IsImmortal(res)) {
inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_BORROW_WITH_NULL : _LOAD_CONST_INLINE_BORROW;
inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE;
}
inst->operand = (uint64_t)res;
+ return res;
}
static int
break;
case _LOAD_GLOBAL_BUILTINS:
if (globals_checked & builtins_checked & globals_watched & builtins_watched & 1) {
- global_to_const(inst, builtins);
+ convert_global_to_const(inst, builtins);
}
break;
case _LOAD_GLOBAL_MODULE:
if (globals_checked & globals_watched & 1) {
- global_to_const(inst, globals);
+ convert_global_to_const(inst, globals);
}
break;
case _PUSH_FRAME:
PyCodeObject *co,
_PyUOpInstruction *trace,
int trace_len,
- int curr_stacklen
+ int curr_stacklen,
+ _PyBloomFilter *dependencies
)
{
* could error. _CHECK_VALIDITY is needed if the previous
* instruction could have escaped. */
int last_set_ip = -1;
- bool may_have_escaped = false;
+ bool may_have_escaped = true;
for (int pc = 0; pc < buffer_size; pc++) {
int opcode = buffer[pc].opcode;
switch (opcode) {
}
last_set_ip = pc;
break;
+ case _POP_TOP:
+ {
+ _PyUOpInstruction *last = &buffer[pc-1];
+ while (last->opcode == _NOP) {
+ last--;
+ }
+ if (last->opcode == _LOAD_CONST_INLINE ||
+ last->opcode == _LOAD_CONST_INLINE_BORROW ||
+ last->opcode == _LOAD_FAST ||
+ last->opcode == _COPY
+ ) {
+ last->opcode = _NOP;
+ buffer[pc].opcode = NOP;
+ }
+ break;
+ }
case _JUMP_TO_TOP:
case _EXIT_TRACE:
return;
if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) {
needs_ip = true;
}
- if (opcode == _PUSH_FRAME) {
- needs_ip = true;
- }
if (needs_ip && last_set_ip >= 0) {
if (buffer[last_set_ip].opcode == _CHECK_VALIDITY) {
buffer[last_set_ip].opcode = _CHECK_VALIDITY_AND_SET_IP;
err = uop_redundancy_eliminator(
(PyCodeObject *)frame->f_executable, buffer,
- buffer_size, curr_stacklen);
+ buffer_size, curr_stacklen, dependencies);
if (err == 0) {
goto not_ready;
#include "Python.h"
#include "pycore_uops.h"
#include "pycore_uop_ids.h"
+#include "internal/pycore_moduleobject.h"
#define op(name, ...) /* NAME is ignored */
}
op(_BINARY_OP_ADD_INT, (left, right -- res)) {
- if (is_const(left) && is_const(right)) {
- assert(PyLong_CheckExact(get_const(left)));
- assert(PyLong_CheckExact(get_const(right)));
- PyObject *temp = _PyLong_Add((PyLongObject *)get_const(left),
- (PyLongObject *)get_const(right));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyLong_CheckExact(sym_get_const(left)));
+ assert(PyLong_CheckExact(sym_get_const(right)));
+ PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left),
+ (PyLongObject *)sym_get_const(right));
if (temp == NULL) {
goto error;
}
}
op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) {
- if (is_const(left) && is_const(right)) {
- assert(PyLong_CheckExact(get_const(left)));
- assert(PyLong_CheckExact(get_const(right)));
- PyObject *temp = _PyLong_Subtract((PyLongObject *)get_const(left),
- (PyLongObject *)get_const(right));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyLong_CheckExact(sym_get_const(left)));
+ assert(PyLong_CheckExact(sym_get_const(right)));
+ PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left),
+ (PyLongObject *)sym_get_const(right));
if (temp == NULL) {
goto error;
}
}
op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) {
- if (is_const(left) && is_const(right)) {
- assert(PyLong_CheckExact(get_const(left)));
- assert(PyLong_CheckExact(get_const(right)));
- PyObject *temp = _PyLong_Multiply((PyLongObject *)get_const(left),
- (PyLongObject *)get_const(right));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyLong_CheckExact(sym_get_const(left)));
+ assert(PyLong_CheckExact(sym_get_const(right)));
+ PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left),
+ (PyLongObject *)sym_get_const(right));
if (temp == NULL) {
goto error;
}
}
op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) {
- if (is_const(left) && is_const(right)) {
- assert(PyFloat_CheckExact(get_const(left)));
- assert(PyFloat_CheckExact(get_const(right)));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyFloat_CheckExact(sym_get_const(left)));
+ assert(PyFloat_CheckExact(sym_get_const(right)));
PyObject *temp = PyFloat_FromDouble(
- PyFloat_AS_DOUBLE(get_const(left)) +
- PyFloat_AS_DOUBLE(get_const(right)));
+ PyFloat_AS_DOUBLE(sym_get_const(left)) +
+ PyFloat_AS_DOUBLE(sym_get_const(right)));
if (temp == NULL) {
goto error;
}
}
op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) {
- if (is_const(left) && is_const(right)) {
- assert(PyFloat_CheckExact(get_const(left)));
- assert(PyFloat_CheckExact(get_const(right)));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyFloat_CheckExact(sym_get_const(left)));
+ assert(PyFloat_CheckExact(sym_get_const(right)));
PyObject *temp = PyFloat_FromDouble(
- PyFloat_AS_DOUBLE(get_const(left)) -
- PyFloat_AS_DOUBLE(get_const(right)));
+ PyFloat_AS_DOUBLE(sym_get_const(left)) -
+ PyFloat_AS_DOUBLE(sym_get_const(right)));
if (temp == NULL) {
goto error;
}
}
op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) {
- if (is_const(left) && is_const(right)) {
- assert(PyFloat_CheckExact(get_const(left)));
- assert(PyFloat_CheckExact(get_const(right)));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyFloat_CheckExact(sym_get_const(left)));
+ assert(PyFloat_CheckExact(sym_get_const(right)));
PyObject *temp = PyFloat_FromDouble(
- PyFloat_AS_DOUBLE(get_const(left)) *
- PyFloat_AS_DOUBLE(get_const(right)));
+ PyFloat_AS_DOUBLE(sym_get_const(left)) *
+ PyFloat_AS_DOUBLE(sym_get_const(right)));
if (temp == NULL) {
goto error;
}
(void)owner;
}
+ op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) {
+ (void)dict_version;
+ if (sym_is_const(owner)) {
+ PyObject *cnst = sym_get_const(owner);
+ if (PyModule_CheckExact(cnst)) {
+ PyModuleObject *mod = (PyModuleObject *)cnst;
+ PyObject *dict = mod->md_dict;
+ uint64_t watched_mutations = get_mutations(dict);
+ if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) {
+ PyDict_Watch(GLOBALS_WATCHER_ID, dict);
+ _Py_BloomFilter_Add(dependencies, dict);
+ this_instr->opcode = _NOP;
+ }
+ }
+ }
+ }
+
op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) {
- _LOAD_ATTR_NOT_NULL
(void)index;
- (void)owner;
+ OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx));
+ attr = NULL;
+ if (this_instr[-1].opcode == _NOP) {
+ // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched.
+ assert(sym_is_const(owner));
+ PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner);
+ assert(PyModule_CheckExact(mod));
+ PyObject *dict = mod->md_dict;
+ PyObject *res = convert_global_to_const(this_instr, dict);
+ if (res != NULL) {
+ this_instr[-1].opcode = _POP_TOP;
+ OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res));
+ }
+ }
+ if (attr == NULL) {
+ /* No conversion made. We don't know what `attr` is. */
+ OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx));
+ }
}
op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) {
// END BYTECODES //
-}
\ No newline at end of file
+}
_Py_UOpsSymType *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (is_const(left) && is_const(right)) {
- assert(PyLong_CheckExact(get_const(left)));
- assert(PyLong_CheckExact(get_const(right)));
- PyObject *temp = _PyLong_Multiply((PyLongObject *)get_const(left),
- (PyLongObject *)get_const(right));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyLong_CheckExact(sym_get_const(left)));
+ assert(PyLong_CheckExact(sym_get_const(right)));
+ PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left),
+ (PyLongObject *)sym_get_const(right));
if (temp == NULL) {
goto error;
}
_Py_UOpsSymType *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (is_const(left) && is_const(right)) {
- assert(PyLong_CheckExact(get_const(left)));
- assert(PyLong_CheckExact(get_const(right)));
- PyObject *temp = _PyLong_Add((PyLongObject *)get_const(left),
- (PyLongObject *)get_const(right));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyLong_CheckExact(sym_get_const(left)));
+ assert(PyLong_CheckExact(sym_get_const(right)));
+ PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left),
+ (PyLongObject *)sym_get_const(right));
if (temp == NULL) {
goto error;
}
_Py_UOpsSymType *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (is_const(left) && is_const(right)) {
- assert(PyLong_CheckExact(get_const(left)));
- assert(PyLong_CheckExact(get_const(right)));
- PyObject *temp = _PyLong_Subtract((PyLongObject *)get_const(left),
- (PyLongObject *)get_const(right));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyLong_CheckExact(sym_get_const(left)));
+ assert(PyLong_CheckExact(sym_get_const(right)));
+ PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left),
+ (PyLongObject *)sym_get_const(right));
if (temp == NULL) {
goto error;
}
_Py_UOpsSymType *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (is_const(left) && is_const(right)) {
- assert(PyFloat_CheckExact(get_const(left)));
- assert(PyFloat_CheckExact(get_const(right)));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyFloat_CheckExact(sym_get_const(left)));
+ assert(PyFloat_CheckExact(sym_get_const(right)));
PyObject *temp = PyFloat_FromDouble(
- PyFloat_AS_DOUBLE(get_const(left)) *
- PyFloat_AS_DOUBLE(get_const(right)));
+ PyFloat_AS_DOUBLE(sym_get_const(left)) *
+ PyFloat_AS_DOUBLE(sym_get_const(right)));
if (temp == NULL) {
goto error;
}
_Py_UOpsSymType *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (is_const(left) && is_const(right)) {
- assert(PyFloat_CheckExact(get_const(left)));
- assert(PyFloat_CheckExact(get_const(right)));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyFloat_CheckExact(sym_get_const(left)));
+ assert(PyFloat_CheckExact(sym_get_const(right)));
PyObject *temp = PyFloat_FromDouble(
- PyFloat_AS_DOUBLE(get_const(left)) +
- PyFloat_AS_DOUBLE(get_const(right)));
+ PyFloat_AS_DOUBLE(sym_get_const(left)) +
+ PyFloat_AS_DOUBLE(sym_get_const(right)));
if (temp == NULL) {
goto error;
}
_Py_UOpsSymType *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (is_const(left) && is_const(right)) {
- assert(PyFloat_CheckExact(get_const(left)));
- assert(PyFloat_CheckExact(get_const(right)));
+ if (sym_is_const(left) && sym_is_const(right)) {
+ assert(PyFloat_CheckExact(sym_get_const(left)));
+ assert(PyFloat_CheckExact(sym_get_const(right)));
PyObject *temp = PyFloat_FromDouble(
- PyFloat_AS_DOUBLE(get_const(left)) -
- PyFloat_AS_DOUBLE(get_const(right)));
+ PyFloat_AS_DOUBLE(sym_get_const(left)) -
+ PyFloat_AS_DOUBLE(sym_get_const(right)));
if (temp == NULL) {
goto error;
}
}
case _CHECK_ATTR_MODULE: {
+ _Py_UOpsSymType *owner;
+ owner = stack_pointer[-1];
+ uint32_t dict_version = (uint32_t)this_instr->operand;
+ (void)dict_version;
+ if (sym_is_const(owner)) {
+ PyObject *cnst = sym_get_const(owner);
+ if (PyModule_CheckExact(cnst)) {
+ PyModuleObject *mod = (PyModuleObject *)cnst;
+ PyObject *dict = mod->md_dict;
+ uint64_t watched_mutations = get_mutations(dict);
+ if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) {
+ PyDict_Watch(GLOBALS_WATCHER_ID, dict);
+ _Py_BloomFilter_Add(dependencies, dict);
+ this_instr->opcode = _NOP;
+ }
+ }
+ }
break;
}
_Py_UOpsSymType *null = NULL;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)this_instr->operand;
- _LOAD_ATTR_NOT_NULL
(void)index;
- (void)owner;
+ OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx));
+ attr = NULL;
+ if (this_instr[-1].opcode == _NOP) {
+ // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched.
+ assert(sym_is_const(owner));
+ PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner);
+ assert(PyModule_CheckExact(mod));
+ PyObject *dict = mod->md_dict;
+ PyObject *res = convert_global_to_const(this_instr, dict);
+ if (res != NULL) {
+ this_instr[-1].opcode = _POP_TOP;
+ OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res));
+ }
+ }
+ if (attr == NULL) {
+ /* No conversion made. We don't know what `attr` is. */
+ OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx));
+ }
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);