Finish specialization for LOAD_ATTR in the free-threaded build by adding support for class and instance receivers.
#define PYSTATS_MAX_UOP_ID 512
-#define SPECIALIZATION_FAILURE_KINDS 36
+#define SPECIALIZATION_FAILURE_KINDS 37
/* Stats for determining who is calling PyEval_EvalFrame */
#define EVAL_CALL_TOTAL 0
extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *);
extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key);
+
+/* Look up a string key in an all unicode dict keys, assign the keys object a version, and
+ * store it in version.
+ *
+ * Returns DKIX_ERROR if key is not a string or if the keys object is not all
+ * strings.
+ *
+ * Returns DKIX_EMPTY if the key is not present.
+ */
+extern Py_ssize_t _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version);
extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key);
PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *);
PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *);
return 0;
}
case LOAD_ATTR_WITH_HINT: {
- *effect = Py_MAX(0, (oparg & 1));
+ *effect = Py_MAX(1, (oparg & 1));
return 0;
}
case LOAD_BUILD_CLASS: {
case _CHECK_ATTR_WITH_HINT:
return 0;
case _LOAD_ATTR_WITH_HINT:
- return 1;
+ return 2;
case _LOAD_ATTR_SLOT_0:
return 1;
case _LOAD_ATTR_SLOT_1:
-from test.support import import_helper
+from test.support import import_helper, Py_GIL_DISABLED, refleak_helper
import unittest
_testcapi = import_helper.import_module('_testcapi')
# as well
type_freeze(D)
+ @unittest.skipIf(
+ Py_GIL_DISABLED and refleak_helper.hunting_for_refleaks(),
+ "Specialization failure triggers gh-127773")
def test_freeze_meta(self):
"""test PyType_Freeze() with overridden MRO"""
type_freeze = _testcapi.type_freeze
import random
import string
import sys
+import textwrap
import types
import unittest
import warnings
from copy import deepcopy
from contextlib import redirect_stdout
from test import support
+from test.support.script_helper import assert_python_ok
try:
import _testcapi
# Issue #14199: _PyType_Lookup() has to keep a strong reference to
# the type MRO because it may be modified during the lookup, if
# __bases__ is set during the lookup for example.
+ code = textwrap.dedent("""
class MyKey(object):
def __hash__(self):
return hash('mykey')
mykey = 'from Base2'
mykey2 = 'from Base2'
- with self.assertWarnsRegex(RuntimeWarning, 'X'):
- X = type('X', (Base,), {MyKey(): 5})
- # mykey is read from Base
- self.assertEqual(X.mykey, 'from Base')
- # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__
- self.assertEqual(X.mykey2, 'from Base2')
+ X = type('X', (Base,), {MyKey(): 5})
+
+ bases_before = ",".join([c.__name__ for c in X.__bases__])
+ print(f"before={bases_before}")
+
+ # mykey is initially read from Base, however, the lookup will be perfomed
+ # again if specialization fails. The second lookup will use the new
+ # mro set by __eq__.
+ print(X.mykey)
+
+ bases_after = ",".join([c.__name__ for c in X.__bases__])
+ print(f"after={bases_after}")
+
+ # mykey2 is read from Base2 because MyKey.__eq__ has set __bases_
+ print(f"mykey2={X.mykey2}")
+ """)
+ _, out, err = assert_python_ok("-c", code)
+ err = err.decode()
+ self.assertRegex(err, "RuntimeWarning: .*X")
+ out = out.decode()
+ self.assertRegex(out, "before=Base")
+ self.assertRegex(out, "after=Base2")
+ self.assertRegex(out, "mykey2=from Base2")
class PicklingTests(unittest.TestCase):
"""
self.run_cases_test(input, output)
- def test_pop_dead_inputs_all_live(self):
+ def test_pystackref_frompyobject_new_next_to_cmacro(self):
input = """
- inst(OP, (a, b --)) {
- POP_DEAD_INPUTS();
- HAM(a, b);
- INPUTS_DEAD();
+ inst(OP, (-- out1, out2)) {
+ PyObject *obj = SPAM();
+ #ifdef Py_GIL_DISABLED
+ out1 = PyStackRef_FromPyObjectNew(obj);
+ #else
+ out1 = PyStackRef_FromPyObjectNew(obj);
+ #endif
+ out2 = PyStackRef_FromPyObjectNew(obj);
}
"""
output = """
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- _PyStackRef a;
- _PyStackRef b;
- b = stack_pointer[-1];
- a = stack_pointer[-2];
- HAM(a, b);
- stack_pointer += -2;
+ _PyStackRef out1;
+ _PyStackRef out2;
+ PyObject *obj = SPAM();
+ #ifdef Py_GIL_DISABLED
+ out1 = PyStackRef_FromPyObjectNew(obj);
+ #else
+ out1 = PyStackRef_FromPyObjectNew(obj);
+ #endif
+ out2 = PyStackRef_FromPyObjectNew(obj);
+ stack_pointer[0] = out1;
+ stack_pointer[1] = out2;
+ stack_pointer += 2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
"""
self.run_cases_test(input, output)
- def test_pop_dead_inputs_some_live(self):
+ def test_pop_input(self):
input = """
- inst(OP, (a, b, c --)) {
- POP_DEAD_INPUTS();
+ inst(OP, (a, b --)) {
+ POP_INPUT(b);
HAM(a);
INPUTS_DEAD();
}
next_instr += 1;
INSTRUCTION_STATS(OP);
_PyStackRef a;
- a = stack_pointer[-3];
- stack_pointer += -2;
+ _PyStackRef b;
+ b = stack_pointer[-1];
+ a = stack_pointer[-2];
+ stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
HAM(a);
stack_pointer += -1;
"""
self.run_cases_test(input, output)
- def test_pop_dead_inputs_with_output(self):
+ def test_pop_input_with_empty_stack(self):
input = """
- inst(OP, (a, b -- c)) {
- POP_DEAD_INPUTS();
- c = SPAM();
+ inst(OP, (--)) {
+ POP_INPUT(foo);
}
"""
- output = """
- TARGET(OP) {
- frame->instr_ptr = next_instr;
- next_instr += 1;
- INSTRUCTION_STATS(OP);
- _PyStackRef c;
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- c = SPAM();
- stack_pointer[0] = c;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- DISPATCH();
+ with self.assertRaises(SyntaxError):
+ self.run_cases_test(input, "")
+
+ def test_pop_input_with_non_tos(self):
+ input = """
+ inst(OP, (a, b --)) {
+ POP_INPUT(a);
}
"""
- self.run_cases_test(input, output)
+ with self.assertRaises(SyntaxError):
+ self.run_cases_test(input, "")
def test_no_escaping_calls_in_branching_macros(self):
instantiate()
+def make_deferred_ref_count_obj():
+ """Create an object that uses deferred reference counting.
+
+ Only objects that use deferred refence counting may be stored in inline
+ caches in free-threaded builds. This constructs a new class named Foo,
+ which uses deferred reference counting.
+ """
+ return type("Foo", (object,), {})
+
+
@threading_helper.requires_working_threading()
class TestRacesDoNotCrash(TestBase):
# Careful with these. Bigger numbers have a higher chance of catching bugs,
opname = "FOR_ITER_LIST"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_class(self):
def get_items():
class C:
- a = object()
+ a = make_deferred_ref_count_obj()
items = []
for _ in range(self.ITEMS):
del item.a
except AttributeError:
pass
- item.a = object()
+ item.a = make_deferred_ref_count_obj()
opname = "LOAD_ATTR_CLASS"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
+ def test_load_attr_class_with_metaclass_check(self):
+ def get_items():
+ class Meta(type):
+ pass
+
+ class C(metaclass=Meta):
+ a = make_deferred_ref_count_obj()
+
+ items = []
+ for _ in range(self.ITEMS):
+ item = C
+ items.append(item)
+ return items
+
+ def read(items):
+ for item in items:
+ try:
+ item.a
+ except AttributeError:
+ pass
+
+ def write(items):
+ for item in items:
+ try:
+ del item.a
+ except AttributeError:
+ pass
+ item.a = make_deferred_ref_count_obj()
+
+ opname = "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK"
+ self.assert_races_do_not_crash(opname, get_items, read, write)
+
+ @requires_specialization_ft
def test_load_attr_getattribute_overridden(self):
def get_items():
class C:
opname = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_instance_value(self):
def get_items():
class C:
opname = "LOAD_ATTR_INSTANCE_VALUE"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_method_lazy_dict(self):
def get_items():
class C(Exception):
opname = "LOAD_ATTR_METHOD_LAZY_DICT"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_method_no_dict(self):
def get_items():
class C:
opname = "LOAD_ATTR_METHOD_NO_DICT"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_method_with_values(self):
def get_items():
class C:
opname = "LOAD_ATTR_MODULE"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
def test_load_attr_property(self):
def get_items():
class C:
opname = "LOAD_ATTR_PROPERTY"
self.assert_races_do_not_crash(opname, get_items, read, write)
- @requires_specialization
+ @requires_specialization_ft
+ def test_load_attr_slot(self):
+ def get_items():
+ class C:
+ __slots__ = ["a", "b"]
+
+ items = []
+ for i in range(self.ITEMS):
+ item = C()
+ item.a = i
+ item.b = i + self.ITEMS
+ items.append(item)
+ return items
+
+ def read(items):
+ for item in items:
+ item.a
+ item.b
+
+ def write(items):
+ for item in items:
+ item.a = 100
+ item.b = 200
+
+ opname = "LOAD_ATTR_SLOT"
+ self.assert_races_do_not_crash(opname, get_items, read, write)
+
+ @requires_specialization_ft
def test_load_attr_with_hint(self):
def get_items():
class C:
return do_lookup(mp, dk, key, hash, compare_generic);
}
+static bool
+check_keys_unicode(PyDictKeysObject *dk, PyObject *key)
+{
+ return PyUnicode_CheckExact(key) && (dk->dk_kind != DICT_KEYS_GENERAL);
+}
+
+static Py_ssize_t
+hash_unicode_key(PyObject *key)
+{
+ assert(PyUnicode_CheckExact(key));
+ Py_hash_t hash = unicode_get_hash(key);
+ if (hash == -1) {
+ hash = PyUnicode_Type.tp_hash(key);
+ assert(hash != -1);
+ }
+ return hash;
+}
+
#ifdef Py_GIL_DISABLED
static Py_ssize_t
unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key,
Py_ssize_t
_PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key)
{
- DictKeysKind kind = dk->dk_kind;
- if (!PyUnicode_CheckExact(key) || kind == DICT_KEYS_GENERAL) {
+ if (!check_keys_unicode(dk, key)) {
return DKIX_ERROR;
}
- Py_hash_t hash = unicode_get_hash(key);
- if (hash == -1) {
- hash = PyUnicode_Type.tp_hash(key);
- if (hash == -1) {
- PyErr_Clear();
- return DKIX_ERROR;
- }
- }
+ Py_hash_t hash = hash_unicode_key(key);
return unicodekeys_lookup_unicode(dk, key, hash);
}
+Py_ssize_t
+_PyDictKeys_StringLookupAndVersion(PyDictKeysObject *dk, PyObject *key, uint32_t *version)
+{
+ if (!check_keys_unicode(dk, key)) {
+ return DKIX_ERROR;
+ }
+ Py_ssize_t ix;
+ Py_hash_t hash = hash_unicode_key(key);
+ LOCK_KEYS(dk);
+ ix = unicodekeys_lookup_unicode(dk, key, hash);
+ *version = _PyDictKeys_GetVersionForCurrentState(_PyInterpreterState_GET(), dk);
+ UNLOCK_KEYS(dk);
+ return ix;
+}
+
/* Like _PyDictKeys_StringLookup() but only works on split keys. Note
* that in free-threaded builds this locks the keys object as required.
*/
}
}
+static void
+invalidate_and_clear_inline_values(PyDictValues *values)
+{
+ assert(values->embedded);
+ FT_ATOMIC_STORE_UINT8(values->valid, 0);
+ for (int i = 0; i < values->capacity; i++) {
+ FT_ATOMIC_STORE_PTR_RELEASE(values->values[i], NULL);
+ }
+}
+
/*
Restructure the table by allocating a new table and reinserting all
items again. When entries have been deleted, the new table may
if (oldvalues->embedded) {
assert(oldvalues->embedded == 1);
assert(oldvalues->valid == 1);
- FT_ATOMIC_STORE_UINT8(oldvalues->valid, 0);
+ invalidate_and_clear_inline_values(oldvalues);
}
else {
free_values(oldvalues, IS_DICT_SHARED(mp));
#ifdef Py_GIL_DISABLED
PyObject *value = _Py_atomic_load_ptr_acquire(&values->values[ix]);
- if (value == NULL || _Py_TryIncrefCompare(&values->values[ix], value)) {
+ if (value == NULL) {
+ if (FT_ATOMIC_LOAD_UINT8(values->valid)) {
+ *attr = NULL;
+ return true;
+ }
+ }
+ else if (_Py_TryIncrefCompare(&values->values[ix], value)) {
*attr = value;
return true;
}
}
mp->ma_values = values;
- FT_ATOMIC_STORE_UINT8(_PyObject_InlineValues(obj)->valid, 0);
+ invalidate_and_clear_inline_values(_PyObject_InlineValues(obj));
assert(_PyObject_InlineValuesConsistencyCheck(obj));
ASSERT_CONSISTENT(mp);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid));
}
split op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
DEOPT_IF(attr_o == NULL);
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ DEOPT_IF(true);
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
DECREF_INPUTS();
}
assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries));
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
- DEAD(mod_keys);
// Clear mod_keys from stack in case we need to deopt
- POP_DEAD_INPUTS();
+ POP_INPUT(mod_keys);
DEOPT_IF(attr_o == NULL);
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr);
_LOAD_ATTR_MODULE_FROM_KEYS +
unused/5;
- op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) {
+ op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict: PyDictObject *)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- EXIT_IF(dict == NULL);
- assert(PyDict_CheckExact((PyObject *)dict));
+ PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
+ EXIT_IF(dict_o == NULL);
+ assert(PyDict_CheckExact((PyObject *)dict_o));
+ dict = dict_o;
}
- op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) {
- PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr, null if (oparg & 1))) {
PyObject *attr_o;
+ if (!LOCK_OBJECT(dict)) {
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
+ if (hint >= (size_t)dict->ma_keys->dk_nentries) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
- DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys));
+ if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
- DEOPT_IF(ep->me_key != name);
+ if (ep->me_key != name) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
attr_o = ep->me_value;
- DEOPT_IF(attr_o == NULL);
+ if (attr_o == NULL) {
+ UNLOCK_OBJECT(dict);
+ POP_INPUT(dict);
+ DEOPT_IF(true);
+ }
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ UNLOCK_OBJECT(dict);
+ DEAD(dict);
null = PyStackRef_NULL;
DECREF_INPUTS();
}
split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
DEOPT_IF(attr_o == NULL);
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
+ DEOPT_IF(!increfed);
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
DECREF_INPUTS();
}
EXIT_IF(!PyType_Check(owner_o));
assert(type_version != 0);
- EXIT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version);
+ EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version);
}
split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) {
DEOPT_IF(tstate->interp->eval_frame);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
- DEOPT_IF(cls->tp_version_tag != type_version);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version);
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
assert(func_version != 0);
op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid);
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid));
}
op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) {
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
- DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version);
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version);
}
split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) {
op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) {
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
- PyObject *dict = *(PyObject **)ptr;
+ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL);
}
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- if (!_PyObject_InlineValues(owner_o)->valid) {
+ if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
case _CHECK_ATTR_WITH_HINT: {
_PyStackRef owner;
+ PyDictObject *dict;
owner = stack_pointer[-1];
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- if (dict == NULL) {
+ PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
+ if (dict_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- assert(PyDict_CheckExact((PyObject *)dict));
+ assert(PyDict_CheckExact((PyObject *)dict_o));
+ dict = dict_o;
+ stack_pointer[0].bits = (uintptr_t)dict;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR_WITH_HINT: {
+ PyDictObject *dict;
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
- owner = stack_pointer[-1];
+ dict = (PyDictObject *)stack_pointer[-1].bits;
+ owner = stack_pointer[-2];
uint16_t hint = (uint16_t)CURRENT_OPERAND0();
- PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject *attr_o;
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
+ if (!LOCK_OBJECT(dict)) {
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ }
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
- if (!DK_IS_UNICODE(dict->ma_keys)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
attr_o = ep->me_value;
if (attr_o == NULL) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
+ UNLOCK_OBJECT(dict);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ if (true) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
}
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ UNLOCK_OBJECT(dict);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
- stack_pointer[-1] = attr;
- if (oparg & 1) stack_pointer[0] = null;
- stack_pointer += (oparg & 1);
+ stack_pointer[-2] = attr;
+ if (oparg & 1) stack_pointer[-1] = null;
+ stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
+ if (!increfed) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
+ if (!increfed) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
JUMP_TO_JUMP_TARGET();
}
assert(type_version != 0);
- if (((PyTypeObject *)owner_o)->tp_version_tag != type_version) {
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
owner = stack_pointer[-1];
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- if (!_PyObject_InlineValues(owner_o)->valid) {
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0();
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
- if (owner_heap_type->ht_cached_keys->dk_version != keys_version) {
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
owner = stack_pointer[-1];
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0();
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
- PyObject *dict = *(PyObject **)ptr;
+ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
if (dict != NULL) {
UOP_STAT_INC(uopcode, miss);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
assert(type_version != 0);
- DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_CLASS
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
assert(type_version != 0);
- DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR);
}
// _GUARD_TYPE_VERSION
{
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
- DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR);
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
assert(func_version != 0);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR);
}
// _LOAD_ATTR_INSTANCE_VALUE
{
uint16_t offset = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
- PyObject *attr_o = *value_ptr;
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ #ifdef Py_GIL_DISABLED
+ if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
+ DEOPT_IF(true, LOAD_ATTR);
+ }
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectSteal(attr_o);
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
{
uint16_t dictoffset = read_u16(&this_instr[4].cache);
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
- PyObject *dict = *(PyObject **)ptr;
+ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_ATTR);
}
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
- DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_METHOD_WITH_VALUES
{
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
+ PyDictValues *ivs = _PyObject_InlineValues(owner_o);
+ DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
- DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
+ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
+ DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
{
{
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- char *addr = (char *)owner_o + index;
- PyObject *attr_o = *(PyObject **)addr;
+ PyObject **addr = (PyObject **)((char *)owner_o + index);
+ PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ #ifdef Py_GIL_DISABLED
+ int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
+ DEOPT_IF(!increfed, LOAD_ATTR);
+ #else
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ #endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
- attr = PyStackRef_FromPyObjectNew(attr_o);
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
+ PyDictObject *dict;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(dict == NULL, LOAD_ATTR);
- assert(PyDict_CheckExact((PyObject *)dict));
+ PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
+ DEOPT_IF(dict_o == NULL, LOAD_ATTR);
+ assert(PyDict_CheckExact((PyObject *)dict_o));
+ dict = dict_o;
}
// _LOAD_ATTR_WITH_HINT
{
uint16_t hint = read_u16(&this_instr[4].cache);
- PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject *attr_o;
- PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
+ if (!LOCK_OBJECT(dict)) {
+ DEOPT_IF(true, LOAD_ATTR);
+ }
+ if (hint >= (size_t)dict->ma_keys->dk_nentries) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
- DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR);
+ if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
- DEOPT_IF(ep->me_key != name, LOAD_ATTR);
+ if (ep->me_key != name) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
attr_o = ep->me_value;
- DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ if (attr_o == NULL) {
+ UNLOCK_OBJECT(dict);
+ DEOPT_IF(true, LOAD_ATTR);
+ }
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr_o);
- attr = PyStackRef_FromPyObjectSteal(attr_o);
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ UNLOCK_OBJECT(dict);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
}
}
- op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) {
+ op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict)) {
+ dict = sym_new_not_null(ctx);
+ (void)owner;
+ }
+
+ op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr, null if (oparg & 1))) {
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)hint;
(void)owner;
+ (void)dict;
}
op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) {
}
case _CHECK_ATTR_WITH_HINT: {
+ _Py_UopsSymbol *owner;
+ _Py_UopsSymbol *dict;
+ owner = stack_pointer[-1];
+ dict = sym_new_not_null(ctx);
+ (void)owner;
+ stack_pointer[0] = dict;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR_WITH_HINT: {
+ _Py_UopsSymbol *dict;
_Py_UopsSymbol *owner;
_Py_UopsSymbol *attr;
_Py_UopsSymbol *null = NULL;
- owner = stack_pointer[-1];
+ dict = stack_pointer[-1];
+ owner = stack_pointer[-2];
uint16_t hint = (uint16_t)this_instr->operand0;
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)hint;
(void)owner;
- stack_pointer[-1] = attr;
- if (oparg & 1) stack_pointer[0] = null;
- stack_pointer += (oparg & 1);
+ (void)dict;
+ stack_pointer[-2] = attr;
+ if (oparg & 1) stack_pointer[-1] = null;
+ stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
#include "opcode.h"
#include "pycore_code.h"
+#include "pycore_critical_section.h"
#include "pycore_descrobject.h" // _PyMethodWrapper_Type
#include "pycore_dict.h" // DICT_KEYS_UNICODE
#include "pycore_function.h" // _PyFunction_GetVersionForCurrentState()
#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33
#define SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN 34
#define SPEC_FAIL_ATTR_SPLIT_DICT 35
+#define SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED 36
/* Binary subscr and store subscr */
}
static int function_kind(PyCodeObject *code);
-#ifndef Py_GIL_DISABLED
static bool function_check_args(PyObject *o, int expected_argcount, int opcode);
static uint32_t function_get_version(PyObject *o, int opcode);
-static uint32_t type_get_version(PyTypeObject *t, int opcode);
-#endif
static int
specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name)
(descriptor == _PyType_Lookup(&PyBaseObject_Type, name)));
}
-#ifndef Py_GIL_DISABLED
static DescriptorClassification
-analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) {
+analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) {
bool has_getattr = false;
+ bool have_ga_version = false;
+ unsigned int ga_version;
getattrofunc getattro_slot = type->tp_getattro;
if (getattro_slot == PyObject_GenericGetAttr) {
/* Normal attribute lookup; */
getattro_slot == _Py_slot_tp_getattro) {
/* One or both of __getattribute__ or __getattr__ may have been
overridden See typeobject.c for why these functions are special. */
- PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__));
+ PyObject *getattribute = _PyType_LookupRefAndVersion(type,
+ &_Py_ID(__getattribute__), &ga_version);
+ have_ga_version = true;
PyInterpreterState *interp = _PyInterpreterState_GET();
bool has_custom_getattribute = getattribute != NULL &&
getattribute != interp->callable_cache.object__getattribute__;
- PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__));
+ PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__));
has_getattr = getattr != NULL;
- Py_XDECREF(getattr);
if (has_custom_getattribute) {
if (getattro_slot == _Py_slot_tp_getattro &&
!has_getattr &&
Py_IS_TYPE(getattribute, &PyFunction_Type)) {
*descr = getattribute;
+ *tp_version = ga_version;
return GETATTRIBUTE_IS_PYTHON_FUNCTION;
}
/* Potentially both __getattr__ and __getattribute__ are set.
Too complicated */
Py_DECREF(getattribute);
*descr = NULL;
+ *tp_version = ga_version;
return GETSET_OVERRIDDEN;
}
/* Potentially has __getattr__ but no custom __getattribute__.
}
else {
*descr = NULL;
+ *tp_version = FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag);
return GETSET_OVERRIDDEN;
}
- PyObject *descriptor = _PyType_LookupRef(type, name);
+ unsigned int descr_version;
+ PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, &descr_version);
*descr = descriptor;
+ *tp_version = have_ga_version ? ga_version : descr_version;
if (descriptor_is_class(descriptor, name)) {
return DUNDER_CLASS;
}
return classify_descriptor(descriptor, has_getattr);
}
-#endif //!Py_GIL_DISABLED
static DescriptorClassification
analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version)
static int
specialize_dict_access_inline(
PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
- DescriptorClassification kind, PyObject *name, unsigned int tp_version,
+ PyObject *name, unsigned int tp_version,
int base_op, int values_op)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(PyUnicode_CheckExact(name));
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(owner);
Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name);
assert (index != DKIX_ERROR);
if (index == DKIX_EMPTY) {
return 0;
}
assert(index >= 0);
+ assert(_PyObject_InlineValues(owner)->valid);
char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index];
Py_ssize_t offset = value_addr - (char *)owner;
if (offset != (uint16_t)offset) {
static int
specialize_dict_access_hint(
PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type,
- DescriptorClassification kind, PyObject *name, unsigned int tp_version,
+ PyObject *name, unsigned int tp_version,
int base_op, int hint_op)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
+
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict);
+
// We found an instance with a __dict__.
if (_PyDict_HasSplitTable(dict)) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (dict == NULL) {
// managed dict, not materialized, inline values valid
- res = specialize_dict_access_inline(owner, instr, type, kind, name,
+ res = specialize_dict_access_inline(owner, instr, type, name,
tp_version, base_op, values_op);
}
else {
int res;
Py_BEGIN_CRITICAL_SECTION(dict);
// materialized managed dict
- res = specialize_dict_access_hint(dict, instr, type, kind, name,
+ res = specialize_dict_access_hint(dict, instr, type, name,
tp_version, base_op, hint_op);
Py_END_CRITICAL_SECTION();
return res;
}
}
-#ifndef Py_GIL_DISABLED
-static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
- PyObject* descr, DescriptorClassification kind, bool is_method);
+static int
+specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
+ PyObject *name, PyObject *descr,
+ unsigned int tp_version,
+ DescriptorClassification kind, bool is_method,
+ uint32_t shared_keys_version);
static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name);
/* Returns true if instances of obj's class are
* For other objects, we check their actual dictionary.
*/
static bool
-instance_has_key(PyObject *obj, PyObject* name)
+instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version)
{
PyTypeObject *cls = Py_TYPE(obj);
if ((cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
}
if (cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
PyDictKeysObject *keys = ((PyHeapTypeObject *)cls)->ht_cached_keys;
- Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
+ Py_ssize_t index =
+ _PyDictKeys_StringLookupAndVersion(keys, name, shared_keys_version);
return index >= 0;
}
PyDictObject *dict = _PyObject_GetManagedDict(obj);
if (dict == NULL || !PyDict_CheckExact(dict)) {
return false;
}
+ bool result;
+ Py_BEGIN_CRITICAL_SECTION(dict);
if (dict->ma_values) {
- return false;
+ result = false;
}
- Py_ssize_t index = _PyDict_LookupIndex(dict, name);
- if (index < 0) {
- return false;
+ else {
+ result = (_PyDict_LookupIndex(dict, name) >= 0);
}
- return true;
+ Py_END_CRITICAL_SECTION();
+ return result;
}
static int
-specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name)
+do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
+ bool shadow, uint32_t shared_keys_version,
+ DescriptorClassification kind, PyObject *descr, unsigned int tp_version)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyTypeObject *type = Py_TYPE(owner);
- bool shadow = instance_has_key(owner, name);
- PyObject *descr = NULL;
- DescriptorClassification kind = analyze_descriptor_load(type, name, &descr);
- Py_XDECREF(descr); // turn strong ref into a borrowed ref
- assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
- if (type_get_version(type, LOAD_ATTR) == 0) {
+ if (tp_version == 0) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
return -1;
}
+ uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg);
switch(kind) {
case OVERRIDING:
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
if (shadow) {
goto try_instance;
}
- int oparg = instr->op.arg;
if (oparg & 1) {
- if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, true)) {
+ if (specialize_attr_loadclassattr(owner, instr, name, descr,
+ tp_version, kind, true,
+ shared_keys_version)) {
return 0;
}
else {
if (!function_check_args(fget, 1, LOAD_ATTR)) {
return -1;
}
- if (instr->op.arg & 1) {
+ if (oparg & 1) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
return -1;
}
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
return -1;
}
- assert(type->tp_version_tag != 0);
- write_u32(lm_cache->type_version, type->tp_version_tag);
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(fget)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ return -1;
+ }
+ #endif
+ assert(tp_version != 0);
+ write_u32(lm_cache->type_version, tp_version);
/* borrowed */
write_obj(lm_cache->descr, fget);
specialize(instr, LOAD_ATTR_PROPERTY);
assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT);
assert(offset > 0);
cache->index = (uint16_t)offset;
- write_u32(cache->version, type->tp_version_tag);
+ write_u32(cache->version, tp_version);
specialize(instr, LOAD_ATTR_SLOT);
return 0;
}
Py_ssize_t offset = offsetof(PyObject, ob_type);
assert(offset == (uint16_t)offset);
cache->index = (uint16_t)offset;
- write_u32(cache->version, type->tp_version_tag);
+ write_u32(cache->version, tp_version);
specialize(instr, LOAD_ATTR_SLOT);
return 0;
}
return -1;
case GETATTRIBUTE_IS_PYTHON_FUNCTION:
{
+ #ifndef Py_GIL_DISABLED
+ // In free-threaded builds it's possible for tp_getattro to change
+ // after the call to analyze_descriptor. That is fine: the version
+ // guard will fail.
assert(type->tp_getattro == _Py_slot_tp_getattro);
+ #endif
assert(Py_IS_TYPE(descr, &PyFunction_Type));
_PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);
if (!function_check_args(descr, 2, LOAD_ATTR)) {
return -1;
}
- if (instr->op.arg & 1) {
+ if (oparg & 1) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
return -1;
}
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
return -1;
}
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(descr)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ return -1;
+ }
+ #endif
write_u32(lm_cache->keys_version, version);
/* borrowed */
write_obj(lm_cache->descr, descr);
- write_u32(lm_cache->type_version, type->tp_version_tag);
+ write_u32(lm_cache->type_version, tp_version);
specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
return 0;
}
if (shadow) {
goto try_instance;
}
- if ((instr->op.arg & 1) == 0) {
- if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, false)) {
+ if ((oparg & 1) == 0) {
+ if (specialize_attr_loadclassattr(owner, instr, name, descr,
+ tp_version, kind, false,
+ shared_keys_version)) {
return 0;
}
}
}
Py_UNREACHABLE();
try_instance:
- if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag,
+ if (specialize_dict_access(owner, instr, type, kind, name, tp_version,
LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
{
return 0;
}
return -1;
}
-#endif // Py_GIL_DISABLED
+
+static int
+specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name)
+{
+ // 0 is not a valid version
+ uint32_t shared_keys_version = 0;
+ bool shadow = instance_has_key(owner, name, &shared_keys_version);
+ PyObject *descr = NULL;
+ unsigned int tp_version = 0;
+ PyTypeObject *type = Py_TYPE(owner);
+ DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version);
+ int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version);
+ Py_XDECREF(descr);
+ return result;
+}
void
_Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name)
fail = specialize_module_load_attr(owner, instr, name);
}
else if (PyType_Check(owner)) {
- #ifdef Py_GIL_DISABLED
- SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
- fail = true;
- #else
fail = specialize_class_load_attr(owner, instr, name);
- #endif
}
else {
- #ifdef Py_GIL_DISABLED
- SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
- fail = true;
- #else
fail = specialize_instance_load_attr(owner, instr, name);
- #endif
}
if (fail) {
return;
}
-#ifndef Py_GIL_DISABLED
-
#ifdef Py_STATS
static int
load_attr_fail_kind(DescriptorClassification kind)
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN);
return -1;
}
- PyObject *metadescriptor = _PyType_Lookup(Py_TYPE(cls), name);
+ unsigned int meta_version = 0;
+ PyObject *metadescriptor = _PyType_LookupRefAndVersion(Py_TYPE(cls), name, &meta_version);
DescriptorClassification metakind = classify_descriptor(metadescriptor, false);
+ Py_XDECREF(metadescriptor);
switch (metakind) {
case METHOD:
case NON_DESCRIPTOR:
}
PyObject *descr = NULL;
DescriptorClassification kind = 0;
- kind = analyze_descriptor_load(cls, name, &descr);
- Py_XDECREF(descr); // turn strong ref into a borrowed ref
- if (type_get_version(cls, LOAD_ATTR) == 0) {
+ unsigned int tp_version = 0;
+ kind = analyze_descriptor_load(cls, name, &descr, &tp_version);
+ if (tp_version == 0) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
+ Py_XDECREF(descr);
return -1;
}
bool metaclass_check = false;
if ((Py_TYPE(cls)->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) == 0) {
metaclass_check = true;
- if (type_get_version(Py_TYPE(cls), LOAD_ATTR) == 0) {
+ if (meta_version == 0) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
+ Py_XDECREF(descr);
return -1;
}
}
switch (kind) {
case METHOD:
case NON_DESCRIPTOR:
- write_u32(cache->type_version, cls->tp_version_tag);
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(descr)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ Py_XDECREF(descr);
+ return -1;
+ }
+ #endif
+ write_u32(cache->type_version, tp_version);
write_obj(cache->descr, descr);
if (metaclass_check) {
- write_u32(cache->keys_version, Py_TYPE(cls)->tp_version_tag);
+ write_u32(cache->keys_version, meta_version);
specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK);
}
else {
specialize(instr, LOAD_ATTR_CLASS);
}
+ Py_XDECREF(descr);
return 0;
#ifdef Py_STATS
case ABSENT:
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
+ Py_XDECREF(descr);
return -1;
#endif
default:
SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind));
+ Py_XDECREF(descr);
return -1;
}
}
// can cause a significant drop in cache hits. A possible test is
// python.exe -m test_typing test_re test_dis test_zlib.
static int
-specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
-PyObject *descr, DescriptorClassification kind, bool is_method)
+specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
+ PyObject *name, PyObject *descr,
+ unsigned int tp_version,
+ DescriptorClassification kind, bool is_method,
+ uint32_t shared_keys_version)
{
_PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
PyTypeObject *owner_cls = Py_TYPE(owner);
assert(descr != NULL);
assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR));
- if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
- PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys;
- assert(_PyDictKeys_StringLookup(keys, name) < 0);
- uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(
- _PyInterpreterState_GET(), keys);
- if (keys_version == 0) {
+
+ #ifdef Py_GIL_DISABLED
+ if (!_PyObject_HasDeferredRefcount(descr)) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
+ return 0;
+ }
+ #endif
+
+ unsigned long tp_flags = PyType_GetFlags(owner_cls);
+ if (tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ #ifndef Py_GIL_DISABLED
+ assert(_PyDictKeys_StringLookup(
+ ((PyHeapTypeObject *)owner_cls)->ht_cached_keys, name) < 0);
+ #endif
+ if (shared_keys_version == 0) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
return 0;
}
- write_u32(cache->keys_version, keys_version);
+ write_u32(cache->keys_version, shared_keys_version);
specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
}
else {
Py_ssize_t dictoffset;
- if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+ if (tp_flags & Py_TPFLAGS_MANAGED_DICT) {
dictoffset = MANAGED_DICT_OFFSET;
}
else {
* PyType_Modified usages in typeobject.c). The MCACHE has been
* working since Python 2.6 and it's battle-tested.
*/
- write_u32(cache->type_version, owner_cls->tp_version_tag);
+ write_u32(cache->type_version, tp_version);
write_obj(cache->descr, descr);
return 1;
}
-#endif // Py_GIL_DISABLED
-
static void
specialize_load_global_lock_held(
return SIMPLE_FUNCTION;
}
-#ifndef Py_GIL_DISABLED
/* Returning false indicates a failure. */
static bool
function_check_args(PyObject *o, int expected_argcount, int opcode)
return version;
}
-/* Returning 0 indicates a failure. */
-static uint32_t
-type_get_version(PyTypeObject *t, int opcode)
-{
- uint32_t version = t->tp_version_tag;
- if (version == 0) {
- SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
- return 0;
- }
- return version;
-}
-#endif // Py_GIL_DISABLED
-
void
_Py_Specialize_BinarySubscr(
_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr)
"""Find the tokens that make up the left-hand side of an assignment"""
offset = 0
for tkn in reversed(node.block.tokens[: idx]):
- if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}:
+ if tkn.kind in {"SEMI", "LBRACE", "RBRACE", "CMACRO"}:
return node.block.tokens[idx - offset : idx]
offset += 1
return []
"PyStackRef_AsPyObjectSteal": self.stackref_steal,
"DISPATCH": self.dispatch,
"INSTRUCTION_SIZE": self.instruction_size,
- "POP_DEAD_INPUTS": self.pop_dead_inputs,
+ "POP_INPUT": self.pop_input,
}
self.out = out
self.emit_save(storage)
return True
- def pop_dead_inputs(
+ def pop_input(
self,
tkn: Token,
tkn_iter: TokenIterator,
inst: Instruction | None,
) -> bool:
next(tkn_iter)
+ name_tkn = next(tkn_iter)
+ name = name_tkn.text
next(tkn_iter)
next(tkn_iter)
- storage.pop_dead_inputs(self.out)
+ if not storage.inputs:
+ raise analysis_error("stack is empty", tkn)
+ tos = storage.inputs[-1]
+ if tos.name != name:
+ raise analysis_error(f"'{name} is not top of stack", name_tkn)
+ tos.defined = False
+ storage.clear_dead_inputs()
+ storage.flush(self.out)
return True
def emit_reload(self, storage: Storage) -> None:
self._push_defined_outputs()
self.stack.flush(out, cast_type, extract_bits)
- def pop_dead_inputs(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = True) -> None:
- self.clear_dead_inputs()
- self.stack.flush(out, cast_type, extract_bits)
-
def save(self, out: CWriter) -> None:
assert self.spilled >= 0
if self.spilled == 0:
projects / thirdparty / Python / cpython.git / commitdiff
