Make instance attributes stored in inline "dict" thread safe on free-threaded builds
PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj);
PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg);
+PyAPI_FUNC(void) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict);
PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj);
#define TYPE_MAX_WATCHERS 8
-
#ifndef Py_INTERNAL_DICT_H
#define Py_INTERNAL_DICT_H
#ifdef __cplusplus
# error "this header requires Py_BUILD_CORE define"
#endif
-#include "pycore_freelist.h" // _PyFreeListState
-#include "pycore_identifier.h" // _Py_Identifier
-#include "pycore_object.h" // PyManagedDictPointer
+#include "pycore_freelist.h" // _PyFreeListState
+#include "pycore_identifier.h" // _Py_Identifier
+#include "pycore_object.h" // PyManagedDictPointer
+#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE
// Unsafe flavor of PyDict_GetItemWithError(): no error checking
extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK);
}
-extern PyDictObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj);
+extern PyDictObject *_PyObject_MaterializeManagedDict(PyObject *obj);
PyAPI_FUNC(PyObject *)_PyDict_FromItems(
PyObject *const *keys, Py_ssize_t keys_offset,
static inline size_t
shared_keys_usable_size(PyDictKeysObject *keys)
{
-#ifdef Py_GIL_DISABLED
// dk_usable will decrease for each instance that is created and each
// value that is added. dk_nentries will increase for each value that
// is added. We want to always return the right value or larger.
// second, and conversely here we read dk_usable first and dk_entries
// second (to avoid the case where we read entries before the increment
// and read usable after the decrement)
- return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
- _Py_atomic_load_ssize_acquire(&keys->dk_nentries));
-#else
- return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
-#endif
+ Py_ssize_t dk_usable = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_usable);
+ Py_ssize_t dk_nentries = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_nentries);
+ return dk_nentries + dk_usable;
}
static inline size_t
#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED()
#include "pycore_emscripten_trampoline.h" // _PyCFunction_TrampolineCall()
#include "pycore_interp.h" // PyInterpreterState.gc
+#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED
#include "pycore_pystate.h" // _PyInterpreterState_GET()
/* Check if an object is consistent. For example, ensure that the reference
extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int);
void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
-extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
- PyObject *name, PyObject *value);
-PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
- PyObject *name);
+extern int _PyObject_StoreInstanceAttribute(PyObject *obj,
+ PyObject *name, PyObject *value);
+extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name,
+ PyObject **attr);
#ifdef Py_GIL_DISABLED
# define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1)
return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET);
}
+static inline PyDictObject *
+_PyObject_GetManagedDict(PyObject *obj)
+{
+ PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj);
+ return (PyDictObject *)FT_ATOMIC_LOAD_PTR_RELAXED(dorv->dict);
+}
+
static inline PyDictValues *
_PyObject_InlineValues(PyObject *obj)
{
#ifdef Py_GIL_DISABLED
#define FT_ATOMIC_LOAD_PTR(value) _Py_atomic_load_ptr(&value)
+#define FT_ATOMIC_STORE_PTR(value, new_value) _Py_atomic_store_ptr(&value, new_value)
#define FT_ATOMIC_LOAD_SSIZE(value) _Py_atomic_load_ssize(&value)
+#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) \
+ _Py_atomic_load_ssize_acquire(&value)
#define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) \
_Py_atomic_load_ssize_relaxed(&value)
#define FT_ATOMIC_STORE_PTR(value, new_value) \
_Py_atomic_load_ptr_acquire(&value)
#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \
_Py_atomic_load_uintptr_acquire(&value)
+#define FT_ATOMIC_LOAD_PTR_RELAXED(value) \
+ _Py_atomic_load_ptr_relaxed(&value)
+#define FT_ATOMIC_LOAD_UINT8(value) \
+ _Py_atomic_load_uint8(&value)
+#define FT_ATOMIC_STORE_UINT8(value, new_value) \
+ _Py_atomic_store_uint8(&value, new_value)
#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \
_Py_atomic_store_ptr_relaxed(&value, new_value)
#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \
#else
#define FT_ATOMIC_LOAD_PTR(value) value
+#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value
#define FT_ATOMIC_LOAD_SSIZE(value) value
+#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value
#define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value
#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value
#define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value
#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value
+#define FT_ATOMIC_LOAD_PTR_RELAXED(value) value
+#define FT_ATOMIC_LOAD_UINT8(value) value
+#define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value
#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value
#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value
#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value
obj.foo = None # Aborted here
self.assertEqual(obj.__dict__, {"foo":None})
+ def test_store_attr_deleted_dict(self):
+ class Foo:
+ pass
+
+ f = Foo()
+ del f.__dict__
+ f.a = 3
+ self.assertEqual(f.a, 3)
+
if __name__ == '__main__':
unittest.main()
uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_MODIFIED, mp, key, value);
if (_PyDict_HasSplitTable(mp)) {
- mp->ma_values->values[ix] = value;
+ STORE_SPLIT_VALUE(mp, ix, value);
if (old_value == NULL) {
_PyDictValues_AddToInsertionOrder(mp->ma_values, ix);
mp->ma_used++;
mp->ma_version_tag = new_version;
if (_PyDict_HasSplitTable(mp)) {
assert(old_value == mp->ma_values->values[ix]);
- mp->ma_values->values[ix] = NULL;
+ STORE_SPLIT_VALUE(mp, ix, NULL);
assert(ix < SHARED_KEYS_MAX_SIZE);
/* Update order */
delete_index_from_values(mp->ma_values, ix);
assert(_PyDict_HasSplitTable(mp));
assert(mp->ma_values->values[ix] == NULL);
MAINTAIN_TRACKING(mp, key, value);
- mp->ma_values->values[ix] = Py_NewRef(value);
+ STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value));
_PyDictValues_AddToInsertionOrder(mp->ma_values, ix);
mp->ma_used++;
mp->ma_version_tag = new_version;
return res;
}
-PyDictObject *
-_PyObject_MakeDictFromInstanceAttributes(PyObject *obj)
+static PyDictObject *
+materialize_managed_dict_lock_held(PyObject *obj)
{
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj);
+
+ OBJECT_STAT_INC(dict_materialized_on_request);
+
PyDictValues *values = _PyObject_InlineValues(obj);
PyInterpreterState *interp = _PyInterpreterState_GET();
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
OBJECT_STAT_INC(dict_materialized_on_request);
- return make_dict_from_instance_attributes(interp, keys, values);
+ PyDictObject *dict = make_dict_from_instance_attributes(interp, keys, values);
+ FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict,
+ (PyDictObject *)dict);
+ return dict;
}
+PyDictObject *
+_PyObject_MaterializeManagedDict(PyObject *obj)
+{
+ PyDictObject *dict = _PyObject_GetManagedDict(obj);
+ if (dict != NULL) {
+ return dict;
+ }
+ Py_BEGIN_CRITICAL_SECTION(obj);
-int
-_PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
+#ifdef Py_GIL_DISABLED
+ dict = _PyObject_GetManagedDict(obj);
+ if (dict != NULL) {
+ // We raced with another thread creating the dict
+ goto exit;
+ }
+#endif
+ dict = materialize_managed_dict_lock_held(obj);
+
+#ifdef Py_GIL_DISABLED
+exit:
+#endif
+ Py_END_CRITICAL_SECTION();
+ return dict;
+}
+
+static int
+set_or_del_lock_held(PyDictObject *dict, PyObject *name, PyObject *value)
+{
+ if (value == NULL) {
+ Py_hash_t hash;
+ if (!PyUnicode_CheckExact(name) || (hash = unicode_get_hash(name)) == -1) {
+ hash = PyObject_Hash(name);
+ if (hash == -1)
+ return -1;
+ }
+ return delitem_knownhash_lock_held((PyObject *)dict, name, hash);
+ } else {
+ return setitem_lock_held(dict, name, value);
+ }
+}
+
+// Called with either the object's lock or the dict's lock held
+// depending on whether or not a dict has been materialized for
+// the object.
+static int
+store_instance_attr_lock_held(PyObject *obj, PyDictValues *values,
PyObject *name, PyObject *value)
{
- PyInterpreterState *interp = _PyInterpreterState_GET();
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
assert(keys != NULL);
assert(values != NULL);
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
Py_ssize_t ix = DKIX_EMPTY;
+ PyDictObject *dict = _PyObject_GetManagedDict(obj);
+ assert(dict == NULL || ((PyDictObject *)dict)->ma_values == values);
if (PyUnicode_CheckExact(name)) {
Py_hash_t hash = unicode_get_hash(name);
if (hash == -1) {
}
#endif
}
- PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict;
+
if (ix == DKIX_EMPTY) {
+ int res;
if (dict == NULL) {
- dict = make_dict_from_instance_attributes(
- interp, keys, values);
- if (dict == NULL) {
+ // Make the dict but don't publish it in the object
+ // so that no one else will see it.
+ dict = make_dict_from_instance_attributes(PyInterpreterState_Get(), keys, values);
+ if (dict == NULL ||
+ set_or_del_lock_held(dict, name, value) < 0) {
+ Py_XDECREF(dict);
return -1;
}
- _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict;
- }
- if (value == NULL) {
- return PyDict_DelItem((PyObject *)dict, name);
- }
- else {
- return PyDict_SetItem((PyObject *)dict, name, value);
+
+ FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict,
+ (PyDictObject *)dict);
+ return 0;
}
+
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict);
+
+ res = set_or_del_lock_held (dict, name, value);
+ return res;
}
+
PyObject *old_value = values->values[ix];
- values->values[ix] = Py_XNewRef(value);
+ FT_ATOMIC_STORE_PTR_RELEASE(values->values[ix], Py_XNewRef(value));
+
if (old_value == NULL) {
if (value == NULL) {
PyErr_Format(PyExc_AttributeError,
return 0;
}
+static inline int
+store_instance_attr_dict(PyObject *obj, PyDictObject *dict, PyObject *name, PyObject *value)
+{
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ int res;
+ Py_BEGIN_CRITICAL_SECTION(dict);
+ if (dict->ma_values == values) {
+ res = store_instance_attr_lock_held(obj, values, name, value);
+ }
+ else {
+ res = set_or_del_lock_held(dict, name, value);
+ }
+ Py_END_CRITICAL_SECTION();
+ return res;
+}
+
+int
+_PyObject_StoreInstanceAttribute(PyObject *obj, PyObject *name, PyObject *value)
+{
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ if (!FT_ATOMIC_LOAD_UINT8(values->valid)) {
+ PyDictObject *dict = _PyObject_GetManagedDict(obj);
+ if (dict == NULL) {
+ dict = (PyDictObject *)PyObject_GenericGetDict(obj, NULL);
+ if (dict == NULL) {
+ return -1;
+ }
+ int res = store_instance_attr_dict(obj, dict, name, value);
+ Py_DECREF(dict);
+ return res;
+ }
+ return store_instance_attr_dict(obj, dict, name, value);
+ }
+
+#ifdef Py_GIL_DISABLED
+ // We have a valid inline values, at least for now... There are two potential
+ // races with having the values become invalid. One is the dictionary
+ // being detached from the object. The other is if someone is inserting
+ // into the dictionary directly and therefore causing it to resize.
+ //
+ // If we haven't materialized the dictionary yet we lock on the object, which
+ // will also be used to prevent the dictionary from being materialized while
+ // we're doing the insertion. If we race and the dictionary gets created
+ // then we'll need to release the object lock and lock the dictionary to
+ // prevent resizing.
+ PyDictObject *dict = _PyObject_GetManagedDict(obj);
+ if (dict == NULL) {
+ int res;
+ Py_BEGIN_CRITICAL_SECTION(obj);
+ dict = _PyObject_GetManagedDict(obj);
+
+ if (dict == NULL) {
+ res = store_instance_attr_lock_held(obj, values, name, value);
+ }
+ Py_END_CRITICAL_SECTION();
+
+ if (dict == NULL) {
+ return res;
+ }
+ }
+ return store_instance_attr_dict(obj, dict, name, value);
+#else
+ return store_instance_attr_lock_held(obj, values, name, value);
+#endif
+}
+
/* Sanity check for managed dicts */
#if 0
#define CHECK(val) assert(val); if (!(val)) { return 0; }
}
#endif
-PyObject *
-_PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
- PyObject *name)
+// Attempts to get an instance attribute from the inline values. Returns true
+// if successful, or false if the caller needs to lookup in the dictionary.
+bool
+_PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr)
{
assert(PyUnicode_CheckExact(name));
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ if (!FT_ATOMIC_LOAD_UINT8(values->valid)) {
+ return false;
+ }
+
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
assert(keys != NULL);
Py_ssize_t ix = _PyDictKeys_StringLookup(keys, name);
if (ix == DKIX_EMPTY) {
- return NULL;
+ *attr = NULL;
+ return true;
+ }
+
+#ifdef Py_GIL_DISABLED
+ PyObject *value = _Py_atomic_load_ptr_relaxed(&values->values[ix]);
+ if (value == NULL || _Py_TryIncrefCompare(&values->values[ix], value)) {
+ *attr = value;
+ return true;
+ }
+
+ PyDictObject *dict = _PyObject_GetManagedDict(obj);
+ if (dict == NULL) {
+ // No dict, lock the object to prevent one from being
+ // materialized...
+ bool success = false;
+ Py_BEGIN_CRITICAL_SECTION(obj);
+
+ dict = _PyObject_GetManagedDict(obj);
+ if (dict == NULL) {
+ // Still no dict, we can read from the values
+ assert(values->valid);
+ value = values->values[ix];
+ *attr = Py_XNewRef(value);
+ success = true;
+ }
+
+ Py_END_CRITICAL_SECTION();
+
+ if (success) {
+ return true;
+ }
}
+
+ // We have a dictionary, we'll need to lock it to prevent
+ // the values from being resized.
+ assert(dict != NULL);
+
+ bool success;
+ Py_BEGIN_CRITICAL_SECTION(dict);
+
+ if (dict->ma_values == values && FT_ATOMIC_LOAD_UINT8(values->valid)) {
+ value = _Py_atomic_load_ptr_relaxed(&values->values[ix]);
+ *attr = Py_XNewRef(value);
+ success = true;
+ } else {
+ // Caller needs to lookup from the dictionary
+ success = false;
+ }
+
+ Py_END_CRITICAL_SECTION();
+
+ return success;
+#else
PyObject *value = values->values[ix];
- return Py_XNewRef(value);
+ *attr = Py_XNewRef(value);
+ return true;
+#endif
}
int
PyDictObject *dict;
if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
PyDictValues *values = _PyObject_InlineValues(obj);
- if (values->valid) {
+ if (FT_ATOMIC_LOAD_UINT8(values->valid)) {
PyDictKeysObject *keys = CACHED_KEYS(tp);
for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
- if (values->values[i] != NULL) {
+ if (FT_ATOMIC_LOAD_PTR_RELAXED(values->values[i]) != NULL) {
return 0;
}
}
return 1;
}
- dict = _PyObject_ManagedDictPointer(obj)->dict;
+ dict = _PyObject_GetManagedDict(obj);
}
else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
- PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
- dict = managed_dict->dict;
+ dict = _PyObject_GetManagedDict(obj);
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
return 0;
}
+static void
+set_dict_inline_values(PyObject *obj, PyDictObject *new_dict)
+{
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj);
+
+ PyDictValues *values = _PyObject_InlineValues(obj);
+
+ Py_XINCREF(new_dict);
+ FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, new_dict);
+
+ if (values->valid) {
+ FT_ATOMIC_STORE_UINT8(values->valid, 0);
+ for (Py_ssize_t i = 0; i < values->capacity; i++) {
+ Py_CLEAR(values->values[i]);
+ }
+ }
+}
+
void
-PyObject_ClearManagedDict(PyObject *obj)
+_PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict)
{
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
assert(_PyObject_InlineValuesConsistencyCheck(obj));
PyTypeObject *tp = Py_TYPE(obj);
if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
- PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict;
- if (dict) {
- _PyDict_DetachFromObject(dict, obj);
- _PyObject_ManagedDictPointer(obj)->dict = NULL;
- Py_DECREF(dict);
- }
- else {
- PyDictValues *values = _PyObject_InlineValues(obj);
- if (values->valid) {
- for (Py_ssize_t i = 0; i < values->capacity; i++) {
- Py_CLEAR(values->values[i]);
- }
- values->valid = 0;
+ PyDictObject *dict = _PyObject_GetManagedDict(obj);
+ if (dict == NULL) {
+#ifdef Py_GIL_DISABLED
+ Py_BEGIN_CRITICAL_SECTION(obj);
+
+ dict = _PyObject_ManagedDictPointer(obj)->dict;
+ if (dict == NULL) {
+ set_dict_inline_values(obj, (PyDictObject *)new_dict);
+ }
+
+ Py_END_CRITICAL_SECTION();
+
+ if (dict == NULL) {
+ return;
}
+#else
+ set_dict_inline_values(obj, (PyDictObject *)new_dict);
+ return;
+#endif
}
+
+ Py_BEGIN_CRITICAL_SECTION2(dict, obj);
+
+ // We've locked dict, but the actual dict could have changed
+ // since we locked it.
+ dict = _PyObject_ManagedDictPointer(obj)->dict;
+
+ FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict,
+ (PyDictObject *)Py_XNewRef(new_dict));
+
+ _PyDict_DetachFromObject(dict, obj);
+
+ Py_END_CRITICAL_SECTION2();
+
+ Py_XDECREF(dict);
}
else {
- Py_CLEAR(_PyObject_ManagedDictPointer(obj)->dict);
+ PyDictObject *dict;
+
+ Py_BEGIN_CRITICAL_SECTION(obj);
+
+ dict = _PyObject_ManagedDictPointer(obj)->dict;
+
+ FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict,
+ (PyDictObject *)Py_XNewRef(new_dict));
+
+ Py_END_CRITICAL_SECTION();
+
+ Py_XDECREF(dict);
}
assert(_PyObject_InlineValuesConsistencyCheck(obj));
}
+void
+PyObject_ClearManagedDict(PyObject *obj)
+{
+ _PyObject_SetManagedDict(obj, NULL);
+}
+
int
_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj)
{
- assert(_PyObject_ManagedDictPointer(obj)->dict == mp);
- assert(_PyObject_InlineValuesConsistencyCheck(obj));
- if (mp->ma_values == NULL || mp->ma_values != _PyObject_InlineValues(obj)) {
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj);
+
+ if (FT_ATOMIC_LOAD_PTR_RELAXED(mp->ma_values) != _PyObject_InlineValues(obj)) {
return 0;
}
+
+ // We could be called with an unlocked dict when the caller knows the
+ // values are already detached, so we assert after inline values check.
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp);
assert(mp->ma_values->embedded == 1);
assert(mp->ma_values->valid == 1);
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- Py_BEGIN_CRITICAL_SECTION(mp);
- mp->ma_values = copy_values(mp->ma_values);
- _PyObject_InlineValues(obj)->valid = 0;
- Py_END_CRITICAL_SECTION();
- if (mp->ma_values == NULL) {
+
+ PyDictValues *values = copy_values(mp->ma_values);
+
+ if (values == NULL) {
return -1;
}
+ mp->ma_values = values;
+
+ FT_ATOMIC_STORE_UINT8(_PyObject_InlineValues(obj)->valid, 0);
+
assert(_PyObject_InlineValuesConsistencyCheck(obj));
ASSERT_CONSISTENT(mp);
return 0;
{
PyInterpreterState *interp = _PyInterpreterState_GET();
PyTypeObject *tp = Py_TYPE(obj);
+ PyDictObject *dict;
if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) {
- PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
- PyDictObject *dict = managed_dict->dict;
+ dict = _PyObject_GetManagedDict(obj);
if (dict == NULL &&
(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
- _PyObject_InlineValues(obj)->valid
- ) {
- PyDictValues *values = _PyObject_InlineValues(obj);
- OBJECT_STAT_INC(dict_materialized_on_request);
- dict = make_dict_from_instance_attributes(
- interp, CACHED_KEYS(tp), values);
- if (dict != NULL) {
- managed_dict->dict = (PyDictObject *)dict;
- }
+ FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(obj)->valid)) {
+ dict = _PyObject_MaterializeManagedDict(obj);
}
- else {
- dict = managed_dict->dict;
+ else if (dict == NULL) {
+ Py_BEGIN_CRITICAL_SECTION(obj);
+
+ // Check again that we're not racing with someone else creating the dict
+ dict = _PyObject_GetManagedDict(obj);
if (dict == NULL) {
- dictkeys_incref(CACHED_KEYS(tp));
OBJECT_STAT_INC(dict_materialized_on_request);
+ dictkeys_incref(CACHED_KEYS(tp));
dict = (PyDictObject *)new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
- managed_dict->dict = (PyDictObject *)dict;
+ FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict,
+ (PyDictObject *)dict);
}
+
+ Py_END_CRITICAL_SECTION();
}
return Py_XNewRef((PyObject *)dict);
}
return 1;
}
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = (PyDictObject *)_PyObject_ManagedDictPointer(obj)->dict;
+ PyDictObject *dict = _PyObject_GetManagedDict(obj);
if (dict == NULL) {
return 1;
}
#include "pycore_call.h" // _PyObject_CallNoArgs()
#include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate()
#include "pycore_context.h" // _PyContextTokenMissing_Type
+#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION
#include "pycore_descrobject.h" // _PyMethodWrapper_Type
#include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes()
#include "pycore_floatobject.h" // _PyFloat_DebugMallocStats()
#include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic
#include "pycore_unionobject.h" // _PyUnion_Type
+
#ifdef Py_LIMITED_API
// Prevent recursive call _Py_IncRef() <=> Py_INCREF()
# error "Py_LIMITED_API macro must not be defined"
if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
return _PyObject_ComputedDictPointer(obj);
}
- PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
- if (managed_dict->dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
- PyDictObject *dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(obj);
+ PyDictObject *dict = _PyObject_GetManagedDict(obj);
+ if (dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ dict = _PyObject_MaterializeManagedDict(obj);
if (dict == NULL) {
PyErr_Clear();
return NULL;
}
- managed_dict->dict = dict;
}
- return (PyObject **)&managed_dict->dict;
+ return (PyObject **)&_PyObject_ManagedDictPointer(obj)->dict;
}
PyObject *
}
}
}
- PyObject *dict;
- if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
- PyDictValues *values = _PyObject_InlineValues(obj);
- PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name);
+ PyObject *dict, *attr;
+ if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
+ _PyObject_TryGetInstanceAttribute(obj, name, &attr)) {
if (attr != NULL) {
*method = attr;
Py_XDECREF(descr);
dict = NULL;
}
else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
- PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
- dict = (PyObject *)managed_dict->dict;
+ dict = (PyObject *)_PyObject_GetManagedDict(obj);
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
}
}
if (dict == NULL) {
- if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
- PyDictValues *values = _PyObject_InlineValues(obj);
- if (PyUnicode_CheckExact(name)) {
- res = _PyObject_GetInstanceAttribute(obj, values, name);
+ if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) {
+ if (PyUnicode_CheckExact(name) &&
+ _PyObject_TryGetInstanceAttribute(obj, name, &res)) {
if (res != NULL) {
goto done;
}
}
else {
- dict = (PyObject *)_PyObject_MakeDictFromInstanceAttributes(obj);
+ dict = (PyObject *)_PyObject_MaterializeManagedDict(obj);
if (dict == NULL) {
res = NULL;
goto done;
}
- _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict;
}
}
else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
- PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
- dict = (PyObject *)managed_dict->dict;
+ dict = (PyObject *)_PyObject_GetManagedDict(obj);
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
if (dict == NULL) {
PyObject **dictptr;
- if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
- res = _PyObject_StoreInstanceAttribute(
- obj, _PyObject_InlineValues(obj), name, value);
+
+ if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) {
+ res = _PyObject_StoreInstanceAttribute(obj, name, value);
goto error_check;
}
- else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
+
+ if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
dictptr = (PyObject **)&managed_dict->dict;
}
PyObject **dictptr = _PyObject_GetDictPtr(obj);
if (dictptr == NULL) {
if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) &&
- _PyObject_ManagedDictPointer(obj)->dict == NULL
+ _PyObject_GetManagedDict(obj) == NULL
) {
/* Was unable to convert to dict */
PyErr_NoMemory();
"not a '%.200s'", Py_TYPE(value)->tp_name);
return -1;
}
+
if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
- PyObject_ClearManagedDict(obj);
- _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)Py_XNewRef(value);
+ _PyObject_SetManagedDict(obj, value);
}
else {
dictptr = _PyObject_ComputedDictPointer(obj);
/* Changing the class will change the implicit dict keys,
* so we must materialize the dictionary first. */
if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
- PyDictObject *dict = _PyObject_ManagedDictPointer(self)->dict;
+ PyDictObject *dict = _PyObject_MaterializeManagedDict(self);
if (dict == NULL) {
- dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(self);
- if (dict == NULL) {
- return -1;
- }
- _PyObject_ManagedDictPointer(self)->dict = dict;
+ return -1;
+ }
+
+ bool error = false;
+
+ Py_BEGIN_CRITICAL_SECTION2(self, dict);
+
+ // If we raced after materialization and replaced the dict
+ // then the materialized dict should no longer have the
+ // inline values in which case detach is a nop.
+ assert(_PyObject_GetManagedDict(self) == dict ||
+ dict->ma_values != _PyObject_InlineValues(self));
+
+ if (_PyDict_DetachFromObject(dict, self) < 0) {
+ error = true;
}
- if (_PyDict_DetachFromObject(dict, self)) {
+
+ Py_END_CRITICAL_SECTION2();
+ if (error) {
return -1;
}
}
op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
- PyDictObject *dict = managed_dict->dict;
+ PyDictObject *dict = _PyObject_GetManagedDict(owner);
DEOPT_IF(dict == NULL);
assert(PyDict_CheckExact((PyObject *)dict));
}
op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) {
- PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
- PyDictObject *dict = managed_dict->dict;
+ PyDictObject *dict = _PyObject_GetManagedDict(owner);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
op(_GUARD_DORV_NO_DICT, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict);
+ DEOPT_IF(_PyObject_GetManagedDict(owner));
DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0);
}
op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) {
STAT_INC(STORE_ATTR, hit);
- assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
+ assert(_PyObject_GetManagedDict(owner) == NULL);
PyDictValues *values = _PyObject_InlineValues(owner);
+
PyObject *old_value = values->values[index];
values->values[index] = value;
if (old_value == NULL) {
else {
Py_DECREF(old_value);
}
+
Py_DECREF(owner);
}
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
- PyDictObject *dict = managed_dict->dict;
+ PyDictObject *dict = _PyObject_GetManagedDict(owner);
DEOPT_IF(dict == NULL);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
- PyDictObject *dict = managed_dict->dict;
+ PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (dict == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
uint16_t hint = (uint16_t)CURRENT_OPERAND();
- PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
- PyDictObject *dict = managed_dict->dict;
+ PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- if (_PyObject_ManagedDictPointer(owner)->dict) {
+ if (_PyObject_GetManagedDict(owner)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
value = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND();
STAT_INC(STORE_ATTR, hit);
- assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
+ assert(_PyObject_GetManagedDict(owner) == NULL);
PyDictValues *values = _PyObject_InlineValues(owner);
PyObject *old_value = values->values[index];
values->values[index] = value;
// _CHECK_ATTR_WITH_HINT
{
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
- PyDictObject *dict = managed_dict->dict;
+ PyDictObject *dict = _PyObject_GetManagedDict(owner);
DEOPT_IF(dict == NULL, LOAD_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
}
// _LOAD_ATTR_WITH_HINT
{
uint16_t hint = read_u16(&this_instr[4].cache);
- PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
- PyDictObject *dict = managed_dict->dict;
+ PyDictObject *dict = _PyObject_GetManagedDict(owner);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
{
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict, STORE_ATTR);
+ DEOPT_IF(_PyObject_GetManagedDict(owner), STORE_ATTR);
DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0, STORE_ATTR);
}
// _STORE_ATTR_INSTANCE_VALUE
{
uint16_t index = read_u16(&this_instr[4].cache);
STAT_INC(STORE_ATTR, hit);
- assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
+ assert(_PyObject_GetManagedDict(owner) == NULL);
PyDictValues *values = _PyObject_InlineValues(owner);
PyObject *old_value = values->values[index];
values->values[index] = value;
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
- PyDictObject *dict = managed_dict->dict;
+ PyDictObject *dict = _PyObject_GetManagedDict(owner);
DEOPT_IF(dict == NULL, STORE_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
instr->op.code = values_op;
}
else {
- PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
- PyDictObject *dict = managed_dict->dict;
+ PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (dict == NULL || !PyDict_CheckExact(dict)) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
return 0;
NON_ESCAPING_FUNCTIONS = (
"Py_INCREF",
"_PyManagedDictPointer_IsValues",
+ "_PyObject_GetManagedDict",
"_PyObject_ManagedDictPointer",
"_PyObject_InlineValues",
"_PyDictValues_AddToInsertionOrder",
projects / thirdparty / Python / cpython.git / commitdiff
