PublicArchive/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-07-26 20:03:16 -04:00
Code Issues Packages Projects Releases Wiki Activity

gh-150490: Raise PyType_Modified for insertion into split dictionary (take 2) (#152914)

Browse Source 
Raise PyType_Modified for insertion into split dictionary - now with better locking discipline in free-threaded builds
This commit is contained in:
Dino Viehland
2026-07-06 12:56:32 -07:00
committed by GitHub
parent 28aa32d852
commit 20921922fa
14 changed files with 1396 additions and 1420 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Include/internal/pycore_dict.h
+12
View File
@@ -90,6 +90,7 @@ typedef struct {
} PyDictUnicodeEntry;
extern PyDictKeysObject *_PyDict_NewKeysForClass(PyHeapTypeObject *);
extern void _PyDict_RemoveKeysForClass(PyHeapTypeObject *);
extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *);
/* Implementations of the `|` and `|=` operators for dict, used by the
@@ -239,6 +240,17 @@ struct _dictkeysobject {
see the DK_ENTRIES() / DK_UNICODE_ENTRIES() functions below */
};
struct _instancekeysobject {
PyTypeObject* dsk_owning_type;
struct _dictkeysobject dsk_keys;
};
static inline struct _instancekeysobject *_PyDictKeys_AsSharedKeys(struct _dictkeysobject *keys)
{
assert(keys->dk_kind == DICT_KEYS_SPLIT);
return _Py_CONTAINER_OF(keys, struct _instancekeysobject, dsk_keys);
}
/* This must be no more than 250, for the prefix size to fit in one byte. */
#define SHARED_KEYS_MAX_SIZE 30
#define NEXT_LOG2_SHARED_KEYS_MAX_SIZE 6
Include/internal/pycore_opcode_metadata.h
Generated
+2 -2
View File
@@ -1458,10 +1458,10 @@ _PyOpcode_macro_expansion[256] = {
[LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } },
[LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } },
[LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } },
[LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } },
[LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } },
[LOAD_ATTR_MODULE] = { .nuops = 4, .uops = { { _LOAD_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, OPERAND1_1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } },
[LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } },
[LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } },
[LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } },
[LOAD_ATTR_PROPERTY] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, 2, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } },
[LOAD_ATTR_SLOT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } },
[LOAD_ATTR_WITH_HINT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } },
Include/internal/pycore_typeobject.h
+3
View File
@@ -148,6 +148,9 @@ extern int _PyType_AddMethod(PyTypeObject *, PyMethodDef *);
extern void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask,
unsigned long flags);
// Raise PyType_Modified with the type lock already held.
extern void _PyType_Modified_Unlocked(PyTypeObject *type);
PyAPI_FUNC(void) _PyType_SetVersion(PyTypeObject *tp, unsigned int version);
PyTypeObject *_PyType_LookupByVersion(unsigned int version);
Include/internal/pycore_uop_ids.h
Generated
+1186 -1191
View File
File diff suppressed because it is too large Load Diff
Include/internal/pycore_uop_metadata.h
Generated
-21
View File
@@ -283,7 +283,6 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_PUSH_EXC_INFO] = 0,
[_GUARD_KEYS_VERSION] = HAS_EXIT_FLAG,
[_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG,
[_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG,
[_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
@@ -2677,15 +2676,6 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = {
{ -1, -1, -1 },
},
},
[_GUARD_KEYS_VERSION] = {
.best = { 0, 1, 2, 3 },
.entries = {
{ 1, 0, _GUARD_KEYS_VERSION_r01 },
{ 1, 1, _GUARD_KEYS_VERSION_r11 },
{ 2, 2, _GUARD_KEYS_VERSION_r22 },
{ 3, 3, _GUARD_KEYS_VERSION_r33 },
},
},
[_LOAD_ATTR_METHOD_WITH_VALUES] = {
.best = { 0, 1, 2, 2 },
.entries = {
@@ -4496,10 +4486,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = {
[_PUSH_EXC_INFO_r02] = _PUSH_EXC_INFO,
[_PUSH_EXC_INFO_r12] = _PUSH_EXC_INFO,
[_PUSH_EXC_INFO_r23] = _PUSH_EXC_INFO,
[_GUARD_KEYS_VERSION_r01] = _GUARD_KEYS_VERSION,
[_GUARD_KEYS_VERSION_r11] = _GUARD_KEYS_VERSION,
[_GUARD_KEYS_VERSION_r22] = _GUARD_KEYS_VERSION,
[_GUARD_KEYS_VERSION_r33] = _GUARD_KEYS_VERSION,
[_LOAD_ATTR_METHOD_WITH_VALUES_r02] = _LOAD_ATTR_METHOD_WITH_VALUES,
[_LOAD_ATTR_METHOD_WITH_VALUES_r12] = _LOAD_ATTR_METHOD_WITH_VALUES,
[_LOAD_ATTR_METHOD_WITH_VALUES_r23] = _LOAD_ATTR_METHOD_WITH_VALUES,
@@ -5415,11 +5401,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = {
[_GUARD_ITER_VIRTUAL_r11] = "_GUARD_ITER_VIRTUAL_r11",
[_GUARD_ITER_VIRTUAL_r22] = "_GUARD_ITER_VIRTUAL_r22",
[_GUARD_ITER_VIRTUAL_r33] = "_GUARD_ITER_VIRTUAL_r33",
[_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION",
[_GUARD_KEYS_VERSION_r01] = "_GUARD_KEYS_VERSION_r01",
[_GUARD_KEYS_VERSION_r11] = "_GUARD_KEYS_VERSION_r11",
[_GUARD_KEYS_VERSION_r22] = "_GUARD_KEYS_VERSION_r22",
[_GUARD_KEYS_VERSION_r33] = "_GUARD_KEYS_VERSION_r33",
[_GUARD_LOAD_SUPER_ATTR_METHOD] = "_GUARD_LOAD_SUPER_ATTR_METHOD",
[_GUARD_LOAD_SUPER_ATTR_METHOD_r03] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r03",
[_GUARD_LOAD_SUPER_ATTR_METHOD_r13] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r13",
@@ -6662,8 +6643,6 @@ int _PyUop_num_popped(int opcode, int oparg)
return 0;
case _PUSH_EXC_INFO:
return 1;
case _GUARD_KEYS_VERSION:
return 0;
case _LOAD_ATTR_METHOD_WITH_VALUES:
return 1;
case _LOAD_ATTR_METHOD_NO_DICT:
Lib/test/test_free_threading/test_type.py
+79 -1
View File
@@ -2,7 +2,7 @@ import threading
import unittest
from concurrent.futures import ThreadPoolExecutor
from threading import Thread
from threading import Barrier, Thread
from unittest import TestCase
from test.support import threading_helper
@@ -160,6 +160,84 @@ class TestType(TestCase):
self.run_one(writer, reader)
def test_race_type_attr_added(self):
NROUNDS = 50
NSTOPPERS = 4
NWRITERS = 4
WARM = 8
KEY = "foo"
def make_reader():
ns = {}
exec(
"def read(o):\n return o.%s\n" % KEY, ns
) # fresh code object per round
return ns["read"]
stop_all = [False]
def stopper():
class Dummy:
pass
Dummy()
while not stop_all[0]:
try:
Dummy.__abstractmethods__ = frozenset()
except Exception:
pass
box = {}
bugs = [] # (round, tid, stored, read_back)
def writer(tid):
for _ in range(NROUNDS):
box["start"].wait()
sentinel = box["sentinel"]
reader = box["reader"]
obj = box["objs"][tid]
val = box["vals"][tid]
# 1) warm THIS thread's own copy while KEY is absent -> NDV cached at V
for _ in range(WARM):
reader(obj)
box["race"].wait()
# 2) race: store, then read back through our own NDV site
setattr(obj, KEY, val)
got = reader(obj)
if got is sentinel:
bugs.append((box["round"], tid, val, got))
box["end"].wait()
box["start"] = Barrier(NWRITERS + 1)
box["race"] = Barrier(NWRITERS)
box["end"] = Barrier(NWRITERS + 1)
stoppers = [Thread(target=stopper, daemon=True) for _ in range(NSTOPPERS)]
writers = [Thread(target=writer, args=(i,)) for i in range(NWRITERS)]
for t in stoppers + writers:
t.start()
for r in range(NROUNDS):
sentinel = type(
"SENTINEL_%d" % r, (), {}
) # non-descriptor, deferred refcount
C = type("C", (), {KEY: sentinel})
box["round"] = r
box["sentinel"] = sentinel
box["reader"] = make_reader()
box["objs"] = [C() for _ in range(NWRITERS)]
box["vals"] = [["value-%d-%d" % (r, i)] for i in range(NWRITERS)]
box["start"].wait()
box["end"].wait()
stop_all[0] = True
for t in writers:
t.join()
self.assertFalse(bugs)
def run_one(self, writer_func, reader_func):
barrier = threading.Barrier(NTHREADS)
Modules/_testinternalcapi/test_cases.c.h
Generated
+2 -24
View File
@@ -9166,18 +9166,7 @@
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
/* Skip 2 cache entries */
// _LOAD_ATTR_METHOD_WITH_VALUES
{
PyObject *descr = read_obj(&this_instr[6].cache);
@@ -9362,18 +9351,7 @@
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
/* Skip 2 cache entries */
// _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
{
PyObject *descr = read_obj(&this_instr[6].cache);
Objects/dictobject.c
+90 -31
View File
@@ -799,16 +799,12 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
}
static PyDictKeysObject*
new_keys_object(uint8_t log2_size, bool unicode)
static inline int
get_log2_bytes(uint8_t log2_size)
{
Py_ssize_t usable;
int log2_bytes;
size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry);
assert(log2_size >= PyDict_LOG_MINSIZE);
usable = USABLE_FRACTION((size_t)1<<log2_size);
if (log2_size < 8) {
log2_bytes = log2_size;
}
@@ -824,6 +820,38 @@ new_keys_object(uint8_t log2_size, bool unicode)
log2_bytes = log2_size + 2;
}
return log2_bytes;
}
static inline void
init_keys_object(PyDictKeysObject* dk, uint8_t log2_size, int log2_bytes, int kind,
Py_ssize_t usable, Py_ssize_t entry_size)
{
#ifdef Py_REF_DEBUG
_Py_IncRefTotal(_PyThreadState_GET());
#endif
dk->dk_refcnt = 1;
dk->dk_log2_size = log2_size;
dk->dk_log2_index_bytes = log2_bytes;
dk->dk_kind = kind;
#ifdef Py_GIL_DISABLED
dk->dk_mutex = (PyMutex){0};
#endif
dk->dk_nentries = 0;
dk->dk_usable = usable;
dk->dk_version = 0;
memset(&dk->dk_indices[0], 0xff, ((size_t)1 << log2_bytes));
memset(&dk->dk_indices[(size_t)1 << log2_bytes], 0, entry_size * usable);
}
static PyDictKeysObject*
new_keys_object(uint8_t log2_size, bool unicode)
{
Py_ssize_t usable = USABLE_FRACTION((size_t)1<<log2_size);
size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry);
int log2_bytes = get_log2_bytes(log2_size);
PyDictKeysObject *dk = NULL;
if (log2_size == PyDict_LOG_MINSIZE && unicode) {
dk = _Py_FREELIST_POP_MEM(dictkeys);
@@ -837,30 +865,28 @@ new_keys_object(uint8_t log2_size, bool unicode)
return NULL;
}
}
#ifdef Py_REF_DEBUG
_Py_IncRefTotal(_PyThreadState_GET());
#endif
dk->dk_refcnt = 1;
dk->dk_log2_size = log2_size;
dk->dk_log2_index_bytes = log2_bytes;
dk->dk_kind = unicode ? DICT_KEYS_UNICODE : DICT_KEYS_GENERAL;
#ifdef Py_GIL_DISABLED
dk->dk_mutex = (PyMutex){0};
#endif
dk->dk_nentries = 0;
dk->dk_usable = usable;
dk->dk_version = 0;
memset(&dk->dk_indices[0], 0xff, ((size_t)1 << log2_bytes));
memset(&dk->dk_indices[(size_t)1 << log2_bytes], 0, entry_size * usable);
init_keys_object(dk, log2_size, log2_bytes,
unicode ? DICT_KEYS_UNICODE : DICT_KEYS_GENERAL,
usable, entry_size);
return dk;
}
static void
free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
{
void *ptr = keys;
#ifdef Py_GIL_DISABLED
size_t size = _PyDict_KeysSize(keys);
#endif
if (keys->dk_kind == DICT_KEYS_SPLIT) {
ptr = _PyDictKeys_AsSharedKeys(keys);
#ifdef Py_GIL_DISABLED
size += offsetof(struct _instancekeysobject, dsk_keys);
#endif
}
#ifdef Py_GIL_DISABLED
if (use_qsbr) {
_PyMem_FreeDelayed(keys, _PyDict_KeysSize(keys));
_PyMem_FreeDelayed(ptr, size);
return;
}
#endif
@@ -868,7 +894,7 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
_Py_FREELIST_FREE(dictkeys, keys, PyMem_Free);
}
else {
PyMem_Free(keys);
PyMem_Free(ptr);
}
}
@@ -1925,6 +1951,12 @@ insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash)
if (ix >= 0) {
return ix;
}
// We need to acquire the type lock before the keys mutex. Another lock
// is never acquired below the keys mutex but a keys mutex can be acquired
// elsewhere while we hold the types lock. To avoid deadlocks we must always
// acquire the type lock first.
Py_BEGIN_CRITICAL_SECTION_MUTEX(&_PyInterpreterState_GET()->types.mutex);
#endif
LOCK_KEYS(keys);
@@ -1932,6 +1964,12 @@ insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash)
if (ix == DKIX_EMPTY && keys->dk_usable > 0) {
// Insert into new slot
FT_ATOMIC_STORE_UINT32_RELAXED(keys->dk_version, 0);
struct _instancekeysobject *shared_keys = _PyDictKeys_AsSharedKeys(keys);
PyTypeObject *type = FT_ATOMIC_LOAD_PTR_ACQUIRE(shared_keys->dsk_owning_type);
if (type) {
// we acquired the type lock above
_PyType_Modified_Unlocked(type);
}
Py_ssize_t hashpos = find_empty_slot(keys, hash);
ix = keys->dk_nentries;
dictkeys_set_index(keys, hashpos, ix);
@@ -1941,6 +1979,10 @@ insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash)
}
assert (ix < SHARED_KEYS_MAX_SIZE);
UNLOCK_KEYS(keys);
#ifdef Py_GIL_DISABLED
Py_END_CRITICAL_SECTION();
#endif
return ix;
}
@@ -7216,16 +7258,24 @@ dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored))
PyDictKeysObject *
_PyDict_NewKeysForClass(PyHeapTypeObject *cls)
{
PyDictKeysObject *keys = new_keys_object(NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1);
if (keys == NULL) {
int log2_bytes = get_log2_bytes(NEXT_LOG2_SHARED_KEYS_MAX_SIZE);
Py_ssize_t usable = USABLE_FRACTION((size_t)1<<NEXT_LOG2_SHARED_KEYS_MAX_SIZE);
struct _instancekeysobject *shared_keys =
PyMem_Malloc(sizeof(struct _instancekeysobject)
+ ((size_t)1 << log2_bytes)
+ sizeof(PyDictUnicodeEntry) * usable);
if (shared_keys == NULL) {
PyErr_Clear();
return NULL;
}
else {
assert(keys->dk_nentries == 0);
/* Set to max size+1 as it will shrink by one before each new object */
keys->dk_usable = SHARED_KEYS_MAX_SIZE;
keys->dk_kind = DICT_KEYS_SPLIT;
}
shared_keys->dsk_owning_type = (PyTypeObject *)cls;
PyDictKeysObject* keys = &shared_keys->dsk_keys;
init_keys_object(keys, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, log2_bytes, DICT_KEYS_SPLIT,
SHARED_KEYS_MAX_SIZE, sizeof(PyDictUnicodeEntry));
assert(keys->dk_nentries == 0);
/* Set to max size+1 as it will shrink by one before each new object */
if (cls->ht_type.tp_dict) {
PyObject *attrs = PyDict_GetItem(cls->ht_type.tp_dict, &_Py_ID(__static_attributes__));
if (attrs != NULL && PyTuple_Check(attrs)) {
@@ -7243,6 +7293,15 @@ _PyDict_NewKeysForClass(PyHeapTypeObject *cls)
return keys;
}
void
_PyDict_RemoveKeysForClass(PyHeapTypeObject *cls)
{
struct _instancekeysobject *shared_keys = _PyDictKeys_AsSharedKeys(cls->ht_cached_keys);
FT_ATOMIC_STORE_PTR_RELEASE(shared_keys->dsk_owning_type, NULL);
_PyDictKeys_DecRef(cls->ht_cached_keys);
}
void
_PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp)
{
Objects/typeobject.c
+13 -13
View File
@@ -1162,8 +1162,8 @@ set_version_unlocked(PyTypeObject *tp, unsigned int version)
#endif
}
static void
type_modified_unlocked(PyTypeObject *type)
void
_PyType_Modified_Unlocked(PyTypeObject *type)
{
/* Invalidate any cached data for the specified type and all
subclasses. This function is called after the base
@@ -1203,7 +1203,7 @@ type_modified_unlocked(PyTypeObject *type)
if (subclass == NULL) {
continue;
}
type_modified_unlocked(subclass);
_PyType_Modified_Unlocked(subclass);
Py_DECREF(subclass);
}
}
@@ -1248,7 +1248,7 @@ PyType_Modified(PyTypeObject *type)
}
BEGIN_TYPE_LOCK();
type_modified_unlocked(type);
_PyType_Modified_Unlocked(type);
END_TYPE_LOCK();
}
@@ -1741,7 +1741,7 @@ type_set_abstractmethods(PyObject *tp, PyObject *value, void *Py_UNUSED(closure)
}
BEGIN_TYPE_LOCK();
type_modified_unlocked(type);
_PyType_Modified_Unlocked(type);
types_stop_world();
if (abstract)
type_add_flags(type, Py_TPFLAGS_IS_ABSTRACT);
@@ -1968,7 +1968,7 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, PyTypeObject *b
goto bail;
}
/* Clear the VALID_VERSION flag of 'type' and all its subclasses. */
type_modified_unlocked(type);
_PyType_Modified_Unlocked(type);
}
else {
res = 0;
@@ -3684,7 +3684,7 @@ mro_internal(PyTypeObject *type, int initial, PyObject **p_old_mro)
// XXX Expand this to Py_TPFLAGS_IMMUTABLETYPE?
if (!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)) {
type_modified_unlocked(type);
_PyType_Modified_Unlocked(type);
}
else {
/* For static builtin types, this is only called during init
@@ -6500,14 +6500,14 @@ _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long
{
BEGIN_TYPE_LOCK();
/* Ideally, changing flags and invalidating the old version tag would
happen in one step. But type_modified_unlocked() is re-entrant and
happen in one step. But _PyType_Modified_Unlocked() is re-entrant and
cannot run with the world stopped, so we must invalidate first.
Immutable/static-builtin types are skipped because
set_flags_recursive() does not modify them. */
if (!PyType_HasFeature(self, Py_TPFLAGS_IMMUTABLETYPE) &&
(self->tp_flags & mask) != flags)
{
type_modified_unlocked(self);
_PyType_Modified_Unlocked(self);
}
/* Keep TYPE_LOCK held while waiting for stop-the-world so no thread
can reassign a version tag before the flag update. */
@@ -6688,7 +6688,7 @@ type_update_dict(PyTypeObject *type, PyDictObject *dict, PyObject *name,
PyObject *value, PyObject **old_value)
{
// We don't want any re-entrancy between when we update the dict
// and call type_modified_unlocked, including running the destructor
// and call _PyType_Modified_Unlocked, including running the destructor
// of the current value as it can observe the cache in an inconsistent
// state. Because we have an exact unicode and our dict has exact
// unicodes we know that this will all complete without releasing
@@ -6702,7 +6702,7 @@ type_update_dict(PyTypeObject *type, PyDictObject *dict, PyObject *name,
update_subclasses() recursion in update_slot(), but carefully:
they each have their own conditions on which to stop
recursing into subclasses. */
type_modified_unlocked(type);
_PyType_Modified_Unlocked(type);
if (_PyDict_SetItem_LockHeld(dict, name, value) < 0) {
PyErr_Format(PyExc_AttributeError,
@@ -7026,7 +7026,7 @@ type_dealloc(PyObject *self)
Py_XDECREF(et->ht_qualname);
Py_XDECREF(et->ht_slots);
if (et->ht_cached_keys) {
_PyDictKeys_DecRef(et->ht_cached_keys);
_PyDict_RemoveKeysForClass(et);
}
Py_XDECREF(et->ht_module);
PyMem_Free(et->_ht_tpname);
@@ -12517,7 +12517,7 @@ PyType_Freeze(PyTypeObject *type)
type_add_flags(type, Py_TPFLAGS_IMMUTABLETYPE);
types_start_world();
ASSERT_TYPE_LOCK_HELD();
type_modified_unlocked(type);
_PyType_Modified_Unlocked(type);
END_TYPE_LOCK();
return 0;
Python/bytecodes.c
+2 -9
View File
@@ -4231,13 +4231,6 @@ dummy_func(
DEAD(exc);
}
op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) {
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
EXIT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version);
}
op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) {
assert(oparg & 1);
/* Cached method object */
@@ -4254,7 +4247,7 @@ dummy_func(
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_CHECK_MANAGED_OBJECT_HAS_VALUES +
_GUARD_KEYS_VERSION +
unused/2 +
_LOAD_ATTR_METHOD_WITH_VALUES;
op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) {
@@ -4288,7 +4281,7 @@ dummy_func(
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_CHECK_MANAGED_OBJECT_HAS_VALUES +
_GUARD_KEYS_VERSION +
unused/2 +
_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES;
op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) {
Python/executor_cases.c.h
Generated
-97
View File
@@ -16306,103 +16306,6 @@
break;
}
case _GUARD_KEYS_VERSION_r01: {
CHECK_CURRENT_CACHED_VALUES(0);
ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__);
_PyStackRef owner;
owner = stack_pointer[-1];
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache0 = owner;
SET_CURRENT_CACHED_VALUES(1);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__);
break;
}
case _GUARD_KEYS_VERSION_r11: {
CHECK_CURRENT_CACHED_VALUES(1);
ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__);
_PyStackRef owner;
_PyStackRef _stack_item_0 = _tos_cache0;
owner = _stack_item_0;
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = owner;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache0 = owner;
SET_CURRENT_CACHED_VALUES(1);
ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__);
break;
}
case _GUARD_KEYS_VERSION_r22: {
CHECK_CURRENT_CACHED_VALUES(2);
ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__);
_PyStackRef owner;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
owner = _stack_item_1;
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = owner;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = owner;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__);
break;
}
case _GUARD_KEYS_VERSION_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__);
_PyStackRef owner;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
owner = _stack_item_2;
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = owner;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = owner;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__);
break;
}
case _LOAD_ATTR_METHOD_WITH_VALUES_r02: {
CHECK_CURRENT_CACHED_VALUES(0);
ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__);
Python/generated_cases.c.h
Generated
+2 -24
View File
@@ -9165,18 +9165,7 @@
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
/* Skip 2 cache entries */
// _LOAD_ATTR_METHOD_WITH_VALUES
{
PyObject *descr = read_obj(&this_instr[6].cache);
@@ -9361,18 +9350,7 @@
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
/* Skip 2 cache entries */
// _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
{
PyObject *descr = read_obj(&this_instr[6].cache);
Python/optimizer_cases.c.h
Generated
-4
View File
@@ -3922,10 +3922,6 @@
break;
}
case _GUARD_KEYS_VERSION: {
break;
}
case _LOAD_ATTR_METHOD_WITH_VALUES: {
JitOptRef owner;
JitOptRef attr;
Python/specialize.c
+5 -3
View File
@@ -980,12 +980,15 @@ static int
specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name)
{
// 0 is not a valid version
uint32_t shared_keys_version = 0;
bool shadow = instance_has_key(owner, name, &shared_keys_version);
PyObject *descr = NULL;
unsigned int tp_version = 0;
PyTypeObject *type = Py_TYPE(owner);
// Read the type version before the keys version, we could have a concurrent
// modification of the split keys in which case we update the keys version and
// then the type version, this ensures we will still deopt if that happens.
DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version);
uint32_t shared_keys_version = 0;
bool shadow = instance_has_key(owner, name, &shared_keys_version);
int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version);
Py_XDECREF(descr);
return result;
@@ -1286,7 +1289,6 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
return 0;
}
write_u32(cache->keys_version, shared_keys_version);
specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
}
else {