gh-142889: Improve layout of dictionary keys

Include/internal/pycore_dict.h

@@ -219,24 +219,10 @@ struct _dictkeysobject {
     /* Number of used entries in dk_entries. */
     Py_ssize_t dk_nentries;
 
-
-    /* Actual hash table of dk_size entries. It holds indices in dk_entries,
-       or DKIX_EMPTY(-1) or DKIX_DUMMY(-2).
-
-       Indices must be: 0 <= indice < USABLE_FRACTION(dk_size).
-
-       The size in bytes of an indice depends on dk_size:
-
-       - 1 byte if dk_size <= 0xff (char*)
-       - 2 bytes if dk_size <= 0xffff (int16_t*)
-       - 4 bytes if dk_size <= 0xffffffff (int32_t*)
-       - 8 bytes otherwise (int64_t*)
-
-       Dynamically sized, SIZEOF_VOID_P is minimum. */
-    char dk_indices[];  /* char is required to avoid strict aliasing. */
-
-    /* "PyDictKeyEntry or PyDictUnicodeEntry dk_entries[USABLE_FRACTION(DK_SIZE(dk))];" array follows:
-       see the DK_ENTRIES() / DK_UNICODE_ENTRIES() functions below */
+     union {
+        PyDictKeyEntry entries[1];
+        PyDictUnicodeEntry unicode_entries[1];
+    } dk_entries;
 };
 
 /* This must be no more than 250, for the prefix size to fit in one byte. */
@@ -258,25 +244,32 @@ struct _dictvalues {
 };
 
 #define DK_LOG_SIZE(dk)  _Py_RVALUE((dk)->dk_log2_size)
+#define DK_LOG_INDEX_BYTES(dk) \
+    _Py_RVALUE((dk)->dk_log2_index_bytes)
 #if SIZEOF_VOID_P > 4
 #define DK_SIZE(dk)      (((int64_t)1)<<DK_LOG_SIZE(dk))
+#define DK_INDEX_BYTES(dk) \
+    (((int64_t)1)<<DK_LOG_INDEX_BYTES(dk))
 #else
 #define DK_SIZE(dk)      (1<<DK_LOG_SIZE(dk))
+#define DK_INDEX_BYTES(dk) (1<<DK_LOG_INDEX_BYTES(dk))
 #endif
+#define _DK_INDICES(dk) ((char *)(dk) - DK_INDEX_BYTES(dk))
+#define _DK_FROM_BASE(base, index_size) \
+    ((PyDictKeysObject *)((char *)(base) + ((size_t) index_size)))
 
 static inline void* _DK_ENTRIES(PyDictKeysObject *dk) {
-    int8_t *indices = (int8_t*)(dk->dk_indices);
-    size_t index = (size_t)1 << dk->dk_log2_index_bytes;
-    return (&indices[index]);
+    return (void *)(&dk->dk_entries);
 }
 
 static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) {
     assert(dk->dk_kind == DICT_KEYS_GENERAL);
-    return (PyDictKeyEntry*)_DK_ENTRIES(dk);
+    return dk->dk_entries.entries;
 }
+
 static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
     assert(dk->dk_kind != DICT_KEYS_GENERAL);
-    return (PyDictUnicodeEntry*)_DK_ENTRIES(dk);
+    return dk->dk_entries.unicode_entries;
 }
 
 #define DK_IS_UNICODE(dk) ((dk)->dk_kind != DICT_KEYS_GENERAL)

Lib/test/test_dict.py

@@ -1794,6 +1794,21 @@ def __hash__(self):
         self.assertEqual(dict_getitem_knownhash(d, k1, hash(k1)), 1)
         self.assertRaises(Exc, dict_getitem_knownhash, d, k2, hash(k2))
 
+    @support.cpython_only
+    def test_dict_keys_layout(self):
+        _testinternalcapi = import_helper.import_module('_testinternalcapi')
+        check_layout = _testinternalcapi.dict_keys_layout
+
+        for i in range(4):
+            self.assertTrue(check_layout({j: j for j in range(10**i)}))
+
+    @support.cpython_only
+    def test_dict_keys_to_base(self):
+        _testinternalcapi = import_helper.import_module('_testinternalcapi')
+        check_base = _testinternalcapi.dict_keys_to_base
+
+        for i in range(4):
+            self.assertTrue(check_base({j: j for j in range(10**i)}))
 
 from test import mapping_tests
 

Misc/NEWS.d/next/Core_and_Builtins/2026-05-30-21-37-54.gh-issue-142889.LvkzYw.rst

@@ -0,0 +1 @@
+Update the memory layout of how :class:`dict` handles the memory of the keys to improve performance.

Modules/Setup.stdlib.in

@@ -172,7 +172,7 @@
 @MODULE_XXSUBTYPE_TRUE@xxsubtype xxsubtype.c
 @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
-@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c
+@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c _testinternalcapi/dict.c
 @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c
 @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c
 @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c

Modules/_testinternalcapi.c

@@ -3352,6 +3352,9 @@ module_exec(PyObject *module)
     if (_PyTestInternalCapi_Init_Tuple(module) < 0) {
         return 1;
     }
+    if (_PyTestInternalCapi_Init_Dict(module) < 0) {
+        return 1;
+    }
 
     Py_ssize_t sizeof_gc_head = 0;
 #ifndef Py_GIL_DISABLED

Modules/_testinternalcapi/dict.c

@@ -0,0 +1,50 @@
+#include "parts.h"
+
+#include "pycore_dict.h"
+
+static PyObject*
+dict_keys_layout(PyObject *self, PyObject *arg)
+{
+    PyDictObject *mp = (PyDictObject *)arg;
+    PyDictKeysObject *keys = mp->ma_keys;
+
+    size_t indices_size = DK_INDEX_BYTES(keys);
+
+    char *base = _DK_INDICES(keys);
+    char *header = (char *)keys;
+    char *entries = (char *)_DK_ENTRIES(keys);
+
+    bool ok = true;
+    ok &= (header == base + indices_size);
+    ok &= (entries == header + offsetof(PyDictKeysObject, dk_entries));
+
+    return PyBool_FromLong(ok);
+}
+
+static PyObject*
+dict_keys_to_base(PyObject *self, PyObject *arg)
+{
+    PyDictObject *mp = (PyDictObject *)arg;
+    PyDictKeysObject *keys = mp->ma_keys;
+
+    void *base = _DK_INDICES(keys);
+    size_t indices_size = DK_INDEX_BYTES(keys);
+    bool ok = _DK_FROM_BASE(base, indices_size) == keys;
+
+    return PyBool_FromLong(ok);
+}
+
+static PyMethodDef test_methods[] = {
+    {"dict_keys_layout", dict_keys_layout, METH_O},
+    {"dict_keys_to_base", dict_keys_to_base, METH_O},
+    {NULL},
+};
+
+int
+_PyTestInternalCapi_Init_Dict(PyObject *m)
+{
+    if (PyModule_AddFunctions(m, test_methods) < 0) {
+        return -1;
+    }
+    return 0;
+}

Modules/_testinternalcapi/parts.h

@@ -16,5 +16,6 @@ int _PyTestInternalCapi_Init_Set(PyObject *module);
 int _PyTestInternalCapi_Init_Complex(PyObject *module);
 int _PyTestInternalCapi_Init_CriticalSection(PyObject *module);
 int _PyTestInternalCapi_Init_Tuple(PyObject *module);
+int _PyTestInternalCapi_Init_Dict(PyObject *module);
 
 #endif // Py_TESTINTERNALCAPI_PARTS_H

Objects/dictobject.c

@@ -17,6 +17,9 @@ As of Python 3.6, this is compact and ordered. Basic idea is described here:
 layout:
 
 +---------------------+
+| dk_indices[]        |
+|                     |
++---------------------+         <---- PyDictKeysObject* pointer points here
 | dk_refcnt           |
 | dk_log2_size        |
 | dk_log2_index_bytes |
@@ -25,15 +28,18 @@ As of Python 3.6, this is compact and ordered. Basic idea is described here:
 | dk_usable           |
 | dk_nentries         |
 +---------------------+
-| dk_indices[]        |
-|                     |
-+---------------------+
 | dk_entries[]        |
 |                     |
 +---------------------+
 
-dk_indices is actual hashtable.  It holds index in entries, or DKIX_EMPTY(-1)
-or DKIX_DUMMY(-2).
+
+
+The layout of the dictionary keys consists of three sections:
+the indicies (the actual hashtable),  the header, and the entries.
+PyDictKeysObject * points to the header of the struct.
+
+The dk_indices table is stored immediately before the header in memory (see gh-142889).
+It holds index in entries, or DKIX_EMPTY(-1) or DKIX_DUMMY(-2).
 Size of indices is dk_size.  Type of each index in indices varies with dk_size:
 
 * int8  for          dk_size <= 128
@@ -177,8 +183,10 @@ ASSERT_DICT_LOCKED(PyObject *op)
 
 #define IS_DICT_SHARED(mp) _PyObject_GC_IS_SHARED(mp)
 #define SET_DICT_SHARED(mp) _PyObject_GC_SET_SHARED(mp)
-#define LOAD_INDEX(keys, size, idx) _Py_atomic_load_int##size##_relaxed(&((const int##size##_t*)keys->dk_indices)[idx]);
-#define STORE_INDEX(keys, size, idx, value) _Py_atomic_store_int##size##_relaxed(&((int##size##_t*)keys->dk_indices)[idx], (int##size##_t)value);
+#define LOAD_INDEX(keys, size, idx) \
+    _Py_atomic_load_int##size##_relaxed(&((const int##size##_t*)(_DK_INDICES(keys)))[idx]);
+#define STORE_INDEX(keys, size, idx, value) \
+    _Py_atomic_store_int##size##_relaxed(&((int##size##_t*)(_DK_INDICES(keys)))[idx], (int##size##_t)value);
 #define ASSERT_OWNED_OR_SHARED(mp) \
     assert(_Py_IsOwnedByCurrentThread((PyObject *)mp) || IS_DICT_SHARED(mp));
 
@@ -257,8 +265,8 @@ static inline void split_keys_entry_added(PyDictKeysObject *keys)
 #define UNLOCK_KEYS_IF_SPLIT(keys, kind)
 #define IS_DICT_SHARED(mp) (false)
 #define SET_DICT_SHARED(mp)
-#define LOAD_INDEX(keys, size, idx) ((const int##size##_t*)(keys->dk_indices))[idx]
-#define STORE_INDEX(keys, size, idx, value) ((int##size##_t*)(keys->dk_indices))[idx] = (int##size##_t)value
+#define LOAD_INDEX(keys, size, idx) ((const int##size##_t*)(_DK_INDICES(keys)))[idx]
+#define STORE_INDEX(keys, size, idx, value) ((int##size##_t*)(_DK_INDICES(keys)))[idx] = (int##size##_t)value
 
 static inline void split_keys_entry_added(PyDictKeysObject *keys)
 {
@@ -627,22 +635,31 @@ estimate_log2_keysize(Py_ssize_t n)
  * See https://github.com/python/cpython/pull/127568#discussion_r1868070614
  * for the rationale of using dk_log2_index_bytes=3 instead of 0.
  */
-static PyDictKeysObject empty_keys_struct = {
-        _Py_DICT_IMMORTAL_INITIAL_REFCNT, /* dk_refcnt */
-        0, /* dk_log2_size */
-        3, /* dk_log2_index_bytes */
-        DICT_KEYS_UNICODE, /* dk_kind */
+typedef struct {
+    int8_t indices[8];
+    PyDictKeysObject keys;
+} _PyDict_EmptyKeys;
+
+static _PyDict_EmptyKeys empty_keys = {
+    .indices = {
+        DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY,
+        DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY,
+    },
+    .keys = {
+        .dk_refcnt = _Py_DICT_IMMORTAL_INITIAL_REFCNT,
+        .dk_log2_size = 0,
+        .dk_log2_index_bytes = 3,
+        .dk_kind = DICT_KEYS_UNICODE,
 #ifdef Py_GIL_DISABLED
-        {0}, /* dk_mutex */
+        .dk_mutex = {0},
 #endif
-        1, /* dk_version */
-        0, /* dk_usable (immutable) */
-        0, /* dk_nentries */
-        {DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY,
-         DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY}, /* dk_indices */
+        .dk_version = 1,
+        .dk_usable = 0,  /* immutable */
+        .dk_nentries = 0,
+    },
 };
 
-#define Py_EMPTY_KEYS &empty_keys_struct
+#define Py_EMPTY_KEYS &empty_keys.keys
 
 /* Uncomment to check the dict content in _PyDict_CheckConsistency() */
 // #define DEBUG_PYDICT
@@ -809,19 +826,22 @@ new_keys_object(uint8_t log2_size, bool unicode)
         log2_bytes = log2_size + 2;
     }
 
-    PyDictKeysObject *dk = NULL;
+    void *base = NULL;
+    size_t indices_size = (size_t)1 << log2_bytes;
     if (log2_size == PyDict_LOG_MINSIZE && unicode) {
-        dk = _Py_FREELIST_POP_MEM(dictkeys);
+        base = _Py_FREELIST_POP_MEM(dictkeys);
     }
-    if (dk == NULL) {
-        dk = PyMem_Malloc(sizeof(PyDictKeysObject)
-                          + ((size_t)1 << log2_bytes)
-                          + entry_size * usable);
-        if (dk == NULL) {
+    if (base == NULL) {
+        base = PyMem_Malloc(sizeof(PyDictKeysObject)
+                          + indices_size + entry_size * usable);
+        if (base == NULL) {
             PyErr_NoMemory();
             return NULL;
         }
     }
+
+    PyDictKeysObject *dk = _DK_FROM_BASE(base, indices_size);
+
 #ifdef Py_REF_DEBUG
     _Py_IncRefTotal(_PyThreadState_GET());
 #endif
@@ -835,25 +855,26 @@ new_keys_object(uint8_t log2_size, bool unicode)
     dk->dk_nentries = 0;
     dk->dk_usable = usable;
     dk->dk_version = 0;
-    memset(&dk->dk_indices[0], 0xff, ((size_t)1 << log2_bytes));
-    memset(&dk->dk_indices[(size_t)1 << log2_bytes], 0, entry_size * usable);
+    memset(_DK_INDICES(dk), 0xff, indices_size);
+    memset(&dk->dk_entries, 0, entry_size * usable);
     return dk;
 }
 
 static void
 free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
 {
+    void *base = _DK_INDICES(keys);
 #ifdef Py_GIL_DISABLED
     if (use_qsbr) {
-        _PyMem_FreeDelayed(keys, _PyDict_KeysSize(keys));
+        _PyMem_FreeDelayed(base, _PyDict_KeysSize(keys));
         return;
     }
 #endif
     if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && keys->dk_kind == DICT_KEYS_UNICODE) {
-        _Py_FREELIST_FREE(dictkeys, keys, PyMem_Free);
+        _Py_FREELIST_FREE(dictkeys, base, PyMem_Free);
     }
     else {
-        PyMem_Free(keys);
+        PyMem_Free(base);
     }
 }
 
@@ -975,14 +996,18 @@ clone_combined_dict_keys(PyDictObject *orig)
         ASSERT_DICT_LOCKED(orig);
     }
 
-    size_t keys_size = _PyDict_KeysSize(orig->ma_keys);
-    PyDictKeysObject *keys = PyMem_Malloc(keys_size);
-    if (keys == NULL) {
+    PyDictKeysObject *orig_keys = orig->ma_keys;
+    size_t keys_size = _PyDict_KeysSize(orig_keys);
+    size_t indices_size = DK_INDEX_BYTES(orig_keys);
+
+    void *base = PyMem_Malloc(keys_size);
+    if (base == NULL) {
         PyErr_NoMemory();
         return NULL;
     }
 
-    memcpy(keys, orig->ma_keys, keys_size);
+    PyDictKeysObject *keys = _DK_FROM_BASE(base, indices_size);
+    memcpy(base, _DK_INDICES(orig_keys), keys_size);
 
     /* After copying key/value pairs, we need to incref all
        keys and values and they are about to be co-owned by a
@@ -5033,7 +5058,7 @@ _PyDict_KeysSize(PyDictKeysObject *keys)
 {
     size_t es = (keys->dk_kind == DICT_KEYS_GENERAL
                  ? sizeof(PyDictKeyEntry) : sizeof(PyDictUnicodeEntry));
-    size_t size = sizeof(PyDictKeysObject);
+    size_t size = sizeof(PyDictKeysObject) - sizeof(PyDictKeyEntry);
     size += (size_t)1 << keys->dk_log2_index_bytes;
     size += USABLE_FRACTION((size_t)DK_SIZE(keys)) * es;
     return size;

PCbuild/_testinternalcapi.vcxproj

@@ -101,6 +101,7 @@
     <ClCompile Include="..\Modules\_testinternalcapi\complex.c" />
     <ClCompile Include="..\Modules\_testinternalcapi\interpreter.c" />
     <ClCompile Include="..\Modules\_testinternalcapi\tuple.c" />
+    <ClCompile Include="..\Modules\_testinternalcapi\dict.c" />
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="..\PC\python_nt.rc" />

PCbuild/_testinternalcapi.vcxproj.filters

@@ -30,6 +30,9 @@
     <ClCompile Include="..\Modules\_testinternalcapi\tuple.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Modules\_testinternalcapi\dict.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="..\PC\python_nt.rc">