diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 2d33aa76d78229f..f099f02c15d4c97 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -209,7 +209,7 @@ extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate); // Functions to clear types free lists extern void _PyTuple_ClearFreeList(PyInterpreterState *interp); extern void _PyFloat_ClearFreeList(PyInterpreterState *interp); -extern void _PyList_ClearFreeList(PyInterpreterState *interp); +extern void _PyList_ClearFreeList(PyFreeListState *state); extern void _PyDict_ClearFreeList(PyInterpreterState *interp); extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp); extern void _PyContext_ClearFreeList(PyInterpreterState *interp); diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 04d7a6a615e370f..ac05dd00d7f4af1 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -174,6 +174,9 @@ struct _is { // One bit is set for each non-NULL entry in code_watchers uint8_t active_code_watchers; +#if !defined(Py_GIL_DISABLED) + struct _Py_freelist_state freelist_state; +#endif struct _py_object_state object_state; struct _Py_unicode_state unicode; struct _Py_float_state float_state; @@ -185,7 +188,6 @@ struct _is { PySliceObject *slice_cache; struct _Py_tuple_state tuple; - struct _Py_list_state list; struct _Py_dict_state dict_state; struct _Py_async_gen_state async_gen; struct _Py_context_state context; diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 37b45faf8a74a04..05a0d80a9f74cc0 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_runtime.h" // _PyRuntime +#include "pycore_tstate.h" // _PyThreadStateImpl // Values for PyThreadState.state. A thread must be in the "attached" state @@ -239,6 +240,19 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); // See also PyInterpreterState_Get() and _PyInterpreterState_GET(). extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); +static inline PyFreeListState* _PyFreeListState_GET(void) { + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + _Py_EnsureTstateNotNULL(tstate); +#endif + +#ifdef Py_GIL_DISABLED + return &((_PyThreadStateImpl*)tstate)->freelist_state; +#else + return &tstate->interp->freelist_state; +#endif +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 856ddd5e7e5ff07..11d0ee35315b164 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -11,6 +11,10 @@ extern "C" { #include "pycore_mimalloc.h" // struct _mimalloc_thread_state +typedef struct _Py_freelist_state { + struct _Py_list_state list; +} _Py_freelist_state; + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields // are intended to be private. The _PyThreadStateImpl fields not exposed. @@ -20,6 +24,7 @@ typedef struct _PyThreadStateImpl { #ifdef Py_GIL_DISABLED struct _mimalloc_thread_state mimalloc; + struct _Py_freelist_state freelist_state; #endif } _PyThreadStateImpl; diff --git a/Include/pytypedefs.h b/Include/pytypedefs.h index e78ed56a3b67cd1..f218298f6ef5e18 100644 --- a/Include/pytypedefs.h +++ b/Include/pytypedefs.h @@ -23,6 +23,7 @@ typedef struct _frame PyFrameObject; typedef struct _ts PyThreadState; typedef struct _is PyInterpreterState; +typedef struct _Py_freelist_state PyFreeListState; #ifdef __cplusplus } diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 2d1f381e6222267..ef79a7cab45a931 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1070,14 +1070,33 @@ delete_garbage(PyThreadState *tstate, GCState *gcstate, static void clear_freelists(PyInterpreterState *interp) { + // TODO: Unify with clear_all_freelists _PyTuple_ClearFreeList(interp); _PyFloat_ClearFreeList(interp); - _PyList_ClearFreeList(interp); _PyDict_ClearFreeList(interp); _PyAsyncGen_ClearFreeLists(interp); _PyContext_ClearFreeList(interp); } +static void +clear_all_freelists(PyInterpreterState *interp) +{ + clear_freelists(interp); +#if defined(Py_GIL_DISABLED) + HEAD_LOCK(&_PyRuntime); + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head; + while (tstate != NULL) { + _PyList_ClearFreeList(&tstate->freelist_state); + tstate = tstate->base.next; + } + HEAD_UNLOCK(&_PyRuntime); +#else + // Only free-lists per interpreter are existed. + PyFreeListState* state = _PyFreeListState_GET(); + _PyList_ClearFreeList(state); +#endif +} + // Show stats for objects in each generations static void show_stats_each_generations(GCState *gcstate) @@ -1486,6 +1505,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) * generation */ if (generation == NUM_GENERATIONS-1) { clear_freelists(tstate->interp); + clear_all_freelists(tstate->interp); } if (_PyErr_Occurred(tstate)) { diff --git a/Objects/listobject.c b/Objects/listobject.c index 2d04218439bd202..db06d7597ec21ac 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -24,8 +24,9 @@ _Py_DECLARE_STR(list_err, "list index out of range"); static struct _Py_list_state * get_list_state(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->list; + PyFreeListState *state = _PyFreeListState_GET(); + assert(state != NULL); + return &state->list; } #endif @@ -120,12 +121,12 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) } void -_PyList_ClearFreeList(PyInterpreterState *interp) +_PyList_ClearFreeList(PyFreeListState *state) { #if PyList_MAXFREELIST > 0 - struct _Py_list_state *state = &interp->list; - while (state->numfree) { - PyListObject *op = state->free_list[--state->numfree]; + struct _Py_list_state *list_state = &state->list; + while (list_state->numfree) { + PyListObject *op = list_state->free_list[--list_state->numfree]; assert(PyList_CheckExact(op)); PyObject_GC_Del(op); } @@ -135,10 +136,10 @@ _PyList_ClearFreeList(PyInterpreterState *interp) void _PyList_Fini(PyInterpreterState *interp) { - _PyList_ClearFreeList(interp); + PyFreeListState *state = _PyFreeListState_GET(); + _PyList_ClearFreeList(state); #if defined(Py_DEBUG) && PyList_MAXFREELIST > 0 - struct _Py_list_state *state = &interp->list; - state->numfree = -1; + state->list.numfree = -1; #endif }