From 7e32c029e7d0f6009a9da239fb6def66ac4d8bcd Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Wed, 10 Jan 2024 08:04:41 +0900 Subject: [PATCH] gh-111968: Introduce _PyFreeListState and _PyFreeListState_GET API (gh-113584) --- Include/internal/pycore_freelist.h | 35 ++++++++++++++++++++++++++ Include/internal/pycore_gc.h | 6 ++++- Include/internal/pycore_interp.h | 4 ++- Include/internal/pycore_list.h | 22 ++-------------- Include/internal/pycore_pystate.h | 16 ++++++++++++ Include/internal/pycore_tstate.h | 2 ++ Makefile.pre.in | 3 +++ Objects/listobject.c | 22 ++++++++-------- PCbuild/_freeze_module.vcxproj | 2 ++ PCbuild/_freeze_module.vcxproj.filters | 6 +++++ PCbuild/pythoncore.vcxproj | 3 +++ PCbuild/pythoncore.vcxproj.filters | 12 +++++++++ Python/gc.c | 17 +------------ Python/gc_free_threading.c | 32 +++++++++++++++++++++++ Python/gc_gil.c | 23 +++++++++++++++++ Python/pylifecycle.c | 5 +++- Python/pystate.c | 11 ++++++++ 17 files changed, 171 insertions(+), 50 deletions(-) create mode 100644 Include/internal/pycore_freelist.h create mode 100644 Python/gc_free_threading.c create mode 100644 Python/gc_gil.c diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h new file mode 100644 index 000000000000000..b725986528d864d --- /dev/null +++ b/Include/internal/pycore_freelist.h @@ -0,0 +1,35 @@ +#ifndef Py_INTERNAL_FREELIST_H +#define Py_INTERNAL_FREELIST_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifndef WITH_FREELISTS +// without freelists +# define PyList_MAXFREELIST 0 +#endif + +/* Empty list reuse scheme to save calls to malloc and free */ +#ifndef PyList_MAXFREELIST +# define PyList_MAXFREELIST 80 +#endif + +struct _Py_list_state { +#if PyList_MAXFREELIST > 0 + PyListObject *free_list[PyList_MAXFREELIST]; + int numfree; +#endif +}; + +typedef struct _Py_freelist_state { + struct _Py_list_state list; +} _PyFreeListState; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FREELIST_H */ diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 2a79c403803ed1e..5d90d3a7f865da9 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState + /* GC information is stored BEFORE the object structure. */ typedef struct { // Pointer to next object in the list. @@ -238,9 +240,11 @@ extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generat extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists +extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); +extern void _Py_ClearFreeLists(_PyFreeListState *state, int is_finalization); extern void _PyTuple_ClearFreeList(PyInterpreterState *interp); extern void _PyFloat_ClearFreeList(PyInterpreterState *interp); -extern void _PyList_ClearFreeList(PyInterpreterState *interp); +extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization); extern void _PyDict_ClearFreeList(PyInterpreterState *interp); extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp); extern void _PyContext_ClearFreeList(PyInterpreterState *interp); diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 4512b1edb4b9b35..4d49fa2a51b88cb 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -179,6 +179,9 @@ struct _is { // One bit is set for each non-NULL entry in code_watchers uint8_t active_code_watchers; +#if !defined(Py_GIL_DISABLED) + struct _Py_freelist_state freelist_state; +#endif struct _py_object_state object_state; struct _Py_unicode_state unicode; struct _Py_float_state float_state; @@ -190,7 +193,6 @@ struct _is { PySliceObject *slice_cache; struct _Py_tuple_state tuple; - struct _Py_list_state list; struct _Py_dict_state dict_state; struct _Py_async_gen_state async_gen; struct _Py_context_state context; diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 55d67b32bc8a63e..6c29d882335512e 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState extern PyObject* _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); @@ -15,28 +16,9 @@ extern void _PyList_DebugMallocStats(FILE *out); /* runtime lifecycle */ -extern void _PyList_Fini(PyInterpreterState *); +extern void _PyList_Fini(_PyFreeListState *); -/* other API */ - -#ifndef WITH_FREELISTS -// without freelists -# define PyList_MAXFREELIST 0 -#endif - -/* Empty list reuse scheme to save calls to malloc and free */ -#ifndef PyList_MAXFREELIST -# define PyList_MAXFREELIST 80 -#endif - -struct _Py_list_state { -#if PyList_MAXFREELIST > 0 - PyListObject *free_list[PyList_MAXFREELIST]; - int numfree; -#endif -}; - #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) extern int diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 37b45faf8a74a04..348c5c634284b00 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,7 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState #include "pycore_runtime.h" // _PyRuntime +#include "pycore_tstate.h" // _PyThreadStateImpl // Values for PyThreadState.state. A thread must be in the "attached" state @@ -239,6 +241,20 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); // See also PyInterpreterState_Get() and _PyInterpreterState_GET(). extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); +static inline _PyFreeListState* _PyFreeListState_GET(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + _Py_EnsureTstateNotNULL(tstate); +#endif + +#ifdef Py_GIL_DISABLED + return &((_PyThreadStateImpl*)tstate)->freelist_state; +#else + return &tstate->interp->freelist_state; +#endif +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 856ddd5e7e5ff07..472fa08154e8f92 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // struct _Py_freelist_state #include "pycore_mimalloc.h" // struct _mimalloc_thread_state @@ -20,6 +21,7 @@ typedef struct _PyThreadStateImpl { #ifdef Py_GIL_DISABLED struct _mimalloc_thread_state mimalloc; + struct _Py_freelist_state freelist_state; #endif } _PyThreadStateImpl; diff --git a/Makefile.pre.in b/Makefile.pre.in index abbd4b1b1fbd6c1..15d419b930c1818 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -418,6 +418,8 @@ PYTHON_OBJS= \ Python/frozenmain.o \ Python/future.o \ Python/gc.o \ + Python/gc_free_threading.o \ + Python/gc_gil.o \ Python/getargs.o \ Python/getcompiler.o \ Python/getcopyright.o \ @@ -1828,6 +1830,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_floatobject.h \ $(srcdir)/Include/internal/pycore_format.h \ $(srcdir)/Include/internal/pycore_frame.h \ + $(srcdir)/Include/internal/pycore_freelist.h \ $(srcdir)/Include/internal/pycore_function.h \ $(srcdir)/Include/internal/pycore_genobject.h \ $(srcdir)/Include/internal/pycore_getopt.h \ diff --git a/Objects/listobject.c b/Objects/listobject.c index 5cd4a059c668ba6..2fc57e13f632f88 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -24,8 +24,9 @@ _Py_DECLARE_STR(list_err, "list index out of range"); static struct _Py_list_state * get_list_state(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->list; + _PyFreeListState *state = _PyFreeListState_GET(); + assert(state != NULL); + return &state->list; } #endif @@ -120,26 +121,25 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) } void -_PyList_ClearFreeList(PyInterpreterState *interp) +_PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) { #if PyList_MAXFREELIST > 0 - struct _Py_list_state *state = &interp->list; - while (state->numfree) { + struct _Py_list_state *state = &freelist_state->list; + while (state->numfree > 0) { PyListObject *op = state->free_list[--state->numfree]; assert(PyList_CheckExact(op)); PyObject_GC_Del(op); } + if (is_finalization) { + state->numfree = -1; + } #endif } void -_PyList_Fini(PyInterpreterState *interp) +_PyList_Fini(_PyFreeListState *state) { - _PyList_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyList_MAXFREELIST > 0 - struct _Py_list_state *state = &interp->list; - state->numfree = -1; -#endif + _PyList_ClearFreeList(state, 1); } /* Print summary info about the state of the optimized allocator */ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index f16a763772e42ec..610581bc96cb1a2 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -208,6 +208,8 @@ + + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 7f03cfea1b3e6f5..3141913c0438695 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -169,6 +169,12 @@ Source Files + + Source Files + + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c599b68e0d8e66d..a8b753ca489ab78 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -231,6 +231,7 @@ + @@ -568,6 +569,8 @@ + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 13582dff3f40c46..965efa2e3d34b99 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -618,6 +618,12 @@ Include\internal + + Include\internal + + + Include\internal + Include\internal @@ -1286,6 +1292,12 @@ Python + + Python + + + Python + Python diff --git a/Python/gc.c b/Python/gc.c index f47c74f87a91661..9f9a755f6ac95e7 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1019,21 +1019,6 @@ delete_garbage(PyThreadState *tstate, GCState *gcstate, } } -/* Clear all free lists - * All free lists are cleared during the collection of the highest generation. - * Allocated items in the free list may keep a pymalloc arena occupied. - * Clearing the free lists may give back memory to the OS earlier. - */ -static void -clear_freelists(PyInterpreterState *interp) -{ - _PyTuple_ClearFreeList(interp); - _PyFloat_ClearFreeList(interp); - _PyList_ClearFreeList(interp); - _PyDict_ClearFreeList(interp); - _PyAsyncGen_ClearFreeLists(interp); - _PyContext_ClearFreeList(interp); -} // Show stats for objects in each generations static void @@ -1449,7 +1434,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) /* Clear free list only during the collection of the highest * generation */ if (generation == NUM_GENERATIONS-1) { - clear_freelists(tstate->interp); + _PyGC_ClearAllFreeLists(tstate->interp); } if (_PyErr_Occurred(tstate)) { diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c new file mode 100644 index 000000000000000..aea272840f950db --- /dev/null +++ b/Python/gc_free_threading.c @@ -0,0 +1,32 @@ +#include "Python.h" +#include "pycore_pystate.h" // _PyFreeListState_GET() +#include "pycore_tstate.h" // _PyThreadStateImpl + +#ifdef Py_GIL_DISABLED + +/* Clear all free lists + * All free lists are cleared during the collection of the highest generation. + * Allocated items in the free list may keep a pymalloc arena occupied. + * Clearing the free lists may give back memory to the OS earlier. + * Free-threading version: Since freelists are managed per thread, + * GC should clear all freelists by traversing all threads. + */ +void +_PyGC_ClearAllFreeLists(PyInterpreterState *interp) +{ + _PyTuple_ClearFreeList(interp); + _PyFloat_ClearFreeList(interp); + _PyDict_ClearFreeList(interp); + _PyAsyncGen_ClearFreeLists(interp); + _PyContext_ClearFreeList(interp); + + HEAD_LOCK(&_PyRuntime); + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head; + while (tstate != NULL) { + _Py_ClearFreeLists(&tstate->freelist_state, 0); + tstate = (_PyThreadStateImpl *)tstate->base.next; + } + HEAD_UNLOCK(&_PyRuntime); +} + +#endif diff --git a/Python/gc_gil.c b/Python/gc_gil.c new file mode 100644 index 000000000000000..b0961cdbee904f9 --- /dev/null +++ b/Python/gc_gil.c @@ -0,0 +1,23 @@ +#include "Python.h" +#include "pycore_pystate.h" // _Py_ClearFreeLists() + +#ifndef Py_GIL_DISABLED + +/* Clear all free lists + * All free lists are cleared during the collection of the highest generation. + * Allocated items in the free list may keep a pymalloc arena occupied. + * Clearing the free lists may give back memory to the OS earlier. + */ +void +_PyGC_ClearAllFreeLists(PyInterpreterState *interp) +{ + _PyTuple_ClearFreeList(interp); + _PyFloat_ClearFreeList(interp); + _PyDict_ClearFreeList(interp); + _PyAsyncGen_ClearFreeLists(interp); + _PyContext_ClearFreeList(interp); + + _Py_ClearFreeLists(&interp->freelist_state, 0); +} + +#endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 1d8af26e4a1cb7f..bd6475f99e464bb 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1752,13 +1752,16 @@ finalize_interp_types(PyInterpreterState *interp) _PyUnicode_ClearInterned(interp); _PyDict_Fini(interp); - _PyList_Fini(interp); _PyTuple_Fini(interp); _PySlice_Fini(interp); _PyUnicode_Fini(interp); _PyFloat_Fini(interp); + + _PyFreeListState *state = _PyFreeListState_GET(); + _PyList_Fini(state); + #ifdef Py_DEBUG _PyStaticObjects_CheckRefcnt(interp); #endif diff --git a/Python/pystate.c b/Python/pystate.c index 21f16b7bcdff0d0..ddd57f75f3f3633 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1455,6 +1455,12 @@ clear_datastack(PyThreadState *tstate) } } +void +_Py_ClearFreeLists(_PyFreeListState *state, int is_finalization) +{ + _PyList_ClearFreeList(state, is_finalization); +} + void PyThreadState_Clear(PyThreadState *tstate) { @@ -1537,6 +1543,11 @@ PyThreadState_Clear(PyThreadState *tstate) // don't call _PyInterpreterState_SetNotRunningMain() yet. tstate->on_delete(tstate->on_delete_data); } +#ifdef Py_GIL_DISABLED + // Each thread should clear own freelists in free-threading builds. + _PyFreeListState *freelist_state = &((_PyThreadStateImpl*)tstate)->freelist_state; + _Py_ClearFreeLists(freelist_state, 0); +#endif _PyThreadState_ClearMimallocHeaps(tstate);