From: H.J. Lu Date: Sat, 18 Apr 2026 03:36:41 +0000 (+0800) Subject: malloc: Don't call __get_thp_mode/__get_thp_size twice X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ccd4cd5324caaaea1bb12bc8cef073170da51a19;p=thirdparty%2Fglibc.git malloc: Don't call __get_thp_mode/__get_thp_size twice Both ld.so and malloc track kernel THP mode and THP page size when THP in ld.so is enabled by GLIBC_TUNABLES=glibc.elf.thp=1 and THP in malloc is enabled by GLIBC_TUNABLES=glibc.malloc.hugetlb=1 But DL_MAP_DEFAULT_THP_PAGESIZE and MALLOC_DEFAULT_THP_PAGESIZE may be different when they are defined in . If THP in ld.so is enabled, change malloc to use kernel THP mode from ld.so, instead of calling __get_thp_mode, and use THP page size from ld.so if it came from __get_thp_size. This avoids calling __get_thp_mode and __get_thp_size again, which are quite expensive: 1. Initialize mp_.thp_mode and GL(dl_thp_mode) to thp_mode_unknown. 2. Set mp_.thp_mode to GL(dl_thp_mode) if GL(dl_thp_mode) isn't thp_mode_unknown. Otherwise call __get_thp_mode to set mp_.thp_mode. 3. GL(dl_elf_thp_pagesize) is set to DL_MAP_DEFAULT_THP_PAGESIZE without calling __get_thp_size and THP page size for malloc may be different from THP page size for ld.so. Set mp_.thp_pagesize to GL(dl_elf_thp_pagesize) if DL_MAP_DEFAULT_THP_PAGESIZE is defined. Otherwise call __get_thp_size to set mp_.thp_pagesize. 4. THP page size in malloc is capped to MAX_THP_PAGESIZE. If THP page size is above MAX_THP_PAGESIZE, THP in malloc is disabled. These result in when glibc.elf.thp is set to 1, malloc uses the actual kernel THP mode instead of defaulting to madvise mode and madvise_thp will stop issuing MADV_HUGEPAGE if kernel THP mode is always. This fixes BZ #34083. Signed-off-by: H.J. Lu --- diff --git a/NEWS b/NEWS index 41b59da04b..f9d90c5194 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,15 @@ Version 2.44 Major new features: +* A new tunable, glibc.elf.thp, is added to map read-only segments with + Transparent Huge Pages (THP) if THP isn't disable in kernel. When + glibc.elf.thp is set to 1, malloc uses the actual kernel THP mode + instead of defaulting to madvise mode and madvise_thp will stop issuing + MADV_HUGEPAGE if kernel THP mode is always. + +* THP page size in malloc is capped to MAX_THP_PAGESIZE. If THP page + size is above MAX_THP_PAGESIZE, THP in malloc is disabled. + * Additional optimized and correctly rounded mathematical functions have been imported from the CORE-MATH project, in particular cosh, sinh, and tanh. diff --git a/malloc/malloc.c b/malloc/malloc.c index a354576aca..c39d60b509 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -1529,7 +1529,7 @@ static struct malloc_par mp_ = .mmap_threshold = DEFAULT_MMAP_THRESHOLD, .trim_threshold = DEFAULT_TRIM_THRESHOLD, .arena_test = sizeof (long) == 4 ? 2 : 8, - .thp_mode = thp_mode_not_supported + .thp_mode = thp_mode_unknown #if USE_TCACHE , .tcache_count = TCACHE_FILL_COUNT, @@ -4657,13 +4657,41 @@ do_set_mxfast (size_t value) return 1; } +#ifdef HAVE_THP +static __always_inline enum thp_mode_t +get_dl_thp_mode (void) +{ + return GL(dl_thp_mode); +} + +static __always_inline unsigned long int +get_dl_elf_thp_pagesize (void) +{ + return GL(dl_elf_thp_pagesize); +} +#else +# define get_dl_thp_mode() __get_thp_mode () +# define get_dl_elf_thp_pagesize() __get_thp_size () +#endif + static __always_inline int do_set_hugetlb (size_t value) { + /* If __get_thp_mode and __get_thp_size have been called during + startup, don't call them again here. */ + enum thp_mode_t thp_mode = get_dl_thp_mode (); + /* Enable THP if MALLOC_DEFAULT_THP_PAGESIZE is non-zero. */ if (MALLOC_DEFAULT_THP_PAGESIZE > 0) { - mp_.thp_mode = thp_mode_madvise; + /* If thp_mode is unknown, THP segment load is disabled by + GLIBC_TUNABLES=glibc.elf.thp=0. In this case, set + mp_.thp_mode to madvise. Otherwise, set it to thp_mode to + keep mp_.thp_mode in sync with GL(dl_thp_mode). */ + if (thp_mode == thp_mode_unknown) + mp_.thp_mode = thp_mode_madvise; + else + mp_.thp_mode = thp_mode; mp_.thp_pagesize = MALLOC_DEFAULT_THP_PAGESIZE; } @@ -4680,9 +4708,27 @@ do_set_hugetlb (size_t value) if (MALLOC_DEFAULT_THP_PAGESIZE > 0) return 0; - mp_.thp_mode = __get_thp_mode (); - if (mp_.thp_mode == thp_mode_madvise || mp_.thp_mode == thp_mode_always) - mp_.thp_pagesize = __get_thp_size (); + if (thp_mode == thp_mode_unknown) + { + /* Call __get_thp_mode and __get_thp_size when THP segment load + is disabled. */ + mp_.thp_mode = __get_thp_mode (); + if (mp_.thp_mode == thp_mode_madvise + || mp_.thp_mode == thp_mode_always) + mp_.thp_pagesize = __get_thp_size (); + } + else + { + /* THP segment load is enabled. GL(dl_elf_thp_pagesize) is + set to DL_MAP_DEFAULT_THP_PAGESIZE if it isn't zero. In + this case, call get_capped_thp_size () instead of using + DL_MAP_DEFAULT_THP_PAGESIZE for malloc. */ + mp_.thp_mode = thp_mode; + if (DL_MAP_DEFAULT_THP_PAGESIZE != 0) + mp_.thp_pagesize = get_capped_thp_size (); + else + mp_.thp_pagesize = get_dl_elf_thp_pagesize (); + } } else if (value >= 2) __get_hugepage_config (value == 2 ? 0 : value, &mp_.hp_pagesize, diff --git a/manual/tunables.texi b/manual/tunables.texi index e0f141077b..e1d9fbae9c 100644 --- a/manual/tunables.texi +++ b/manual/tunables.texi @@ -255,6 +255,10 @@ Setting its value to @code{1} enables the use of @code{madvise} with only if the system supports Transparent Huge Page (currently only on Linux). This is the default used for AArch64. +When @code{glibc.elf.thp} set to 1, malloc uses the actual kernel THP mode +instead of defaulting to madvise mode and madvise_thp will stop issuing +MADV_HUGEPAGE if kernel THP mode is always. + Setting its value to @code{2} enables the use of Huge Page directly with @code{mmap} with the use of @code{MAP_HUGETLB} flag. The huge page size to use will be the default one provided by the system. A value larger than diff --git a/sysdeps/generic/hugepages.h b/sysdeps/generic/hugepages.h index 8e54661527..e8fc70fbf9 100644 --- a/sysdeps/generic/hugepages.h +++ b/sysdeps/generic/hugepages.h @@ -37,7 +37,8 @@ unsigned long int __get_thp_size (void) attribute_hidden; enum thp_mode_t { - thp_mode_not_supported = 0, + thp_mode_unknown = 0, + thp_mode_not_supported, thp_mode_always, thp_mode_madvise, thp_mode_never @@ -64,4 +65,17 @@ void __get_hugepage_config (size_t requested, size_t *pagesize, int *flags) # define MAX_THP_PAGESIZE (32 * 1024 * 1024) #endif +/* If THP page size is above MAX_THP_PAGESIZE, return 0 to cap the THP + size at MAX_THP_PAGESIZE to avoid over-aligning on systems with very + large normal pages (like 64K pages with 512M huge pages). */ + +static inline unsigned long int +get_capped_thp_size (void) +{ + unsigned long int size = __get_thp_size (); + if (size > MAX_THP_PAGESIZE) + size = 0; + return size; +} + #endif /* _HUGEPAGES_H */ diff --git a/sysdeps/unix/sysv/linux/dl-exec-post.h b/sysdeps/unix/sysv/linux/dl-exec-post.h index dd8a379809..9a49486db2 100644 --- a/sysdeps/unix/sysv/linux/dl-exec-post.h +++ b/sysdeps/unix/sysv/linux/dl-exec-post.h @@ -48,14 +48,7 @@ _dl_get_thp_config (void) GL(dl_thp_mode) = __get_thp_mode (); if (GL(dl_thp_mode) == thp_mode_always || GL(dl_thp_mode) == thp_mode_madvise) - { - GL(dl_elf_thp_pagesize) = __get_thp_size (); - /* We cap the huge page size at MAX_THP_PAGESIZE to avoid - over-aligning on systems with very large normal pages - (like 64K pages with 512M huge pages). */ - if (GL(dl_elf_thp_pagesize) > MAX_THP_PAGESIZE) - GL(dl_elf_thp_pagesize) = 0; - } + GL(dl_elf_thp_pagesize) = get_capped_thp_size (); else GL(dl_elf_thp_pagesize) = 0;