From: DJ Delorie Date: Fri, 13 Jan 2017 00:24:10 +0000 (-0500) Subject: Support new tunables system X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b13e136a9838e3df35c7b17809f3cd52b1ca2093;p=thirdparty%2Fglibc.git Support new tunables system Add new tunables: malloc_tcache_max malloc_tcache_count malloc_tcache_unsorted_limit (the last is new, it limits how deeply we scan the unsorted list looking for chunks to move into the tcache) --- diff --git a/elf/dl-tunables.list b/elf/dl-tunables.list index d8cd912559f..3e498759cbe 100644 --- a/elf/dl-tunables.list +++ b/elf/dl-tunables.list @@ -64,5 +64,17 @@ glibc { env_alias: MALLOC_ARENA_TEST minval: 1 } + tcache_max { + type: SIZE_T + env_alias: MALLOC_TCACHE_MAX + } + tcache_count { + type: SIZE_T + env_alias: MALLOC_TCACHE_COUNT + } + tcache_unsorted_limit { + type: SIZE_T + env_alias: MALLOC_TCACHE_UNSORTED_LIMIT + } } } diff --git a/malloc/arena.c b/malloc/arena.c index b91d7d6b16c..74616df110e 100644 --- a/malloc/arena.c +++ b/malloc/arena.c @@ -236,6 +236,9 @@ DL_TUNABLE_CALLBACK_FNDECL (set_perturb_byte, int32_t) DL_TUNABLE_CALLBACK_FNDECL (set_trim_threshold, size_t) DL_TUNABLE_CALLBACK_FNDECL (set_arena_max, size_t) DL_TUNABLE_CALLBACK_FNDECL (set_arena_test, size_t) +DL_TUNABLE_CALLBACK_FNDECL (set_tcache_max, size_t) +DL_TUNABLE_CALLBACK_FNDECL (set_tcache_count, size_t) +DL_TUNABLE_CALLBACK_FNDECL (set_tcache_unsorted_limit, size_t) #else /* Initialization routine. */ #include @@ -322,6 +325,11 @@ ptmalloc_init (void) TUNABLE_SET_VAL_WITH_CALLBACK (mmap_max, NULL, set_mmaps_max); TUNABLE_SET_VAL_WITH_CALLBACK (arena_max, NULL, set_arena_max); TUNABLE_SET_VAL_WITH_CALLBACK (arena_test, NULL, set_arena_test); +#if USE_TCACHE + TUNABLE_SET_VAL_WITH_CALLBACK (tcache_max, NULL, set_tcache_max); + TUNABLE_SET_VAL_WITH_CALLBACK (tcache_count, NULL, set_tcache_count); + TUNABLE_SET_VAL_WITH_CALLBACK (tcache_unsorted_limit, NULL, set_tcache_unsorted_limit); +#endif __libc_lock_unlock (main_arena.mutex); #else const char *s = NULL; @@ -371,7 +379,23 @@ ptmalloc_init (void) if (memcmp (envline, "ARENA_TEST", 10) == 0) __libc_mallopt (M_ARENA_TEST, atoi (&envline[11])); } +#if USE_TCACHE + if (!__builtin_expect (__libc_enable_secure, 0)) + { + if (memcmp (envline, "TCACHE_MAX", 10) == 0) + __libc_mallopt (M_TCACHE_MAX, atoi (&envline[11])); + } +#endif break; +#if USE_TCACHE + case 12: + if (!__builtin_expect (__libc_enable_secure, 0)) + { + if (memcmp (envline, "TCACHE_COUNT", 12) == 0) + __libc_mallopt (M_TCACHE_COUNT, atoi (&envline[13])); + } + break; +#endif case 15: if (!__builtin_expect (__libc_enable_secure, 0)) { @@ -381,6 +405,15 @@ ptmalloc_init (void) __libc_mallopt (M_MMAP_THRESHOLD, atoi (&envline[16])); } break; +#if USE_TCACHE + case 21: + if (!__builtin_expect (__libc_enable_secure, 0)) + { + if (memcmp (envline, "TCACHE_UNSORTED_LIMIT", 21) == 0) + __libc_mallopt (M_TCACHE_UNSORTED_LIMIT, atoi (&envline[22])); + } + break; +#endif default: break; } diff --git a/malloc/malloc.c b/malloc/malloc.c index 1621fb36fd9..a6536077f02 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -1736,6 +1736,9 @@ struct malloc_par size_t tcache_max; /* Maximum number of chunks in each bucket. */ size_t tcache_count; + /* Maximum number of chunks to remove from the unsorted list, which + don't match. */ + size_t tcache_unsorted_limit; #endif }; @@ -1778,7 +1781,8 @@ static struct malloc_par mp_ = #if USE_TCACHE , .tcache_count = TCACHE_FILL_COUNT, - .tcache_max = TCACHE_IDX-1 + .tcache_max = TCACHE_IDX-1, + .tcache_unsorted_limit = 0 /* No limit */ #endif }; @@ -1786,6 +1790,7 @@ static struct malloc_par mp_ = #if USE_TCACHE #define M_TCACHE_COUNT -9 #define M_TCACHE_MAX -10 +#define M_TCACHE_UNSORTED_LIMIT -11 #endif /* Maximum size of memory handled in fastbins. */ @@ -3437,6 +3442,10 @@ _int_malloc (mstate av, size_t bytes) mchunkptr fwd; /* misc temp for linking */ mchunkptr bck; /* misc temp for linking */ +#if USE_TCACHE + size_t tcache_unsorted_count; /* count of unsorted chunks processed */ +#endif + const char *errstr = NULL; /* @@ -3637,6 +3646,8 @@ _int_malloc (mstate av, size_t bytes) tcache_nb = nb; size_t tc_idx = size2tidx (nb-SIZE_SZ); int return_cached = 0; + + tcache_unsorted_count = 0; #endif for (;; ) @@ -3788,6 +3799,21 @@ _int_malloc (mstate av, size_t bytes) fwd->bk = victim; bck->fd = victim; +#if USE_TCACHE + /* If we've processed as many chunks as we're allowed while + filling the cache, return one of the cached ones. */ + tcache_unsorted_count ++; + if (return_cached + && mp_.tcache_unsorted_limit > 0 + && tcache_unsorted_count > mp_.tcache_unsorted_limit) + { + TCacheEntry *e = tcache.entries[tc_idx]; + tcache.entries[tc_idx] = e->next; + tcache.counts[tc_idx] --; + return (void *) e; + } +#endif + #define MAX_ITERS 10000 if (++iters >= MAX_ITERS) break; @@ -5067,6 +5093,34 @@ do_set_arena_max (size_t value) return 1; } +#ifdef USE_TCACHE +static inline int +__always_inline +do_set_tcache_max (size_t value) +{ + LIBC_PROBE (memory_mallopt_tcache_max, 2, value, mp_.tcache_max); + mp_.tcache_max = value; + return 1; +} + +static inline int +__always_inline +do_set_tcache_count (size_t value) +{ + LIBC_PROBE (memory_mallopt_tcache_count, 2, value, mp_.tcache_count); + mp_.tcache_count = value; + return 1; +} + +static inline int +__always_inline +do_set_tcache_unsorted_limit (size_t value) +{ + LIBC_PROBE (memory_mallopt_tcache_unsorted_limit, 2, value, mp_.tcache_unsorted_limit); + mp_.tcache_unsorted_limit = value; + return 1; +} +#endif int __libc_mallopt (int param_number, int value) @@ -5130,22 +5184,20 @@ __libc_mallopt (int param_number, int value) #if USE_TCACHE case M_TCACHE_COUNT: if (value >= 0) - { - LIBC_PROBE (memory_mallopt_tcache_count, 2, value, mp_.tcache_count); - mp_.tcache_count = value; - } + do_set_tcache_max (value); break; case M_TCACHE_MAX: if (value >= 0) { value = size2tidx (value); if (value < TCACHE_IDX) - { - LIBC_PROBE (memory_mallopt_tcache_max, 2, value, mp_.tcache_max); - mp_.tcache_max = value; - } + do_set_tcache_max (value); } break; + case M_TCACHE_UNSORTED_LIMIT: + if (value >= 0) + do_set_tcache_unsorted_limit (value); + break; #endif } __libc_lock_unlock (av->mutex);