MEDIUM: pools: make CONFIG_HAP_POOLS control both local and shared pools

Continuing the unification of local and shared pools, now the usage of
pools is governed by CONFIG_HAP_POOLS without which allocations and
releases are performed directly from the OS using pool_alloc_nocache()
and pool_free_nocache().

diff --git a/include/haproxy/pool-t.h b/include/haproxy/pool-t.h
index bbdc5365a0e0333c818785af4c04d0ff0a30f746..d49982165f333112d7ff7bd1b2ec4d35fe86362f 100644 (file)
--- a/include/haproxy/pool-t.h
+++ b/include/haproxy/pool-t.h
@@ -36,7 +36,7 @@
 /* On architectures supporting threads and double-word CAS, we can implement
  * lock-less memory pools. This isn't supported for debugging modes however.
  */
-#if defined(USE_THREAD) && defined(HA_HAVE_CAS_DW) && !defined(DEBUG_NO_LOCKLESS_POOLS) && !defined(DEBUG_UAF) && !defined(DEBUG_FAIL_ALLOC)
+#if defined(USE_THREAD) && defined(HA_HAVE_CAS_DW) && defined(CONFIG_HAP_POOLS) && !defined(DEBUG_NO_LOCKLESS_POOLS)
 #define CONFIG_HAP_LOCKLESS_POOLS
 #endif
 

diff --git a/include/haproxy/pool.h b/include/haproxy/pool.h
index c1faa279374c436b28a6ed9ac03c275a0b0665a2..df0199196afdcc95ae93b8e059ce33fde2f5d4be 100644 (file)
--- a/include/haproxy/pool.h
+++ b/include/haproxy/pool.h
@@ -65,14 +65,6 @@ void *pool_destroy(struct pool_head *pool);
 void pool_destroy_all();
 
 
-/* returns true if the pool is considered to have too many free objects */
-static inline int pool_is_crowded(const struct pool_head *pool)
-{
-       return pool->allocated >= swrate_avg(pool->needed_avg + pool->needed_avg / 4, POOL_AVG_SAMPLES) &&
-              (int)(pool->allocated - pool->used) >= pool->minavail;
-}
-
-
 #ifdef CONFIG_HAP_POOLS
 
 /****************** Thread-local cache management ******************/
@@ -82,6 +74,13 @@ extern THREAD_LOCAL size_t pool_cache_count;   /* #cache objects   */
 
 void pool_evict_from_local_caches();
 
+/* returns true if the pool is considered to have too many free objects */
+static inline int pool_is_crowded(const struct pool_head *pool)
+{
+       return pool->allocated >= swrate_avg(pool->needed_avg + pool->needed_avg / 4, POOL_AVG_SAMPLES) &&
+              (int)(pool->allocated - pool->used) >= pool->minavail;
+}
+
 /* Tries to retrieve an object from the local pool cache corresponding to pool
  * <pool>. Returns NULL if none is available.
  */
@@ -125,8 +124,6 @@ static inline void pool_put_to_local_cache(struct pool_head *pool, void *ptr)
                pool_evict_from_local_caches();
 }
 
-#endif // CONFIG_HAP_POOLS
-
 
 #if defined(CONFIG_HAP_NO_GLOBAL_POOLS)
 
@@ -263,6 +260,8 @@ static inline void pool_put_to_shared_cache(struct pool_head *pool, void *ptr)
 
 #endif /* CONFIG_HAP_LOCKLESS_POOLS */
 
+#endif /* CONFIG_HAP_POOLS */
+
 
 /****************** Common high-level code ******************/
 
@@ -274,7 +273,7 @@ static inline void pool_put_to_shared_cache(struct pool_head *pool, void *ptr)
  */
 static inline void *__pool_alloc(struct pool_head *pool, unsigned int flags)
 {
-       void *p;
+       void *p = NULL;
 
 #ifdef DEBUG_FAIL_ALLOC
        if (!(flags & POOL_F_NO_FAIL) && mem_should_fail(pool))
@@ -282,14 +281,15 @@ static inline void *__pool_alloc(struct pool_head *pool, unsigned int flags)
 #endif
 
 #ifdef CONFIG_HAP_POOLS
-       if (likely(p = pool_get_from_local_cache(pool)))
-               goto ret;
-#endif
+       if (!p)
+               p = pool_get_from_local_cache(pool);
 
-       p = pool_get_from_shared_cache(pool);
+       if (!p)
+               p = pool_get_from_shared_cache(pool);
+#endif
        if (!p)
                p = pool_alloc_nocache(pool);
- ret:
+
        if (p) {
                if (flags & POOL_F_MUST_ZERO)
                        memset(p, 0, pool->size);
@@ -347,8 +347,10 @@ static inline void pool_free(struct pool_head *pool, void *ptr)
                        pool_put_to_local_cache(pool, ptr);
                        return;
                }
-#endif
                pool_put_to_shared_cache(pool, ptr);
+#else
+               pool_free_nocache(pool, ptr);
+#endif
        }
 }
 

diff --git a/src/pool.c b/src/pool.c
index 0956976eea13677e680b0e5d7f4875f6ce97fb2d..487e0da28bc10eebc1bfaa8f34fc3c2b64a995ee 100644 (file)
--- a/src/pool.c
+++ b/src/pool.c
@@ -148,33 +148,6 @@ void pool_put_to_os(struct pool_head *pool, void *ptr)
        _HA_ATOMIC_DEC(&pool->allocated);
 }
 
-#ifdef CONFIG_HAP_POOLS
-/* Evicts some of the oldest objects from the local cache, pushing them to the
- * global pool.
- */
-void pool_evict_from_local_caches()
-{
-       struct pool_cache_item *item;
-       struct pool_cache_head *ph;
-       struct pool_head *pool;
-
-       do {
-               item = LIST_PREV(&ti->pool_lru_head, struct pool_cache_item *, by_lru);
-               /* note: by definition we remove oldest objects so they also are the
-                * oldest in their own pools, thus their next is the pool's head.
-                */
-               ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
-               pool = container_of(ph - tid, struct pool_head, cache);
-               LIST_DEL(&item->by_pool);
-               LIST_DEL(&item->by_lru);
-               ph->count--;
-               pool_cache_count--;
-               pool_cache_bytes -= pool->size;
-               pool_put_to_shared_cache(pool, item);
-       } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
-}
-#endif
-
 /* Tries to allocate an object for the pool <pool> using the system's allocator
  * and directly returns it. The pool's counters are updated but the object is
  * never cached, so this is usable with and without local or shared caches.
@@ -211,6 +184,33 @@ void pool_free_nocache(struct pool_head *pool, void *ptr)
 }
 
 
+#ifdef CONFIG_HAP_POOLS
+
+/* Evicts some of the oldest objects from the local cache, pushing them to the
+ * global pool.
+ */
+void pool_evict_from_local_caches()
+{
+       struct pool_cache_item *item;
+       struct pool_cache_head *ph;
+       struct pool_head *pool;
+
+       do {
+               item = LIST_PREV(&ti->pool_lru_head, struct pool_cache_item *, by_lru);
+               /* note: by definition we remove oldest objects so they also are the
+                * oldest in their own pools, thus their next is the pool's head.
+                */
+               ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
+               pool = container_of(ph - tid, struct pool_head, cache);
+               LIST_DEL(&item->by_pool);
+               LIST_DEL(&item->by_lru);
+               ph->count--;
+               pool_cache_count--;
+               pool_cache_bytes -= pool->size;
+               pool_put_to_shared_cache(pool, item);
+       } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
+}
+
 #if defined(CONFIG_HAP_NO_GLOBAL_POOLS)
 
 /* legacy stuff */
@@ -349,7 +349,24 @@ void pool_gc(struct pool_head *pool_ctx)
        if (!isolated)
                thread_release();
 }
+#endif /* CONFIG_HAP_LOCKLESS_POOLS */
+
+#else  /* CONFIG_HAP_POOLS */
+
+/* legacy stuff */
+void pool_flush(struct pool_head *pool)
+{
+}
+
+/* This function might ask the malloc library to trim its buffers. */
+void pool_gc(struct pool_head *pool_ctx)
+{
+#if defined(HA_HAVE_MALLOC_TRIM)
+       malloc_trim(0);
 #endif
+}
+
+#endif /* CONFIG_HAP_POOLS */
 
 /*
  * This function destroys a pool by freeing it completely, unless it's still