MEDIUM: pools: make CONFIG_HAP_POOLS control both local and shared pools
Continuing the unification of local and shared pools, now the usage of
pools is governed by CONFIG_HAP_POOLS without which allocations and
releases are performed directly from the OS using pool_alloc_nocache()
and pool_free_nocache().
diff --git a/include/haproxy/pool-t.h b/include/haproxy/pool-t.h
index bbdc536..d499821 100644
--- a/include/haproxy/pool-t.h
+++ b/include/haproxy/pool-t.h
@@ -36,7 +36,7 @@
/* On architectures supporting threads and double-word CAS, we can implement
* lock-less memory pools. This isn't supported for debugging modes however.
*/
-#if defined(USE_THREAD) && defined(HA_HAVE_CAS_DW) && !defined(DEBUG_NO_LOCKLESS_POOLS) && !defined(DEBUG_UAF) && !defined(DEBUG_FAIL_ALLOC)
+#if defined(USE_THREAD) && defined(HA_HAVE_CAS_DW) && defined(CONFIG_HAP_POOLS) && !defined(DEBUG_NO_LOCKLESS_POOLS)
#define CONFIG_HAP_LOCKLESS_POOLS
#endif
diff --git a/include/haproxy/pool.h b/include/haproxy/pool.h
index c1faa27..df01991 100644
--- a/include/haproxy/pool.h
+++ b/include/haproxy/pool.h
@@ -65,14 +65,6 @@
void pool_destroy_all();
-/* returns true if the pool is considered to have too many free objects */
-static inline int pool_is_crowded(const struct pool_head *pool)
-{
- return pool->allocated >= swrate_avg(pool->needed_avg + pool->needed_avg / 4, POOL_AVG_SAMPLES) &&
- (int)(pool->allocated - pool->used) >= pool->minavail;
-}
-
-
#ifdef CONFIG_HAP_POOLS
/****************** Thread-local cache management ******************/
@@ -82,6 +74,13 @@
void pool_evict_from_local_caches();
+/* returns true if the pool is considered to have too many free objects */
+static inline int pool_is_crowded(const struct pool_head *pool)
+{
+ return pool->allocated >= swrate_avg(pool->needed_avg + pool->needed_avg / 4, POOL_AVG_SAMPLES) &&
+ (int)(pool->allocated - pool->used) >= pool->minavail;
+}
+
/* Tries to retrieve an object from the local pool cache corresponding to pool
* <pool>. Returns NULL if none is available.
*/
@@ -125,8 +124,6 @@
pool_evict_from_local_caches();
}
-#endif // CONFIG_HAP_POOLS
-
#if defined(CONFIG_HAP_NO_GLOBAL_POOLS)
@@ -263,6 +260,8 @@
#endif /* CONFIG_HAP_LOCKLESS_POOLS */
+#endif /* CONFIG_HAP_POOLS */
+
/****************** Common high-level code ******************/
@@ -274,7 +273,7 @@
*/
static inline void *__pool_alloc(struct pool_head *pool, unsigned int flags)
{
- void *p;
+ void *p = NULL;
#ifdef DEBUG_FAIL_ALLOC
if (!(flags & POOL_F_NO_FAIL) && mem_should_fail(pool))
@@ -282,14 +281,15 @@
#endif
#ifdef CONFIG_HAP_POOLS
- if (likely(p = pool_get_from_local_cache(pool)))
- goto ret;
-#endif
+ if (!p)
+ p = pool_get_from_local_cache(pool);
- p = pool_get_from_shared_cache(pool);
+ if (!p)
+ p = pool_get_from_shared_cache(pool);
+#endif
if (!p)
p = pool_alloc_nocache(pool);
- ret:
+
if (p) {
if (flags & POOL_F_MUST_ZERO)
memset(p, 0, pool->size);
@@ -347,8 +347,10 @@
pool_put_to_local_cache(pool, ptr);
return;
}
-#endif
pool_put_to_shared_cache(pool, ptr);
+#else
+ pool_free_nocache(pool, ptr);
+#endif
}
}
diff --git a/src/pool.c b/src/pool.c
index 0956976..487e0da 100644
--- a/src/pool.c
+++ b/src/pool.c
@@ -148,33 +148,6 @@
_HA_ATOMIC_DEC(&pool->allocated);
}
-#ifdef CONFIG_HAP_POOLS
-/* Evicts some of the oldest objects from the local cache, pushing them to the
- * global pool.
- */
-void pool_evict_from_local_caches()
-{
- struct pool_cache_item *item;
- struct pool_cache_head *ph;
- struct pool_head *pool;
-
- do {
- item = LIST_PREV(&ti->pool_lru_head, struct pool_cache_item *, by_lru);
- /* note: by definition we remove oldest objects so they also are the
- * oldest in their own pools, thus their next is the pool's head.
- */
- ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
- pool = container_of(ph - tid, struct pool_head, cache);
- LIST_DEL(&item->by_pool);
- LIST_DEL(&item->by_lru);
- ph->count--;
- pool_cache_count--;
- pool_cache_bytes -= pool->size;
- pool_put_to_shared_cache(pool, item);
- } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
-}
-#endif
-
/* Tries to allocate an object for the pool <pool> using the system's allocator
* and directly returns it. The pool's counters are updated but the object is
* never cached, so this is usable with and without local or shared caches.
@@ -210,6 +183,33 @@
pool_put_to_os(pool, ptr);
}
+
+#ifdef CONFIG_HAP_POOLS
+
+/* Evicts some of the oldest objects from the local cache, pushing them to the
+ * global pool.
+ */
+void pool_evict_from_local_caches()
+{
+ struct pool_cache_item *item;
+ struct pool_cache_head *ph;
+ struct pool_head *pool;
+
+ do {
+ item = LIST_PREV(&ti->pool_lru_head, struct pool_cache_item *, by_lru);
+ /* note: by definition we remove oldest objects so they also are the
+ * oldest in their own pools, thus their next is the pool's head.
+ */
+ ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
+ pool = container_of(ph - tid, struct pool_head, cache);
+ LIST_DEL(&item->by_pool);
+ LIST_DEL(&item->by_lru);
+ ph->count--;
+ pool_cache_count--;
+ pool_cache_bytes -= pool->size;
+ pool_put_to_shared_cache(pool, item);
+ } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
+}
#if defined(CONFIG_HAP_NO_GLOBAL_POOLS)
@@ -349,7 +349,24 @@
if (!isolated)
thread_release();
}
+#endif /* CONFIG_HAP_LOCKLESS_POOLS */
+
+#else /* CONFIG_HAP_POOLS */
+
+/* legacy stuff */
+void pool_flush(struct pool_head *pool)
+{
+}
+
+/* This function might ask the malloc library to trim its buffers. */
+void pool_gc(struct pool_head *pool_ctx)
+{
+#if defined(HA_HAVE_MALLOC_TRIM)
+ malloc_trim(0);
#endif
+}
+
+#endif /* CONFIG_HAP_POOLS */
/*
* This function destroys a pool by freeing it completely, unless it's still