MEDIUM: memory: make pool_gc() run under thread isolation
pool_gc() causes quite some stress on the memory allocator because
it calls a lot of free() calls while other threads are using malloc().
In addition, pool_gc() needs to take care of possible locking because
it may be called from pool allocators. One way to avoid all this is to
use thread_isolate() to make sure the gc runs alone. By putting less
pressure on the pools and getting rid of the locks, it may even take
less time to complete.
(cherry picked from commit c0e2ff202bda535aa60f3dc272ccf19a7b4f5094)
[wt: this is required for subsequent lockless pool fixes]
Signed-off-by: Willy Tarreau <w@1wt.eu>
diff --git a/src/memory.c b/src/memory.c
index a9e5f47..320b32a 100644
--- a/src/memory.c
+++ b/src/memory.c
@@ -251,22 +251,18 @@
/*
* This function frees whatever can be freed in all pools, but respecting
- * the minimum thresholds imposed by owners. It takes care of avoiding
- * recursion because it may be called from a signal handler.
- *
- * <pool_ctx> is unused
+ * the minimum thresholds imposed by owners. It makes sure to be alone to
+ * run by using thread_isolate(). <pool_ctx> is unused.
*/
void pool_gc(struct pool_head *pool_ctx)
{
- static int recurse;
- int cur_recurse = 0;
struct pool_head *entry;
+ int isolated = thread_isolated();
- if (recurse || !_HA_ATOMIC_CAS(&recurse, &cur_recurse, 1))
- return;
+ if (!isolated)
+ thread_isolate();
list_for_each_entry(entry, &pools, list) {
- HA_SPIN_LOCK(POOL_LOCK, &entry->flush_lock);
while ((int)((volatile int)entry->allocated - (volatile int)entry->used) > (int)entry->minavail) {
struct pool_free_list cmp, new;
@@ -283,10 +279,10 @@
free(cmp.free_list);
_HA_ATOMIC_SUB(&entry->allocated, 1);
}
- HA_SPIN_UNLOCK(POOL_LOCK, &entry->flush_lock);
}
- _HA_ATOMIC_STORE(&recurse, 0);
+ if (!isolated)
+ thread_release();
}
/* frees an object to the local cache, possibly pushing oldest objects to the
@@ -414,27 +410,20 @@
/*
* This function frees whatever can be freed in all pools, but respecting
- * the minimum thresholds imposed by owners. It takes care of avoiding
- * recursion because it may be called from a signal handler.
- *
- * <pool_ctx> is used when pool_gc is called to release resources to allocate
- * an element in __pool_refill_alloc. It is important because <pool_ctx> is
- * already locked, so we need to skip the lock here.
+ * the minimum thresholds imposed by owners. It makes sure to be alone to
+ * run by using thread_isolate(). <pool_ctx> is unused.
*/
void pool_gc(struct pool_head *pool_ctx)
{
- static int recurse;
- int cur_recurse = 0;
struct pool_head *entry;
+ int isolated = thread_isolated();
- if (recurse || !_HA_ATOMIC_CAS(&recurse, &cur_recurse, 1))
- return;
+ if (!isolated)
+ thread_isolate();
list_for_each_entry(entry, &pools, list) {
void *temp;
//qfprintf(stderr, "Flushing pool %s\n", entry->name);
- if (entry != pool_ctx)
- HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
while (entry->free_list &&
(int)(entry->allocated - entry->used) > (int)entry->minavail) {
temp = entry->free_list;
@@ -442,11 +431,10 @@
entry->allocated--;
pool_free_area(temp, entry->size + POOL_EXTRA);
}
- if (entry != pool_ctx)
- HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
}
- _HA_ATOMIC_STORE(&recurse, 0);
+ if (!isolated)
+ thread_release();
}
#endif