blob: b0512aaece5ddfc2a1f93086240243e2bc25021b [file] [log] [blame]
Willy Tarreau50e608d2007-05-13 18:26:08 +02001/*
2 * Memory management functions.
3 *
4 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
Willy Tarreau7107c8b2018-11-26 11:44:35 +010012#include <errno.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020013
William Lallemande7ed8852016-11-19 02:25:36 +010014#include <types/applet.h>
15#include <types/cli.h>
Willy Tarreau12833bb2014-01-28 16:49:56 +010016#include <types/global.h>
William Lallemande7ed8852016-11-19 02:25:36 +010017#include <types/stats.h>
18
Olivier Houcharddc21ff72019-01-29 15:20:16 +010019#include <common/cfgparse.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020020#include <common/config.h>
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +020021#include <common/debug.h>
Willy Tarreaue18db9e2018-10-16 10:28:54 +020022#include <common/hathreads.h>
Willy Tarreau0108d902018-11-25 19:14:37 +010023#include <common/initcall.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020024#include <common/memory.h>
25#include <common/mini-clist.h>
26#include <common/standard.h>
27
Willy Tarreaua8b2ce02019-05-28 17:04:16 +020028#include <types/activity.h>
29
William Lallemande7ed8852016-11-19 02:25:36 +010030#include <proto/applet.h>
31#include <proto/cli.h>
32#include <proto/channel.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020033#include <proto/log.h>
William Lallemande7ed8852016-11-19 02:25:36 +010034#include <proto/stream_interface.h>
35#include <proto/stats.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020036
Willy Tarreau0a93b642018-10-16 07:58:39 +020037/* These are the most common pools, expected to be initialized first. These
38 * ones are allocated from an array, allowing to map them to an index.
39 */
40struct pool_head pool_base_start[MAX_BASE_POOLS] = { };
41unsigned int pool_base_count = 0;
42
Willy Tarreau7f0165e2018-11-26 17:09:46 +010043/* These ones are initialized per-thread on startup by init_pools() */
44struct pool_cache_head pool_cache[MAX_THREADS][MAX_BASE_POOLS];
45static struct list pool_lru_head[MAX_THREADS]; /* oldest objects */
Willy Tarreaue18db9e2018-10-16 10:28:54 +020046THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
47THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
48
Willy Tarreau50e608d2007-05-13 18:26:08 +020049static struct list pools = LIST_HEAD_INIT(pools);
Willy Tarreau067ac9f2015-10-08 14:12:13 +020050int mem_poison_byte = -1;
Willy Tarreau50e608d2007-05-13 18:26:08 +020051
Olivier Houcharddc21ff72019-01-29 15:20:16 +010052#ifdef DEBUG_FAIL_ALLOC
53static int mem_fail_rate = 0;
54static int mem_should_fail(const struct pool_head *);
55#endif
56
Willy Tarreau50e608d2007-05-13 18:26:08 +020057/* Try to find an existing shared pool with the same characteristics and
58 * returns it, otherwise creates this one. NULL is returned if no memory
Willy Tarreau581bf812016-01-25 02:19:13 +010059 * is available for a new creation. Two flags are supported :
60 * - MEM_F_SHARED to indicate that the pool may be shared with other users
61 * - MEM_F_EXACT to indicate that the size must not be rounded up
Willy Tarreau50e608d2007-05-13 18:26:08 +020062 */
63struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
64{
65 struct pool_head *pool;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020066 struct pool_head *entry;
67 struct list *start;
Willy Tarreau50e608d2007-05-13 18:26:08 +020068 unsigned int align;
Christopher Faulet2f6d3c02019-06-25 21:45:59 +020069 int thr, idx;
Willy Tarreau50e608d2007-05-13 18:26:08 +020070
Willy Tarreauac421112015-10-28 15:09:29 +010071 /* We need to store a (void *) at the end of the chunks. Since we know
Willy Tarreau50e608d2007-05-13 18:26:08 +020072 * that the malloc() function will never return such a small size,
73 * let's round the size up to something slightly bigger, in order to
74 * ease merging of entries. Note that the rounding is a power of two.
Willy Tarreauac421112015-10-28 15:09:29 +010075 * This extra (void *) is not accounted for in the size computation
76 * so that the visible parts outside are not affected.
Willy Tarreau30f931e2018-10-23 14:40:23 +020077 *
78 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
Willy Tarreau50e608d2007-05-13 18:26:08 +020079 */
80
Willy Tarreau581bf812016-01-25 02:19:13 +010081 if (!(flags & MEM_F_EXACT)) {
Willy Tarreau30f931e2018-10-23 14:40:23 +020082 align = 4 * sizeof(void *); // 2 lists = 4 pointers min
Willy Tarreau581bf812016-01-25 02:19:13 +010083 size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA;
84 }
Willy Tarreau50e608d2007-05-13 18:26:08 +020085
Christopher Fauletb349e482017-08-29 09:52:38 +020086 /* TODO: thread: we do not lock pool list for now because all pools are
87 * created during HAProxy startup (so before threads creation) */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020088 start = &pools;
Willy Tarreau50e608d2007-05-13 18:26:08 +020089 pool = NULL;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020090
91 list_for_each_entry(entry, &pools, list) {
92 if (entry->size == size) {
93 /* either we can share this place and we take it, or
94 * we look for a sharable one or for the next position
95 * before which we will insert a new one.
96 */
97 if (flags & entry->flags & MEM_F_SHARED) {
98 /* we can share this one */
Willy Tarreau50e608d2007-05-13 18:26:08 +020099 pool = entry;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200100 DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
Willy Tarreau50e608d2007-05-13 18:26:08 +0200101 break;
102 }
103 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200104 else if (entry->size > size) {
105 /* insert before this one */
106 start = &entry->list;
107 break;
108 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200109 }
110
111 if (!pool) {
Willy Tarreau0a93b642018-10-16 07:58:39 +0200112 if (pool_base_count < MAX_BASE_POOLS)
113 pool = &pool_base_start[pool_base_count++];
114
115 if (!pool) {
116 /* look for a freed entry */
117 for (entry = pool_base_start; entry != pool_base_start + MAX_BASE_POOLS; entry++) {
118 if (!entry->size) {
119 pool = entry;
120 break;
121 }
122 }
123 }
124
125 if (!pool)
126 pool = calloc(1, sizeof(*pool));
127
Willy Tarreau50e608d2007-05-13 18:26:08 +0200128 if (!pool)
129 return NULL;
130 if (name)
131 strlcpy2(pool->name, name, sizeof(pool->name));
132 pool->size = size;
133 pool->flags = flags;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200134 LIST_ADDQ(start, &pool->list);
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200135
136 /* update per-thread pool cache if necessary */
137 idx = pool_get_index(pool);
138 if (idx >= 0) {
139 for (thr = 0; thr < MAX_THREADS; thr++)
140 pool_cache[thr][idx].size = size;
141 }
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100142#ifndef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchard8af97eb2020-02-01 17:45:32 +0100143 HA_SPIN_INIT(&pool->lock);
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100144#else
145 HA_RWLOCK_INIT(&pool->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100146#endif
Olivier Houchard8af97eb2020-02-01 17:45:32 +0100147 }
148 pool->users++;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200149 return pool;
150}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100151
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100152#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +0100153/* Allocates new entries for pool <pool> until there are at least <avail> + 1
154 * available, then returns the last one for immediate use, so that at least
155 * <avail> are left available in the pool upon return. NULL is returned if the
156 * last entry could not be allocated. It's important to note that at least one
157 * allocation is always performed even if there are enough entries in the pool.
158 * A call to the garbage collector is performed at most once in case malloc()
159 * returns an error, before returning NULL.
160 */
161void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
162{
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200163 void *ptr = NULL, **free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100164 int failed = 0;
165 int size = pool->size;
166 int limit = pool->limit;
167 int allocated = pool->allocated, allocated_orig = allocated;
168
169 /* stop point */
170 avail += pool->used;
171
172 while (1) {
173 if (limit && allocated >= limit) {
Olivier Houchard20872762019-03-08 18:53:35 +0100174 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200175 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100176 return NULL;
177 }
178
179 ptr = malloc(size + POOL_EXTRA);
180 if (!ptr) {
Olivier Houchard20872762019-03-08 18:53:35 +0100181 _HA_ATOMIC_ADD(&pool->failed, 1);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200182 if (failed) {
183 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100184 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200185 }
Olivier Houchardcf975d42018-01-24 18:38:31 +0100186 failed++;
187 pool_gc(pool);
188 continue;
189 }
190 if (++allocated > avail)
191 break;
192
193 free_list = pool->free_list;
194 do {
195 *POOL_LINK(pool, ptr) = free_list;
196 __ha_barrier_store();
Olivier Houchard20872762019-03-08 18:53:35 +0100197 } while (_HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr) == 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100198 }
Olivier Houchard20872762019-03-08 18:53:35 +0100199 __ha_barrier_atomic_store();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100200
Olivier Houchard20872762019-03-08 18:53:35 +0100201 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
202 _HA_ATOMIC_ADD(&pool->used, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100203
204#ifdef DEBUG_MEMORY_POOLS
205 /* keep track of where the element was allocated from */
206 *POOL_LINK(pool, ptr) = (void *)pool;
207#endif
208 return ptr;
209}
210void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
211{
212 void *ptr;
213
214 ptr = __pool_refill_alloc(pool, avail);
215 return ptr;
216}
217/*
218 * This function frees whatever can be freed in pool <pool>.
219 */
220void pool_flush(struct pool_head *pool)
221{
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100222 struct pool_free_list cmp, new;
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200223 void **next, *temp;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100224 int removed = 0;
225
226 if (!pool)
227 return;
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100228 HA_RWLOCK_WRLOCK(POOL_LOCK, &pool->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100229 do {
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100230 cmp.free_list = pool->free_list;
231 cmp.seq = pool->seq;
232 new.free_list = NULL;
233 new.seq = cmp.seq + 1;
234 } while (!_HA_ATOMIC_DWCAS(&pool->free_list, &cmp, &new));
Olivier Houchard20872762019-03-08 18:53:35 +0100235 __ha_barrier_atomic_store();
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100236 HA_RWLOCK_WRUNLOCK(POOL_LOCK, &pool->flush_lock);
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100237 next = cmp.free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100238 while (next) {
239 temp = next;
240 next = *POOL_LINK(pool, temp);
241 removed++;
242 free(temp);
243 }
244 pool->free_list = next;
Olivier Houchard20872762019-03-08 18:53:35 +0100245 _HA_ATOMIC_SUB(&pool->allocated, removed);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100246 /* here, we should have pool->allocate == pool->used */
247}
248
249/*
250 * This function frees whatever can be freed in all pools, but respecting
251 * the minimum thresholds imposed by owners. It takes care of avoiding
252 * recursion because it may be called from a signal handler.
253 *
254 * <pool_ctx> is unused
255 */
256void pool_gc(struct pool_head *pool_ctx)
257{
258 static int recurse;
259 int cur_recurse = 0;
260 struct pool_head *entry;
261
Olivier Houchard20872762019-03-08 18:53:35 +0100262 if (recurse || !_HA_ATOMIC_CAS(&recurse, &cur_recurse, 1))
Olivier Houchardcf975d42018-01-24 18:38:31 +0100263 return;
264
265 list_for_each_entry(entry, &pools, list) {
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100266 HA_RWLOCK_WRLOCK(POOL_LOCK, &entry->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100267 while ((int)((volatile int)entry->allocated - (volatile int)entry->used) > (int)entry->minavail) {
268 struct pool_free_list cmp, new;
269
270 cmp.seq = entry->seq;
271 __ha_barrier_load();
272 cmp.free_list = entry->free_list;
273 __ha_barrier_load();
274 if (cmp.free_list == NULL)
275 break;
276 new.free_list = *POOL_LINK(entry, cmp.free_list);
277 new.seq = cmp.seq + 1;
Willy Tarreau6a38b322019-05-11 18:04:24 +0200278 if (HA_ATOMIC_DWCAS(&entry->free_list, &cmp, &new) == 0)
Olivier Houchardcf975d42018-01-24 18:38:31 +0100279 continue;
280 free(cmp.free_list);
Olivier Houchard20872762019-03-08 18:53:35 +0100281 _HA_ATOMIC_SUB(&entry->allocated, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100282 }
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100283 HA_RWLOCK_WRUNLOCK(POOL_LOCK, &entry->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100284 }
285
Olivier Houchard20872762019-03-08 18:53:35 +0100286 _HA_ATOMIC_STORE(&recurse, 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100287}
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200288
289/* frees an object to the local cache, possibly pushing oldest objects to the
290 * global pool. Must not be called directly.
291 */
292void __pool_put_to_cache(struct pool_head *pool, void *ptr, ssize_t idx)
293{
294 struct pool_cache_item *item = (struct pool_cache_item *)ptr;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100295 struct pool_cache_head *ph = &pool_cache[tid][idx];
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200296
297 LIST_ADD(&ph->list, &item->by_pool);
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100298 LIST_ADD(&pool_lru_head[tid], &item->by_lru);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200299 ph->count++;
300 pool_cache_count++;
301 pool_cache_bytes += ph->size;
302
303 if (pool_cache_bytes <= CONFIG_HAP_POOL_CACHE_SIZE)
304 return;
305
306 do {
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100307 item = LIST_PREV(&pool_lru_head[tid], struct pool_cache_item *, by_lru);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200308 /* note: by definition we remove oldest objects so they also are the
309 * oldest in their own pools, thus their next is the pool's head.
310 */
311 ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
312 LIST_DEL(&item->by_pool);
313 LIST_DEL(&item->by_lru);
314 ph->count--;
315 pool_cache_count--;
316 pool_cache_bytes -= ph->size;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100317 __pool_free(pool_base_start + (ph - pool_cache[tid]), item);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200318 } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
319}
320
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100321#else /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200322
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100323/* Allocates new entries for pool <pool> until there are at least <avail> + 1
324 * available, then returns the last one for immediate use, so that at least
325 * <avail> are left available in the pool upon return. NULL is returned if the
326 * last entry could not be allocated. It's important to note that at least one
327 * allocation is always performed even if there are enough entries in the pool.
328 * A call to the garbage collector is performed at most once in case malloc()
329 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200330 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200331void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200332{
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100333 void *ptr = NULL;
334 int failed = 0;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200335
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100336#ifdef DEBUG_FAIL_ALLOC
337 if (mem_should_fail(pool))
338 return NULL;
339#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100340 /* stop point */
341 avail += pool->used;
342
343 while (1) {
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200344 if (pool->limit && pool->allocated >= pool->limit) {
345 activity[tid].pool_fail++;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200346 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200347 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100348
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200349 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreauf13322e2017-11-22 10:50:54 +0100350 ptr = pool_alloc_area(pool->size + POOL_EXTRA);
Willy Tarreau82867542019-07-04 11:48:16 +0200351#ifdef DEBUG_MEMORY_POOLS
352 /* keep track of where the element was allocated from. This
353 * is done out of the lock so that the system really allocates
354 * the data without harming other threads waiting on the lock.
355 */
356 if (ptr)
357 *POOL_LINK(pool, ptr) = (void *)pool;
358#endif
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200359 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100360 if (!ptr) {
Willy Tarreau58102cf2015-10-28 16:24:21 +0100361 pool->failed++;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200362 if (failed) {
363 activity[tid].pool_fail++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100364 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200365 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100366 failed++;
Willy Tarreaubafbe012017-11-24 17:34:44 +0100367 pool_gc(pool);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100368 continue;
369 }
370 if (++pool->allocated > avail)
371 break;
372
Willy Tarreauac421112015-10-28 15:09:29 +0100373 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100374 pool->free_list = ptr;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200375 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200376 pool->used++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100377 return ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200378}
Christopher Fauletb349e482017-08-29 09:52:38 +0200379void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
380{
381 void *ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200382
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100383 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200384 ptr = __pool_refill_alloc(pool, avail);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100385 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200386 return ptr;
387}
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200388/*
389 * This function frees whatever can be freed in pool <pool>.
390 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100391void pool_flush(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200392{
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200393 void *temp;
394
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200395 if (!pool)
396 return;
397
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200398 while (1) {
399 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
400 temp = pool->free_list;
401 if (!temp) {
402 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
403 break;
404 }
405 pool->free_list = *POOL_LINK(pool, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200406 pool->allocated--;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200407 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreauf13322e2017-11-22 10:50:54 +0100408 pool_free_area(temp, pool->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200409 }
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200410 /* here, we should have pool->allocated == pool->used */
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200411}
412
413/*
414 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200415 * the minimum thresholds imposed by owners. It takes care of avoiding
416 * recursion because it may be called from a signal handler.
Christopher Fauletb349e482017-08-29 09:52:38 +0200417 *
Willy Tarreaubafbe012017-11-24 17:34:44 +0100418 * <pool_ctx> is used when pool_gc is called to release resources to allocate
Christopher Fauletb349e482017-08-29 09:52:38 +0200419 * an element in __pool_refill_alloc. It is important because <pool_ctx> is
420 * already locked, so we need to skip the lock here.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200421 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100422void pool_gc(struct pool_head *pool_ctx)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200423{
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200424 static int recurse;
Christopher Fauletb349e482017-08-29 09:52:38 +0200425 int cur_recurse = 0;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200426 struct pool_head *entry;
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200427
Olivier Houchard20872762019-03-08 18:53:35 +0100428 if (recurse || !_HA_ATOMIC_CAS(&recurse, &cur_recurse, 1))
Christopher Fauletb349e482017-08-29 09:52:38 +0200429 return;
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200430
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200431 list_for_each_entry(entry, &pools, list) {
Olivier Houchard51d93392020-03-12 19:05:39 +0100432 void *temp;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200433 //qfprintf(stderr, "Flushing pool %s\n", entry->name);
Christopher Fauletb349e482017-08-29 09:52:38 +0200434 if (entry != pool_ctx)
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100435 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Olivier Houchard51d93392020-03-12 19:05:39 +0100436 while (entry->free_list &&
Willy Tarreau57767b82014-12-22 21:40:55 +0100437 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
Olivier Houchard51d93392020-03-12 19:05:39 +0100438 temp = entry->free_list;
439 entry->free_list = *POOL_LINK(entry, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200440 entry->allocated--;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200441 if (entry != pool_ctx)
442 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Willy Tarreauf13322e2017-11-22 10:50:54 +0100443 pool_free_area(temp, entry->size + POOL_EXTRA);
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200444 if (entry != pool_ctx)
445 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200446 }
Christopher Fauletb349e482017-08-29 09:52:38 +0200447 if (entry != pool_ctx)
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100448 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200449 }
Christopher Fauletb349e482017-08-29 09:52:38 +0200450
Olivier Houchard20872762019-03-08 18:53:35 +0100451 _HA_ATOMIC_STORE(&recurse, 0);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200452}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100453#endif
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200454
455/*
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200456 * This function destroys a pool by freeing it completely, unless it's still
457 * in use. This should be called only under extreme circumstances. It always
458 * returns NULL if the resulting pool is empty, easing the clearing of the old
459 * pointer, otherwise it returns the pool.
460 * .
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200461 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100462void *pool_destroy(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200463{
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200464 if (pool) {
Willy Tarreaubafbe012017-11-24 17:34:44 +0100465 pool_flush(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200466 if (pool->used)
467 return pool;
468 pool->users--;
469 if (!pool->users) {
470 LIST_DEL(&pool->list);
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100471#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100472 HA_SPIN_DESTROY(&pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100473#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200474 if ((pool - pool_base_start) < MAX_BASE_POOLS)
475 memset(pool, 0, sizeof(*pool));
476 else
477 free(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200478 }
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200479 }
480 return NULL;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200481}
482
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100483/* This destroys all pools on exit. It is *not* thread safe. */
484void pool_destroy_all()
485{
486 struct pool_head *entry, *back;
487
488 list_for_each_entry_safe(entry, back, &pools, list)
489 pool_destroy(entry);
490}
491
Willy Tarreau12833bb2014-01-28 16:49:56 +0100492/* This function dumps memory usage information into the trash buffer. */
493void dump_pools_to_trash()
Willy Tarreau50e608d2007-05-13 18:26:08 +0200494{
495 struct pool_head *entry;
496 unsigned long allocated, used;
497 int nbpools;
498
499 allocated = used = nbpools = 0;
Willy Tarreau12833bb2014-01-28 16:49:56 +0100500 chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200501 list_for_each_entry(entry, &pools, list) {
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100502#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100503 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100504#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200505 chunk_appendf(&trash, " - Pool %s (%d bytes) : %d allocated (%u bytes), %d used, %d failures, %d users, @%p=%02d%s\n",
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200506 entry->name, entry->size, entry->allocated,
Willy Tarreau58102cf2015-10-28 16:24:21 +0100507 entry->size * entry->allocated, entry->used, entry->failed,
Willy Tarreau0a93b642018-10-16 07:58:39 +0200508 entry->users, entry, (int)pool_get_index(entry),
509 (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200510
511 allocated += entry->allocated * entry->size;
512 used += entry->used * entry->size;
513 nbpools++;
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100514#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100515 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100516#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +0200517 }
Willy Tarreau12833bb2014-01-28 16:49:56 +0100518 chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n",
Willy Tarreau50e608d2007-05-13 18:26:08 +0200519 nbpools, allocated, used);
520}
521
Willy Tarreau12833bb2014-01-28 16:49:56 +0100522/* Dump statistics on pools usage. */
523void dump_pools(void)
524{
525 dump_pools_to_trash();
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200526 qfprintf(stderr, "%s", trash.area);
Willy Tarreau12833bb2014-01-28 16:49:56 +0100527}
528
Willy Tarreau58102cf2015-10-28 16:24:21 +0100529/* This function returns the total number of failed pool allocations */
530int pool_total_failures()
531{
532 struct pool_head *entry;
533 int failed = 0;
534
535 list_for_each_entry(entry, &pools, list)
536 failed += entry->failed;
537 return failed;
538}
539
540/* This function returns the total amount of memory allocated in pools (in bytes) */
541unsigned long pool_total_allocated()
542{
543 struct pool_head *entry;
544 unsigned long allocated = 0;
545
546 list_for_each_entry(entry, &pools, list)
547 allocated += entry->allocated * entry->size;
548 return allocated;
549}
550
551/* This function returns the total amount of memory used in pools (in bytes) */
552unsigned long pool_total_used()
553{
554 struct pool_head *entry;
555 unsigned long used = 0;
556
557 list_for_each_entry(entry, &pools, list)
558 used += entry->used * entry->size;
559 return used;
560}
561
William Lallemande7ed8852016-11-19 02:25:36 +0100562/* This function dumps memory usage information onto the stream interface's
563 * read buffer. It returns 0 as long as it does not complete, non-zero upon
564 * completion. No state is used.
565 */
566static int cli_io_handler_dump_pools(struct appctx *appctx)
567{
568 struct stream_interface *si = appctx->owner;
569
570 dump_pools_to_trash();
Willy Tarreau06d80a92017-10-19 14:32:15 +0200571 if (ci_putchk(si_ic(si), &trash) == -1) {
Willy Tarreaudb398432018-11-15 11:08:52 +0100572 si_rx_room_blk(si);
William Lallemande7ed8852016-11-19 02:25:36 +0100573 return 0;
574 }
575 return 1;
576}
577
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100578/* callback used to create early pool <name> of size <size> and store the
579 * resulting pointer into <ptr>. If the allocation fails, it quits with after
580 * emitting an error message.
581 */
582void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
583{
584 *ptr = create_pool(name, size, MEM_F_SHARED);
585 if (!*ptr) {
586 ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
587 name, size, strerror(errno));
588 exit(1);
589 }
590}
591
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100592/* Initializes all per-thread arrays on startup */
593static void init_pools()
594{
595 int thr, idx;
596
597 for (thr = 0; thr < MAX_THREADS; thr++) {
598 for (idx = 0; idx < MAX_BASE_POOLS; idx++) {
599 LIST_INIT(&pool_cache[thr][idx].list);
600 pool_cache[thr][idx].size = 0;
601 }
602 LIST_INIT(&pool_lru_head[thr]);
603 }
604}
605
606INITCALL0(STG_PREPARE, init_pools);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100607
William Lallemande7ed8852016-11-19 02:25:36 +0100608/* register cli keywords */
609static struct cli_kw_list cli_kws = {{ },{
Willy Tarreaue9ecec82016-12-16 18:55:23 +0100610 { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
William Lallemande7ed8852016-11-19 02:25:36 +0100611 {{},}
612}};
613
Willy Tarreau0108d902018-11-25 19:14:37 +0100614INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
William Lallemande7ed8852016-11-19 02:25:36 +0100615
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100616#ifdef DEBUG_FAIL_ALLOC
617#define MEM_FAIL_MAX_CHAR 32
618#define MEM_FAIL_MAX_STR 128
619static int mem_fail_cur_idx;
620static char mem_fail_str[MEM_FAIL_MAX_CHAR * MEM_FAIL_MAX_STR];
621__decl_hathreads(static HA_SPINLOCK_T mem_fail_lock);
622
623int mem_should_fail(const struct pool_head *pool)
624{
Olivier Houchard9c4f08a2019-02-01 16:28:04 +0100625 int ret = 0;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100626 int n;
627
628 if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
Willy Tarreau52bf8392020-03-08 00:42:37 +0100629 int randnb = ha_random() % 100;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100630
631 if (mem_fail_rate > randnb)
632 ret = 1;
633 else
634 ret = 0;
635 }
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100636 HA_SPIN_LOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100637 n = snprintf(&mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR],
638 MEM_FAIL_MAX_CHAR - 2,
639 "%d %.18s %d %d", mem_fail_cur_idx, pool->name, ret, tid);
640 while (n < MEM_FAIL_MAX_CHAR - 1)
641 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n++] = ' ';
642 if (mem_fail_cur_idx < MEM_FAIL_MAX_STR - 1)
643 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = '\n';
644 else
645 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = 0;
646 mem_fail_cur_idx++;
647 if (mem_fail_cur_idx == MEM_FAIL_MAX_STR)
648 mem_fail_cur_idx = 0;
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100649 HA_SPIN_UNLOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100650 return ret;
651
652}
653
654/* config parser for global "tune.fail-alloc" */
655static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
656 struct proxy *defpx, const char *file, int line,
657 char **err)
658{
659 if (too_many_args(1, args, err, NULL))
660 return -1;
661 mem_fail_rate = atoi(args[1]);
662 if (mem_fail_rate < 0 || mem_fail_rate > 100) {
663 memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
664 return -1;
665 }
666 return 0;
667}
668#endif
669
670/* register global config keywords */
671static struct cfg_kw_list mem_cfg_kws = {ILH, {
672#ifdef DEBUG_FAIL_ALLOC
673 { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
674#endif
675 { 0, NULL, NULL }
676}};
677
678INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);
679
Willy Tarreau50e608d2007-05-13 18:26:08 +0200680/*
681 * Local variables:
682 * c-indent-level: 8
683 * c-basic-offset: 8
684 * End:
685 */