blob: 5ea2df074c55723ab48db344f7274a73d2853df0 [file] [log] [blame]
Willy Tarreau50e608d2007-05-13 18:26:08 +02001/*
2 * Memory management functions.
3 *
4 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
Willy Tarreau7107c8b2018-11-26 11:44:35 +010012#include <errno.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020013
Willy Tarreaub2551052020-06-09 09:07:15 +020014#include <haproxy/activity-t.h>
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020015#include <haproxy/api.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020016#include <haproxy/applet-t.h>
Willy Tarreau6be78492020-06-05 00:00:29 +020017#include <haproxy/cfgparse.h>
Willy Tarreauf1d32c42020-06-04 21:07:02 +020018#include <haproxy/channel.h>
Willy Tarreau83487a82020-06-04 20:19:54 +020019#include <haproxy/cli.h>
Willy Tarreau36979d92020-06-05 17:27:29 +020020#include <haproxy/errors.h>
Willy Tarreauf268ee82020-06-04 17:05:57 +020021#include <haproxy/global.h>
Willy Tarreau853b2972020-05-27 18:01:47 +020022#include <haproxy/list.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020023#include <haproxy/pool.h>
Willy Tarreau2eec9b52020-06-04 19:58:55 +020024#include <haproxy/stats-t.h>
Willy Tarreau5e539c92020-06-04 20:45:39 +020025#include <haproxy/stream_interface.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020026#include <haproxy/thread.h>
Willy Tarreau48fbcae2020-06-03 18:09:46 +020027#include <haproxy/tools.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020028
Willy Tarreau50e608d2007-05-13 18:26:08 +020029
Willy Tarreaued891fd2020-06-01 19:00:28 +020030#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau0a93b642018-10-16 07:58:39 +020031/* These are the most common pools, expected to be initialized first. These
32 * ones are allocated from an array, allowing to map them to an index.
33 */
34struct pool_head pool_base_start[MAX_BASE_POOLS] = { };
35unsigned int pool_base_count = 0;
36
Willy Tarreau7f0165e2018-11-26 17:09:46 +010037/* These ones are initialized per-thread on startup by init_pools() */
38struct pool_cache_head pool_cache[MAX_THREADS][MAX_BASE_POOLS];
Willy Tarreaue18db9e2018-10-16 10:28:54 +020039THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
40THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
Willy Tarreaued891fd2020-06-01 19:00:28 +020041#endif
Willy Tarreaue18db9e2018-10-16 10:28:54 +020042
Willy Tarreau50e608d2007-05-13 18:26:08 +020043static struct list pools = LIST_HEAD_INIT(pools);
Willy Tarreau067ac9f2015-10-08 14:12:13 +020044int mem_poison_byte = -1;
Willy Tarreau50e608d2007-05-13 18:26:08 +020045
Olivier Houcharddc21ff72019-01-29 15:20:16 +010046#ifdef DEBUG_FAIL_ALLOC
47static int mem_fail_rate = 0;
48static int mem_should_fail(const struct pool_head *);
49#endif
50
Willy Tarreau50e608d2007-05-13 18:26:08 +020051/* Try to find an existing shared pool with the same characteristics and
52 * returns it, otherwise creates this one. NULL is returned if no memory
Willy Tarreau581bf812016-01-25 02:19:13 +010053 * is available for a new creation. Two flags are supported :
54 * - MEM_F_SHARED to indicate that the pool may be shared with other users
55 * - MEM_F_EXACT to indicate that the size must not be rounded up
Willy Tarreau50e608d2007-05-13 18:26:08 +020056 */
57struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
58{
59 struct pool_head *pool;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020060 struct pool_head *entry;
61 struct list *start;
Willy Tarreau50e608d2007-05-13 18:26:08 +020062 unsigned int align;
Willy Tarreaued891fd2020-06-01 19:00:28 +020063 int idx __maybe_unused;
Willy Tarreau50e608d2007-05-13 18:26:08 +020064
Willy Tarreauac421112015-10-28 15:09:29 +010065 /* We need to store a (void *) at the end of the chunks. Since we know
Willy Tarreau50e608d2007-05-13 18:26:08 +020066 * that the malloc() function will never return such a small size,
67 * let's round the size up to something slightly bigger, in order to
68 * ease merging of entries. Note that the rounding is a power of two.
Willy Tarreauac421112015-10-28 15:09:29 +010069 * This extra (void *) is not accounted for in the size computation
70 * so that the visible parts outside are not affected.
Willy Tarreau30f931e2018-10-23 14:40:23 +020071 *
72 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
Willy Tarreau50e608d2007-05-13 18:26:08 +020073 */
74
Willy Tarreau581bf812016-01-25 02:19:13 +010075 if (!(flags & MEM_F_EXACT)) {
Willy Tarreau30f931e2018-10-23 14:40:23 +020076 align = 4 * sizeof(void *); // 2 lists = 4 pointers min
Willy Tarreau581bf812016-01-25 02:19:13 +010077 size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA;
78 }
Willy Tarreau50e608d2007-05-13 18:26:08 +020079
Christopher Fauletb349e482017-08-29 09:52:38 +020080 /* TODO: thread: we do not lock pool list for now because all pools are
81 * created during HAProxy startup (so before threads creation) */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020082 start = &pools;
Willy Tarreau50e608d2007-05-13 18:26:08 +020083 pool = NULL;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020084
85 list_for_each_entry(entry, &pools, list) {
86 if (entry->size == size) {
87 /* either we can share this place and we take it, or
Ilya Shipitsin47d17182020-06-21 21:42:57 +050088 * we look for a shareable one or for the next position
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020089 * before which we will insert a new one.
90 */
Willy Tarreau0b155162021-05-05 07:29:01 +020091 if ((flags & entry->flags & MEM_F_SHARED)
92#ifdef DEBUG_DONT_SHARE_POOLS
93 && strcmp(name, entry->name) == 0
94#endif
95 ) {
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020096 /* we can share this one */
Willy Tarreau50e608d2007-05-13 18:26:08 +020097 pool = entry;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +020098 DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
Willy Tarreau50e608d2007-05-13 18:26:08 +020099 break;
100 }
101 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200102 else if (entry->size > size) {
103 /* insert before this one */
104 start = &entry->list;
105 break;
106 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200107 }
108
109 if (!pool) {
Willy Tarreaued891fd2020-06-01 19:00:28 +0200110#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau0a93b642018-10-16 07:58:39 +0200111 if (pool_base_count < MAX_BASE_POOLS)
112 pool = &pool_base_start[pool_base_count++];
113
114 if (!pool) {
115 /* look for a freed entry */
116 for (entry = pool_base_start; entry != pool_base_start + MAX_BASE_POOLS; entry++) {
117 if (!entry->size) {
118 pool = entry;
119 break;
120 }
121 }
122 }
Willy Tarreaued891fd2020-06-01 19:00:28 +0200123#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200124
125 if (!pool)
126 pool = calloc(1, sizeof(*pool));
127
Willy Tarreau50e608d2007-05-13 18:26:08 +0200128 if (!pool)
129 return NULL;
130 if (name)
131 strlcpy2(pool->name, name, sizeof(pool->name));
132 pool->size = size;
133 pool->flags = flags;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200134 LIST_ADDQ(start, &pool->list);
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200135
Willy Tarreaued891fd2020-06-01 19:00:28 +0200136#ifdef CONFIG_HAP_LOCAL_POOLS
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200137 /* update per-thread pool cache if necessary */
138 idx = pool_get_index(pool);
139 if (idx >= 0) {
Willy Tarreaued891fd2020-06-01 19:00:28 +0200140 int thr;
141
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200142 for (thr = 0; thr < MAX_THREADS; thr++)
143 pool_cache[thr][idx].size = size;
144 }
Willy Tarreaued891fd2020-06-01 19:00:28 +0200145#endif
Olivier Houchard8af97eb2020-02-01 17:45:32 +0100146 HA_SPIN_INIT(&pool->lock);
Olivier Houchard8af97eb2020-02-01 17:45:32 +0100147 }
148 pool->users++;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200149 return pool;
150}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100151
Willy Tarreaued891fd2020-06-01 19:00:28 +0200152#ifdef CONFIG_HAP_LOCAL_POOLS
153/* Evicts some of the oldest objects from the local cache, pushing them to the
154 * global pool.
155 */
156void pool_evict_from_cache()
157{
158 struct pool_cache_item *item;
159 struct pool_cache_head *ph;
160
161 do {
Willy Tarreau20dc3cd2020-06-28 00:54:27 +0200162 item = LIST_PREV(&ti->pool_lru_head, struct pool_cache_item *, by_lru);
Willy Tarreaued891fd2020-06-01 19:00:28 +0200163 /* note: by definition we remove oldest objects so they also are the
164 * oldest in their own pools, thus their next is the pool's head.
165 */
166 ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
167 LIST_DEL(&item->by_pool);
168 LIST_DEL(&item->by_lru);
169 ph->count--;
170 pool_cache_count--;
171 pool_cache_bytes -= ph->size;
172 __pool_free(pool_base_start + (ph - pool_cache[tid]), item);
173 } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
174}
175#endif
176
Willy Tarreauc0457dc2021-03-02 20:05:09 +0100177#if defined(CONFIG_HAP_NO_GLOBAL_POOLS)
178
179/* simply fall back on the default OS' allocator */
180
181void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
182{
183 int allocated = pool->allocated;
184 int limit = pool->limit;
185 void *ptr = NULL;
186
187 if (limit && allocated >= limit) {
188 _HA_ATOMIC_ADD(&pool->allocated, 1);
189 activity[tid].pool_fail++;
190 return NULL;
191 }
192
193 ptr = pool_alloc_area(pool->size + POOL_EXTRA);
194 if (!ptr) {
195 _HA_ATOMIC_ADD(&pool->failed, 1);
196 activity[tid].pool_fail++;
197 return NULL;
198 }
199
200 _HA_ATOMIC_ADD(&pool->allocated, 1);
201 _HA_ATOMIC_ADD(&pool->used, 1);
202
203#ifdef DEBUG_MEMORY_POOLS
204 /* keep track of where the element was allocated from */
205 *POOL_LINK(pool, ptr) = (void *)pool;
206#endif
207 return ptr;
208}
209
210/* legacy stuff */
211void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
212{
213 void *ptr;
214
215 ptr = __pool_refill_alloc(pool, avail);
216 return ptr;
217}
218
219/* legacy stuff */
220void pool_flush(struct pool_head *pool)
221{
222}
223
224/* This function might ask the malloc library to trim its buffers. */
225void pool_gc(struct pool_head *pool_ctx)
226{
227#if defined(HA_HAVE_MALLOC_TRIM)
228 malloc_trim(0);
229#endif
230}
231
232#elif defined(CONFIG_HAP_LOCKLESS_POOLS)
233
Olivier Houchardcf975d42018-01-24 18:38:31 +0100234/* Allocates new entries for pool <pool> until there are at least <avail> + 1
235 * available, then returns the last one for immediate use, so that at least
236 * <avail> are left available in the pool upon return. NULL is returned if the
237 * last entry could not be allocated. It's important to note that at least one
238 * allocation is always performed even if there are enough entries in the pool.
239 * A call to the garbage collector is performed at most once in case malloc()
240 * returns an error, before returning NULL.
241 */
242void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
243{
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200244 void *ptr = NULL, **free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100245 int failed = 0;
246 int size = pool->size;
247 int limit = pool->limit;
248 int allocated = pool->allocated, allocated_orig = allocated;
249
250 /* stop point */
251 avail += pool->used;
252
253 while (1) {
254 if (limit && allocated >= limit) {
Olivier Houchard20872762019-03-08 18:53:35 +0100255 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200256 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100257 return NULL;
258 }
259
Willy Tarreau606135a2020-06-01 12:35:03 +0200260 swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->allocated, POOL_AVG_SAMPLES/4);
Willy Tarreaua1e4f8c2020-05-08 08:31:56 +0200261
Willy Tarreau24aa1ee2020-05-30 18:56:17 +0200262 ptr = pool_alloc_area(size + POOL_EXTRA);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100263 if (!ptr) {
Olivier Houchard20872762019-03-08 18:53:35 +0100264 _HA_ATOMIC_ADD(&pool->failed, 1);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200265 if (failed) {
266 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100267 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200268 }
Olivier Houchardcf975d42018-01-24 18:38:31 +0100269 failed++;
270 pool_gc(pool);
271 continue;
272 }
273 if (++allocated > avail)
274 break;
275
276 free_list = pool->free_list;
277 do {
278 *POOL_LINK(pool, ptr) = free_list;
279 __ha_barrier_store();
Olivier Houchard20872762019-03-08 18:53:35 +0100280 } while (_HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr) == 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100281 }
Olivier Houchard20872762019-03-08 18:53:35 +0100282 __ha_barrier_atomic_store();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100283
Olivier Houchard20872762019-03-08 18:53:35 +0100284 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
285 _HA_ATOMIC_ADD(&pool->used, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100286
287#ifdef DEBUG_MEMORY_POOLS
288 /* keep track of where the element was allocated from */
289 *POOL_LINK(pool, ptr) = (void *)pool;
290#endif
291 return ptr;
292}
293void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
294{
295 void *ptr;
296
297 ptr = __pool_refill_alloc(pool, avail);
298 return ptr;
299}
300/*
301 * This function frees whatever can be freed in pool <pool>.
302 */
303void pool_flush(struct pool_head *pool)
304{
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100305 struct pool_free_list cmp, new;
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200306 void **next, *temp;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100307 int removed = 0;
308
309 if (!pool)
310 return;
Willy Tarreau21072b92020-05-29 17:23:05 +0200311 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100312 do {
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100313 cmp.free_list = pool->free_list;
314 cmp.seq = pool->seq;
315 new.free_list = NULL;
316 new.seq = cmp.seq + 1;
317 } while (!_HA_ATOMIC_DWCAS(&pool->free_list, &cmp, &new));
Olivier Houchard20872762019-03-08 18:53:35 +0100318 __ha_barrier_atomic_store();
Willy Tarreau21072b92020-05-29 17:23:05 +0200319 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100320 next = cmp.free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100321 while (next) {
322 temp = next;
323 next = *POOL_LINK(pool, temp);
324 removed++;
Willy Tarreau24aa1ee2020-05-30 18:56:17 +0200325 pool_free_area(temp, pool->size + POOL_EXTRA);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100326 }
Olivier Houchard20872762019-03-08 18:53:35 +0100327 _HA_ATOMIC_SUB(&pool->allocated, removed);
Willy Tarreaud5c27172021-06-10 06:54:22 +0200328 /* here, we should have pool->allocated == pool->used */
Olivier Houchardcf975d42018-01-24 18:38:31 +0100329}
330
331/*
332 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200333 * the minimum thresholds imposed by owners. It makes sure to be alone to
334 * run by using thread_isolate(). <pool_ctx> is unused.
Olivier Houchardcf975d42018-01-24 18:38:31 +0100335 */
336void pool_gc(struct pool_head *pool_ctx)
337{
Olivier Houchardcf975d42018-01-24 18:38:31 +0100338 struct pool_head *entry;
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200339 int isolated = thread_isolated();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100340
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200341 if (!isolated)
342 thread_isolate();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100343
344 list_for_each_entry(entry, &pools, list) {
345 while ((int)((volatile int)entry->allocated - (volatile int)entry->used) > (int)entry->minavail) {
346 struct pool_free_list cmp, new;
347
348 cmp.seq = entry->seq;
349 __ha_barrier_load();
350 cmp.free_list = entry->free_list;
351 __ha_barrier_load();
352 if (cmp.free_list == NULL)
353 break;
354 new.free_list = *POOL_LINK(entry, cmp.free_list);
355 new.seq = cmp.seq + 1;
Willy Tarreau6a38b322019-05-11 18:04:24 +0200356 if (HA_ATOMIC_DWCAS(&entry->free_list, &cmp, &new) == 0)
Olivier Houchardcf975d42018-01-24 18:38:31 +0100357 continue;
Willy Tarreau24aa1ee2020-05-30 18:56:17 +0200358 pool_free_area(cmp.free_list, entry->size + POOL_EXTRA);
Olivier Houchard20872762019-03-08 18:53:35 +0100359 _HA_ATOMIC_SUB(&entry->allocated, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100360 }
361 }
362
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200363 if (!isolated)
364 thread_release();
Willy Tarreau3652d602020-11-03 15:53:34 +0100365
366#if defined(HA_HAVE_MALLOC_TRIM)
367 malloc_trim(0);
368#endif
Olivier Houchardcf975d42018-01-24 18:38:31 +0100369}
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200370
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100371#else /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200372
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100373/* Allocates new entries for pool <pool> until there are at least <avail> + 1
374 * available, then returns the last one for immediate use, so that at least
375 * <avail> are left available in the pool upon return. NULL is returned if the
376 * last entry could not be allocated. It's important to note that at least one
377 * allocation is always performed even if there are enough entries in the pool.
378 * A call to the garbage collector is performed at most once in case malloc()
379 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200380 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200381void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200382{
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100383 void *ptr = NULL;
384 int failed = 0;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200385
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100386#ifdef DEBUG_FAIL_ALLOC
387 if (mem_should_fail(pool))
388 return NULL;
389#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100390 /* stop point */
391 avail += pool->used;
392
393 while (1) {
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200394 if (pool->limit && pool->allocated >= pool->limit) {
395 activity[tid].pool_fail++;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200396 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200397 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100398
Willy Tarreau606135a2020-06-01 12:35:03 +0200399 swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->allocated, POOL_AVG_SAMPLES/4);
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200400 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreauf13322e2017-11-22 10:50:54 +0100401 ptr = pool_alloc_area(pool->size + POOL_EXTRA);
Willy Tarreau82867542019-07-04 11:48:16 +0200402#ifdef DEBUG_MEMORY_POOLS
403 /* keep track of where the element was allocated from. This
404 * is done out of the lock so that the system really allocates
405 * the data without harming other threads waiting on the lock.
406 */
407 if (ptr)
408 *POOL_LINK(pool, ptr) = (void *)pool;
409#endif
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200410 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100411 if (!ptr) {
Willy Tarreau58102cf2015-10-28 16:24:21 +0100412 pool->failed++;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200413 if (failed) {
414 activity[tid].pool_fail++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100415 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200416 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100417 failed++;
Willy Tarreaubafbe012017-11-24 17:34:44 +0100418 pool_gc(pool);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100419 continue;
420 }
421 if (++pool->allocated > avail)
422 break;
423
Willy Tarreauac421112015-10-28 15:09:29 +0100424 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100425 pool->free_list = ptr;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200426 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200427 pool->used++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100428 return ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200429}
Christopher Fauletb349e482017-08-29 09:52:38 +0200430void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
431{
432 void *ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200433
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100434 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200435 ptr = __pool_refill_alloc(pool, avail);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100436 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200437 return ptr;
438}
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200439/*
440 * This function frees whatever can be freed in pool <pool>.
441 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100442void pool_flush(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200443{
Willy Tarreau027eb8b2021-06-10 07:13:04 +0200444 void *temp, **next;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200445
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200446 if (!pool)
447 return;
448
Willy Tarreau027eb8b2021-06-10 07:13:04 +0200449 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
450 next = pool->free_list;
451 while (next) {
452 temp = next;
453 next = *POOL_LINK(pool, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200454 pool->allocated--;
Willy Tarreau027eb8b2021-06-10 07:13:04 +0200455 }
456
457 next = pool->free_list;
458 pool->free_list = NULL;
459 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
460
461 while (next) {
462 temp = next;
463 next = *POOL_LINK(pool, temp);
Willy Tarreauf13322e2017-11-22 10:50:54 +0100464 pool_free_area(temp, pool->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200465 }
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200466 /* here, we should have pool->allocated == pool->used */
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200467}
468
469/*
470 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200471 * the minimum thresholds imposed by owners. It makes sure to be alone to
472 * run by using thread_isolate(). <pool_ctx> is unused.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200473 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100474void pool_gc(struct pool_head *pool_ctx)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200475{
476 struct pool_head *entry;
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200477 int isolated = thread_isolated();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200478
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200479 if (!isolated)
480 thread_isolate();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200481
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200482 list_for_each_entry(entry, &pools, list) {
Olivier Houchard51d93392020-03-12 19:05:39 +0100483 void *temp;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200484 //qfprintf(stderr, "Flushing pool %s\n", entry->name);
Olivier Houchard51d93392020-03-12 19:05:39 +0100485 while (entry->free_list &&
Willy Tarreau57767b82014-12-22 21:40:55 +0100486 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
Olivier Houchard51d93392020-03-12 19:05:39 +0100487 temp = entry->free_list;
488 entry->free_list = *POOL_LINK(entry, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200489 entry->allocated--;
Willy Tarreauf13322e2017-11-22 10:50:54 +0100490 pool_free_area(temp, entry->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200491 }
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200492 }
Christopher Fauletb349e482017-08-29 09:52:38 +0200493
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200494 if (!isolated)
495 thread_release();
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200496}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100497#endif
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200498
499/*
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200500 * This function destroys a pool by freeing it completely, unless it's still
501 * in use. This should be called only under extreme circumstances. It always
502 * returns NULL if the resulting pool is empty, easing the clearing of the old
503 * pointer, otherwise it returns the pool.
504 * .
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200505 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100506void *pool_destroy(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200507{
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200508 if (pool) {
Willy Tarreaubafbe012017-11-24 17:34:44 +0100509 pool_flush(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200510 if (pool->used)
511 return pool;
512 pool->users--;
513 if (!pool->users) {
514 LIST_DEL(&pool->list);
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100515#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100516 HA_SPIN_DESTROY(&pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100517#endif
Willy Tarreaued891fd2020-06-01 19:00:28 +0200518
519#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau0a93b642018-10-16 07:58:39 +0200520 if ((pool - pool_base_start) < MAX_BASE_POOLS)
521 memset(pool, 0, sizeof(*pool));
522 else
Willy Tarreaued891fd2020-06-01 19:00:28 +0200523#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200524 free(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200525 }
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200526 }
527 return NULL;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200528}
529
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100530/* This destroys all pools on exit. It is *not* thread safe. */
531void pool_destroy_all()
532{
533 struct pool_head *entry, *back;
534
535 list_for_each_entry_safe(entry, back, &pools, list)
536 pool_destroy(entry);
537}
538
Willy Tarreau12833bb2014-01-28 16:49:56 +0100539/* This function dumps memory usage information into the trash buffer. */
540void dump_pools_to_trash()
Willy Tarreau50e608d2007-05-13 18:26:08 +0200541{
542 struct pool_head *entry;
543 unsigned long allocated, used;
544 int nbpools;
545
546 allocated = used = nbpools = 0;
Willy Tarreau12833bb2014-01-28 16:49:56 +0100547 chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200548 list_for_each_entry(entry, &pools, list) {
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100549#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100550 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100551#endif
Willy Tarreaua1e4f8c2020-05-08 08:31:56 +0200552 chunk_appendf(&trash, " - Pool %s (%u bytes) : %u allocated (%u bytes), %u used, needed_avg %u, %u failures, %u users, @%p=%02d%s\n",
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200553 entry->name, entry->size, entry->allocated,
Willy Tarreaua1e4f8c2020-05-08 08:31:56 +0200554 entry->size * entry->allocated, entry->used,
Willy Tarreau606135a2020-06-01 12:35:03 +0200555 swrate_avg(entry->needed_avg, POOL_AVG_SAMPLES), entry->failed,
Willy Tarreau0a93b642018-10-16 07:58:39 +0200556 entry->users, entry, (int)pool_get_index(entry),
557 (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200558
559 allocated += entry->allocated * entry->size;
560 used += entry->used * entry->size;
561 nbpools++;
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100562#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100563 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100564#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +0200565 }
Willy Tarreau12833bb2014-01-28 16:49:56 +0100566 chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n",
Willy Tarreau50e608d2007-05-13 18:26:08 +0200567 nbpools, allocated, used);
568}
569
Willy Tarreau12833bb2014-01-28 16:49:56 +0100570/* Dump statistics on pools usage. */
571void dump_pools(void)
572{
573 dump_pools_to_trash();
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200574 qfprintf(stderr, "%s", trash.area);
Willy Tarreau12833bb2014-01-28 16:49:56 +0100575}
576
Willy Tarreau58102cf2015-10-28 16:24:21 +0100577/* This function returns the total number of failed pool allocations */
578int pool_total_failures()
579{
580 struct pool_head *entry;
581 int failed = 0;
582
583 list_for_each_entry(entry, &pools, list)
584 failed += entry->failed;
585 return failed;
586}
587
588/* This function returns the total amount of memory allocated in pools (in bytes) */
589unsigned long pool_total_allocated()
590{
591 struct pool_head *entry;
592 unsigned long allocated = 0;
593
594 list_for_each_entry(entry, &pools, list)
595 allocated += entry->allocated * entry->size;
596 return allocated;
597}
598
599/* This function returns the total amount of memory used in pools (in bytes) */
600unsigned long pool_total_used()
601{
602 struct pool_head *entry;
603 unsigned long used = 0;
604
605 list_for_each_entry(entry, &pools, list)
606 used += entry->used * entry->size;
607 return used;
608}
609
William Lallemande7ed8852016-11-19 02:25:36 +0100610/* This function dumps memory usage information onto the stream interface's
611 * read buffer. It returns 0 as long as it does not complete, non-zero upon
612 * completion. No state is used.
613 */
614static int cli_io_handler_dump_pools(struct appctx *appctx)
615{
616 struct stream_interface *si = appctx->owner;
617
618 dump_pools_to_trash();
Willy Tarreau06d80a92017-10-19 14:32:15 +0200619 if (ci_putchk(si_ic(si), &trash) == -1) {
Willy Tarreaudb398432018-11-15 11:08:52 +0100620 si_rx_room_blk(si);
William Lallemande7ed8852016-11-19 02:25:36 +0100621 return 0;
622 }
623 return 1;
624}
625
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100626/* callback used to create early pool <name> of size <size> and store the
627 * resulting pointer into <ptr>. If the allocation fails, it quits with after
628 * emitting an error message.
629 */
630void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
631{
632 *ptr = create_pool(name, size, MEM_F_SHARED);
633 if (!*ptr) {
634 ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
635 name, size, strerror(errno));
636 exit(1);
637 }
638}
639
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100640/* Initializes all per-thread arrays on startup */
641static void init_pools()
642{
Willy Tarreaued891fd2020-06-01 19:00:28 +0200643#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100644 int thr, idx;
645
646 for (thr = 0; thr < MAX_THREADS; thr++) {
647 for (idx = 0; idx < MAX_BASE_POOLS; idx++) {
648 LIST_INIT(&pool_cache[thr][idx].list);
649 pool_cache[thr][idx].size = 0;
650 }
Willy Tarreau20dc3cd2020-06-28 00:54:27 +0200651 LIST_INIT(&ha_thread_info[thr].pool_lru_head);
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100652 }
Willy Tarreaued891fd2020-06-01 19:00:28 +0200653#endif
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100654}
655
656INITCALL0(STG_PREPARE, init_pools);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100657
William Lallemande7ed8852016-11-19 02:25:36 +0100658/* register cli keywords */
659static struct cli_kw_list cli_kws = {{ },{
Willy Tarreaue9ecec82016-12-16 18:55:23 +0100660 { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
William Lallemande7ed8852016-11-19 02:25:36 +0100661 {{},}
662}};
663
Willy Tarreau0108d902018-11-25 19:14:37 +0100664INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
William Lallemande7ed8852016-11-19 02:25:36 +0100665
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100666#ifdef DEBUG_FAIL_ALLOC
667#define MEM_FAIL_MAX_CHAR 32
668#define MEM_FAIL_MAX_STR 128
669static int mem_fail_cur_idx;
670static char mem_fail_str[MEM_FAIL_MAX_CHAR * MEM_FAIL_MAX_STR];
Willy Tarreauaf613e82020-06-05 08:40:51 +0200671__decl_thread(static HA_SPINLOCK_T mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100672
673int mem_should_fail(const struct pool_head *pool)
674{
Olivier Houchard9c4f08a2019-02-01 16:28:04 +0100675 int ret = 0;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100676 int n;
677
678 if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
Willy Tarreau52bf8392020-03-08 00:42:37 +0100679 int randnb = ha_random() % 100;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100680
681 if (mem_fail_rate > randnb)
682 ret = 1;
683 else
684 ret = 0;
685 }
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100686 HA_SPIN_LOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100687 n = snprintf(&mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR],
688 MEM_FAIL_MAX_CHAR - 2,
689 "%d %.18s %d %d", mem_fail_cur_idx, pool->name, ret, tid);
690 while (n < MEM_FAIL_MAX_CHAR - 1)
691 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n++] = ' ';
692 if (mem_fail_cur_idx < MEM_FAIL_MAX_STR - 1)
693 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = '\n';
694 else
695 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = 0;
696 mem_fail_cur_idx++;
697 if (mem_fail_cur_idx == MEM_FAIL_MAX_STR)
698 mem_fail_cur_idx = 0;
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100699 HA_SPIN_UNLOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100700 return ret;
701
702}
703
704/* config parser for global "tune.fail-alloc" */
705static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
706 struct proxy *defpx, const char *file, int line,
707 char **err)
708{
709 if (too_many_args(1, args, err, NULL))
710 return -1;
711 mem_fail_rate = atoi(args[1]);
712 if (mem_fail_rate < 0 || mem_fail_rate > 100) {
713 memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
714 return -1;
715 }
716 return 0;
717}
718#endif
719
720/* register global config keywords */
721static struct cfg_kw_list mem_cfg_kws = {ILH, {
722#ifdef DEBUG_FAIL_ALLOC
723 { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
724#endif
725 { 0, NULL, NULL }
726}};
727
728INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);
729
Willy Tarreau50e608d2007-05-13 18:26:08 +0200730/*
731 * Local variables:
732 * c-indent-level: 8
733 * c-basic-offset: 8
734 * End:
735 */