blob: 41d9c449a227ef9f0365adf2718e313fb01f66ac [file] [log] [blame]
Willy Tarreau50e608d2007-05-13 18:26:08 +02001/*
2 * Memory management functions.
3 *
4 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
Willy Tarreau7107c8b2018-11-26 11:44:35 +010012#include <errno.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020013
Willy Tarreau3f0f82e2020-06-04 19:42:41 +020014#include <haproxy/applet-t.h>
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020015#include <haproxy/api.h>
Willy Tarreauf1d32c42020-06-04 21:07:02 +020016#include <haproxy/channel.h>
Willy Tarreau83487a82020-06-04 20:19:54 +020017#include <haproxy/cli.h>
Willy Tarreauf268ee82020-06-04 17:05:57 +020018#include <haproxy/global.h>
William Lallemande7ed8852016-11-19 02:25:36 +010019
Olivier Houcharddc21ff72019-01-29 15:20:16 +010020#include <common/cfgparse.h>
Willy Tarreau3f567e42020-05-28 15:29:19 +020021#include <haproxy/thread.h>
Willy Tarreaud0ef4392020-06-02 09:38:52 +020022#include <haproxy/pool.h>
Willy Tarreau853b2972020-05-27 18:01:47 +020023#include <haproxy/list.h>
Willy Tarreau2eec9b52020-06-04 19:58:55 +020024#include <haproxy/stats-t.h>
Willy Tarreau5e539c92020-06-04 20:45:39 +020025#include <haproxy/stream_interface.h>
Willy Tarreau48fbcae2020-06-03 18:09:46 +020026#include <haproxy/tools.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020027
Willy Tarreaua04ded52020-06-02 10:29:48 +020028#include <haproxy/activity-t.h>
Willy Tarreaua8b2ce02019-05-28 17:04:16 +020029
Willy Tarreau50e608d2007-05-13 18:26:08 +020030#include <proto/log.h>
31
Willy Tarreaued891fd2020-06-01 19:00:28 +020032#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau0a93b642018-10-16 07:58:39 +020033/* These are the most common pools, expected to be initialized first. These
34 * ones are allocated from an array, allowing to map them to an index.
35 */
36struct pool_head pool_base_start[MAX_BASE_POOLS] = { };
37unsigned int pool_base_count = 0;
38
Willy Tarreau7f0165e2018-11-26 17:09:46 +010039/* These ones are initialized per-thread on startup by init_pools() */
40struct pool_cache_head pool_cache[MAX_THREADS][MAX_BASE_POOLS];
Willy Tarreaufb117e62020-06-01 18:16:57 +020041struct list pool_lru_head[MAX_THREADS]; /* oldest objects */
Willy Tarreaue18db9e2018-10-16 10:28:54 +020042THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
43THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
Willy Tarreaued891fd2020-06-01 19:00:28 +020044#endif
Willy Tarreaue18db9e2018-10-16 10:28:54 +020045
Willy Tarreau50e608d2007-05-13 18:26:08 +020046static struct list pools = LIST_HEAD_INIT(pools);
Willy Tarreau067ac9f2015-10-08 14:12:13 +020047int mem_poison_byte = -1;
Willy Tarreau50e608d2007-05-13 18:26:08 +020048
Olivier Houcharddc21ff72019-01-29 15:20:16 +010049#ifdef DEBUG_FAIL_ALLOC
50static int mem_fail_rate = 0;
51static int mem_should_fail(const struct pool_head *);
52#endif
53
Willy Tarreau50e608d2007-05-13 18:26:08 +020054/* Try to find an existing shared pool with the same characteristics and
55 * returns it, otherwise creates this one. NULL is returned if no memory
Willy Tarreau581bf812016-01-25 02:19:13 +010056 * is available for a new creation. Two flags are supported :
57 * - MEM_F_SHARED to indicate that the pool may be shared with other users
58 * - MEM_F_EXACT to indicate that the size must not be rounded up
Willy Tarreau50e608d2007-05-13 18:26:08 +020059 */
60struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
61{
62 struct pool_head *pool;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020063 struct pool_head *entry;
64 struct list *start;
Willy Tarreau50e608d2007-05-13 18:26:08 +020065 unsigned int align;
Willy Tarreaued891fd2020-06-01 19:00:28 +020066 int idx __maybe_unused;
Willy Tarreau50e608d2007-05-13 18:26:08 +020067
Willy Tarreauac421112015-10-28 15:09:29 +010068 /* We need to store a (void *) at the end of the chunks. Since we know
Willy Tarreau50e608d2007-05-13 18:26:08 +020069 * that the malloc() function will never return such a small size,
70 * let's round the size up to something slightly bigger, in order to
71 * ease merging of entries. Note that the rounding is a power of two.
Willy Tarreauac421112015-10-28 15:09:29 +010072 * This extra (void *) is not accounted for in the size computation
73 * so that the visible parts outside are not affected.
Willy Tarreau30f931e2018-10-23 14:40:23 +020074 *
75 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
Willy Tarreau50e608d2007-05-13 18:26:08 +020076 */
77
Willy Tarreau581bf812016-01-25 02:19:13 +010078 if (!(flags & MEM_F_EXACT)) {
Willy Tarreau30f931e2018-10-23 14:40:23 +020079 align = 4 * sizeof(void *); // 2 lists = 4 pointers min
Willy Tarreau581bf812016-01-25 02:19:13 +010080 size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA;
81 }
Willy Tarreau50e608d2007-05-13 18:26:08 +020082
Christopher Fauletb349e482017-08-29 09:52:38 +020083 /* TODO: thread: we do not lock pool list for now because all pools are
84 * created during HAProxy startup (so before threads creation) */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020085 start = &pools;
Willy Tarreau50e608d2007-05-13 18:26:08 +020086 pool = NULL;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020087
88 list_for_each_entry(entry, &pools, list) {
89 if (entry->size == size) {
90 /* either we can share this place and we take it, or
91 * we look for a sharable one or for the next position
92 * before which we will insert a new one.
93 */
94 if (flags & entry->flags & MEM_F_SHARED) {
95 /* we can share this one */
Willy Tarreau50e608d2007-05-13 18:26:08 +020096 pool = entry;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +020097 DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
Willy Tarreau50e608d2007-05-13 18:26:08 +020098 break;
99 }
100 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200101 else if (entry->size > size) {
102 /* insert before this one */
103 start = &entry->list;
104 break;
105 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200106 }
107
108 if (!pool) {
Willy Tarreaued891fd2020-06-01 19:00:28 +0200109#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau0a93b642018-10-16 07:58:39 +0200110 if (pool_base_count < MAX_BASE_POOLS)
111 pool = &pool_base_start[pool_base_count++];
112
113 if (!pool) {
114 /* look for a freed entry */
115 for (entry = pool_base_start; entry != pool_base_start + MAX_BASE_POOLS; entry++) {
116 if (!entry->size) {
117 pool = entry;
118 break;
119 }
120 }
121 }
Willy Tarreaued891fd2020-06-01 19:00:28 +0200122#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200123
124 if (!pool)
125 pool = calloc(1, sizeof(*pool));
126
Willy Tarreau50e608d2007-05-13 18:26:08 +0200127 if (!pool)
128 return NULL;
129 if (name)
130 strlcpy2(pool->name, name, sizeof(pool->name));
131 pool->size = size;
132 pool->flags = flags;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200133 LIST_ADDQ(start, &pool->list);
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200134
Willy Tarreaued891fd2020-06-01 19:00:28 +0200135#ifdef CONFIG_HAP_LOCAL_POOLS
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200136 /* update per-thread pool cache if necessary */
137 idx = pool_get_index(pool);
138 if (idx >= 0) {
Willy Tarreaued891fd2020-06-01 19:00:28 +0200139 int thr;
140
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200141 for (thr = 0; thr < MAX_THREADS; thr++)
142 pool_cache[thr][idx].size = size;
143 }
Willy Tarreaued891fd2020-06-01 19:00:28 +0200144#endif
Olivier Houchard8af97eb2020-02-01 17:45:32 +0100145 HA_SPIN_INIT(&pool->lock);
Olivier Houchard8af97eb2020-02-01 17:45:32 +0100146 }
147 pool->users++;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200148 return pool;
149}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100150
Willy Tarreaued891fd2020-06-01 19:00:28 +0200151#ifdef CONFIG_HAP_LOCAL_POOLS
152/* Evicts some of the oldest objects from the local cache, pushing them to the
153 * global pool.
154 */
155void pool_evict_from_cache()
156{
157 struct pool_cache_item *item;
158 struct pool_cache_head *ph;
159
160 do {
161 item = LIST_PREV(&pool_lru_head[tid], struct pool_cache_item *, by_lru);
162 /* note: by definition we remove oldest objects so they also are the
163 * oldest in their own pools, thus their next is the pool's head.
164 */
165 ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
166 LIST_DEL(&item->by_pool);
167 LIST_DEL(&item->by_lru);
168 ph->count--;
169 pool_cache_count--;
170 pool_cache_bytes -= ph->size;
171 __pool_free(pool_base_start + (ph - pool_cache[tid]), item);
172 } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
173}
174#endif
175
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100176#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +0100177/* Allocates new entries for pool <pool> until there are at least <avail> + 1
178 * available, then returns the last one for immediate use, so that at least
179 * <avail> are left available in the pool upon return. NULL is returned if the
180 * last entry could not be allocated. It's important to note that at least one
181 * allocation is always performed even if there are enough entries in the pool.
182 * A call to the garbage collector is performed at most once in case malloc()
183 * returns an error, before returning NULL.
184 */
185void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
186{
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200187 void *ptr = NULL, **free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100188 int failed = 0;
189 int size = pool->size;
190 int limit = pool->limit;
191 int allocated = pool->allocated, allocated_orig = allocated;
192
193 /* stop point */
194 avail += pool->used;
195
196 while (1) {
197 if (limit && allocated >= limit) {
Olivier Houchard20872762019-03-08 18:53:35 +0100198 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200199 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100200 return NULL;
201 }
202
Willy Tarreau606135a2020-06-01 12:35:03 +0200203 swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->allocated, POOL_AVG_SAMPLES/4);
Willy Tarreaua1e4f8c2020-05-08 08:31:56 +0200204
Willy Tarreau24aa1ee2020-05-30 18:56:17 +0200205 ptr = pool_alloc_area(size + POOL_EXTRA);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100206 if (!ptr) {
Olivier Houchard20872762019-03-08 18:53:35 +0100207 _HA_ATOMIC_ADD(&pool->failed, 1);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200208 if (failed) {
209 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100210 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200211 }
Olivier Houchardcf975d42018-01-24 18:38:31 +0100212 failed++;
213 pool_gc(pool);
214 continue;
215 }
216 if (++allocated > avail)
217 break;
218
219 free_list = pool->free_list;
220 do {
221 *POOL_LINK(pool, ptr) = free_list;
222 __ha_barrier_store();
Olivier Houchard20872762019-03-08 18:53:35 +0100223 } while (_HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr) == 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100224 }
Olivier Houchard20872762019-03-08 18:53:35 +0100225 __ha_barrier_atomic_store();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100226
Olivier Houchard20872762019-03-08 18:53:35 +0100227 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
228 _HA_ATOMIC_ADD(&pool->used, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100229
230#ifdef DEBUG_MEMORY_POOLS
231 /* keep track of where the element was allocated from */
232 *POOL_LINK(pool, ptr) = (void *)pool;
233#endif
234 return ptr;
235}
236void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
237{
238 void *ptr;
239
240 ptr = __pool_refill_alloc(pool, avail);
241 return ptr;
242}
243/*
244 * This function frees whatever can be freed in pool <pool>.
245 */
246void pool_flush(struct pool_head *pool)
247{
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100248 struct pool_free_list cmp, new;
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200249 void **next, *temp;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100250 int removed = 0;
251
252 if (!pool)
253 return;
Willy Tarreau21072b92020-05-29 17:23:05 +0200254 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100255 do {
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100256 cmp.free_list = pool->free_list;
257 cmp.seq = pool->seq;
258 new.free_list = NULL;
259 new.seq = cmp.seq + 1;
260 } while (!_HA_ATOMIC_DWCAS(&pool->free_list, &cmp, &new));
Olivier Houchard20872762019-03-08 18:53:35 +0100261 __ha_barrier_atomic_store();
Willy Tarreau21072b92020-05-29 17:23:05 +0200262 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100263 next = cmp.free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100264 while (next) {
265 temp = next;
266 next = *POOL_LINK(pool, temp);
267 removed++;
Willy Tarreau24aa1ee2020-05-30 18:56:17 +0200268 pool_free_area(temp, pool->size + POOL_EXTRA);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100269 }
270 pool->free_list = next;
Olivier Houchard20872762019-03-08 18:53:35 +0100271 _HA_ATOMIC_SUB(&pool->allocated, removed);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100272 /* here, we should have pool->allocate == pool->used */
273}
274
275/*
276 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200277 * the minimum thresholds imposed by owners. It makes sure to be alone to
278 * run by using thread_isolate(). <pool_ctx> is unused.
Olivier Houchardcf975d42018-01-24 18:38:31 +0100279 */
280void pool_gc(struct pool_head *pool_ctx)
281{
Olivier Houchardcf975d42018-01-24 18:38:31 +0100282 struct pool_head *entry;
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200283 int isolated = thread_isolated();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100284
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200285 if (!isolated)
286 thread_isolate();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100287
288 list_for_each_entry(entry, &pools, list) {
289 while ((int)((volatile int)entry->allocated - (volatile int)entry->used) > (int)entry->minavail) {
290 struct pool_free_list cmp, new;
291
292 cmp.seq = entry->seq;
293 __ha_barrier_load();
294 cmp.free_list = entry->free_list;
295 __ha_barrier_load();
296 if (cmp.free_list == NULL)
297 break;
298 new.free_list = *POOL_LINK(entry, cmp.free_list);
299 new.seq = cmp.seq + 1;
Willy Tarreau6a38b322019-05-11 18:04:24 +0200300 if (HA_ATOMIC_DWCAS(&entry->free_list, &cmp, &new) == 0)
Olivier Houchardcf975d42018-01-24 18:38:31 +0100301 continue;
Willy Tarreau24aa1ee2020-05-30 18:56:17 +0200302 pool_free_area(cmp.free_list, entry->size + POOL_EXTRA);
Olivier Houchard20872762019-03-08 18:53:35 +0100303 _HA_ATOMIC_SUB(&entry->allocated, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100304 }
305 }
306
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200307 if (!isolated)
308 thread_release();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100309}
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200310
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100311#else /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200312
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100313/* Allocates new entries for pool <pool> until there are at least <avail> + 1
314 * available, then returns the last one for immediate use, so that at least
315 * <avail> are left available in the pool upon return. NULL is returned if the
316 * last entry could not be allocated. It's important to note that at least one
317 * allocation is always performed even if there are enough entries in the pool.
318 * A call to the garbage collector is performed at most once in case malloc()
319 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200320 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200321void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200322{
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100323 void *ptr = NULL;
324 int failed = 0;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200325
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100326#ifdef DEBUG_FAIL_ALLOC
327 if (mem_should_fail(pool))
328 return NULL;
329#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100330 /* stop point */
331 avail += pool->used;
332
333 while (1) {
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200334 if (pool->limit && pool->allocated >= pool->limit) {
335 activity[tid].pool_fail++;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200336 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200337 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100338
Willy Tarreau606135a2020-06-01 12:35:03 +0200339 swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->allocated, POOL_AVG_SAMPLES/4);
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200340 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreauf13322e2017-11-22 10:50:54 +0100341 ptr = pool_alloc_area(pool->size + POOL_EXTRA);
Willy Tarreau82867542019-07-04 11:48:16 +0200342#ifdef DEBUG_MEMORY_POOLS
343 /* keep track of where the element was allocated from. This
344 * is done out of the lock so that the system really allocates
345 * the data without harming other threads waiting on the lock.
346 */
347 if (ptr)
348 *POOL_LINK(pool, ptr) = (void *)pool;
349#endif
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200350 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100351 if (!ptr) {
Willy Tarreau58102cf2015-10-28 16:24:21 +0100352 pool->failed++;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200353 if (failed) {
354 activity[tid].pool_fail++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100355 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200356 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100357 failed++;
Willy Tarreaubafbe012017-11-24 17:34:44 +0100358 pool_gc(pool);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100359 continue;
360 }
361 if (++pool->allocated > avail)
362 break;
363
Willy Tarreauac421112015-10-28 15:09:29 +0100364 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100365 pool->free_list = ptr;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200366 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200367 pool->used++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100368 return ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200369}
Christopher Fauletb349e482017-08-29 09:52:38 +0200370void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
371{
372 void *ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200373
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100374 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200375 ptr = __pool_refill_alloc(pool, avail);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100376 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200377 return ptr;
378}
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200379/*
380 * This function frees whatever can be freed in pool <pool>.
381 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100382void pool_flush(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200383{
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200384 void *temp;
385
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200386 if (!pool)
387 return;
388
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200389 while (1) {
390 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
391 temp = pool->free_list;
392 if (!temp) {
393 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
394 break;
395 }
396 pool->free_list = *POOL_LINK(pool, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200397 pool->allocated--;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200398 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreauf13322e2017-11-22 10:50:54 +0100399 pool_free_area(temp, pool->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200400 }
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200401 /* here, we should have pool->allocated == pool->used */
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200402}
403
404/*
405 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200406 * the minimum thresholds imposed by owners. It makes sure to be alone to
407 * run by using thread_isolate(). <pool_ctx> is unused.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200408 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100409void pool_gc(struct pool_head *pool_ctx)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200410{
411 struct pool_head *entry;
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200412 int isolated = thread_isolated();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200413
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200414 if (!isolated)
415 thread_isolate();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200416
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200417 list_for_each_entry(entry, &pools, list) {
Olivier Houchard51d93392020-03-12 19:05:39 +0100418 void *temp;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200419 //qfprintf(stderr, "Flushing pool %s\n", entry->name);
Olivier Houchard51d93392020-03-12 19:05:39 +0100420 while (entry->free_list &&
Willy Tarreau57767b82014-12-22 21:40:55 +0100421 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
Olivier Houchard51d93392020-03-12 19:05:39 +0100422 temp = entry->free_list;
423 entry->free_list = *POOL_LINK(entry, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200424 entry->allocated--;
Willy Tarreauf13322e2017-11-22 10:50:54 +0100425 pool_free_area(temp, entry->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200426 }
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200427 }
Christopher Fauletb349e482017-08-29 09:52:38 +0200428
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200429 if (!isolated)
430 thread_release();
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200431}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100432#endif
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200433
434/*
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200435 * This function destroys a pool by freeing it completely, unless it's still
436 * in use. This should be called only under extreme circumstances. It always
437 * returns NULL if the resulting pool is empty, easing the clearing of the old
438 * pointer, otherwise it returns the pool.
439 * .
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200440 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100441void *pool_destroy(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200442{
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200443 if (pool) {
Willy Tarreaubafbe012017-11-24 17:34:44 +0100444 pool_flush(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200445 if (pool->used)
446 return pool;
447 pool->users--;
448 if (!pool->users) {
449 LIST_DEL(&pool->list);
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100450#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100451 HA_SPIN_DESTROY(&pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100452#endif
Willy Tarreaued891fd2020-06-01 19:00:28 +0200453
454#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau0a93b642018-10-16 07:58:39 +0200455 if ((pool - pool_base_start) < MAX_BASE_POOLS)
456 memset(pool, 0, sizeof(*pool));
457 else
Willy Tarreaued891fd2020-06-01 19:00:28 +0200458#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200459 free(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200460 }
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200461 }
462 return NULL;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200463}
464
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100465/* This destroys all pools on exit. It is *not* thread safe. */
466void pool_destroy_all()
467{
468 struct pool_head *entry, *back;
469
470 list_for_each_entry_safe(entry, back, &pools, list)
471 pool_destroy(entry);
472}
473
Willy Tarreau12833bb2014-01-28 16:49:56 +0100474/* This function dumps memory usage information into the trash buffer. */
475void dump_pools_to_trash()
Willy Tarreau50e608d2007-05-13 18:26:08 +0200476{
477 struct pool_head *entry;
478 unsigned long allocated, used;
479 int nbpools;
480
481 allocated = used = nbpools = 0;
Willy Tarreau12833bb2014-01-28 16:49:56 +0100482 chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200483 list_for_each_entry(entry, &pools, list) {
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100484#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100485 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100486#endif
Willy Tarreaua1e4f8c2020-05-08 08:31:56 +0200487 chunk_appendf(&trash, " - Pool %s (%u bytes) : %u allocated (%u bytes), %u used, needed_avg %u, %u failures, %u users, @%p=%02d%s\n",
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200488 entry->name, entry->size, entry->allocated,
Willy Tarreaua1e4f8c2020-05-08 08:31:56 +0200489 entry->size * entry->allocated, entry->used,
Willy Tarreau606135a2020-06-01 12:35:03 +0200490 swrate_avg(entry->needed_avg, POOL_AVG_SAMPLES), entry->failed,
Willy Tarreau0a93b642018-10-16 07:58:39 +0200491 entry->users, entry, (int)pool_get_index(entry),
492 (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200493
494 allocated += entry->allocated * entry->size;
495 used += entry->used * entry->size;
496 nbpools++;
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100497#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100498 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100499#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +0200500 }
Willy Tarreau12833bb2014-01-28 16:49:56 +0100501 chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n",
Willy Tarreau50e608d2007-05-13 18:26:08 +0200502 nbpools, allocated, used);
503}
504
Willy Tarreau12833bb2014-01-28 16:49:56 +0100505/* Dump statistics on pools usage. */
506void dump_pools(void)
507{
508 dump_pools_to_trash();
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200509 qfprintf(stderr, "%s", trash.area);
Willy Tarreau12833bb2014-01-28 16:49:56 +0100510}
511
Willy Tarreau58102cf2015-10-28 16:24:21 +0100512/* This function returns the total number of failed pool allocations */
513int pool_total_failures()
514{
515 struct pool_head *entry;
516 int failed = 0;
517
518 list_for_each_entry(entry, &pools, list)
519 failed += entry->failed;
520 return failed;
521}
522
523/* This function returns the total amount of memory allocated in pools (in bytes) */
524unsigned long pool_total_allocated()
525{
526 struct pool_head *entry;
527 unsigned long allocated = 0;
528
529 list_for_each_entry(entry, &pools, list)
530 allocated += entry->allocated * entry->size;
531 return allocated;
532}
533
534/* This function returns the total amount of memory used in pools (in bytes) */
535unsigned long pool_total_used()
536{
537 struct pool_head *entry;
538 unsigned long used = 0;
539
540 list_for_each_entry(entry, &pools, list)
541 used += entry->used * entry->size;
542 return used;
543}
544
William Lallemande7ed8852016-11-19 02:25:36 +0100545/* This function dumps memory usage information onto the stream interface's
546 * read buffer. It returns 0 as long as it does not complete, non-zero upon
547 * completion. No state is used.
548 */
549static int cli_io_handler_dump_pools(struct appctx *appctx)
550{
551 struct stream_interface *si = appctx->owner;
552
553 dump_pools_to_trash();
Willy Tarreau06d80a92017-10-19 14:32:15 +0200554 if (ci_putchk(si_ic(si), &trash) == -1) {
Willy Tarreaudb398432018-11-15 11:08:52 +0100555 si_rx_room_blk(si);
William Lallemande7ed8852016-11-19 02:25:36 +0100556 return 0;
557 }
558 return 1;
559}
560
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100561/* callback used to create early pool <name> of size <size> and store the
562 * resulting pointer into <ptr>. If the allocation fails, it quits with after
563 * emitting an error message.
564 */
565void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
566{
567 *ptr = create_pool(name, size, MEM_F_SHARED);
568 if (!*ptr) {
569 ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
570 name, size, strerror(errno));
571 exit(1);
572 }
573}
574
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100575/* Initializes all per-thread arrays on startup */
576static void init_pools()
577{
Willy Tarreaued891fd2020-06-01 19:00:28 +0200578#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100579 int thr, idx;
580
581 for (thr = 0; thr < MAX_THREADS; thr++) {
582 for (idx = 0; idx < MAX_BASE_POOLS; idx++) {
583 LIST_INIT(&pool_cache[thr][idx].list);
584 pool_cache[thr][idx].size = 0;
585 }
586 LIST_INIT(&pool_lru_head[thr]);
587 }
Willy Tarreaued891fd2020-06-01 19:00:28 +0200588#endif
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100589}
590
591INITCALL0(STG_PREPARE, init_pools);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100592
William Lallemande7ed8852016-11-19 02:25:36 +0100593/* register cli keywords */
594static struct cli_kw_list cli_kws = {{ },{
Willy Tarreaue9ecec82016-12-16 18:55:23 +0100595 { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
William Lallemande7ed8852016-11-19 02:25:36 +0100596 {{},}
597}};
598
Willy Tarreau0108d902018-11-25 19:14:37 +0100599INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
William Lallemande7ed8852016-11-19 02:25:36 +0100600
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100601#ifdef DEBUG_FAIL_ALLOC
602#define MEM_FAIL_MAX_CHAR 32
603#define MEM_FAIL_MAX_STR 128
604static int mem_fail_cur_idx;
605static char mem_fail_str[MEM_FAIL_MAX_CHAR * MEM_FAIL_MAX_STR];
Willy Tarreauaf613e82020-06-05 08:40:51 +0200606__decl_thread(static HA_SPINLOCK_T mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100607
608int mem_should_fail(const struct pool_head *pool)
609{
Olivier Houchard9c4f08a2019-02-01 16:28:04 +0100610 int ret = 0;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100611 int n;
612
613 if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
Willy Tarreau52bf8392020-03-08 00:42:37 +0100614 int randnb = ha_random() % 100;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100615
616 if (mem_fail_rate > randnb)
617 ret = 1;
618 else
619 ret = 0;
620 }
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100621 HA_SPIN_LOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100622 n = snprintf(&mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR],
623 MEM_FAIL_MAX_CHAR - 2,
624 "%d %.18s %d %d", mem_fail_cur_idx, pool->name, ret, tid);
625 while (n < MEM_FAIL_MAX_CHAR - 1)
626 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n++] = ' ';
627 if (mem_fail_cur_idx < MEM_FAIL_MAX_STR - 1)
628 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = '\n';
629 else
630 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = 0;
631 mem_fail_cur_idx++;
632 if (mem_fail_cur_idx == MEM_FAIL_MAX_STR)
633 mem_fail_cur_idx = 0;
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100634 HA_SPIN_UNLOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100635 return ret;
636
637}
638
639/* config parser for global "tune.fail-alloc" */
640static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
641 struct proxy *defpx, const char *file, int line,
642 char **err)
643{
644 if (too_many_args(1, args, err, NULL))
645 return -1;
646 mem_fail_rate = atoi(args[1]);
647 if (mem_fail_rate < 0 || mem_fail_rate > 100) {
648 memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
649 return -1;
650 }
651 return 0;
652}
653#endif
654
655/* register global config keywords */
656static struct cfg_kw_list mem_cfg_kws = {ILH, {
657#ifdef DEBUG_FAIL_ALLOC
658 { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
659#endif
660 { 0, NULL, NULL }
661}};
662
663INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);
664
Willy Tarreau50e608d2007-05-13 18:26:08 +0200665/*
666 * Local variables:
667 * c-indent-level: 8
668 * c-basic-offset: 8
669 * End:
670 */