blob: b74ffd2b5d790c4f562c4dec76e2277a6ac11f4d [file] [log] [blame]
Willy Tarreau50e608d2007-05-13 18:26:08 +02001/*
2 * Memory management functions.
3 *
4 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
Willy Tarreau7107c8b2018-11-26 11:44:35 +010012#include <errno.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020013
Willy Tarreaub2551052020-06-09 09:07:15 +020014#include <haproxy/activity-t.h>
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020015#include <haproxy/api.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020016#include <haproxy/applet-t.h>
Willy Tarreau6be78492020-06-05 00:00:29 +020017#include <haproxy/cfgparse.h>
Willy Tarreauf1d32c42020-06-04 21:07:02 +020018#include <haproxy/channel.h>
Willy Tarreau83487a82020-06-04 20:19:54 +020019#include <haproxy/cli.h>
Willy Tarreau36979d92020-06-05 17:27:29 +020020#include <haproxy/errors.h>
Willy Tarreauf268ee82020-06-04 17:05:57 +020021#include <haproxy/global.h>
Willy Tarreau853b2972020-05-27 18:01:47 +020022#include <haproxy/list.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020023#include <haproxy/pool.h>
Willy Tarreau2eec9b52020-06-04 19:58:55 +020024#include <haproxy/stats-t.h>
Willy Tarreau5e539c92020-06-04 20:45:39 +020025#include <haproxy/stream_interface.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020026#include <haproxy/thread.h>
Willy Tarreau48fbcae2020-06-03 18:09:46 +020027#include <haproxy/tools.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020028
Willy Tarreau50e608d2007-05-13 18:26:08 +020029
Willy Tarreau2d6f6282021-04-15 16:24:00 +020030#ifdef CONFIG_HAP_POOLS
Willy Tarreau7f0165e2018-11-26 17:09:46 +010031/* These ones are initialized per-thread on startup by init_pools() */
Willy Tarreaue18db9e2018-10-16 10:28:54 +020032THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
33THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
Willy Tarreaued891fd2020-06-01 19:00:28 +020034#endif
Willy Tarreaue18db9e2018-10-16 10:28:54 +020035
Willy Tarreau50e608d2007-05-13 18:26:08 +020036static struct list pools = LIST_HEAD_INIT(pools);
Willy Tarreau067ac9f2015-10-08 14:12:13 +020037int mem_poison_byte = -1;
Willy Tarreau50e608d2007-05-13 18:26:08 +020038
Olivier Houcharddc21ff72019-01-29 15:20:16 +010039#ifdef DEBUG_FAIL_ALLOC
40static int mem_fail_rate = 0;
Olivier Houcharddc21ff72019-01-29 15:20:16 +010041#endif
42
Willy Tarreau33298de2021-09-15 10:38:21 +020043#if defined(HA_HAVE_MALLOC_TRIM)
Willy Tarreau562bf202021-09-15 10:05:48 +020044static int using_libc_allocator = 0;
Willy Tarreau85dd5212022-03-08 10:41:40 +010045static int disable_trim __read_mostly = 0;
Willy Tarreau562bf202021-09-15 10:05:48 +020046
Willy Tarreau619a5112021-12-23 09:26:30 +010047/* ask the allocator to trim memory pools.
48 * This must run under thread isolation so that competing threads trying to
49 * allocate or release memory do not prevent the allocator from completing
50 * its job. We just have to be careful as callers might already be isolated
51 * themselves.
52 */
Willy Tarreau33298de2021-09-15 10:38:21 +020053static void trim_all_pools(void)
54{
Willy Tarreau619a5112021-12-23 09:26:30 +010055 int isolated = thread_isolated();
56
Willy Tarreau85dd5212022-03-08 10:41:40 +010057 if (disable_trim)
58 return;
59
Willy Tarreau619a5112021-12-23 09:26:30 +010060 if (!isolated)
61 thread_isolate();
62
Willy Tarreau33298de2021-09-15 10:38:21 +020063 if (using_libc_allocator)
64 malloc_trim(0);
Willy Tarreau619a5112021-12-23 09:26:30 +010065
66 if (!isolated)
67 thread_release();
Willy Tarreau33298de2021-09-15 10:38:21 +020068}
69
Willy Tarreau562bf202021-09-15 10:05:48 +020070/* check if we're using the same allocator as the one that provides
71 * malloc_trim() and mallinfo(). The principle is that on glibc, both
72 * malloc_trim() and mallinfo() are provided, and using mallinfo() we
73 * can check if malloc() is performed through glibc or any other one
74 * the executable was linked against (e.g. jemalloc).
75 */
76static void detect_allocator(void)
77{
Willy Tarreau3bf75692021-09-16 09:18:21 +020078#ifdef HA_HAVE_MALLINFO2
79 struct mallinfo2 mi1, mi2;
80#else
Willy Tarreau562bf202021-09-15 10:05:48 +020081 struct mallinfo mi1, mi2;
Willy Tarreau3bf75692021-09-16 09:18:21 +020082#endif
Willy Tarreau562bf202021-09-15 10:05:48 +020083 void *ptr;
84
Willy Tarreau3bf75692021-09-16 09:18:21 +020085#ifdef HA_HAVE_MALLINFO2
86 mi1 = mallinfo2();
87#else
Willy Tarreau562bf202021-09-15 10:05:48 +020088 mi1 = mallinfo();
Willy Tarreau3bf75692021-09-16 09:18:21 +020089#endif
Willy Tarreau562bf202021-09-15 10:05:48 +020090 ptr = DISGUISE(malloc(1));
Willy Tarreau3bf75692021-09-16 09:18:21 +020091#ifdef HA_HAVE_MALLINFO2
92 mi2 = mallinfo2();
93#else
Willy Tarreau562bf202021-09-15 10:05:48 +020094 mi2 = mallinfo();
Willy Tarreau3bf75692021-09-16 09:18:21 +020095#endif
Willy Tarreau562bf202021-09-15 10:05:48 +020096 free(DISGUISE(ptr));
97
98 using_libc_allocator = !!memcmp(&mi1, &mi2, sizeof(mi1));
99}
Willy Tarreau33298de2021-09-15 10:38:21 +0200100#else
101
102static void trim_all_pools(void)
103{
104}
Willy Tarreau562bf202021-09-15 10:05:48 +0200105
106static void detect_allocator(void)
107{
108}
Willy Tarreau33298de2021-09-15 10:38:21 +0200109#endif
110
Willy Tarreau50e608d2007-05-13 18:26:08 +0200111/* Try to find an existing shared pool with the same characteristics and
112 * returns it, otherwise creates this one. NULL is returned if no memory
Willy Tarreau581bf812016-01-25 02:19:13 +0100113 * is available for a new creation. Two flags are supported :
114 * - MEM_F_SHARED to indicate that the pool may be shared with other users
115 * - MEM_F_EXACT to indicate that the size must not be rounded up
Willy Tarreau50e608d2007-05-13 18:26:08 +0200116 */
117struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
118{
119 struct pool_head *pool;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200120 struct pool_head *entry;
121 struct list *start;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200122 unsigned int align;
Willy Tarreau9f3129e2021-04-17 00:31:38 +0200123 int thr __maybe_unused;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200124
Willy Tarreauac421112015-10-28 15:09:29 +0100125 /* We need to store a (void *) at the end of the chunks. Since we know
Willy Tarreau50e608d2007-05-13 18:26:08 +0200126 * that the malloc() function will never return such a small size,
127 * let's round the size up to something slightly bigger, in order to
128 * ease merging of entries. Note that the rounding is a power of two.
Willy Tarreauac421112015-10-28 15:09:29 +0100129 * This extra (void *) is not accounted for in the size computation
130 * so that the visible parts outside are not affected.
Willy Tarreau30f931e2018-10-23 14:40:23 +0200131 *
132 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200133 */
134
Willy Tarreau581bf812016-01-25 02:19:13 +0100135 if (!(flags & MEM_F_EXACT)) {
Willy Tarreau30f931e2018-10-23 14:40:23 +0200136 align = 4 * sizeof(void *); // 2 lists = 4 pointers min
Willy Tarreau581bf812016-01-25 02:19:13 +0100137 size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA;
138 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200139
Christopher Fauletb349e482017-08-29 09:52:38 +0200140 /* TODO: thread: we do not lock pool list for now because all pools are
141 * created during HAProxy startup (so before threads creation) */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200142 start = &pools;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200143 pool = NULL;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200144
145 list_for_each_entry(entry, &pools, list) {
146 if (entry->size == size) {
147 /* either we can share this place and we take it, or
Ilya Shipitsin47d17182020-06-21 21:42:57 +0500148 * we look for a shareable one or for the next position
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200149 * before which we will insert a new one.
150 */
Willy Tarreau1ab6c0b2021-05-05 07:29:01 +0200151 if ((flags & entry->flags & MEM_F_SHARED)
152#ifdef DEBUG_DONT_SHARE_POOLS
153 && strcmp(name, entry->name) == 0
154#endif
155 ) {
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200156 /* we can share this one */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200157 pool = entry;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200158 DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
Willy Tarreau50e608d2007-05-13 18:26:08 +0200159 break;
160 }
161 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200162 else if (entry->size > size) {
163 /* insert before this one */
164 start = &entry->list;
165 break;
166 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200167 }
168
169 if (!pool) {
Willy Tarreauceee2382022-03-02 17:59:04 +0100170 void *pool_addr;
Willy Tarreau0a93b642018-10-16 07:58:39 +0200171
Willy Tarreauceee2382022-03-02 17:59:04 +0100172 pool_addr = calloc(1, sizeof(*pool) + __alignof__(*pool));
173 if (!pool_addr)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200174 return NULL;
Willy Tarreauceee2382022-03-02 17:59:04 +0100175
176 /* always provide an aligned pool */
177 pool = (struct pool_head*)((((size_t)pool_addr) + __alignof__(*pool)) & -(size_t)__alignof__(*pool));
178 pool->base_addr = pool_addr; // keep it, it's the address to free later
179
Willy Tarreau50e608d2007-05-13 18:26:08 +0200180 if (name)
181 strlcpy2(pool->name, name, sizeof(pool->name));
182 pool->size = size;
183 pool->flags = flags;
Willy Tarreau2b718102021-04-21 07:32:39 +0200184 LIST_APPEND(start, &pool->list);
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200185
Willy Tarreau2d6f6282021-04-15 16:24:00 +0200186#ifdef CONFIG_HAP_POOLS
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200187 /* update per-thread pool cache if necessary */
Willy Tarreau9f3129e2021-04-17 00:31:38 +0200188 for (thr = 0; thr < MAX_THREADS; thr++) {
189 LIST_INIT(&pool->cache[thr].list);
Willy Tarreau674843a2022-02-09 16:33:22 +0100190 pool->cache[thr].tid = thr;
191 pool->cache[thr].pool = pool;
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200192 }
Willy Tarreaued891fd2020-06-01 19:00:28 +0200193#endif
Olivier Houchard8af97eb2020-02-01 17:45:32 +0100194 HA_SPIN_INIT(&pool->lock);
Olivier Houchard8af97eb2020-02-01 17:45:32 +0100195 }
196 pool->users++;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200197 return pool;
198}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100199
Willy Tarreau13843642021-04-17 16:57:25 +0200200/* Tries to allocate an object for the pool <pool> using the system's allocator
201 * and directly returns it. The pool's allocated counter is checked and updated,
202 * but no other checks are performed. The pool's lock is not used and is not a
203 * problem either.
204 */
205void *pool_get_from_os(struct pool_head *pool)
206{
207 if (!pool->limit || pool->allocated < pool->limit) {
208 void *ptr = pool_alloc_area(pool->size + POOL_EXTRA);
209 if (ptr) {
210 _HA_ATOMIC_INC(&pool->allocated);
211 return ptr;
212 }
213 _HA_ATOMIC_INC(&pool->failed);
214 }
215 activity[tid].pool_fail++;
216 return NULL;
217
218}
219
Willy Tarreau45e4e282021-04-17 17:48:40 +0200220/* Releases a pool item back to the operating system and atomically updates
221 * the allocation counter.
222 */
223void pool_put_to_os(struct pool_head *pool, void *ptr)
224{
Willy Tarreau11e7af32021-06-10 17:20:19 +0200225#ifdef DEBUG_UAF
226 /* This object will be released for real in order to detect a use after
227 * free. We also force a write to the area to ensure we crash on double
228 * free or free of a const area.
229 */
230 *(uint32_t *)ptr = 0xDEADADD4;
231#endif /* DEBUG_UAF */
232
Willy Tarreau45e4e282021-04-17 17:48:40 +0200233 pool_free_area(ptr, pool->size + POOL_EXTRA);
234 _HA_ATOMIC_DEC(&pool->allocated);
235}
236
Willy Tarreau8fe726f2021-04-15 18:20:12 +0200237/* Tries to allocate an object for the pool <pool> using the system's allocator
238 * and directly returns it. The pool's counters are updated but the object is
239 * never cached, so this is usable with and without local or shared caches.
240 * This may be called with or without the pool lock held, so it must not use
241 * the pool's lock.
242 */
243void *pool_alloc_nocache(struct pool_head *pool)
Willy Tarreau0bae0752021-03-02 20:05:09 +0100244{
Willy Tarreau0bae0752021-03-02 20:05:09 +0100245 void *ptr = NULL;
246
Willy Tarreau13843642021-04-17 16:57:25 +0200247 ptr = pool_get_from_os(pool);
248 if (!ptr)
Willy Tarreau0bae0752021-03-02 20:05:09 +0100249 return NULL;
Willy Tarreau0bae0752021-03-02 20:05:09 +0100250
Willy Tarreau13843642021-04-17 16:57:25 +0200251 swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used, POOL_AVG_SAMPLES/4);
Willy Tarreau4781b152021-04-06 13:53:36 +0200252 _HA_ATOMIC_INC(&pool->used);
Willy Tarreau0bae0752021-03-02 20:05:09 +0100253
254#ifdef DEBUG_MEMORY_POOLS
255 /* keep track of where the element was allocated from */
256 *POOL_LINK(pool, ptr) = (void *)pool;
257#endif
258 return ptr;
259}
260
Willy Tarreau45e4e282021-04-17 17:48:40 +0200261/* Release a pool item back to the OS and keeps the pool's counters up to date.
262 * This is always defined even when pools are not enabled (their usage stats
263 * are maintained).
264 */
265void pool_free_nocache(struct pool_head *pool, void *ptr)
266{
267 _HA_ATOMIC_DEC(&pool->used);
268 swrate_add(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used);
269 pool_put_to_os(pool, ptr);
270}
271
Willy Tarreaub8498e92021-04-18 10:23:02 +0200272
273#ifdef CONFIG_HAP_POOLS
274
Willy Tarreau87212032021-04-19 08:14:03 +0200275/* Evicts some of the oldest objects from one local cache, until its number of
276 * objects is no more than 16+1/8 of the total number of locally cached objects
277 * or the total size of the local cache is no more than 75% of its maximum (i.e.
278 * we don't want a single cache to use all the cache for itself). For this, the
279 * list is scanned in reverse.
280 */
Willy Tarreau9043da72022-02-09 16:19:24 +0100281void pool_evict_from_local_cache(struct pool_head *pool, int full)
Willy Tarreau87212032021-04-19 08:14:03 +0200282{
283 struct pool_cache_head *ph = &pool->cache[tid];
284 struct pool_cache_item *item;
Willy Tarreau87212032021-04-19 08:14:03 +0200285
Willy Tarreau9043da72022-02-09 16:19:24 +0100286 while ((ph->count && full) ||
287 (ph->count >= 16 + pool_cache_count / 8 &&
288 pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4)) {
Willy Tarreau87212032021-04-19 08:14:03 +0200289 item = LIST_NEXT(&ph->list, typeof(item), by_pool);
Willy Tarreau9a3cd532022-02-09 16:23:55 +0100290 BUG_ON(&item->by_pool == &ph->list);
Willy Tarreau87212032021-04-19 08:14:03 +0200291 ph->count--;
292 pool_cache_bytes -= pool->size;
293 pool_cache_count--;
Willy Tarreauc2418432022-01-21 19:00:25 +0100294 pool_check_pattern(ph, item, pool->size);
Willy Tarreau2b718102021-04-21 07:32:39 +0200295 LIST_DELETE(&item->by_pool);
296 LIST_DELETE(&item->by_lru);
Willy Tarreau87212032021-04-19 08:14:03 +0200297 pool_put_to_shared_cache(pool, item);
298 }
299}
300
Willy Tarreaub8498e92021-04-18 10:23:02 +0200301/* Evicts some of the oldest objects from the local cache, pushing them to the
302 * global pool.
303 */
304void pool_evict_from_local_caches()
305{
306 struct pool_cache_item *item;
307 struct pool_cache_head *ph;
308 struct pool_head *pool;
309
310 do {
311 item = LIST_PREV(&ti->pool_lru_head, struct pool_cache_item *, by_lru);
Willy Tarreau9a3cd532022-02-09 16:23:55 +0100312 BUG_ON(&item->by_lru == &ti->pool_lru_head);
Willy Tarreaub8498e92021-04-18 10:23:02 +0200313 /* note: by definition we remove oldest objects so they also are the
314 * oldest in their own pools, thus their next is the pool's head.
315 */
316 ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
Willy Tarreau674843a2022-02-09 16:33:22 +0100317 BUG_ON(ph->tid != tid);
318
Willy Tarreaub8498e92021-04-18 10:23:02 +0200319 pool = container_of(ph - tid, struct pool_head, cache);
Willy Tarreau674843a2022-02-09 16:33:22 +0100320 BUG_ON(pool != ph->pool);
321
Willy Tarreauc2418432022-01-21 19:00:25 +0100322 pool_check_pattern(ph, item, pool->size);
Willy Tarreau2b718102021-04-21 07:32:39 +0200323 LIST_DELETE(&item->by_pool);
324 LIST_DELETE(&item->by_lru);
Willy Tarreaub8498e92021-04-18 10:23:02 +0200325 ph->count--;
326 pool_cache_count--;
327 pool_cache_bytes -= pool->size;
328 pool_put_to_shared_cache(pool, item);
329 } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
330}
Willy Tarreau0bae0752021-03-02 20:05:09 +0100331
Willy Tarreaub2a853d2021-04-19 11:49:26 +0200332/* Frees an object to the local cache, possibly pushing oldest objects to the
333 * shared cache, which itself may decide to release some of them to the OS.
334 * While it is unspecified what the object becomes past this point, it is
335 * guaranteed to be released from the users' perpective.
336 */
337void pool_put_to_cache(struct pool_head *pool, void *ptr)
338{
339 struct pool_cache_item *item = (struct pool_cache_item *)ptr;
340 struct pool_cache_head *ph = &pool->cache[tid];
341
Willy Tarreau2b718102021-04-21 07:32:39 +0200342 LIST_INSERT(&ph->list, &item->by_pool);
343 LIST_INSERT(&ti->pool_lru_head, &item->by_lru);
Willy Tarreaub2a853d2021-04-19 11:49:26 +0200344 ph->count++;
Willy Tarreauc2418432022-01-21 19:00:25 +0100345 pool_fill_pattern(ph, item, pool->size);
Willy Tarreaub2a853d2021-04-19 11:49:26 +0200346 pool_cache_count++;
347 pool_cache_bytes += pool->size;
348
349 if (unlikely(pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4)) {
350 if (ph->count >= 16 + pool_cache_count / 8)
Willy Tarreau9043da72022-02-09 16:19:24 +0100351 pool_evict_from_local_cache(pool, 0);
Willy Tarreaub2a853d2021-04-19 11:49:26 +0200352 if (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE)
353 pool_evict_from_local_caches();
354 }
355}
356
Willy Tarreaueb3cc292021-04-15 18:13:13 +0200357#if defined(CONFIG_HAP_NO_GLOBAL_POOLS)
358
Willy Tarreau0bae0752021-03-02 20:05:09 +0100359/* legacy stuff */
360void pool_flush(struct pool_head *pool)
361{
362}
363
364/* This function might ask the malloc library to trim its buffers. */
365void pool_gc(struct pool_head *pool_ctx)
366{
Willy Tarreau33298de2021-09-15 10:38:21 +0200367 trim_all_pools();
Willy Tarreau0bae0752021-03-02 20:05:09 +0100368}
369
Willy Tarreaua206cf92021-06-10 10:21:35 +0200370#else /* CONFIG_HAP_NO_GLOBAL_POOLS */
371
372#if defined(CONFIG_HAP_LOCKLESS_POOLS)
Willy Tarreau0bae0752021-03-02 20:05:09 +0100373
Olivier Houchardcf975d42018-01-24 18:38:31 +0100374/*
375 * This function frees whatever can be freed in pool <pool>.
376 */
377void pool_flush(struct pool_head *pool)
378{
Willy Tarreaud0cc3762021-06-09 18:59:58 +0200379 void *next, *temp;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100380
381 if (!pool)
382 return;
Willy Tarreaud0cc3762021-06-09 18:59:58 +0200383
384 /* The loop below atomically detaches the head of the free list and
385 * replaces it with a NULL. Then the list can be released.
386 */
387 next = pool->free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100388 do {
Willy Tarreaud0cc3762021-06-09 18:59:58 +0200389 while (unlikely(next == POOL_BUSY)) {
390 __ha_cpu_relax();
391 next = _HA_ATOMIC_LOAD(&pool->free_list);
392 }
393 if (next == NULL)
394 return;
395 } while (unlikely((next = _HA_ATOMIC_XCHG(&pool->free_list, POOL_BUSY)) == POOL_BUSY));
396 _HA_ATOMIC_STORE(&pool->free_list, NULL);
Olivier Houchard20872762019-03-08 18:53:35 +0100397 __ha_barrier_atomic_store();
Willy Tarreaud0cc3762021-06-09 18:59:58 +0200398
Olivier Houchardcf975d42018-01-24 18:38:31 +0100399 while (next) {
400 temp = next;
401 next = *POOL_LINK(pool, temp);
Willy Tarreau45e4e282021-04-17 17:48:40 +0200402 pool_put_to_os(pool, temp);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100403 }
Willy Tarreaucaf65552021-06-10 06:54:22 +0200404 /* here, we should have pool->allocated == pool->used */
Olivier Houchardcf975d42018-01-24 18:38:31 +0100405}
406
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100407#else /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200408
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200409/*
410 * This function frees whatever can be freed in pool <pool>.
411 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100412void pool_flush(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200413{
Willy Tarreauaf8120a2021-06-10 07:13:04 +0200414 void *temp, **next;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200415
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200416 if (!pool)
417 return;
418
Willy Tarreauaf8120a2021-06-10 07:13:04 +0200419 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
420 next = pool->free_list;
421 pool->free_list = NULL;
422 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
423
424 while (next) {
425 temp = next;
426 next = *POOL_LINK(pool, temp);
Willy Tarreau45e4e282021-04-17 17:48:40 +0200427 pool_put_to_os(pool, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200428 }
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200429 /* here, we should have pool->allocated == pool->used */
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200430}
431
Willy Tarreaua206cf92021-06-10 10:21:35 +0200432#endif /* CONFIG_HAP_LOCKLESS_POOLS */
433
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200434/*
435 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200436 * the minimum thresholds imposed by owners. It makes sure to be alone to
437 * run by using thread_isolate(). <pool_ctx> is unused.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200438 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100439void pool_gc(struct pool_head *pool_ctx)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200440{
441 struct pool_head *entry;
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200442 int isolated = thread_isolated();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200443
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200444 if (!isolated)
445 thread_isolate();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200446
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200447 list_for_each_entry(entry, &pools, list) {
Olivier Houchard51d93392020-03-12 19:05:39 +0100448 void *temp;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200449 //qfprintf(stderr, "Flushing pool %s\n", entry->name);
Olivier Houchard51d93392020-03-12 19:05:39 +0100450 while (entry->free_list &&
Willy Tarreau57767b82014-12-22 21:40:55 +0100451 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
Olivier Houchard51d93392020-03-12 19:05:39 +0100452 temp = entry->free_list;
453 entry->free_list = *POOL_LINK(entry, temp);
Willy Tarreau45e4e282021-04-17 17:48:40 +0200454 pool_put_to_os(entry, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200455 }
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200456 }
Christopher Fauletb349e482017-08-29 09:52:38 +0200457
Willy Tarreau33298de2021-09-15 10:38:21 +0200458 trim_all_pools();
Willy Tarreau46b515c2021-06-10 08:40:16 +0200459
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200460 if (!isolated)
461 thread_release();
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200462}
Willy Tarreaua206cf92021-06-10 10:21:35 +0200463#endif /* CONFIG_HAP_NO_GLOBAL_POOLS */
Willy Tarreaub8498e92021-04-18 10:23:02 +0200464
465#else /* CONFIG_HAP_POOLS */
466
467/* legacy stuff */
468void pool_flush(struct pool_head *pool)
469{
470}
471
472/* This function might ask the malloc library to trim its buffers. */
473void pool_gc(struct pool_head *pool_ctx)
474{
Willy Tarreau33298de2021-09-15 10:38:21 +0200475 trim_all_pools();
Willy Tarreaub8498e92021-04-18 10:23:02 +0200476}
477
478#endif /* CONFIG_HAP_POOLS */
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200479
480/*
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200481 * This function destroys a pool by freeing it completely, unless it's still
482 * in use. This should be called only under extreme circumstances. It always
483 * returns NULL if the resulting pool is empty, easing the clearing of the old
484 * pointer, otherwise it returns the pool.
485 * .
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200486 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100487void *pool_destroy(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200488{
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200489 if (pool) {
Willy Tarreau9043da72022-02-09 16:19:24 +0100490#ifdef CONFIG_HAP_POOLS
491 pool_evict_from_local_cache(pool, 1);
492#endif
Willy Tarreaubafbe012017-11-24 17:34:44 +0100493 pool_flush(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200494 if (pool->used)
495 return pool;
496 pool->users--;
497 if (!pool->users) {
Willy Tarreau2b718102021-04-21 07:32:39 +0200498 LIST_DELETE(&pool->list);
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100499#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100500 HA_SPIN_DESTROY(&pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100501#endif
Willy Tarreau9f3129e2021-04-17 00:31:38 +0200502 /* note that if used == 0, the cache is empty */
Willy Tarreau337899f2022-03-03 18:31:54 +0100503 free(pool->base_addr);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200504 }
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200505 }
506 return NULL;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200507}
508
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100509/* This destroys all pools on exit. It is *not* thread safe. */
510void pool_destroy_all()
511{
512 struct pool_head *entry, *back;
513
Willy Tarreau857357e2022-04-27 11:33:13 +0200514 list_for_each_entry_safe(entry, back, &pools, list) {
515 /* there's only one occurrence of each pool in the list,
516 * and we're existing instead of looping on the whole
517 * list just to decrement users, force it to 1 here.
518 */
519 entry->users = 1;
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100520 pool_destroy(entry);
Willy Tarreau857357e2022-04-27 11:33:13 +0200521 }
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100522}
523
Willy Tarreau12833bb2014-01-28 16:49:56 +0100524/* This function dumps memory usage information into the trash buffer. */
525void dump_pools_to_trash()
Willy Tarreau50e608d2007-05-13 18:26:08 +0200526{
527 struct pool_head *entry;
528 unsigned long allocated, used;
529 int nbpools;
530
531 allocated = used = nbpools = 0;
Willy Tarreau12833bb2014-01-28 16:49:56 +0100532 chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200533 list_for_each_entry(entry, &pools, list) {
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100534#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100535 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100536#endif
Willy Tarreau9f3129e2021-04-17 00:31:38 +0200537 chunk_appendf(&trash, " - Pool %s (%u bytes) : %u allocated (%u bytes), %u used, needed_avg %u, %u failures, %u users, @%p%s\n",
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200538 entry->name, entry->size, entry->allocated,
Willy Tarreaua1e4f8c2020-05-08 08:31:56 +0200539 entry->size * entry->allocated, entry->used,
Willy Tarreau606135a2020-06-01 12:35:03 +0200540 swrate_avg(entry->needed_avg, POOL_AVG_SAMPLES), entry->failed,
Willy Tarreau9f3129e2021-04-17 00:31:38 +0200541 entry->users, entry,
Willy Tarreau0a93b642018-10-16 07:58:39 +0200542 (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200543
544 allocated += entry->allocated * entry->size;
545 used += entry->used * entry->size;
546 nbpools++;
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100547#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100548 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100549#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +0200550 }
Willy Tarreau12833bb2014-01-28 16:49:56 +0100551 chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n",
Willy Tarreau50e608d2007-05-13 18:26:08 +0200552 nbpools, allocated, used);
553}
554
Willy Tarreau12833bb2014-01-28 16:49:56 +0100555/* Dump statistics on pools usage. */
556void dump_pools(void)
557{
558 dump_pools_to_trash();
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200559 qfprintf(stderr, "%s", trash.area);
Willy Tarreau12833bb2014-01-28 16:49:56 +0100560}
561
Willy Tarreau58102cf2015-10-28 16:24:21 +0100562/* This function returns the total number of failed pool allocations */
563int pool_total_failures()
564{
565 struct pool_head *entry;
566 int failed = 0;
567
568 list_for_each_entry(entry, &pools, list)
569 failed += entry->failed;
570 return failed;
571}
572
573/* This function returns the total amount of memory allocated in pools (in bytes) */
574unsigned long pool_total_allocated()
575{
576 struct pool_head *entry;
577 unsigned long allocated = 0;
578
579 list_for_each_entry(entry, &pools, list)
580 allocated += entry->allocated * entry->size;
581 return allocated;
582}
583
584/* This function returns the total amount of memory used in pools (in bytes) */
585unsigned long pool_total_used()
586{
587 struct pool_head *entry;
588 unsigned long used = 0;
589
590 list_for_each_entry(entry, &pools, list)
591 used += entry->used * entry->size;
592 return used;
593}
594
William Lallemande7ed8852016-11-19 02:25:36 +0100595/* This function dumps memory usage information onto the stream interface's
596 * read buffer. It returns 0 as long as it does not complete, non-zero upon
597 * completion. No state is used.
598 */
599static int cli_io_handler_dump_pools(struct appctx *appctx)
600{
601 struct stream_interface *si = appctx->owner;
602
603 dump_pools_to_trash();
Willy Tarreau06d80a92017-10-19 14:32:15 +0200604 if (ci_putchk(si_ic(si), &trash) == -1) {
Willy Tarreaudb398432018-11-15 11:08:52 +0100605 si_rx_room_blk(si);
William Lallemande7ed8852016-11-19 02:25:36 +0100606 return 0;
607 }
608 return 1;
609}
610
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100611/* callback used to create early pool <name> of size <size> and store the
612 * resulting pointer into <ptr>. If the allocation fails, it quits with after
613 * emitting an error message.
614 */
615void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
616{
617 *ptr = create_pool(name, size, MEM_F_SHARED);
618 if (!*ptr) {
619 ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
620 name, size, strerror(errno));
621 exit(1);
622 }
623}
624
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100625/* Initializes all per-thread arrays on startup */
626static void init_pools()
627{
Willy Tarreau2d6f6282021-04-15 16:24:00 +0200628#ifdef CONFIG_HAP_POOLS
Willy Tarreau9f3129e2021-04-17 00:31:38 +0200629 int thr;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100630
631 for (thr = 0; thr < MAX_THREADS; thr++) {
Willy Tarreau20dc3cd2020-06-28 00:54:27 +0200632 LIST_INIT(&ha_thread_info[thr].pool_lru_head);
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100633 }
Willy Tarreaued891fd2020-06-01 19:00:28 +0200634#endif
Willy Tarreau562bf202021-09-15 10:05:48 +0200635 detect_allocator();
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100636}
637
638INITCALL0(STG_PREPARE, init_pools);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100639
William Lallemande7ed8852016-11-19 02:25:36 +0100640/* register cli keywords */
641static struct cli_kw_list cli_kws = {{ },{
Willy Tarreaub205bfd2021-05-07 11:38:37 +0200642 { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
William Lallemande7ed8852016-11-19 02:25:36 +0100643 {{},}
644}};
645
Willy Tarreau0108d902018-11-25 19:14:37 +0100646INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
William Lallemande7ed8852016-11-19 02:25:36 +0100647
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100648#ifdef DEBUG_FAIL_ALLOC
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100649
650int mem_should_fail(const struct pool_head *pool)
651{
Olivier Houchard9c4f08a2019-02-01 16:28:04 +0100652 int ret = 0;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100653
654 if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
Willy Tarreau20f88ab2021-04-17 15:50:28 +0200655 if (mem_fail_rate > statistical_prng_range(100))
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100656 ret = 1;
657 else
658 ret = 0;
659 }
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100660 return ret;
661
662}
663
664/* config parser for global "tune.fail-alloc" */
665static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
Amaury Denoyelle3b1c9a32021-03-22 11:21:36 +0100666 const struct proxy *defpx, const char *file, int line,
667 char **err)
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100668{
669 if (too_many_args(1, args, err, NULL))
670 return -1;
671 mem_fail_rate = atoi(args[1]);
672 if (mem_fail_rate < 0 || mem_fail_rate > 100) {
673 memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
674 return -1;
675 }
676 return 0;
677}
678#endif
679
Willy Tarreaua891e572022-03-08 15:10:05 +0100680#if defined(HA_HAVE_MALLOC_TRIM)
Willy Tarreau85dd5212022-03-08 10:41:40 +0100681/* config parser for global "no-memory-trimming" */
682static int mem_parse_global_no_mem_trim(char **args, int section_type, struct proxy *curpx,
683 const struct proxy *defpx, const char *file, int line,
684 char **err)
685{
686 if (too_many_args(0, args, err, NULL))
687 return -1;
688 disable_trim = 1;
689 return 0;
690}
Willy Tarreaua891e572022-03-08 15:10:05 +0100691#endif
Willy Tarreau85dd5212022-03-08 10:41:40 +0100692
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100693/* register global config keywords */
694static struct cfg_kw_list mem_cfg_kws = {ILH, {
695#ifdef DEBUG_FAIL_ALLOC
696 { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
697#endif
Willy Tarreaua891e572022-03-08 15:10:05 +0100698#if defined(HA_HAVE_MALLOC_TRIM)
Willy Tarreau85dd5212022-03-08 10:41:40 +0100699 { CFG_GLOBAL, "no-memory-trimming", mem_parse_global_no_mem_trim },
Willy Tarreaua891e572022-03-08 15:10:05 +0100700#endif
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100701 { 0, NULL, NULL }
702}};
703
704INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);
705
Willy Tarreau50e608d2007-05-13 18:26:08 +0200706/*
707 * Local variables:
708 * c-indent-level: 8
709 * c-basic-offset: 8
710 * End:
711 */