blob: 83525311f1741ea6851cbc6fe0633ac33fa3610d [file] [log] [blame]
Willy Tarreau50e608d2007-05-13 18:26:08 +02001/*
2 * Memory management functions.
3 *
4 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
Willy Tarreauf14d1902021-10-05 18:14:11 +020012
13#include <sys/mman.h>
Willy Tarreau7107c8b2018-11-26 11:44:35 +010014#include <errno.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020015
Willy Tarreau5d9ddc52021-10-06 19:54:09 +020016#include <haproxy/activity.h>
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020017#include <haproxy/api.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020018#include <haproxy/applet-t.h>
Willy Tarreau6be78492020-06-05 00:00:29 +020019#include <haproxy/cfgparse.h>
Willy Tarreauf1d32c42020-06-04 21:07:02 +020020#include <haproxy/channel.h>
Willy Tarreau83487a82020-06-04 20:19:54 +020021#include <haproxy/cli.h>
Christopher Faulet908628c2022-03-25 16:43:49 +010022#include <haproxy/conn_stream.h>
23#include <haproxy/cs_utils.h>
Willy Tarreau36979d92020-06-05 17:27:29 +020024#include <haproxy/errors.h>
Willy Tarreauf268ee82020-06-04 17:05:57 +020025#include <haproxy/global.h>
Willy Tarreau853b2972020-05-27 18:01:47 +020026#include <haproxy/list.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020027#include <haproxy/pool.h>
Willy Tarreau2eec9b52020-06-04 19:58:55 +020028#include <haproxy/stats-t.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020029#include <haproxy/thread.h>
Willy Tarreau48fbcae2020-06-03 18:09:46 +020030#include <haproxy/tools.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020031
Willy Tarreau50e608d2007-05-13 18:26:08 +020032
Willy Tarreau7f0165e2018-11-26 17:09:46 +010033/* These ones are initialized per-thread on startup by init_pools() */
Willy Tarreaue18db9e2018-10-16 10:28:54 +020034THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
35THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
36
Willy Tarreau9f699952022-02-18 18:31:53 +010037static struct list pools __read_mostly = LIST_HEAD_INIT(pools);
Willy Tarreauef301b72022-02-23 14:15:18 +010038int mem_poison_byte __read_mostly = 'P';
Willy Tarreau8d0273e2022-02-21 17:16:22 +010039uint pool_debugging __read_mostly = /* set of POOL_DBG_* flags */
Olivier Houcharddc21ff72019-01-29 15:20:16 +010040#ifdef DEBUG_FAIL_ALLOC
Willy Tarreau8d0273e2022-02-21 17:16:22 +010041 POOL_DBG_FAIL_ALLOC |
Olivier Houcharddc21ff72019-01-29 15:20:16 +010042#endif
Willy Tarreaufd8b7372022-02-21 17:31:50 +010043#ifdef DEBUG_DONT_SHARE_POOLS
44 POOL_DBG_DONT_MERGE |
45#endif
Willy Tarreaud3470e12022-02-21 18:30:25 +010046#ifdef DEBUG_POOL_INTEGRITY
47 POOL_DBG_COLD_FIRST |
48#endif
Willy Tarreau6f3c7f62022-02-21 18:42:53 +010049#ifdef DEBUG_POOL_INTEGRITY
50 POOL_DBG_INTEGRITY |
51#endif
Willy Tarreaudff3b062022-02-22 09:21:13 +010052#ifdef CONFIG_HAP_NO_GLOBAL_POOLS
53 POOL_DBG_NO_GLOBAL |
54#endif
Willy Tarreaue9816312022-02-22 16:23:09 +010055#ifndef CONFIG_HAP_POOLS
56 POOL_DBG_NO_CACHE |
57#endif
Willy Tarreau02718222022-02-23 10:10:33 +010058#if defined(DEBUG_POOL_TRACING)
59 POOL_DBG_CALLER |
60#endif
Willy Tarreau13d77752022-02-23 10:20:37 +010061#if defined(DEBUG_MEMORY_POOLS)
62 POOL_DBG_TAG |
63#endif
Willy Tarreau8d0273e2022-02-21 17:16:22 +010064 0;
Olivier Houcharddc21ff72019-01-29 15:20:16 +010065
Willy Tarreauf4b79c42022-02-23 15:20:53 +010066static const struct {
67 uint flg;
68 const char *set;
69 const char *clr;
70 const char *hlp;
71} dbg_options[] = {
72 /* flg, set, clr, hlp */
73 { POOL_DBG_FAIL_ALLOC, "fail", "no-fail", "randomly fail allocations" },
74 { POOL_DBG_DONT_MERGE, "no-merge", "merge", "disable merging of similar pools" },
75 { POOL_DBG_COLD_FIRST, "cold-first", "hot-first", "pick cold objects first" },
76 { POOL_DBG_INTEGRITY, "integrity", "no-integrity", "enable cache integrity checks" },
77 { POOL_DBG_NO_GLOBAL, "no-global", "global", "disable global shared cache" },
78 { POOL_DBG_NO_CACHE, "no-cache", "cache", "disable thread-local cache" },
79 { POOL_DBG_CALLER, "caller", "no-caller", "save caller information in cache" },
80 { POOL_DBG_TAG, "tag", "no-tag", "add tag at end of allocated objects" },
81 { POOL_DBG_POISON, "poison", "no-poison", "poison newly allocated objects" },
82 { 0 /* end */ }
83};
84
Willy Tarreau8d0273e2022-02-21 17:16:22 +010085static int mem_fail_rate __read_mostly = 0;
Willy Tarreau9f699952022-02-18 18:31:53 +010086static int using_default_allocator __read_mostly = 1;
Willy Tarreauc4e56dc2022-03-08 10:41:40 +010087static int disable_trim __read_mostly = 0;
David Carliered232142021-11-25 16:09:45 +000088static int(*my_mallctl)(const char *, void *, size_t *, void *, size_t) = NULL;
Willy Tarreau157e3932021-09-15 10:05:48 +020089
Willy Tarreau0d93a812021-12-23 09:26:30 +010090/* ask the allocator to trim memory pools.
91 * This must run under thread isolation so that competing threads trying to
92 * allocate or release memory do not prevent the allocator from completing
93 * its job. We just have to be careful as callers might already be isolated
94 * themselves.
95 */
Willy Tarreauea3323f2021-09-15 10:38:21 +020096static void trim_all_pools(void)
97{
Willy Tarreau0d93a812021-12-23 09:26:30 +010098 int isolated = thread_isolated();
99
Willy Tarreauc4e56dc2022-03-08 10:41:40 +0100100 if (disable_trim)
101 return;
102
Willy Tarreau0d93a812021-12-23 09:26:30 +0100103 if (!isolated)
104 thread_isolate();
105
David Carlierd450ff62021-11-25 16:14:38 +0000106 if (my_mallctl) {
107 unsigned int i, narenas = 0;
108 size_t len = sizeof(narenas);
109
110 if (my_mallctl("arenas.narenas", &narenas, &len, NULL, 0) == 0) {
111 for (i = 0; i < narenas; i ++) {
112 char mib[32] = {0};
113 snprintf(mib, sizeof(mib), "arena.%u.purge", i);
114 (void)my_mallctl(mib, NULL, NULL, NULL, 0);
115 }
116 }
117 } else {
David Carliered232142021-11-25 16:09:45 +0000118#if defined(HA_HAVE_MALLOC_TRIM)
David Carlierd450ff62021-11-25 16:14:38 +0000119 if (using_default_allocator)
120 malloc_trim(0);
David CARLIERb1e190a2021-11-26 20:44:44 +0000121#elif defined(HA_HAVE_MALLOC_ZONE)
122 if (using_default_allocator) {
123 vm_address_t *zones;
124 unsigned int i, nzones;
125
126 if (malloc_get_all_zones(0, NULL, &zones, &nzones) == KERN_SUCCESS) {
127 for (i = 0; i < nzones; i ++) {
128 malloc_zone_t *zone = (malloc_zone_t *)zones[i];
129
130 /* we cannot purge anonymous zones */
131 if (zone->zone_name)
132 malloc_zone_pressure_relief(zone, 0);
133 }
134 }
135 }
David Carliered232142021-11-25 16:09:45 +0000136#endif
David Carlierd450ff62021-11-25 16:14:38 +0000137 }
Willy Tarreau0d93a812021-12-23 09:26:30 +0100138
139 if (!isolated)
140 thread_release();
Willy Tarreauea3323f2021-09-15 10:38:21 +0200141}
142
Willy Tarreau157e3932021-09-15 10:05:48 +0200143/* check if we're using the same allocator as the one that provides
144 * malloc_trim() and mallinfo(). The principle is that on glibc, both
145 * malloc_trim() and mallinfo() are provided, and using mallinfo() we
146 * can check if malloc() is performed through glibc or any other one
David Carliered232142021-11-25 16:09:45 +0000147 * the executable was linked against (e.g. jemalloc). Prior to this we
148 * have to check whether we're running on jemalloc by verifying if the
149 * mallctl() function is provided. Its pointer will be used later.
Willy Tarreau157e3932021-09-15 10:05:48 +0200150 */
151static void detect_allocator(void)
152{
Willy Tarreau781f07a2021-11-26 15:55:55 +0100153#if defined(__ELF__)
David Carliered232142021-11-25 16:09:45 +0000154 extern int mallctl(const char *, void *, size_t *, void *, size_t) __attribute__((weak));
155
156 my_mallctl = mallctl;
Willy Tarreau781f07a2021-11-26 15:55:55 +0100157#endif
David Carliered232142021-11-25 16:09:45 +0000158
159 if (!my_mallctl) {
160 my_mallctl = get_sym_curr_addr("mallctl");
161 using_default_allocator = (my_mallctl == NULL);
162 }
163
164 if (!my_mallctl) {
165#if defined(HA_HAVE_MALLOC_TRIM)
Willy Tarreauc2afb862021-09-16 09:18:21 +0200166#ifdef HA_HAVE_MALLINFO2
David Carliered232142021-11-25 16:09:45 +0000167 struct mallinfo2 mi1, mi2;
Willy Tarreauc2afb862021-09-16 09:18:21 +0200168#else
David Carliered232142021-11-25 16:09:45 +0000169 struct mallinfo mi1, mi2;
Willy Tarreauc2afb862021-09-16 09:18:21 +0200170#endif
David Carliered232142021-11-25 16:09:45 +0000171 void *ptr;
Willy Tarreau157e3932021-09-15 10:05:48 +0200172
Willy Tarreauc2afb862021-09-16 09:18:21 +0200173#ifdef HA_HAVE_MALLINFO2
David Carliered232142021-11-25 16:09:45 +0000174 mi1 = mallinfo2();
Willy Tarreauc2afb862021-09-16 09:18:21 +0200175#else
David Carliered232142021-11-25 16:09:45 +0000176 mi1 = mallinfo();
Willy Tarreauc2afb862021-09-16 09:18:21 +0200177#endif
David Carliered232142021-11-25 16:09:45 +0000178 ptr = DISGUISE(malloc(1));
Willy Tarreauc2afb862021-09-16 09:18:21 +0200179#ifdef HA_HAVE_MALLINFO2
David Carliered232142021-11-25 16:09:45 +0000180 mi2 = mallinfo2();
Willy Tarreauc2afb862021-09-16 09:18:21 +0200181#else
David Carliered232142021-11-25 16:09:45 +0000182 mi2 = mallinfo();
Willy Tarreauc2afb862021-09-16 09:18:21 +0200183#endif
David Carliered232142021-11-25 16:09:45 +0000184 free(DISGUISE(ptr));
Willy Tarreauea3323f2021-09-15 10:38:21 +0200185
David Carliered232142021-11-25 16:09:45 +0000186 using_default_allocator = !!memcmp(&mi1, &mi2, sizeof(mi1));
David CARLIERb1e190a2021-11-26 20:44:44 +0000187#elif defined(HA_HAVE_MALLOC_ZONE)
188 using_default_allocator = (malloc_default_zone() != NULL);
David Carliered232142021-11-25 16:09:45 +0000189#endif
190 }
Willy Tarreau845b5602021-09-15 10:41:24 +0200191}
192
193static int is_trim_enabled(void)
194{
David Carliered232142021-11-25 16:09:45 +0000195 return using_default_allocator;
Willy Tarreau157e3932021-09-15 10:05:48 +0200196}
Willy Tarreauea3323f2021-09-15 10:38:21 +0200197
Willy Tarreau8d0273e2022-02-21 17:16:22 +0100198static int mem_should_fail(const struct pool_head *pool)
199{
200 int ret = 0;
201
202 if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
203 if (mem_fail_rate > statistical_prng_range(100))
204 ret = 1;
205 else
206 ret = 0;
207 }
208 return ret;
209}
210
Willy Tarreau50e608d2007-05-13 18:26:08 +0200211/* Try to find an existing shared pool with the same characteristics and
212 * returns it, otherwise creates this one. NULL is returned if no memory
Willy Tarreau581bf812016-01-25 02:19:13 +0100213 * is available for a new creation. Two flags are supported :
214 * - MEM_F_SHARED to indicate that the pool may be shared with other users
215 * - MEM_F_EXACT to indicate that the size must not be rounded up
Willy Tarreau50e608d2007-05-13 18:26:08 +0200216 */
217struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
218{
Willy Tarreau42705d02022-02-23 10:03:11 +0100219 unsigned int extra_mark, extra_caller, extra;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200220 struct pool_head *pool;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200221 struct pool_head *entry;
222 struct list *start;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200223 unsigned int align;
Willy Tarreau9f3129e2021-04-17 00:31:38 +0200224 int thr __maybe_unused;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200225
Willy Tarreauac421112015-10-28 15:09:29 +0100226 /* We need to store a (void *) at the end of the chunks. Since we know
Willy Tarreau50e608d2007-05-13 18:26:08 +0200227 * that the malloc() function will never return such a small size,
228 * let's round the size up to something slightly bigger, in order to
229 * ease merging of entries. Note that the rounding is a power of two.
Willy Tarreauac421112015-10-28 15:09:29 +0100230 * This extra (void *) is not accounted for in the size computation
231 * so that the visible parts outside are not affected.
Willy Tarreau30f931e2018-10-23 14:40:23 +0200232 *
233 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200234 */
235
Willy Tarreau13d77752022-02-23 10:20:37 +0100236 extra_mark = (pool_debugging & POOL_DBG_TAG) ? POOL_EXTRA_MARK : 0;
Willy Tarreau02718222022-02-23 10:10:33 +0100237 extra_caller = (pool_debugging & POOL_DBG_CALLER) ? POOL_EXTRA_CALLER : 0;
Willy Tarreau42705d02022-02-23 10:03:11 +0100238 extra = extra_mark + extra_caller;
239
Willy Tarreau581bf812016-01-25 02:19:13 +0100240 if (!(flags & MEM_F_EXACT)) {
Willy Tarreau30f931e2018-10-23 14:40:23 +0200241 align = 4 * sizeof(void *); // 2 lists = 4 pointers min
Willy Tarreau42705d02022-02-23 10:03:11 +0100242 size = ((size + extra + align - 1) & -align) - extra;
Willy Tarreau581bf812016-01-25 02:19:13 +0100243 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200244
Willy Tarreaue9816312022-02-22 16:23:09 +0100245 if (!(pool_debugging & POOL_DBG_NO_CACHE)) {
246 /* we'll store two lists there, we need the room for this. This is
247 * guaranteed by the test above, except if MEM_F_EXACT is set, or if
248 * the only EXTRA part is in fact the one that's stored in the cache
249 * in addition to the pci struct.
250 */
Willy Tarreau42705d02022-02-23 10:03:11 +0100251 if (size + extra - extra_caller < sizeof(struct pool_cache_item))
252 size = sizeof(struct pool_cache_item) + extra_caller - extra;
Willy Tarreaue9816312022-02-22 16:23:09 +0100253 }
254
Christopher Fauletb349e482017-08-29 09:52:38 +0200255 /* TODO: thread: we do not lock pool list for now because all pools are
256 * created during HAProxy startup (so before threads creation) */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200257 start = &pools;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200258 pool = NULL;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200259
260 list_for_each_entry(entry, &pools, list) {
261 if (entry->size == size) {
262 /* either we can share this place and we take it, or
Ilya Shipitsin47d17182020-06-21 21:42:57 +0500263 * we look for a shareable one or for the next position
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200264 * before which we will insert a new one.
265 */
Willy Tarreaufd8b7372022-02-21 17:31:50 +0100266 if ((flags & entry->flags & MEM_F_SHARED) &&
267 (!(pool_debugging & POOL_DBG_DONT_MERGE) ||
268 strcmp(name, entry->name) == 0)) {
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200269 /* we can share this one */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200270 pool = entry;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200271 DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
Willy Tarreau50e608d2007-05-13 18:26:08 +0200272 break;
273 }
274 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200275 else if (entry->size > size) {
276 /* insert before this one */
277 start = &entry->list;
278 break;
279 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200280 }
281
282 if (!pool) {
Willy Tarreaue81248c2022-03-02 17:59:04 +0100283 void *pool_addr;
Willy Tarreau0a93b642018-10-16 07:58:39 +0200284
Willy Tarreaue81248c2022-03-02 17:59:04 +0100285 pool_addr = calloc(1, sizeof(*pool) + __alignof__(*pool));
286 if (!pool_addr)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200287 return NULL;
Willy Tarreaue81248c2022-03-02 17:59:04 +0100288
289 /* always provide an aligned pool */
290 pool = (struct pool_head*)((((size_t)pool_addr) + __alignof__(*pool)) & -(size_t)__alignof__(*pool));
291 pool->base_addr = pool_addr; // keep it, it's the address to free later
292
Willy Tarreau50e608d2007-05-13 18:26:08 +0200293 if (name)
294 strlcpy2(pool->name, name, sizeof(pool->name));
Willy Tarreau42705d02022-02-23 10:03:11 +0100295 pool->alloc_sz = size + extra;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200296 pool->size = size;
297 pool->flags = flags;
Willy Tarreau2b718102021-04-21 07:32:39 +0200298 LIST_APPEND(start, &pool->list);
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200299
Willy Tarreaue9816312022-02-22 16:23:09 +0100300 if (!(pool_debugging & POOL_DBG_NO_CACHE)) {
301 /* update per-thread pool cache if necessary */
302 for (thr = 0; thr < MAX_THREADS; thr++) {
303 LIST_INIT(&pool->cache[thr].list);
304 pool->cache[thr].tid = thr;
305 pool->cache[thr].pool = pool;
306 }
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200307 }
Olivier Houchard8af97eb2020-02-01 17:45:32 +0100308 }
309 pool->users++;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200310 return pool;
311}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100312
Willy Tarreau13843642021-04-17 16:57:25 +0200313/* Tries to allocate an object for the pool <pool> using the system's allocator
314 * and directly returns it. The pool's allocated counter is checked and updated,
Willy Tarreau8715dec2021-06-10 17:31:48 +0200315 * but no other checks are performed.
Willy Tarreau13843642021-04-17 16:57:25 +0200316 */
317void *pool_get_from_os(struct pool_head *pool)
318{
319 if (!pool->limit || pool->allocated < pool->limit) {
Willy Tarreau96d5bc72022-02-23 08:57:59 +0100320 void *ptr = pool_alloc_area(pool->alloc_sz);
Willy Tarreau13843642021-04-17 16:57:25 +0200321 if (ptr) {
322 _HA_ATOMIC_INC(&pool->allocated);
323 return ptr;
324 }
325 _HA_ATOMIC_INC(&pool->failed);
326 }
327 activity[tid].pool_fail++;
328 return NULL;
329
330}
331
Willy Tarreau45e4e282021-04-17 17:48:40 +0200332/* Releases a pool item back to the operating system and atomically updates
333 * the allocation counter.
334 */
335void pool_put_to_os(struct pool_head *pool, void *ptr)
336{
Willy Tarreau9a7aa3b2021-06-10 17:20:19 +0200337#ifdef DEBUG_UAF
338 /* This object will be released for real in order to detect a use after
339 * free. We also force a write to the area to ensure we crash on double
340 * free or free of a const area.
341 */
342 *(uint32_t *)ptr = 0xDEADADD4;
343#endif /* DEBUG_UAF */
344
Willy Tarreau96d5bc72022-02-23 08:57:59 +0100345 pool_free_area(ptr, pool->alloc_sz);
Willy Tarreau45e4e282021-04-17 17:48:40 +0200346 _HA_ATOMIC_DEC(&pool->allocated);
347}
348
Willy Tarreau8fe726f2021-04-15 18:20:12 +0200349/* Tries to allocate an object for the pool <pool> using the system's allocator
350 * and directly returns it. The pool's counters are updated but the object is
351 * never cached, so this is usable with and without local or shared caches.
Willy Tarreau8fe726f2021-04-15 18:20:12 +0200352 */
353void *pool_alloc_nocache(struct pool_head *pool)
Willy Tarreau0bae0752021-03-02 20:05:09 +0100354{
Willy Tarreau0bae0752021-03-02 20:05:09 +0100355 void *ptr = NULL;
356
Willy Tarreau13843642021-04-17 16:57:25 +0200357 ptr = pool_get_from_os(pool);
358 if (!ptr)
Willy Tarreau0bae0752021-03-02 20:05:09 +0100359 return NULL;
Willy Tarreau0bae0752021-03-02 20:05:09 +0100360
Willy Tarreau13843642021-04-17 16:57:25 +0200361 swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used, POOL_AVG_SAMPLES/4);
Willy Tarreau4781b152021-04-06 13:53:36 +0200362 _HA_ATOMIC_INC(&pool->used);
Willy Tarreau0bae0752021-03-02 20:05:09 +0100363
Willy Tarreau0bae0752021-03-02 20:05:09 +0100364 /* keep track of where the element was allocated from */
Willy Tarreau8c492702022-01-01 17:10:50 +0100365 POOL_DEBUG_SET_MARK(pool, ptr);
Willy Tarreauf70fdde2022-01-25 15:56:50 +0100366 POOL_DEBUG_TRACE_CALLER(pool, (struct pool_cache_item *)ptr, NULL);
Willy Tarreau0bae0752021-03-02 20:05:09 +0100367 return ptr;
368}
369
Willy Tarreau45e4e282021-04-17 17:48:40 +0200370/* Release a pool item back to the OS and keeps the pool's counters up to date.
371 * This is always defined even when pools are not enabled (their usage stats
372 * are maintained).
373 */
374void pool_free_nocache(struct pool_head *pool, void *ptr)
375{
376 _HA_ATOMIC_DEC(&pool->used);
377 swrate_add(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used);
378 pool_put_to_os(pool, ptr);
379}
380
Willy Tarreaub8498e92021-04-18 10:23:02 +0200381
Willy Tarreau6f3c7f62022-02-21 18:42:53 +0100382/* Updates <pch>'s fill_pattern and fills the free area after <item> with it,
383 * up to <size> bytes. The item part is left untouched.
384 */
385void pool_fill_pattern(struct pool_cache_head *pch, struct pool_cache_item *item, uint size)
386{
387 ulong *ptr = (ulong *)item;
388 uint ofs;
389 ulong u;
390
391 if (size <= sizeof(*item))
392 return;
393
394 /* Upgrade the fill_pattern to change about half of the bits
395 * (to be sure to catch static flag corruption), and apply it.
396 */
397 u = pch->fill_pattern += ~0UL / 3; // 0x55...55
398 ofs = sizeof(*item) / sizeof(*ptr);
399 while (ofs < size / sizeof(*ptr))
400 ptr[ofs++] = u;
401}
402
403/* check for a pool_cache_item integrity after extracting it from the cache. It
404 * must have been previously initialized using pool_fill_pattern(). If any
405 * corruption is detected, the function provokes an immediate crash.
406 */
407void pool_check_pattern(struct pool_cache_head *pch, struct pool_cache_item *item, uint size)
408{
409 const ulong *ptr = (const ulong *)item;
410 uint ofs;
411 ulong u;
412
413 if (size <= sizeof(*item))
414 return;
415
416 /* let's check that all words past *item are equal */
417 ofs = sizeof(*item) / sizeof(*ptr);
418 u = ptr[ofs++];
419 while (ofs < size / sizeof(*ptr)) {
420 if (unlikely(ptr[ofs] != u))
421 ABORT_NOW();
422 ofs++;
423 }
424}
425
Willy Tarreaua0b58312022-01-02 17:19:14 +0100426/* removes up to <count> items from the end of the local pool cache <ph> for
427 * pool <pool>. The shared pool is refilled with these objects in the limit
428 * of the number of acceptable objects, and the rest will be released to the
429 * OS. It is not a problem is <count> is larger than the number of objects in
Willy Tarreaue9816312022-02-22 16:23:09 +0100430 * the local cache. The counters are automatically updated. Must not be used
431 * with pools disabled.
Willy Tarreau87212032021-04-19 08:14:03 +0200432 */
Willy Tarreaua0b58312022-01-02 17:19:14 +0100433static void pool_evict_last_items(struct pool_head *pool, struct pool_cache_head *ph, uint count)
Willy Tarreau87212032021-04-19 08:14:03 +0200434{
Willy Tarreau87212032021-04-19 08:14:03 +0200435 struct pool_cache_item *item;
Willy Tarreau1513c542022-01-02 17:53:02 +0100436 struct pool_item *pi, *head = NULL;
Willy Tarreaua0b58312022-01-02 17:19:14 +0100437 uint released = 0;
Willy Tarreau1513c542022-01-02 17:53:02 +0100438 uint cluster = 0;
Willy Tarreau361e31e2022-01-02 00:27:06 +0100439 uint to_free_max;
440
Willy Tarreaue9816312022-02-22 16:23:09 +0100441 BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);
442
Willy Tarreaudff3b062022-02-22 09:21:13 +0100443 /* Note: this will be zero when global pools are disabled */
Willy Tarreau361e31e2022-01-02 00:27:06 +0100444 to_free_max = pool_releasable(pool);
Willy Tarreau87212032021-04-19 08:14:03 +0200445
Willy Tarreaua0b58312022-01-02 17:19:14 +0100446 while (released < count && !LIST_ISEMPTY(&ph->list)) {
Willy Tarreaud5ec1002022-01-02 12:40:14 +0100447 item = LIST_PREV(&ph->list, typeof(item), by_pool);
Willy Tarreaue2830ad2022-02-09 16:23:55 +0100448 BUG_ON(&item->by_pool == &ph->list);
Willy Tarreau6f3c7f62022-02-21 18:42:53 +0100449 if (unlikely(pool_debugging & POOL_DBG_INTEGRITY))
450 pool_check_pattern(ph, item, pool->size);
Willy Tarreau2b718102021-04-21 07:32:39 +0200451 LIST_DELETE(&item->by_pool);
452 LIST_DELETE(&item->by_lru);
Willy Tarreaub46674a2021-12-30 17:37:33 +0100453
Willy Tarreau1513c542022-01-02 17:53:02 +0100454 if (to_free_max > released || cluster) {
Willy Tarreaudff3b062022-02-22 09:21:13 +0100455 /* will never match when global pools are disabled */
Willy Tarreau361e31e2022-01-02 00:27:06 +0100456 pi = (struct pool_item *)item;
Willy Tarreau1513c542022-01-02 17:53:02 +0100457 pi->next = NULL;
458 pi->down = head;
459 head = pi;
460 cluster++;
461 if (cluster >= CONFIG_HAP_POOL_CLUSTER_SIZE) {
462 /* enough to make a cluster */
463 pool_put_to_shared_cache(pool, head, cluster);
464 cluster = 0;
465 head = NULL;
466 }
Willy Tarreau361e31e2022-01-02 00:27:06 +0100467 } else
Willy Tarreaub46674a2021-12-30 17:37:33 +0100468 pool_free_nocache(pool, item);
Willy Tarreau1513c542022-01-02 17:53:02 +0100469
470 released++;
Willy Tarreau361e31e2022-01-02 00:27:06 +0100471 }
472
Willy Tarreau1513c542022-01-02 17:53:02 +0100473 /* incomplete cluster left */
474 if (cluster)
475 pool_put_to_shared_cache(pool, head, cluster);
476
Willy Tarreaua0b58312022-01-02 17:19:14 +0100477 ph->count -= released;
478 pool_cache_count -= released;
479 pool_cache_bytes -= released * pool->size;
480}
481
482/* Evicts some of the oldest objects from one local cache, until its number of
483 * objects is no more than 16+1/8 of the total number of locally cached objects
484 * or the total size of the local cache is no more than 75% of its maximum (i.e.
485 * we don't want a single cache to use all the cache for itself). For this, the
Willy Tarreauc895c442022-02-09 16:19:24 +0100486 * list is scanned in reverse. If <full> is non-null, all objects are evicted.
Willy Tarreaue9816312022-02-22 16:23:09 +0100487 * Must not be used when pools are disabled.
Willy Tarreaua0b58312022-01-02 17:19:14 +0100488 */
Willy Tarreauc895c442022-02-09 16:19:24 +0100489void pool_evict_from_local_cache(struct pool_head *pool, int full)
Willy Tarreaua0b58312022-01-02 17:19:14 +0100490{
491 struct pool_cache_head *ph = &pool->cache[tid];
492
Willy Tarreaue9816312022-02-22 16:23:09 +0100493 BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);
494
Willy Tarreauc895c442022-02-09 16:19:24 +0100495 while ((ph->count && full) ||
496 (ph->count >= CONFIG_HAP_POOL_CLUSTER_SIZE &&
497 ph->count >= 16 + pool_cache_count / 8 &&
498 pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4)) {
Willy Tarreau43937e92022-01-02 17:24:55 +0100499 pool_evict_last_items(pool, ph, CONFIG_HAP_POOL_CLUSTER_SIZE);
Willy Tarreau87212032021-04-19 08:14:03 +0200500 }
501}
502
Willy Tarreaub8498e92021-04-18 10:23:02 +0200503/* Evicts some of the oldest objects from the local cache, pushing them to the
Willy Tarreaue9816312022-02-22 16:23:09 +0100504 * global pool. Must not be used when pools are disabled.
Willy Tarreaub8498e92021-04-18 10:23:02 +0200505 */
506void pool_evict_from_local_caches()
507{
508 struct pool_cache_item *item;
509 struct pool_cache_head *ph;
510 struct pool_head *pool;
511
Willy Tarreaue9816312022-02-22 16:23:09 +0100512 BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);
513
Willy Tarreaub8498e92021-04-18 10:23:02 +0200514 do {
Willy Tarreaub4e34762021-09-30 19:02:18 +0200515 item = LIST_PREV(&th_ctx->pool_lru_head, struct pool_cache_item *, by_lru);
Willy Tarreaue2830ad2022-02-09 16:23:55 +0100516 BUG_ON(&item->by_lru == &th_ctx->pool_lru_head);
Willy Tarreaub8498e92021-04-18 10:23:02 +0200517 /* note: by definition we remove oldest objects so they also are the
518 * oldest in their own pools, thus their next is the pool's head.
519 */
520 ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
Willy Tarreau49bb5d42022-02-09 16:33:22 +0100521 BUG_ON(ph->tid != tid);
522
Willy Tarreaub8498e92021-04-18 10:23:02 +0200523 pool = container_of(ph - tid, struct pool_head, cache);
Willy Tarreau49bb5d42022-02-09 16:33:22 +0100524 BUG_ON(pool != ph->pool);
525
Willy Tarreau43937e92022-01-02 17:24:55 +0100526 pool_evict_last_items(pool, ph, CONFIG_HAP_POOL_CLUSTER_SIZE);
Willy Tarreaub8498e92021-04-18 10:23:02 +0200527 } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
528}
Willy Tarreau0bae0752021-03-02 20:05:09 +0100529
Willy Tarreaub2a853d2021-04-19 11:49:26 +0200530/* Frees an object to the local cache, possibly pushing oldest objects to the
531 * shared cache, which itself may decide to release some of them to the OS.
532 * While it is unspecified what the object becomes past this point, it is
Willy Tarreauadd43fa2022-01-24 15:52:51 +0100533 * guaranteed to be released from the users' perpective. A caller address may
Willy Tarreaue9816312022-02-22 16:23:09 +0100534 * be passed and stored into the area when DEBUG_POOL_TRACING is set. Must not
535 * be used with pools disabled.
Willy Tarreaub2a853d2021-04-19 11:49:26 +0200536 */
Willy Tarreau0e2a5b42022-01-24 15:51:50 +0100537void pool_put_to_cache(struct pool_head *pool, void *ptr, const void *caller)
Willy Tarreaub2a853d2021-04-19 11:49:26 +0200538{
539 struct pool_cache_item *item = (struct pool_cache_item *)ptr;
540 struct pool_cache_head *ph = &pool->cache[tid];
541
Willy Tarreaue9816312022-02-22 16:23:09 +0100542 BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);
543
Willy Tarreau2b718102021-04-21 07:32:39 +0200544 LIST_INSERT(&ph->list, &item->by_pool);
Willy Tarreaub4e34762021-09-30 19:02:18 +0200545 LIST_INSERT(&th_ctx->pool_lru_head, &item->by_lru);
Willy Tarreauadd43fa2022-01-24 15:52:51 +0100546 POOL_DEBUG_TRACE_CALLER(pool, item, caller);
Willy Tarreaub2a853d2021-04-19 11:49:26 +0200547 ph->count++;
Willy Tarreau6f3c7f62022-02-21 18:42:53 +0100548 if (unlikely(pool_debugging & POOL_DBG_INTEGRITY))
549 pool_fill_pattern(ph, item, pool->size);
Willy Tarreaub2a853d2021-04-19 11:49:26 +0200550 pool_cache_count++;
551 pool_cache_bytes += pool->size;
552
553 if (unlikely(pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4)) {
Willy Tarreau43937e92022-01-02 17:24:55 +0100554 if (ph->count >= 16 + pool_cache_count / 8 + CONFIG_HAP_POOL_CLUSTER_SIZE)
Willy Tarreauc895c442022-02-09 16:19:24 +0100555 pool_evict_from_local_cache(pool, 0);
Willy Tarreaub2a853d2021-04-19 11:49:26 +0200556 if (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE)
557 pool_evict_from_local_caches();
558 }
559}
560
Willy Tarreauafe2c4a2021-12-30 17:09:31 +0100561/* Tries to refill the local cache <pch> from the shared one for pool <pool>.
562 * This is only used when pools are in use and shared pools are enabled. No
563 * malloc() is attempted, and poisonning is never performed. The purpose is to
564 * get the fastest possible refilling so that the caller can easily check if
Willy Tarreaue9816312022-02-22 16:23:09 +0100565 * the cache has enough objects for its use. Must not be used when pools are
566 * disabled.
Willy Tarreauafe2c4a2021-12-30 17:09:31 +0100567 */
568void pool_refill_local_from_shared(struct pool_head *pool, struct pool_cache_head *pch)
569{
570 struct pool_cache_item *item;
Willy Tarreau148160b2022-01-02 14:35:57 +0100571 struct pool_item *ret, *down;
572 uint count;
Willy Tarreauafe2c4a2021-12-30 17:09:31 +0100573
Willy Tarreaue9816312022-02-22 16:23:09 +0100574 BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);
575
Willy Tarreauafe2c4a2021-12-30 17:09:31 +0100576 /* we'll need to reference the first element to figure the next one. We
577 * must temporarily lock it so that nobody allocates then releases it,
578 * or the dereference could fail.
579 */
580 ret = _HA_ATOMIC_LOAD(&pool->free_list);
581 do {
582 while (unlikely(ret == POOL_BUSY)) {
583 __ha_cpu_relax();
584 ret = _HA_ATOMIC_LOAD(&pool->free_list);
585 }
586 if (ret == NULL)
587 return;
588 } while (unlikely((ret = _HA_ATOMIC_XCHG(&pool->free_list, POOL_BUSY)) == POOL_BUSY));
589
590 if (unlikely(ret == NULL)) {
591 HA_ATOMIC_STORE(&pool->free_list, NULL);
592 return;
593 }
594
595 /* this releases the lock */
Willy Tarreauc16ed3b2022-01-01 18:22:20 +0100596 HA_ATOMIC_STORE(&pool->free_list, ret->next);
Willy Tarreauafe2c4a2021-12-30 17:09:31 +0100597
Willy Tarreau148160b2022-01-02 14:35:57 +0100598 /* now store the retrieved object(s) into the local cache */
599 count = 0;
600 for (; ret; ret = down) {
601 down = ret->down;
Willy Tarreau148160b2022-01-02 14:35:57 +0100602 item = (struct pool_cache_item *)ret;
Willy Tarreauf70fdde2022-01-25 15:56:50 +0100603 POOL_DEBUG_TRACE_CALLER(pool, item, NULL);
Willy Tarreau148160b2022-01-02 14:35:57 +0100604 LIST_INSERT(&pch->list, &item->by_pool);
605 LIST_INSERT(&th_ctx->pool_lru_head, &item->by_lru);
606 count++;
Willy Tarreau6f3c7f62022-02-21 18:42:53 +0100607 if (unlikely(pool_debugging & POOL_DBG_INTEGRITY))
608 pool_fill_pattern(pch, item, pool->size);
Willy Tarreau148160b2022-01-02 14:35:57 +0100609 }
610 HA_ATOMIC_ADD(&pool->used, count);
611 pch->count += count;
612 pool_cache_count += count;
613 pool_cache_bytes += count * pool->size;
Willy Tarreauafe2c4a2021-12-30 17:09:31 +0100614}
615
Willy Tarreau337410c2022-01-02 15:15:54 +0100616/* Adds pool item cluster <item> to the shared cache, which contains <count>
617 * elements. The caller is advised to first check using pool_releasable() if
618 * it's wise to add this series of objects there. Both the pool and the item's
619 * head must be valid.
Willy Tarreaub46674a2021-12-30 17:37:33 +0100620 */
Willy Tarreau337410c2022-01-02 15:15:54 +0100621void pool_put_to_shared_cache(struct pool_head *pool, struct pool_item *item, uint count)
Willy Tarreaub46674a2021-12-30 17:37:33 +0100622{
Willy Tarreauc16ed3b2022-01-01 18:22:20 +0100623 struct pool_item *free_list;
Willy Tarreaub46674a2021-12-30 17:37:33 +0100624
Willy Tarreau337410c2022-01-02 15:15:54 +0100625 _HA_ATOMIC_SUB(&pool->used, count);
Willy Tarreaub46674a2021-12-30 17:37:33 +0100626 free_list = _HA_ATOMIC_LOAD(&pool->free_list);
627 do {
628 while (unlikely(free_list == POOL_BUSY)) {
629 __ha_cpu_relax();
630 free_list = _HA_ATOMIC_LOAD(&pool->free_list);
631 }
Willy Tarreauc16ed3b2022-01-01 18:22:20 +0100632 _HA_ATOMIC_STORE(&item->next, free_list);
Willy Tarreaub46674a2021-12-30 17:37:33 +0100633 __ha_barrier_atomic_store();
634 } while (!_HA_ATOMIC_CAS(&pool->free_list, &free_list, item));
635 __ha_barrier_atomic_store();
636 swrate_add(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used);
637}
638
Olivier Houchardcf975d42018-01-24 18:38:31 +0100639/*
640 * This function frees whatever can be freed in pool <pool>.
641 */
642void pool_flush(struct pool_head *pool)
643{
Willy Tarreau148160b2022-01-02 14:35:57 +0100644 struct pool_item *next, *temp, *down;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100645
Willy Tarreaue9816312022-02-22 16:23:09 +0100646 if (!pool || (pool_debugging & (POOL_DBG_NO_CACHE|POOL_DBG_NO_GLOBAL)))
Olivier Houchardcf975d42018-01-24 18:38:31 +0100647 return;
Willy Tarreau2a4523f2021-06-09 18:59:58 +0200648
649 /* The loop below atomically detaches the head of the free list and
650 * replaces it with a NULL. Then the list can be released.
651 */
652 next = pool->free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100653 do {
Willy Tarreau2a4523f2021-06-09 18:59:58 +0200654 while (unlikely(next == POOL_BUSY)) {
655 __ha_cpu_relax();
656 next = _HA_ATOMIC_LOAD(&pool->free_list);
657 }
658 if (next == NULL)
659 return;
660 } while (unlikely((next = _HA_ATOMIC_XCHG(&pool->free_list, POOL_BUSY)) == POOL_BUSY));
661 _HA_ATOMIC_STORE(&pool->free_list, NULL);
Olivier Houchard20872762019-03-08 18:53:35 +0100662 __ha_barrier_atomic_store();
Willy Tarreau2a4523f2021-06-09 18:59:58 +0200663
Olivier Houchardcf975d42018-01-24 18:38:31 +0100664 while (next) {
665 temp = next;
Willy Tarreauc16ed3b2022-01-01 18:22:20 +0100666 next = temp->next;
Willy Tarreau148160b2022-01-02 14:35:57 +0100667 for (; temp; temp = down) {
668 down = temp->down;
669 pool_put_to_os(pool, temp);
670 }
Olivier Houchardcf975d42018-01-24 18:38:31 +0100671 }
Willy Tarreauc239cde2021-06-10 06:54:22 +0200672 /* here, we should have pool->allocated == pool->used */
Olivier Houchardcf975d42018-01-24 18:38:31 +0100673}
674
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200675/*
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200676 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200677 * the minimum thresholds imposed by owners. It makes sure to be alone to
678 * run by using thread_isolate(). <pool_ctx> is unused.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200679 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100680void pool_gc(struct pool_head *pool_ctx)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200681{
682 struct pool_head *entry;
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200683 int isolated = thread_isolated();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200684
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200685 if (!isolated)
686 thread_isolate();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200687
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200688 list_for_each_entry(entry, &pools, list) {
Willy Tarreau148160b2022-01-02 14:35:57 +0100689 struct pool_item *temp, *down;
Willy Tarreauc16ed3b2022-01-01 18:22:20 +0100690
Olivier Houchard51d93392020-03-12 19:05:39 +0100691 while (entry->free_list &&
Willy Tarreau57767b82014-12-22 21:40:55 +0100692 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
Olivier Houchard51d93392020-03-12 19:05:39 +0100693 temp = entry->free_list;
Willy Tarreauc16ed3b2022-01-01 18:22:20 +0100694 entry->free_list = temp->next;
Willy Tarreau148160b2022-01-02 14:35:57 +0100695 for (; temp; temp = down) {
696 down = temp->down;
697 pool_put_to_os(entry, temp);
698 }
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200699 }
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200700 }
Christopher Fauletb349e482017-08-29 09:52:38 +0200701
Willy Tarreauea3323f2021-09-15 10:38:21 +0200702 trim_all_pools();
Willy Tarreau26ed1832021-06-10 08:40:16 +0200703
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200704 if (!isolated)
705 thread_release();
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200706}
Willy Tarreaub8498e92021-04-18 10:23:02 +0200707
Willy Tarreau15c322c2022-01-24 11:51:43 +0100708/*
Willy Tarreaud3929732022-01-24 16:09:29 +0100709 * Returns a pointer to type <type> taken from the pool <pool_type> or
710 * dynamically allocated. In the first case, <pool_type> is updated to point to
711 * the next element in the list. <flags> is a binary-OR of POOL_F_* flags.
712 * Prefer using pool_alloc() which does the right thing without flags.
713 */
714void *__pool_alloc(struct pool_head *pool, unsigned int flags)
715{
716 void *p = NULL;
Willy Tarreau02718222022-02-23 10:10:33 +0100717 void *caller = __builtin_return_address(0);
Willy Tarreaud3929732022-01-24 16:09:29 +0100718
Willy Tarreau8d0273e2022-02-21 17:16:22 +0100719 if (unlikely(pool_debugging & POOL_DBG_FAIL_ALLOC))
720 if (!(flags & POOL_F_NO_FAIL) && mem_should_fail(pool))
721 return NULL;
Willy Tarreaud3929732022-01-24 16:09:29 +0100722
Willy Tarreaue9816312022-02-22 16:23:09 +0100723 if (likely(!(pool_debugging & POOL_DBG_NO_CACHE)) && !p)
Willy Tarreau0e2a5b42022-01-24 15:51:50 +0100724 p = pool_get_from_cache(pool, caller);
Willy Tarreaue9816312022-02-22 16:23:09 +0100725
Willy Tarreaud3929732022-01-24 16:09:29 +0100726 if (unlikely(!p))
727 p = pool_alloc_nocache(pool);
728
729 if (likely(p)) {
730 if (unlikely(flags & POOL_F_MUST_ZERO))
731 memset(p, 0, pool->size);
Willy Tarreauef301b72022-02-23 14:15:18 +0100732 else if (unlikely(!(flags & POOL_F_NO_POISON) && (pool_debugging & POOL_DBG_POISON)))
Willy Tarreaud3929732022-01-24 16:09:29 +0100733 memset(p, mem_poison_byte, pool->size);
734 }
735 return p;
736}
737
738/*
Willy Tarreau15c322c2022-01-24 11:51:43 +0100739 * Puts a memory area back to the corresponding pool. <ptr> be valid. Using
740 * pool_free() is preferred.
741 */
742void __pool_free(struct pool_head *pool, void *ptr)
743{
Willy Tarreau02718222022-02-23 10:10:33 +0100744 const void *caller = __builtin_return_address(0);
Willy Tarreau0e2a5b42022-01-24 15:51:50 +0100745
Willy Tarreau15c322c2022-01-24 11:51:43 +0100746 /* we'll get late corruption if we refill to the wrong pool or double-free */
747 POOL_DEBUG_CHECK_MARK(pool, ptr);
Willy Tarreau27c8da12022-02-09 16:49:16 +0100748 POOL_DEBUG_RESET_MARK(pool, ptr);
Willy Tarreaue9816312022-02-22 16:23:09 +0100749
750 if (unlikely(pool_debugging & POOL_DBG_NO_CACHE)) {
751 pool_free_nocache(pool, ptr);
752 return;
753 }
754
Willy Tarreau0e2a5b42022-01-24 15:51:50 +0100755 pool_put_to_cache(pool, ptr, caller);
Willy Tarreau15c322c2022-01-24 11:51:43 +0100756}
757
Willy Tarreauf14d1902021-10-05 18:14:11 +0200758
759#ifdef DEBUG_UAF
760
761/************* use-after-free allocator *************/
762
763/* allocates an area of size <size> and returns it. The semantics are similar
764 * to those of malloc(). However the allocation is rounded up to 4kB so that a
765 * full page is allocated. This ensures the object can be freed alone so that
766 * future dereferences are easily detected. The returned object is always
767 * 16-bytes aligned to avoid issues with unaligned structure objects. In case
768 * some padding is added, the area's start address is copied at the end of the
769 * padding to help detect underflows.
770 */
771void *pool_alloc_area_uaf(size_t size)
772{
773 size_t pad = (4096 - size) & 0xFF0;
Willy Tarreauf14d1902021-10-05 18:14:11 +0200774 void *ret;
775
Willy Tarreauf14d1902021-10-05 18:14:11 +0200776 ret = mmap(NULL, (size + 4095) & -4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
777 if (ret != MAP_FAILED) {
778 /* let's dereference the page before returning so that the real
779 * allocation in the system is performed without holding the lock.
780 */
781 *(int *)ret = 0;
782 if (pad >= sizeof(void *))
783 *(void **)(ret + pad - sizeof(void *)) = ret + pad;
784 ret += pad;
785 } else {
786 ret = NULL;
787 }
Willy Tarreauf14d1902021-10-05 18:14:11 +0200788 return ret;
789}
790
791/* frees an area <area> of size <size> allocated by pool_alloc_area(). The
792 * semantics are identical to free() except that the size must absolutely match
793 * the one passed to pool_alloc_area(). In case some padding is added, the
794 * area's start address is compared to the one at the end of the padding, and
795 * a segfault is triggered if they don't match, indicating an underflow.
796 */
797void pool_free_area_uaf(void *area, size_t size)
798{
799 size_t pad = (4096 - size) & 0xFF0;
800
801 if (pad >= sizeof(void *) && *(void **)(area - sizeof(void *)) != area)
802 ABORT_NOW();
803
Willy Tarreauf14d1902021-10-05 18:14:11 +0200804 munmap(area - pad, (size + 4095) & -4096);
Willy Tarreauf14d1902021-10-05 18:14:11 +0200805}
806
807#endif /* DEBUG_UAF */
808
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200809/*
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200810 * This function destroys a pool by freeing it completely, unless it's still
811 * in use. This should be called only under extreme circumstances. It always
812 * returns NULL if the resulting pool is empty, easing the clearing of the old
813 * pointer, otherwise it returns the pool.
814 * .
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200815 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100816void *pool_destroy(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200817{
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200818 if (pool) {
Willy Tarreaue9816312022-02-22 16:23:09 +0100819 if (!(pool_debugging & POOL_DBG_NO_CACHE))
820 pool_evict_from_local_cache(pool, 1);
821
Willy Tarreaubafbe012017-11-24 17:34:44 +0100822 pool_flush(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200823 if (pool->used)
824 return pool;
825 pool->users--;
826 if (!pool->users) {
Willy Tarreau2b718102021-04-21 07:32:39 +0200827 LIST_DELETE(&pool->list);
Willy Tarreau9f3129e2021-04-17 00:31:38 +0200828 /* note that if used == 0, the cache is empty */
Willy Tarreauf9eba782022-03-03 18:31:54 +0100829 free(pool->base_addr);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200830 }
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200831 }
832 return NULL;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200833}
834
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100835/* This destroys all pools on exit. It is *not* thread safe. */
836void pool_destroy_all()
837{
838 struct pool_head *entry, *back;
839
Willy Tarreau243e68b2022-04-27 11:33:13 +0200840 list_for_each_entry_safe(entry, back, &pools, list) {
841 /* there's only one occurrence of each pool in the list,
842 * and we're existing instead of looping on the whole
843 * list just to decrement users, force it to 1 here.
844 */
845 entry->users = 1;
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100846 pool_destroy(entry);
Willy Tarreau243e68b2022-04-27 11:33:13 +0200847 }
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100848}
849
Willy Tarreau12833bb2014-01-28 16:49:56 +0100850/* This function dumps memory usage information into the trash buffer. */
851void dump_pools_to_trash()
Willy Tarreau50e608d2007-05-13 18:26:08 +0200852{
853 struct pool_head *entry;
854 unsigned long allocated, used;
855 int nbpools;
Willy Tarreau1b4a7142021-10-07 16:29:31 +0200856 unsigned long cached_bytes = 0;
857 uint cached = 0;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200858
859 allocated = used = nbpools = 0;
Willy Tarreau12833bb2014-01-28 16:49:56 +0100860 chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200861 list_for_each_entry(entry, &pools, list) {
Willy Tarreaue9816312022-02-22 16:23:09 +0100862 if (!(pool_debugging & POOL_DBG_NO_CACHE)) {
863 int i;
864 for (cached = i = 0; i < global.nbthread; i++)
865 cached += entry->cache[i].count;
866 cached_bytes += cached * entry->size;
867 }
Willy Tarreau1b4a7142021-10-07 16:29:31 +0200868 chunk_appendf(&trash, " - Pool %s (%u bytes) : %u allocated (%u bytes), %u used"
Willy Tarreau1b4a7142021-10-07 16:29:31 +0200869 " (~%u by thread caches)"
Willy Tarreau1b4a7142021-10-07 16:29:31 +0200870 ", needed_avg %u, %u failures, %u users, @%p%s\n",
871 entry->name, entry->size, entry->allocated,
872 entry->size * entry->allocated, entry->used,
Willy Tarreau1b4a7142021-10-07 16:29:31 +0200873 cached,
Willy Tarreau1b4a7142021-10-07 16:29:31 +0200874 swrate_avg(entry->needed_avg, POOL_AVG_SAMPLES), entry->failed,
875 entry->users, entry,
876 (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200877
878 allocated += entry->allocated * entry->size;
879 used += entry->used * entry->size;
880 nbpools++;
881 }
Willy Tarreau1b4a7142021-10-07 16:29:31 +0200882 chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used"
Willy Tarreau1b4a7142021-10-07 16:29:31 +0200883 " (~%lu by thread caches)"
Willy Tarreau1b4a7142021-10-07 16:29:31 +0200884 ".\n",
Willy Tarreaue9816312022-02-22 16:23:09 +0100885 nbpools, allocated, used, cached_bytes
Willy Tarreau1b4a7142021-10-07 16:29:31 +0200886 );
Willy Tarreau50e608d2007-05-13 18:26:08 +0200887}
888
Willy Tarreau12833bb2014-01-28 16:49:56 +0100889/* Dump statistics on pools usage. */
890void dump_pools(void)
891{
892 dump_pools_to_trash();
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200893 qfprintf(stderr, "%s", trash.area);
Willy Tarreau12833bb2014-01-28 16:49:56 +0100894}
895
Willy Tarreau58102cf2015-10-28 16:24:21 +0100896/* This function returns the total number of failed pool allocations */
897int pool_total_failures()
898{
899 struct pool_head *entry;
900 int failed = 0;
901
902 list_for_each_entry(entry, &pools, list)
903 failed += entry->failed;
904 return failed;
905}
906
907/* This function returns the total amount of memory allocated in pools (in bytes) */
908unsigned long pool_total_allocated()
909{
910 struct pool_head *entry;
911 unsigned long allocated = 0;
912
913 list_for_each_entry(entry, &pools, list)
914 allocated += entry->allocated * entry->size;
915 return allocated;
916}
917
918/* This function returns the total amount of memory used in pools (in bytes) */
919unsigned long pool_total_used()
920{
921 struct pool_head *entry;
922 unsigned long used = 0;
923
924 list_for_each_entry(entry, &pools, list)
925 used += entry->used * entry->size;
926 return used;
927}
928
Willy Tarreau1408b1f2022-02-18 18:54:40 +0100929/* This function parses a string made of a set of debugging features as
930 * specified after -dM on the command line, and will set pool_debugging
931 * accordingly. On success it returns a strictly positive value. It may zero
932 * with the first warning in <err>, -1 with a help message in <err>, or -2 with
933 * the first error in <err> return the first error in <err>. <err> is undefined
934 * on success, and will be non-null and locally allocated on help/error/warning.
935 * The caller must free it. Warnings are used to report features that were not
936 * enabled at build time, and errors are used to report unknown features.
937 */
938int pool_parse_debugging(const char *str, char **err)
939{
Willy Tarreauf4b79c42022-02-23 15:20:53 +0100940 struct ist args;
Willy Tarreau1408b1f2022-02-18 18:54:40 +0100941 char *end;
Willy Tarreauf4b79c42022-02-23 15:20:53 +0100942 uint new_dbg;
Willy Tarreau1408b1f2022-02-18 18:54:40 +0100943 int v;
944
945
946 /* if it's empty or starts with a number, it's the mem poisonning byte */
947 v = strtol(str, &end, 0);
948 if (!*end || *end == ',') {
949 mem_poison_byte = *str ? v : 'P';
950 if (mem_poison_byte >= 0)
951 pool_debugging |= POOL_DBG_POISON;
952 else
953 pool_debugging &= ~POOL_DBG_POISON;
954 str = end;
955 }
Willy Tarreauf4b79c42022-02-23 15:20:53 +0100956
957 new_dbg = pool_debugging;
958
959 for (args = ist(str); istlen(args); args = istadv(istfind(args, ','), 1)) {
960 struct ist feat = iststop(args, ',');
961
962 if (!istlen(feat))
963 continue;
964
965 if (isteq(feat, ist("help"))) {
966 ha_free(err);
967 memprintf(err,
968 "-dM alone enables memory poisonning with byte 0x50 on allocation. A numeric\n"
969 "value may be appended immediately after -dM to use another value (0 supported).\n"
970 "Then an optional list of comma-delimited keywords may be appended to set or\n"
971 "clear some debugging options ('*' marks the current setting):\n\n"
972 " set clear description\n"
973 " -----------------+-----------------+-----------------------------------------\n");
974
975 for (v = 0; dbg_options[v].flg; v++) {
976 memprintf(err, "%s %c %-15s|%c %-15s| %s\n",
977 *err,
978 (pool_debugging & dbg_options[v].flg) ? '*' : ' ',
979 dbg_options[v].set,
980 (pool_debugging & dbg_options[v].flg) ? ' ' : '*',
981 dbg_options[v].clr,
982 dbg_options[v].hlp);
983 }
984 return -1;
985 }
986
987 for (v = 0; dbg_options[v].flg; v++) {
988 if (isteq(feat, ist(dbg_options[v].set))) {
989 new_dbg |= dbg_options[v].flg;
990 break;
991 }
992 else if (isteq(feat, ist(dbg_options[v].clr))) {
993 new_dbg &= ~dbg_options[v].flg;
994 break;
995 }
996 }
997
998 if (!dbg_options[v].flg) {
999 memprintf(err, "unknown pool debugging feature <%.*s>", (int)istlen(feat), istptr(feat));
1000 return -2;
1001 }
1002 }
1003
1004 pool_debugging = new_dbg;
Willy Tarreau1408b1f2022-02-18 18:54:40 +01001005 return 1;
1006}
1007
Christopher Faulet6b0a0fb2022-04-04 11:29:28 +02001008/* This function dumps memory usage information onto the conn-stream's
William Lallemande7ed8852016-11-19 02:25:36 +01001009 * read buffer. It returns 0 as long as it does not complete, non-zero upon
1010 * completion. No state is used.
1011 */
1012static int cli_io_handler_dump_pools(struct appctx *appctx)
1013{
Christopher Faulet908628c2022-03-25 16:43:49 +01001014 struct conn_stream *cs = appctx->owner;
William Lallemande7ed8852016-11-19 02:25:36 +01001015
1016 dump_pools_to_trash();
Christopher Faulet908628c2022-03-25 16:43:49 +01001017 if (ci_putchk(cs_ic(cs), &trash) == -1) {
Christopher Fauleta0bdec32022-04-04 07:51:21 +02001018 cs_rx_room_blk(cs);
William Lallemande7ed8852016-11-19 02:25:36 +01001019 return 0;
1020 }
1021 return 1;
1022}
1023
Willy Tarreau7107c8b2018-11-26 11:44:35 +01001024/* callback used to create early pool <name> of size <size> and store the
1025 * resulting pointer into <ptr>. If the allocation fails, it quits with after
1026 * emitting an error message.
1027 */
1028void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
1029{
1030 *ptr = create_pool(name, size, MEM_F_SHARED);
1031 if (!*ptr) {
1032 ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
1033 name, size, strerror(errno));
1034 exit(1);
1035 }
1036}
1037
Willy Tarreau7f0165e2018-11-26 17:09:46 +01001038/* Initializes all per-thread arrays on startup */
1039static void init_pools()
1040{
Willy Tarreau9f3129e2021-04-17 00:31:38 +02001041 int thr;
Willy Tarreau7f0165e2018-11-26 17:09:46 +01001042
1043 for (thr = 0; thr < MAX_THREADS; thr++) {
Willy Tarreaub4e34762021-09-30 19:02:18 +02001044 LIST_INIT(&ha_thread_ctx[thr].pool_lru_head);
Willy Tarreau7f0165e2018-11-26 17:09:46 +01001045 }
Willy Tarreaue9816312022-02-22 16:23:09 +01001046
Willy Tarreau157e3932021-09-15 10:05:48 +02001047 detect_allocator();
Willy Tarreau7f0165e2018-11-26 17:09:46 +01001048}
1049
1050INITCALL0(STG_PREPARE, init_pools);
Willy Tarreau7107c8b2018-11-26 11:44:35 +01001051
Willy Tarreau845b5602021-09-15 10:41:24 +02001052/* Report in build options if trim is supported */
1053static void pools_register_build_options(void)
1054{
1055 if (is_trim_enabled()) {
1056 char *ptr = NULL;
1057 memprintf(&ptr, "Support for malloc_trim() is enabled.");
1058 hap_register_build_opts(ptr, 1);
1059 }
1060}
1061INITCALL0(STG_REGISTER, pools_register_build_options);
1062
William Lallemande7ed8852016-11-19 02:25:36 +01001063/* register cli keywords */
1064static struct cli_kw_list cli_kws = {{ },{
Willy Tarreaub205bfd2021-05-07 11:38:37 +02001065 { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
William Lallemande7ed8852016-11-19 02:25:36 +01001066 {{},}
1067}};
1068
Willy Tarreau0108d902018-11-25 19:14:37 +01001069INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
William Lallemande7ed8852016-11-19 02:25:36 +01001070
Olivier Houcharddc21ff72019-01-29 15:20:16 +01001071
1072/* config parser for global "tune.fail-alloc" */
1073static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
Amaury Denoyelle3b1c9a32021-03-22 11:21:36 +01001074 const struct proxy *defpx, const char *file, int line,
1075 char **err)
Olivier Houcharddc21ff72019-01-29 15:20:16 +01001076{
1077 if (too_many_args(1, args, err, NULL))
1078 return -1;
1079 mem_fail_rate = atoi(args[1]);
1080 if (mem_fail_rate < 0 || mem_fail_rate > 100) {
1081 memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
1082 return -1;
1083 }
1084 return 0;
1085}
Olivier Houcharddc21ff72019-01-29 15:20:16 +01001086
Willy Tarreauc4e56dc2022-03-08 10:41:40 +01001087/* config parser for global "no-memory-trimming" */
1088static int mem_parse_global_no_mem_trim(char **args, int section_type, struct proxy *curpx,
1089 const struct proxy *defpx, const char *file, int line,
1090 char **err)
1091{
1092 if (too_many_args(0, args, err, NULL))
1093 return -1;
1094 disable_trim = 1;
1095 return 0;
1096}
1097
Olivier Houcharddc21ff72019-01-29 15:20:16 +01001098/* register global config keywords */
1099static struct cfg_kw_list mem_cfg_kws = {ILH, {
Olivier Houcharddc21ff72019-01-29 15:20:16 +01001100 { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
Willy Tarreauc4e56dc2022-03-08 10:41:40 +01001101 { CFG_GLOBAL, "no-memory-trimming", mem_parse_global_no_mem_trim },
Olivier Houcharddc21ff72019-01-29 15:20:16 +01001102 { 0, NULL, NULL }
1103}};
1104
1105INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);
1106
Willy Tarreau50e608d2007-05-13 18:26:08 +02001107/*
1108 * Local variables:
1109 * c-indent-level: 8
1110 * c-basic-offset: 8
1111 * End:
1112 */