Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Memory management functions. |
| 3 | * |
| 4 | * Copyright 2000-2007 Willy Tarreau <w@1wt.eu> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
| 10 | * |
| 11 | */ |
Willy Tarreau | 7107c8b | 2018-11-26 11:44:35 +0100 | [diff] [blame] | 12 | #include <errno.h> |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 13 | |
Willy Tarreau | b255105 | 2020-06-09 09:07:15 +0200 | [diff] [blame] | 14 | #include <haproxy/activity-t.h> |
Willy Tarreau | 4c7e4b7 | 2020-05-27 12:58:42 +0200 | [diff] [blame] | 15 | #include <haproxy/api.h> |
Willy Tarreau | b255105 | 2020-06-09 09:07:15 +0200 | [diff] [blame] | 16 | #include <haproxy/applet-t.h> |
Willy Tarreau | 6be7849 | 2020-06-05 00:00:29 +0200 | [diff] [blame] | 17 | #include <haproxy/cfgparse.h> |
Willy Tarreau | f1d32c4 | 2020-06-04 21:07:02 +0200 | [diff] [blame] | 18 | #include <haproxy/channel.h> |
Willy Tarreau | 83487a8 | 2020-06-04 20:19:54 +0200 | [diff] [blame] | 19 | #include <haproxy/cli.h> |
Willy Tarreau | 36979d9 | 2020-06-05 17:27:29 +0200 | [diff] [blame] | 20 | #include <haproxy/errors.h> |
Willy Tarreau | f268ee8 | 2020-06-04 17:05:57 +0200 | [diff] [blame] | 21 | #include <haproxy/global.h> |
Willy Tarreau | 853b297 | 2020-05-27 18:01:47 +0200 | [diff] [blame] | 22 | #include <haproxy/list.h> |
Willy Tarreau | b255105 | 2020-06-09 09:07:15 +0200 | [diff] [blame] | 23 | #include <haproxy/pool.h> |
Willy Tarreau | 2eec9b5 | 2020-06-04 19:58:55 +0200 | [diff] [blame] | 24 | #include <haproxy/stats-t.h> |
Willy Tarreau | 5e539c9 | 2020-06-04 20:45:39 +0200 | [diff] [blame] | 25 | #include <haproxy/stream_interface.h> |
Willy Tarreau | b255105 | 2020-06-09 09:07:15 +0200 | [diff] [blame] | 26 | #include <haproxy/thread.h> |
Willy Tarreau | 48fbcae | 2020-06-03 18:09:46 +0200 | [diff] [blame] | 27 | #include <haproxy/tools.h> |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 28 | |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 29 | |
Willy Tarreau | 2d6f628 | 2021-04-15 16:24:00 +0200 | [diff] [blame] | 30 | #ifdef CONFIG_HAP_POOLS |
Willy Tarreau | 7f0165e | 2018-11-26 17:09:46 +0100 | [diff] [blame] | 31 | /* These ones are initialized per-thread on startup by init_pools() */ |
Willy Tarreau | e18db9e | 2018-10-16 10:28:54 +0200 | [diff] [blame] | 32 | THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */ |
| 33 | THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */ |
Willy Tarreau | ed891fd | 2020-06-01 19:00:28 +0200 | [diff] [blame] | 34 | #endif |
Willy Tarreau | e18db9e | 2018-10-16 10:28:54 +0200 | [diff] [blame] | 35 | |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 36 | static struct list pools = LIST_HEAD_INIT(pools); |
Willy Tarreau | 067ac9f | 2015-10-08 14:12:13 +0200 | [diff] [blame] | 37 | int mem_poison_byte = -1; |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 38 | |
Olivier Houchard | dc21ff7 | 2019-01-29 15:20:16 +0100 | [diff] [blame] | 39 | #ifdef DEBUG_FAIL_ALLOC |
| 40 | static int mem_fail_rate = 0; |
Olivier Houchard | dc21ff7 | 2019-01-29 15:20:16 +0100 | [diff] [blame] | 41 | #endif |
| 42 | |
Willy Tarreau | 33298de | 2021-09-15 10:38:21 +0200 | [diff] [blame] | 43 | #if defined(HA_HAVE_MALLOC_TRIM) |
Willy Tarreau | 562bf20 | 2021-09-15 10:05:48 +0200 | [diff] [blame] | 44 | static int using_libc_allocator = 0; |
| 45 | |
Willy Tarreau | 619a511 | 2021-12-23 09:26:30 +0100 | [diff] [blame] | 46 | /* ask the allocator to trim memory pools. |
| 47 | * This must run under thread isolation so that competing threads trying to |
| 48 | * allocate or release memory do not prevent the allocator from completing |
| 49 | * its job. We just have to be careful as callers might already be isolated |
| 50 | * themselves. |
| 51 | */ |
Willy Tarreau | 33298de | 2021-09-15 10:38:21 +0200 | [diff] [blame] | 52 | static void trim_all_pools(void) |
| 53 | { |
Willy Tarreau | 619a511 | 2021-12-23 09:26:30 +0100 | [diff] [blame] | 54 | int isolated = thread_isolated(); |
| 55 | |
| 56 | if (!isolated) |
| 57 | thread_isolate(); |
| 58 | |
Willy Tarreau | 33298de | 2021-09-15 10:38:21 +0200 | [diff] [blame] | 59 | if (using_libc_allocator) |
| 60 | malloc_trim(0); |
Willy Tarreau | 619a511 | 2021-12-23 09:26:30 +0100 | [diff] [blame] | 61 | |
| 62 | if (!isolated) |
| 63 | thread_release(); |
Willy Tarreau | 33298de | 2021-09-15 10:38:21 +0200 | [diff] [blame] | 64 | } |
| 65 | |
Willy Tarreau | 562bf20 | 2021-09-15 10:05:48 +0200 | [diff] [blame] | 66 | /* check if we're using the same allocator as the one that provides |
| 67 | * malloc_trim() and mallinfo(). The principle is that on glibc, both |
| 68 | * malloc_trim() and mallinfo() are provided, and using mallinfo() we |
| 69 | * can check if malloc() is performed through glibc or any other one |
| 70 | * the executable was linked against (e.g. jemalloc). |
| 71 | */ |
| 72 | static void detect_allocator(void) |
| 73 | { |
Willy Tarreau | 3bf7569 | 2021-09-16 09:18:21 +0200 | [diff] [blame] | 74 | #ifdef HA_HAVE_MALLINFO2 |
| 75 | struct mallinfo2 mi1, mi2; |
| 76 | #else |
Willy Tarreau | 562bf20 | 2021-09-15 10:05:48 +0200 | [diff] [blame] | 77 | struct mallinfo mi1, mi2; |
Willy Tarreau | 3bf7569 | 2021-09-16 09:18:21 +0200 | [diff] [blame] | 78 | #endif |
Willy Tarreau | 562bf20 | 2021-09-15 10:05:48 +0200 | [diff] [blame] | 79 | void *ptr; |
| 80 | |
Willy Tarreau | 3bf7569 | 2021-09-16 09:18:21 +0200 | [diff] [blame] | 81 | #ifdef HA_HAVE_MALLINFO2 |
| 82 | mi1 = mallinfo2(); |
| 83 | #else |
Willy Tarreau | 562bf20 | 2021-09-15 10:05:48 +0200 | [diff] [blame] | 84 | mi1 = mallinfo(); |
Willy Tarreau | 3bf7569 | 2021-09-16 09:18:21 +0200 | [diff] [blame] | 85 | #endif |
Willy Tarreau | 562bf20 | 2021-09-15 10:05:48 +0200 | [diff] [blame] | 86 | ptr = DISGUISE(malloc(1)); |
Willy Tarreau | 3bf7569 | 2021-09-16 09:18:21 +0200 | [diff] [blame] | 87 | #ifdef HA_HAVE_MALLINFO2 |
| 88 | mi2 = mallinfo2(); |
| 89 | #else |
Willy Tarreau | 562bf20 | 2021-09-15 10:05:48 +0200 | [diff] [blame] | 90 | mi2 = mallinfo(); |
Willy Tarreau | 3bf7569 | 2021-09-16 09:18:21 +0200 | [diff] [blame] | 91 | #endif |
Willy Tarreau | 562bf20 | 2021-09-15 10:05:48 +0200 | [diff] [blame] | 92 | free(DISGUISE(ptr)); |
| 93 | |
| 94 | using_libc_allocator = !!memcmp(&mi1, &mi2, sizeof(mi1)); |
| 95 | } |
Willy Tarreau | 33298de | 2021-09-15 10:38:21 +0200 | [diff] [blame] | 96 | #else |
| 97 | |
| 98 | static void trim_all_pools(void) |
| 99 | { |
| 100 | } |
Willy Tarreau | 562bf20 | 2021-09-15 10:05:48 +0200 | [diff] [blame] | 101 | |
| 102 | static void detect_allocator(void) |
| 103 | { |
| 104 | } |
Willy Tarreau | 33298de | 2021-09-15 10:38:21 +0200 | [diff] [blame] | 105 | #endif |
| 106 | |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 107 | /* Try to find an existing shared pool with the same characteristics and |
| 108 | * returns it, otherwise creates this one. NULL is returned if no memory |
Willy Tarreau | 581bf81 | 2016-01-25 02:19:13 +0100 | [diff] [blame] | 109 | * is available for a new creation. Two flags are supported : |
| 110 | * - MEM_F_SHARED to indicate that the pool may be shared with other users |
| 111 | * - MEM_F_EXACT to indicate that the size must not be rounded up |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 112 | */ |
| 113 | struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags) |
| 114 | { |
| 115 | struct pool_head *pool; |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 116 | struct pool_head *entry; |
| 117 | struct list *start; |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 118 | unsigned int align; |
Willy Tarreau | 9f3129e | 2021-04-17 00:31:38 +0200 | [diff] [blame] | 119 | int thr __maybe_unused; |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 120 | |
Willy Tarreau | ac42111 | 2015-10-28 15:09:29 +0100 | [diff] [blame] | 121 | /* We need to store a (void *) at the end of the chunks. Since we know |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 122 | * that the malloc() function will never return such a small size, |
| 123 | * let's round the size up to something slightly bigger, in order to |
| 124 | * ease merging of entries. Note that the rounding is a power of two. |
Willy Tarreau | ac42111 | 2015-10-28 15:09:29 +0100 | [diff] [blame] | 125 | * This extra (void *) is not accounted for in the size computation |
| 126 | * so that the visible parts outside are not affected. |
Willy Tarreau | 30f931e | 2018-10-23 14:40:23 +0200 | [diff] [blame] | 127 | * |
| 128 | * Note: for the LRU cache, we need to store 2 doubly-linked lists. |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 129 | */ |
| 130 | |
Willy Tarreau | 581bf81 | 2016-01-25 02:19:13 +0100 | [diff] [blame] | 131 | if (!(flags & MEM_F_EXACT)) { |
Willy Tarreau | 30f931e | 2018-10-23 14:40:23 +0200 | [diff] [blame] | 132 | align = 4 * sizeof(void *); // 2 lists = 4 pointers min |
Willy Tarreau | 581bf81 | 2016-01-25 02:19:13 +0100 | [diff] [blame] | 133 | size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA; |
| 134 | } |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 135 | |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 136 | /* TODO: thread: we do not lock pool list for now because all pools are |
| 137 | * created during HAProxy startup (so before threads creation) */ |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 138 | start = &pools; |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 139 | pool = NULL; |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 140 | |
| 141 | list_for_each_entry(entry, &pools, list) { |
| 142 | if (entry->size == size) { |
| 143 | /* either we can share this place and we take it, or |
Ilya Shipitsin | 47d1718 | 2020-06-21 21:42:57 +0500 | [diff] [blame] | 144 | * we look for a shareable one or for the next position |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 145 | * before which we will insert a new one. |
| 146 | */ |
Willy Tarreau | 1ab6c0b | 2021-05-05 07:29:01 +0200 | [diff] [blame] | 147 | if ((flags & entry->flags & MEM_F_SHARED) |
| 148 | #ifdef DEBUG_DONT_SHARE_POOLS |
| 149 | && strcmp(name, entry->name) == 0 |
| 150 | #endif |
| 151 | ) { |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 152 | /* we can share this one */ |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 153 | pool = entry; |
Krzysztof Piotr Oledzki | a643baf | 2008-05-29 23:53:44 +0200 | [diff] [blame] | 154 | DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name); |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 155 | break; |
| 156 | } |
| 157 | } |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 158 | else if (entry->size > size) { |
| 159 | /* insert before this one */ |
| 160 | start = &entry->list; |
| 161 | break; |
| 162 | } |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 163 | } |
| 164 | |
| 165 | if (!pool) { |
Willy Tarreau | 0a93b64 | 2018-10-16 07:58:39 +0200 | [diff] [blame] | 166 | if (!pool) |
| 167 | pool = calloc(1, sizeof(*pool)); |
| 168 | |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 169 | if (!pool) |
| 170 | return NULL; |
| 171 | if (name) |
| 172 | strlcpy2(pool->name, name, sizeof(pool->name)); |
| 173 | pool->size = size; |
| 174 | pool->flags = flags; |
Willy Tarreau | 2b71810 | 2021-04-21 07:32:39 +0200 | [diff] [blame] | 175 | LIST_APPEND(start, &pool->list); |
Christopher Faulet | 2f6d3c0 | 2019-06-25 21:45:59 +0200 | [diff] [blame] | 176 | |
Willy Tarreau | 2d6f628 | 2021-04-15 16:24:00 +0200 | [diff] [blame] | 177 | #ifdef CONFIG_HAP_POOLS |
Christopher Faulet | 2f6d3c0 | 2019-06-25 21:45:59 +0200 | [diff] [blame] | 178 | /* update per-thread pool cache if necessary */ |
Willy Tarreau | 9f3129e | 2021-04-17 00:31:38 +0200 | [diff] [blame] | 179 | for (thr = 0; thr < MAX_THREADS; thr++) { |
| 180 | LIST_INIT(&pool->cache[thr].list); |
Willy Tarreau | 674843a | 2022-02-09 16:33:22 +0100 | [diff] [blame] | 181 | pool->cache[thr].tid = thr; |
| 182 | pool->cache[thr].pool = pool; |
Christopher Faulet | 2f6d3c0 | 2019-06-25 21:45:59 +0200 | [diff] [blame] | 183 | } |
Willy Tarreau | ed891fd | 2020-06-01 19:00:28 +0200 | [diff] [blame] | 184 | #endif |
Olivier Houchard | 8af97eb | 2020-02-01 17:45:32 +0100 | [diff] [blame] | 185 | HA_SPIN_INIT(&pool->lock); |
Olivier Houchard | 8af97eb | 2020-02-01 17:45:32 +0100 | [diff] [blame] | 186 | } |
| 187 | pool->users++; |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 188 | return pool; |
| 189 | } |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 190 | |
Willy Tarreau | 1384364 | 2021-04-17 16:57:25 +0200 | [diff] [blame] | 191 | /* Tries to allocate an object for the pool <pool> using the system's allocator |
| 192 | * and directly returns it. The pool's allocated counter is checked and updated, |
| 193 | * but no other checks are performed. The pool's lock is not used and is not a |
| 194 | * problem either. |
| 195 | */ |
| 196 | void *pool_get_from_os(struct pool_head *pool) |
| 197 | { |
| 198 | if (!pool->limit || pool->allocated < pool->limit) { |
| 199 | void *ptr = pool_alloc_area(pool->size + POOL_EXTRA); |
| 200 | if (ptr) { |
| 201 | _HA_ATOMIC_INC(&pool->allocated); |
| 202 | return ptr; |
| 203 | } |
| 204 | _HA_ATOMIC_INC(&pool->failed); |
| 205 | } |
| 206 | activity[tid].pool_fail++; |
| 207 | return NULL; |
| 208 | |
| 209 | } |
| 210 | |
Willy Tarreau | 45e4e28 | 2021-04-17 17:48:40 +0200 | [diff] [blame] | 211 | /* Releases a pool item back to the operating system and atomically updates |
| 212 | * the allocation counter. |
| 213 | */ |
| 214 | void pool_put_to_os(struct pool_head *pool, void *ptr) |
| 215 | { |
Willy Tarreau | 11e7af3 | 2021-06-10 17:20:19 +0200 | [diff] [blame] | 216 | #ifdef DEBUG_UAF |
| 217 | /* This object will be released for real in order to detect a use after |
| 218 | * free. We also force a write to the area to ensure we crash on double |
| 219 | * free or free of a const area. |
| 220 | */ |
| 221 | *(uint32_t *)ptr = 0xDEADADD4; |
| 222 | #endif /* DEBUG_UAF */ |
| 223 | |
Willy Tarreau | 45e4e28 | 2021-04-17 17:48:40 +0200 | [diff] [blame] | 224 | pool_free_area(ptr, pool->size + POOL_EXTRA); |
| 225 | _HA_ATOMIC_DEC(&pool->allocated); |
| 226 | } |
| 227 | |
Willy Tarreau | 8fe726f | 2021-04-15 18:20:12 +0200 | [diff] [blame] | 228 | /* Tries to allocate an object for the pool <pool> using the system's allocator |
| 229 | * and directly returns it. The pool's counters are updated but the object is |
| 230 | * never cached, so this is usable with and without local or shared caches. |
| 231 | * This may be called with or without the pool lock held, so it must not use |
| 232 | * the pool's lock. |
| 233 | */ |
| 234 | void *pool_alloc_nocache(struct pool_head *pool) |
Willy Tarreau | 0bae075 | 2021-03-02 20:05:09 +0100 | [diff] [blame] | 235 | { |
Willy Tarreau | 0bae075 | 2021-03-02 20:05:09 +0100 | [diff] [blame] | 236 | void *ptr = NULL; |
| 237 | |
Willy Tarreau | 1384364 | 2021-04-17 16:57:25 +0200 | [diff] [blame] | 238 | ptr = pool_get_from_os(pool); |
| 239 | if (!ptr) |
Willy Tarreau | 0bae075 | 2021-03-02 20:05:09 +0100 | [diff] [blame] | 240 | return NULL; |
Willy Tarreau | 0bae075 | 2021-03-02 20:05:09 +0100 | [diff] [blame] | 241 | |
Willy Tarreau | 1384364 | 2021-04-17 16:57:25 +0200 | [diff] [blame] | 242 | swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used, POOL_AVG_SAMPLES/4); |
Willy Tarreau | 4781b15 | 2021-04-06 13:53:36 +0200 | [diff] [blame] | 243 | _HA_ATOMIC_INC(&pool->used); |
Willy Tarreau | 0bae075 | 2021-03-02 20:05:09 +0100 | [diff] [blame] | 244 | |
| 245 | #ifdef DEBUG_MEMORY_POOLS |
| 246 | /* keep track of where the element was allocated from */ |
| 247 | *POOL_LINK(pool, ptr) = (void *)pool; |
| 248 | #endif |
| 249 | return ptr; |
| 250 | } |
| 251 | |
Willy Tarreau | 45e4e28 | 2021-04-17 17:48:40 +0200 | [diff] [blame] | 252 | /* Release a pool item back to the OS and keeps the pool's counters up to date. |
| 253 | * This is always defined even when pools are not enabled (their usage stats |
| 254 | * are maintained). |
| 255 | */ |
| 256 | void pool_free_nocache(struct pool_head *pool, void *ptr) |
| 257 | { |
| 258 | _HA_ATOMIC_DEC(&pool->used); |
| 259 | swrate_add(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used); |
| 260 | pool_put_to_os(pool, ptr); |
| 261 | } |
| 262 | |
Willy Tarreau | b8498e9 | 2021-04-18 10:23:02 +0200 | [diff] [blame] | 263 | |
| 264 | #ifdef CONFIG_HAP_POOLS |
| 265 | |
Willy Tarreau | 8721203 | 2021-04-19 08:14:03 +0200 | [diff] [blame] | 266 | /* Evicts some of the oldest objects from one local cache, until its number of |
| 267 | * objects is no more than 16+1/8 of the total number of locally cached objects |
| 268 | * or the total size of the local cache is no more than 75% of its maximum (i.e. |
| 269 | * we don't want a single cache to use all the cache for itself). For this, the |
| 270 | * list is scanned in reverse. |
| 271 | */ |
Willy Tarreau | 9043da7 | 2022-02-09 16:19:24 +0100 | [diff] [blame] | 272 | void pool_evict_from_local_cache(struct pool_head *pool, int full) |
Willy Tarreau | 8721203 | 2021-04-19 08:14:03 +0200 | [diff] [blame] | 273 | { |
| 274 | struct pool_cache_head *ph = &pool->cache[tid]; |
| 275 | struct pool_cache_item *item; |
Willy Tarreau | 8721203 | 2021-04-19 08:14:03 +0200 | [diff] [blame] | 276 | |
Willy Tarreau | 9043da7 | 2022-02-09 16:19:24 +0100 | [diff] [blame] | 277 | while ((ph->count && full) || |
| 278 | (ph->count >= 16 + pool_cache_count / 8 && |
| 279 | pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4)) { |
Willy Tarreau | 8721203 | 2021-04-19 08:14:03 +0200 | [diff] [blame] | 280 | item = LIST_NEXT(&ph->list, typeof(item), by_pool); |
Willy Tarreau | 9a3cd53 | 2022-02-09 16:23:55 +0100 | [diff] [blame] | 281 | BUG_ON(&item->by_pool == &ph->list); |
Willy Tarreau | 8721203 | 2021-04-19 08:14:03 +0200 | [diff] [blame] | 282 | ph->count--; |
| 283 | pool_cache_bytes -= pool->size; |
| 284 | pool_cache_count--; |
Willy Tarreau | c241843 | 2022-01-21 19:00:25 +0100 | [diff] [blame] | 285 | pool_check_pattern(ph, item, pool->size); |
Willy Tarreau | 2b71810 | 2021-04-21 07:32:39 +0200 | [diff] [blame] | 286 | LIST_DELETE(&item->by_pool); |
| 287 | LIST_DELETE(&item->by_lru); |
Willy Tarreau | 8721203 | 2021-04-19 08:14:03 +0200 | [diff] [blame] | 288 | pool_put_to_shared_cache(pool, item); |
| 289 | } |
| 290 | } |
| 291 | |
Willy Tarreau | b8498e9 | 2021-04-18 10:23:02 +0200 | [diff] [blame] | 292 | /* Evicts some of the oldest objects from the local cache, pushing them to the |
| 293 | * global pool. |
| 294 | */ |
| 295 | void pool_evict_from_local_caches() |
| 296 | { |
| 297 | struct pool_cache_item *item; |
| 298 | struct pool_cache_head *ph; |
| 299 | struct pool_head *pool; |
| 300 | |
| 301 | do { |
| 302 | item = LIST_PREV(&ti->pool_lru_head, struct pool_cache_item *, by_lru); |
Willy Tarreau | 9a3cd53 | 2022-02-09 16:23:55 +0100 | [diff] [blame] | 303 | BUG_ON(&item->by_lru == &ti->pool_lru_head); |
Willy Tarreau | b8498e9 | 2021-04-18 10:23:02 +0200 | [diff] [blame] | 304 | /* note: by definition we remove oldest objects so they also are the |
| 305 | * oldest in their own pools, thus their next is the pool's head. |
| 306 | */ |
| 307 | ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list); |
Willy Tarreau | 674843a | 2022-02-09 16:33:22 +0100 | [diff] [blame] | 308 | BUG_ON(ph->tid != tid); |
| 309 | |
Willy Tarreau | b8498e9 | 2021-04-18 10:23:02 +0200 | [diff] [blame] | 310 | pool = container_of(ph - tid, struct pool_head, cache); |
Willy Tarreau | 674843a | 2022-02-09 16:33:22 +0100 | [diff] [blame] | 311 | BUG_ON(pool != ph->pool); |
| 312 | |
Willy Tarreau | c241843 | 2022-01-21 19:00:25 +0100 | [diff] [blame] | 313 | pool_check_pattern(ph, item, pool->size); |
Willy Tarreau | 2b71810 | 2021-04-21 07:32:39 +0200 | [diff] [blame] | 314 | LIST_DELETE(&item->by_pool); |
| 315 | LIST_DELETE(&item->by_lru); |
Willy Tarreau | b8498e9 | 2021-04-18 10:23:02 +0200 | [diff] [blame] | 316 | ph->count--; |
| 317 | pool_cache_count--; |
| 318 | pool_cache_bytes -= pool->size; |
| 319 | pool_put_to_shared_cache(pool, item); |
| 320 | } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8); |
| 321 | } |
Willy Tarreau | 0bae075 | 2021-03-02 20:05:09 +0100 | [diff] [blame] | 322 | |
Willy Tarreau | b2a853d | 2021-04-19 11:49:26 +0200 | [diff] [blame] | 323 | /* Frees an object to the local cache, possibly pushing oldest objects to the |
| 324 | * shared cache, which itself may decide to release some of them to the OS. |
| 325 | * While it is unspecified what the object becomes past this point, it is |
| 326 | * guaranteed to be released from the users' perpective. |
| 327 | */ |
| 328 | void pool_put_to_cache(struct pool_head *pool, void *ptr) |
| 329 | { |
| 330 | struct pool_cache_item *item = (struct pool_cache_item *)ptr; |
| 331 | struct pool_cache_head *ph = &pool->cache[tid]; |
| 332 | |
Willy Tarreau | 2b71810 | 2021-04-21 07:32:39 +0200 | [diff] [blame] | 333 | LIST_INSERT(&ph->list, &item->by_pool); |
| 334 | LIST_INSERT(&ti->pool_lru_head, &item->by_lru); |
Willy Tarreau | b2a853d | 2021-04-19 11:49:26 +0200 | [diff] [blame] | 335 | ph->count++; |
Willy Tarreau | c241843 | 2022-01-21 19:00:25 +0100 | [diff] [blame] | 336 | pool_fill_pattern(ph, item, pool->size); |
Willy Tarreau | b2a853d | 2021-04-19 11:49:26 +0200 | [diff] [blame] | 337 | pool_cache_count++; |
| 338 | pool_cache_bytes += pool->size; |
| 339 | |
| 340 | if (unlikely(pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4)) { |
| 341 | if (ph->count >= 16 + pool_cache_count / 8) |
Willy Tarreau | 9043da7 | 2022-02-09 16:19:24 +0100 | [diff] [blame] | 342 | pool_evict_from_local_cache(pool, 0); |
Willy Tarreau | b2a853d | 2021-04-19 11:49:26 +0200 | [diff] [blame] | 343 | if (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE) |
| 344 | pool_evict_from_local_caches(); |
| 345 | } |
| 346 | } |
| 347 | |
Willy Tarreau | eb3cc29 | 2021-04-15 18:13:13 +0200 | [diff] [blame] | 348 | #if defined(CONFIG_HAP_NO_GLOBAL_POOLS) |
| 349 | |
Willy Tarreau | 0bae075 | 2021-03-02 20:05:09 +0100 | [diff] [blame] | 350 | /* legacy stuff */ |
| 351 | void pool_flush(struct pool_head *pool) |
| 352 | { |
| 353 | } |
| 354 | |
| 355 | /* This function might ask the malloc library to trim its buffers. */ |
| 356 | void pool_gc(struct pool_head *pool_ctx) |
| 357 | { |
Willy Tarreau | 33298de | 2021-09-15 10:38:21 +0200 | [diff] [blame] | 358 | trim_all_pools(); |
Willy Tarreau | 0bae075 | 2021-03-02 20:05:09 +0100 | [diff] [blame] | 359 | } |
| 360 | |
Willy Tarreau | a206cf9 | 2021-06-10 10:21:35 +0200 | [diff] [blame] | 361 | #else /* CONFIG_HAP_NO_GLOBAL_POOLS */ |
| 362 | |
| 363 | #if defined(CONFIG_HAP_LOCKLESS_POOLS) |
Willy Tarreau | 0bae075 | 2021-03-02 20:05:09 +0100 | [diff] [blame] | 364 | |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 365 | /* |
| 366 | * This function frees whatever can be freed in pool <pool>. |
| 367 | */ |
| 368 | void pool_flush(struct pool_head *pool) |
| 369 | { |
Willy Tarreau | d0cc376 | 2021-06-09 18:59:58 +0200 | [diff] [blame] | 370 | void *next, *temp; |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 371 | |
| 372 | if (!pool) |
| 373 | return; |
Willy Tarreau | d0cc376 | 2021-06-09 18:59:58 +0200 | [diff] [blame] | 374 | |
| 375 | /* The loop below atomically detaches the head of the free list and |
| 376 | * replaces it with a NULL. Then the list can be released. |
| 377 | */ |
| 378 | next = pool->free_list; |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 379 | do { |
Willy Tarreau | d0cc376 | 2021-06-09 18:59:58 +0200 | [diff] [blame] | 380 | while (unlikely(next == POOL_BUSY)) { |
| 381 | __ha_cpu_relax(); |
| 382 | next = _HA_ATOMIC_LOAD(&pool->free_list); |
| 383 | } |
| 384 | if (next == NULL) |
| 385 | return; |
| 386 | } while (unlikely((next = _HA_ATOMIC_XCHG(&pool->free_list, POOL_BUSY)) == POOL_BUSY)); |
| 387 | _HA_ATOMIC_STORE(&pool->free_list, NULL); |
Olivier Houchard | 2087276 | 2019-03-08 18:53:35 +0100 | [diff] [blame] | 388 | __ha_barrier_atomic_store(); |
Willy Tarreau | d0cc376 | 2021-06-09 18:59:58 +0200 | [diff] [blame] | 389 | |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 390 | while (next) { |
| 391 | temp = next; |
| 392 | next = *POOL_LINK(pool, temp); |
Willy Tarreau | 45e4e28 | 2021-04-17 17:48:40 +0200 | [diff] [blame] | 393 | pool_put_to_os(pool, temp); |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 394 | } |
Willy Tarreau | caf6555 | 2021-06-10 06:54:22 +0200 | [diff] [blame] | 395 | /* here, we should have pool->allocated == pool->used */ |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 396 | } |
| 397 | |
Willy Tarreau | f161d0f | 2018-02-22 14:05:55 +0100 | [diff] [blame] | 398 | #else /* CONFIG_HAP_LOCKLESS_POOLS */ |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 399 | |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 400 | /* |
| 401 | * This function frees whatever can be freed in pool <pool>. |
| 402 | */ |
Willy Tarreau | bafbe01 | 2017-11-24 17:34:44 +0100 | [diff] [blame] | 403 | void pool_flush(struct pool_head *pool) |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 404 | { |
Willy Tarreau | af8120a | 2021-06-10 07:13:04 +0200 | [diff] [blame] | 405 | void *temp, **next; |
Willy Tarreau | 3e853ea | 2019-07-04 11:30:00 +0200 | [diff] [blame] | 406 | |
Willy Tarreau | 4d2d098 | 2007-05-14 00:39:29 +0200 | [diff] [blame] | 407 | if (!pool) |
| 408 | return; |
| 409 | |
Willy Tarreau | af8120a | 2021-06-10 07:13:04 +0200 | [diff] [blame] | 410 | HA_SPIN_LOCK(POOL_LOCK, &pool->lock); |
| 411 | next = pool->free_list; |
| 412 | pool->free_list = NULL; |
| 413 | HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock); |
| 414 | |
| 415 | while (next) { |
| 416 | temp = next; |
| 417 | next = *POOL_LINK(pool, temp); |
Willy Tarreau | 45e4e28 | 2021-04-17 17:48:40 +0200 | [diff] [blame] | 418 | pool_put_to_os(pool, temp); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 419 | } |
Willy Tarreau | 3e853ea | 2019-07-04 11:30:00 +0200 | [diff] [blame] | 420 | /* here, we should have pool->allocated == pool->used */ |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 421 | } |
| 422 | |
Willy Tarreau | a206cf9 | 2021-06-10 10:21:35 +0200 | [diff] [blame] | 423 | #endif /* CONFIG_HAP_LOCKLESS_POOLS */ |
| 424 | |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 425 | /* |
| 426 | * This function frees whatever can be freed in all pools, but respecting |
Willy Tarreau | c0e2ff2 | 2020-04-24 06:15:24 +0200 | [diff] [blame] | 427 | * the minimum thresholds imposed by owners. It makes sure to be alone to |
| 428 | * run by using thread_isolate(). <pool_ctx> is unused. |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 429 | */ |
Willy Tarreau | bafbe01 | 2017-11-24 17:34:44 +0100 | [diff] [blame] | 430 | void pool_gc(struct pool_head *pool_ctx) |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 431 | { |
| 432 | struct pool_head *entry; |
Willy Tarreau | c0e2ff2 | 2020-04-24 06:15:24 +0200 | [diff] [blame] | 433 | int isolated = thread_isolated(); |
Willy Tarreau | b7f9d12 | 2009-04-21 02:17:45 +0200 | [diff] [blame] | 434 | |
Willy Tarreau | c0e2ff2 | 2020-04-24 06:15:24 +0200 | [diff] [blame] | 435 | if (!isolated) |
| 436 | thread_isolate(); |
Willy Tarreau | b7f9d12 | 2009-04-21 02:17:45 +0200 | [diff] [blame] | 437 | |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 438 | list_for_each_entry(entry, &pools, list) { |
Olivier Houchard | 51d9339 | 2020-03-12 19:05:39 +0100 | [diff] [blame] | 439 | void *temp; |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 440 | //qfprintf(stderr, "Flushing pool %s\n", entry->name); |
Olivier Houchard | 51d9339 | 2020-03-12 19:05:39 +0100 | [diff] [blame] | 441 | while (entry->free_list && |
Willy Tarreau | 57767b8 | 2014-12-22 21:40:55 +0100 | [diff] [blame] | 442 | (int)(entry->allocated - entry->used) > (int)entry->minavail) { |
Olivier Houchard | 51d9339 | 2020-03-12 19:05:39 +0100 | [diff] [blame] | 443 | temp = entry->free_list; |
| 444 | entry->free_list = *POOL_LINK(entry, temp); |
Willy Tarreau | 45e4e28 | 2021-04-17 17:48:40 +0200 | [diff] [blame] | 445 | pool_put_to_os(entry, temp); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 446 | } |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 447 | } |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 448 | |
Willy Tarreau | 33298de | 2021-09-15 10:38:21 +0200 | [diff] [blame] | 449 | trim_all_pools(); |
Willy Tarreau | 46b515c | 2021-06-10 08:40:16 +0200 | [diff] [blame] | 450 | |
Willy Tarreau | c0e2ff2 | 2020-04-24 06:15:24 +0200 | [diff] [blame] | 451 | if (!isolated) |
| 452 | thread_release(); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 453 | } |
Willy Tarreau | a206cf9 | 2021-06-10 10:21:35 +0200 | [diff] [blame] | 454 | #endif /* CONFIG_HAP_NO_GLOBAL_POOLS */ |
Willy Tarreau | b8498e9 | 2021-04-18 10:23:02 +0200 | [diff] [blame] | 455 | |
| 456 | #else /* CONFIG_HAP_POOLS */ |
| 457 | |
| 458 | /* legacy stuff */ |
| 459 | void pool_flush(struct pool_head *pool) |
| 460 | { |
| 461 | } |
| 462 | |
| 463 | /* This function might ask the malloc library to trim its buffers. */ |
| 464 | void pool_gc(struct pool_head *pool_ctx) |
| 465 | { |
Willy Tarreau | 33298de | 2021-09-15 10:38:21 +0200 | [diff] [blame] | 466 | trim_all_pools(); |
Willy Tarreau | b8498e9 | 2021-04-18 10:23:02 +0200 | [diff] [blame] | 467 | } |
| 468 | |
| 469 | #endif /* CONFIG_HAP_POOLS */ |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 470 | |
| 471 | /* |
Willy Tarreau | dae4aa8 | 2007-06-16 23:19:53 +0200 | [diff] [blame] | 472 | * This function destroys a pool by freeing it completely, unless it's still |
| 473 | * in use. This should be called only under extreme circumstances. It always |
| 474 | * returns NULL if the resulting pool is empty, easing the clearing of the old |
| 475 | * pointer, otherwise it returns the pool. |
| 476 | * . |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 477 | */ |
Willy Tarreau | bafbe01 | 2017-11-24 17:34:44 +0100 | [diff] [blame] | 478 | void *pool_destroy(struct pool_head *pool) |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 479 | { |
Willy Tarreau | 4d2d098 | 2007-05-14 00:39:29 +0200 | [diff] [blame] | 480 | if (pool) { |
Willy Tarreau | 9043da7 | 2022-02-09 16:19:24 +0100 | [diff] [blame] | 481 | #ifdef CONFIG_HAP_POOLS |
| 482 | pool_evict_from_local_cache(pool, 1); |
| 483 | #endif |
Willy Tarreau | bafbe01 | 2017-11-24 17:34:44 +0100 | [diff] [blame] | 484 | pool_flush(pool); |
Willy Tarreau | dae4aa8 | 2007-06-16 23:19:53 +0200 | [diff] [blame] | 485 | if (pool->used) |
| 486 | return pool; |
| 487 | pool->users--; |
| 488 | if (!pool->users) { |
Willy Tarreau | 2b71810 | 2021-04-21 07:32:39 +0200 | [diff] [blame] | 489 | LIST_DELETE(&pool->list); |
Willy Tarreau | f161d0f | 2018-02-22 14:05:55 +0100 | [diff] [blame] | 490 | #ifndef CONFIG_HAP_LOCKLESS_POOLS |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 491 | HA_SPIN_DESTROY(&pool->lock); |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 492 | #endif |
Willy Tarreau | 9f3129e | 2021-04-17 00:31:38 +0200 | [diff] [blame] | 493 | /* note that if used == 0, the cache is empty */ |
| 494 | free(pool); |
Willy Tarreau | dae4aa8 | 2007-06-16 23:19:53 +0200 | [diff] [blame] | 495 | } |
Willy Tarreau | 4d2d098 | 2007-05-14 00:39:29 +0200 | [diff] [blame] | 496 | } |
| 497 | return NULL; |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 498 | } |
| 499 | |
Willy Tarreau | 2455ceb | 2018-11-26 15:57:34 +0100 | [diff] [blame] | 500 | /* This destroys all pools on exit. It is *not* thread safe. */ |
| 501 | void pool_destroy_all() |
| 502 | { |
| 503 | struct pool_head *entry, *back; |
| 504 | |
| 505 | list_for_each_entry_safe(entry, back, &pools, list) |
| 506 | pool_destroy(entry); |
| 507 | } |
| 508 | |
Willy Tarreau | 12833bb | 2014-01-28 16:49:56 +0100 | [diff] [blame] | 509 | /* This function dumps memory usage information into the trash buffer. */ |
| 510 | void dump_pools_to_trash() |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 511 | { |
| 512 | struct pool_head *entry; |
| 513 | unsigned long allocated, used; |
| 514 | int nbpools; |
| 515 | |
| 516 | allocated = used = nbpools = 0; |
Willy Tarreau | 12833bb | 2014-01-28 16:49:56 +0100 | [diff] [blame] | 517 | chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n"); |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 518 | list_for_each_entry(entry, &pools, list) { |
Willy Tarreau | f161d0f | 2018-02-22 14:05:55 +0100 | [diff] [blame] | 519 | #ifndef CONFIG_HAP_LOCKLESS_POOLS |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 520 | HA_SPIN_LOCK(POOL_LOCK, &entry->lock); |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 521 | #endif |
Willy Tarreau | 9f3129e | 2021-04-17 00:31:38 +0200 | [diff] [blame] | 522 | chunk_appendf(&trash, " - Pool %s (%u bytes) : %u allocated (%u bytes), %u used, needed_avg %u, %u failures, %u users, @%p%s\n", |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 523 | entry->name, entry->size, entry->allocated, |
Willy Tarreau | a1e4f8c | 2020-05-08 08:31:56 +0200 | [diff] [blame] | 524 | entry->size * entry->allocated, entry->used, |
Willy Tarreau | 606135a | 2020-06-01 12:35:03 +0200 | [diff] [blame] | 525 | swrate_avg(entry->needed_avg, POOL_AVG_SAMPLES), entry->failed, |
Willy Tarreau | 9f3129e | 2021-04-17 00:31:38 +0200 | [diff] [blame] | 526 | entry->users, entry, |
Willy Tarreau | 0a93b64 | 2018-10-16 07:58:39 +0200 | [diff] [blame] | 527 | (entry->flags & MEM_F_SHARED) ? " [SHARED]" : ""); |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 528 | |
| 529 | allocated += entry->allocated * entry->size; |
| 530 | used += entry->used * entry->size; |
| 531 | nbpools++; |
Willy Tarreau | f161d0f | 2018-02-22 14:05:55 +0100 | [diff] [blame] | 532 | #ifndef CONFIG_HAP_LOCKLESS_POOLS |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 533 | HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock); |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 534 | #endif |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 535 | } |
Willy Tarreau | 12833bb | 2014-01-28 16:49:56 +0100 | [diff] [blame] | 536 | chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n", |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 537 | nbpools, allocated, used); |
| 538 | } |
| 539 | |
Willy Tarreau | 12833bb | 2014-01-28 16:49:56 +0100 | [diff] [blame] | 540 | /* Dump statistics on pools usage. */ |
| 541 | void dump_pools(void) |
| 542 | { |
| 543 | dump_pools_to_trash(); |
Willy Tarreau | 843b7cb | 2018-07-13 10:54:26 +0200 | [diff] [blame] | 544 | qfprintf(stderr, "%s", trash.area); |
Willy Tarreau | 12833bb | 2014-01-28 16:49:56 +0100 | [diff] [blame] | 545 | } |
| 546 | |
Willy Tarreau | 58102cf | 2015-10-28 16:24:21 +0100 | [diff] [blame] | 547 | /* This function returns the total number of failed pool allocations */ |
| 548 | int pool_total_failures() |
| 549 | { |
| 550 | struct pool_head *entry; |
| 551 | int failed = 0; |
| 552 | |
| 553 | list_for_each_entry(entry, &pools, list) |
| 554 | failed += entry->failed; |
| 555 | return failed; |
| 556 | } |
| 557 | |
| 558 | /* This function returns the total amount of memory allocated in pools (in bytes) */ |
| 559 | unsigned long pool_total_allocated() |
| 560 | { |
| 561 | struct pool_head *entry; |
| 562 | unsigned long allocated = 0; |
| 563 | |
| 564 | list_for_each_entry(entry, &pools, list) |
| 565 | allocated += entry->allocated * entry->size; |
| 566 | return allocated; |
| 567 | } |
| 568 | |
| 569 | /* This function returns the total amount of memory used in pools (in bytes) */ |
| 570 | unsigned long pool_total_used() |
| 571 | { |
| 572 | struct pool_head *entry; |
| 573 | unsigned long used = 0; |
| 574 | |
| 575 | list_for_each_entry(entry, &pools, list) |
| 576 | used += entry->used * entry->size; |
| 577 | return used; |
| 578 | } |
| 579 | |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 580 | /* This function dumps memory usage information onto the stream interface's |
| 581 | * read buffer. It returns 0 as long as it does not complete, non-zero upon |
| 582 | * completion. No state is used. |
| 583 | */ |
| 584 | static int cli_io_handler_dump_pools(struct appctx *appctx) |
| 585 | { |
| 586 | struct stream_interface *si = appctx->owner; |
| 587 | |
| 588 | dump_pools_to_trash(); |
Willy Tarreau | 06d80a9 | 2017-10-19 14:32:15 +0200 | [diff] [blame] | 589 | if (ci_putchk(si_ic(si), &trash) == -1) { |
Willy Tarreau | db39843 | 2018-11-15 11:08:52 +0100 | [diff] [blame] | 590 | si_rx_room_blk(si); |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 591 | return 0; |
| 592 | } |
| 593 | return 1; |
| 594 | } |
| 595 | |
Willy Tarreau | 7107c8b | 2018-11-26 11:44:35 +0100 | [diff] [blame] | 596 | /* callback used to create early pool <name> of size <size> and store the |
| 597 | * resulting pointer into <ptr>. If the allocation fails, it quits with after |
| 598 | * emitting an error message. |
| 599 | */ |
| 600 | void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size) |
| 601 | { |
| 602 | *ptr = create_pool(name, size, MEM_F_SHARED); |
| 603 | if (!*ptr) { |
| 604 | ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n", |
| 605 | name, size, strerror(errno)); |
| 606 | exit(1); |
| 607 | } |
| 608 | } |
| 609 | |
Willy Tarreau | 7f0165e | 2018-11-26 17:09:46 +0100 | [diff] [blame] | 610 | /* Initializes all per-thread arrays on startup */ |
| 611 | static void init_pools() |
| 612 | { |
Willy Tarreau | 2d6f628 | 2021-04-15 16:24:00 +0200 | [diff] [blame] | 613 | #ifdef CONFIG_HAP_POOLS |
Willy Tarreau | 9f3129e | 2021-04-17 00:31:38 +0200 | [diff] [blame] | 614 | int thr; |
Willy Tarreau | 7f0165e | 2018-11-26 17:09:46 +0100 | [diff] [blame] | 615 | |
| 616 | for (thr = 0; thr < MAX_THREADS; thr++) { |
Willy Tarreau | 20dc3cd | 2020-06-28 00:54:27 +0200 | [diff] [blame] | 617 | LIST_INIT(&ha_thread_info[thr].pool_lru_head); |
Willy Tarreau | 7f0165e | 2018-11-26 17:09:46 +0100 | [diff] [blame] | 618 | } |
Willy Tarreau | ed891fd | 2020-06-01 19:00:28 +0200 | [diff] [blame] | 619 | #endif |
Willy Tarreau | 562bf20 | 2021-09-15 10:05:48 +0200 | [diff] [blame] | 620 | detect_allocator(); |
Willy Tarreau | 7f0165e | 2018-11-26 17:09:46 +0100 | [diff] [blame] | 621 | } |
| 622 | |
| 623 | INITCALL0(STG_PREPARE, init_pools); |
Willy Tarreau | 7107c8b | 2018-11-26 11:44:35 +0100 | [diff] [blame] | 624 | |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 625 | /* register cli keywords */ |
| 626 | static struct cli_kw_list cli_kws = {{ },{ |
Willy Tarreau | b205bfd | 2021-05-07 11:38:37 +0200 | [diff] [blame] | 627 | { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools }, |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 628 | {{},} |
| 629 | }}; |
| 630 | |
Willy Tarreau | 0108d90 | 2018-11-25 19:14:37 +0100 | [diff] [blame] | 631 | INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws); |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 632 | |
Olivier Houchard | dc21ff7 | 2019-01-29 15:20:16 +0100 | [diff] [blame] | 633 | #ifdef DEBUG_FAIL_ALLOC |
Olivier Houchard | dc21ff7 | 2019-01-29 15:20:16 +0100 | [diff] [blame] | 634 | |
| 635 | int mem_should_fail(const struct pool_head *pool) |
| 636 | { |
Olivier Houchard | 9c4f08a | 2019-02-01 16:28:04 +0100 | [diff] [blame] | 637 | int ret = 0; |
Olivier Houchard | dc21ff7 | 2019-01-29 15:20:16 +0100 | [diff] [blame] | 638 | |
| 639 | if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) { |
Willy Tarreau | 20f88ab | 2021-04-17 15:50:28 +0200 | [diff] [blame] | 640 | if (mem_fail_rate > statistical_prng_range(100)) |
Olivier Houchard | dc21ff7 | 2019-01-29 15:20:16 +0100 | [diff] [blame] | 641 | ret = 1; |
| 642 | else |
| 643 | ret = 0; |
| 644 | } |
Olivier Houchard | dc21ff7 | 2019-01-29 15:20:16 +0100 | [diff] [blame] | 645 | return ret; |
| 646 | |
| 647 | } |
| 648 | |
| 649 | /* config parser for global "tune.fail-alloc" */ |
| 650 | static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx, |
Amaury Denoyelle | 3b1c9a3 | 2021-03-22 11:21:36 +0100 | [diff] [blame] | 651 | const struct proxy *defpx, const char *file, int line, |
| 652 | char **err) |
Olivier Houchard | dc21ff7 | 2019-01-29 15:20:16 +0100 | [diff] [blame] | 653 | { |
| 654 | if (too_many_args(1, args, err, NULL)) |
| 655 | return -1; |
| 656 | mem_fail_rate = atoi(args[1]); |
| 657 | if (mem_fail_rate < 0 || mem_fail_rate > 100) { |
| 658 | memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]); |
| 659 | return -1; |
| 660 | } |
| 661 | return 0; |
| 662 | } |
| 663 | #endif |
| 664 | |
| 665 | /* register global config keywords */ |
| 666 | static struct cfg_kw_list mem_cfg_kws = {ILH, { |
| 667 | #ifdef DEBUG_FAIL_ALLOC |
| 668 | { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc }, |
| 669 | #endif |
| 670 | { 0, NULL, NULL } |
| 671 | }}; |
| 672 | |
| 673 | INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws); |
| 674 | |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 675 | /* |
| 676 | * Local variables: |
| 677 | * c-indent-level: 8 |
| 678 | * c-basic-offset: 8 |
| 679 | * End: |
| 680 | */ |