Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Memory management functions. |
| 3 | * |
| 4 | * Copyright 2000-2007 Willy Tarreau <w@1wt.eu> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
| 10 | * |
| 11 | */ |
| 12 | |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 13 | #include <types/applet.h> |
| 14 | #include <types/cli.h> |
Willy Tarreau | 12833bb | 2014-01-28 16:49:56 +0100 | [diff] [blame] | 15 | #include <types/global.h> |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 16 | #include <types/stats.h> |
| 17 | |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 18 | #include <common/config.h> |
Krzysztof Piotr Oledzki | a643baf | 2008-05-29 23:53:44 +0200 | [diff] [blame] | 19 | #include <common/debug.h> |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 20 | #include <common/memory.h> |
| 21 | #include <common/mini-clist.h> |
| 22 | #include <common/standard.h> |
| 23 | |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 24 | #include <proto/applet.h> |
| 25 | #include <proto/cli.h> |
| 26 | #include <proto/channel.h> |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 27 | #include <proto/log.h> |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 28 | #include <proto/stream_interface.h> |
| 29 | #include <proto/stats.h> |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 30 | |
Willy Tarreau | 0a93b64 | 2018-10-16 07:58:39 +0200 | [diff] [blame] | 31 | /* These are the most common pools, expected to be initialized first. These |
| 32 | * ones are allocated from an array, allowing to map them to an index. |
| 33 | */ |
| 34 | struct pool_head pool_base_start[MAX_BASE_POOLS] = { }; |
| 35 | unsigned int pool_base_count = 0; |
| 36 | |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 37 | static struct list pools = LIST_HEAD_INIT(pools); |
Willy Tarreau | 067ac9f | 2015-10-08 14:12:13 +0200 | [diff] [blame] | 38 | int mem_poison_byte = -1; |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 39 | |
| 40 | /* Try to find an existing shared pool with the same characteristics and |
| 41 | * returns it, otherwise creates this one. NULL is returned if no memory |
Willy Tarreau | 581bf81 | 2016-01-25 02:19:13 +0100 | [diff] [blame] | 42 | * is available for a new creation. Two flags are supported : |
| 43 | * - MEM_F_SHARED to indicate that the pool may be shared with other users |
| 44 | * - MEM_F_EXACT to indicate that the size must not be rounded up |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 45 | */ |
| 46 | struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags) |
| 47 | { |
| 48 | struct pool_head *pool; |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 49 | struct pool_head *entry; |
| 50 | struct list *start; |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 51 | unsigned int align; |
| 52 | |
Willy Tarreau | ac42111 | 2015-10-28 15:09:29 +0100 | [diff] [blame] | 53 | /* We need to store a (void *) at the end of the chunks. Since we know |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 54 | * that the malloc() function will never return such a small size, |
| 55 | * let's round the size up to something slightly bigger, in order to |
| 56 | * ease merging of entries. Note that the rounding is a power of two. |
Willy Tarreau | ac42111 | 2015-10-28 15:09:29 +0100 | [diff] [blame] | 57 | * This extra (void *) is not accounted for in the size computation |
| 58 | * so that the visible parts outside are not affected. |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 59 | */ |
| 60 | |
Willy Tarreau | 581bf81 | 2016-01-25 02:19:13 +0100 | [diff] [blame] | 61 | if (!(flags & MEM_F_EXACT)) { |
| 62 | align = 16; |
| 63 | size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA; |
| 64 | } |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 65 | |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 66 | /* TODO: thread: we do not lock pool list for now because all pools are |
| 67 | * created during HAProxy startup (so before threads creation) */ |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 68 | start = &pools; |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 69 | pool = NULL; |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 70 | |
| 71 | list_for_each_entry(entry, &pools, list) { |
| 72 | if (entry->size == size) { |
| 73 | /* either we can share this place and we take it, or |
| 74 | * we look for a sharable one or for the next position |
| 75 | * before which we will insert a new one. |
| 76 | */ |
| 77 | if (flags & entry->flags & MEM_F_SHARED) { |
| 78 | /* we can share this one */ |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 79 | pool = entry; |
Krzysztof Piotr Oledzki | a643baf | 2008-05-29 23:53:44 +0200 | [diff] [blame] | 80 | DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name); |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 81 | break; |
| 82 | } |
| 83 | } |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 84 | else if (entry->size > size) { |
| 85 | /* insert before this one */ |
| 86 | start = &entry->list; |
| 87 | break; |
| 88 | } |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 89 | } |
| 90 | |
| 91 | if (!pool) { |
Willy Tarreau | 0a93b64 | 2018-10-16 07:58:39 +0200 | [diff] [blame] | 92 | if (pool_base_count < MAX_BASE_POOLS) |
| 93 | pool = &pool_base_start[pool_base_count++]; |
| 94 | |
| 95 | if (!pool) { |
| 96 | /* look for a freed entry */ |
| 97 | for (entry = pool_base_start; entry != pool_base_start + MAX_BASE_POOLS; entry++) { |
| 98 | if (!entry->size) { |
| 99 | pool = entry; |
| 100 | break; |
| 101 | } |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | if (!pool) |
| 106 | pool = calloc(1, sizeof(*pool)); |
| 107 | |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 108 | if (!pool) |
| 109 | return NULL; |
| 110 | if (name) |
| 111 | strlcpy2(pool->name, name, sizeof(pool->name)); |
| 112 | pool->size = size; |
| 113 | pool->flags = flags; |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 114 | LIST_ADDQ(start, &pool->list); |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 115 | } |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 116 | pool->users++; |
Willy Tarreau | f161d0f | 2018-02-22 14:05:55 +0100 | [diff] [blame] | 117 | #ifndef CONFIG_HAP_LOCKLESS_POOLS |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 118 | HA_SPIN_INIT(&pool->lock); |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 119 | #endif |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 120 | return pool; |
| 121 | } |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 122 | |
Willy Tarreau | f161d0f | 2018-02-22 14:05:55 +0100 | [diff] [blame] | 123 | #ifdef CONFIG_HAP_LOCKLESS_POOLS |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 124 | /* Allocates new entries for pool <pool> until there are at least <avail> + 1 |
| 125 | * available, then returns the last one for immediate use, so that at least |
| 126 | * <avail> are left available in the pool upon return. NULL is returned if the |
| 127 | * last entry could not be allocated. It's important to note that at least one |
| 128 | * allocation is always performed even if there are enough entries in the pool. |
| 129 | * A call to the garbage collector is performed at most once in case malloc() |
| 130 | * returns an error, before returning NULL. |
| 131 | */ |
| 132 | void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail) |
| 133 | { |
| 134 | void *ptr = NULL, *free_list; |
| 135 | int failed = 0; |
| 136 | int size = pool->size; |
| 137 | int limit = pool->limit; |
| 138 | int allocated = pool->allocated, allocated_orig = allocated; |
| 139 | |
| 140 | /* stop point */ |
| 141 | avail += pool->used; |
| 142 | |
| 143 | while (1) { |
| 144 | if (limit && allocated >= limit) { |
| 145 | HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig); |
| 146 | return NULL; |
| 147 | } |
| 148 | |
| 149 | ptr = malloc(size + POOL_EXTRA); |
| 150 | if (!ptr) { |
| 151 | HA_ATOMIC_ADD(&pool->failed, 1); |
| 152 | if (failed) |
| 153 | return NULL; |
| 154 | failed++; |
| 155 | pool_gc(pool); |
| 156 | continue; |
| 157 | } |
| 158 | if (++allocated > avail) |
| 159 | break; |
| 160 | |
| 161 | free_list = pool->free_list; |
| 162 | do { |
| 163 | *POOL_LINK(pool, ptr) = free_list; |
| 164 | __ha_barrier_store(); |
| 165 | } while (HA_ATOMIC_CAS(&pool->free_list, (void *)&free_list, ptr) == 0); |
| 166 | } |
| 167 | |
| 168 | HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig); |
| 169 | HA_ATOMIC_ADD(&pool->used, 1); |
| 170 | |
| 171 | #ifdef DEBUG_MEMORY_POOLS |
| 172 | /* keep track of where the element was allocated from */ |
| 173 | *POOL_LINK(pool, ptr) = (void *)pool; |
| 174 | #endif |
| 175 | return ptr; |
| 176 | } |
| 177 | void *pool_refill_alloc(struct pool_head *pool, unsigned int avail) |
| 178 | { |
| 179 | void *ptr; |
| 180 | |
| 181 | ptr = __pool_refill_alloc(pool, avail); |
| 182 | return ptr; |
| 183 | } |
| 184 | /* |
| 185 | * This function frees whatever can be freed in pool <pool>. |
| 186 | */ |
| 187 | void pool_flush(struct pool_head *pool) |
| 188 | { |
| 189 | void *next, *temp; |
| 190 | int removed = 0; |
| 191 | |
| 192 | if (!pool) |
| 193 | return; |
| 194 | do { |
| 195 | next = pool->free_list; |
| 196 | } while (!HA_ATOMIC_CAS(&pool->free_list, (void *)&next, NULL)); |
| 197 | while (next) { |
| 198 | temp = next; |
| 199 | next = *POOL_LINK(pool, temp); |
| 200 | removed++; |
| 201 | free(temp); |
| 202 | } |
| 203 | pool->free_list = next; |
| 204 | HA_ATOMIC_SUB(&pool->allocated, removed); |
| 205 | /* here, we should have pool->allocate == pool->used */ |
| 206 | } |
| 207 | |
| 208 | /* |
| 209 | * This function frees whatever can be freed in all pools, but respecting |
| 210 | * the minimum thresholds imposed by owners. It takes care of avoiding |
| 211 | * recursion because it may be called from a signal handler. |
| 212 | * |
| 213 | * <pool_ctx> is unused |
| 214 | */ |
| 215 | void pool_gc(struct pool_head *pool_ctx) |
| 216 | { |
| 217 | static int recurse; |
| 218 | int cur_recurse = 0; |
| 219 | struct pool_head *entry; |
| 220 | |
| 221 | if (recurse || !HA_ATOMIC_CAS(&recurse, &cur_recurse, 1)) |
| 222 | return; |
| 223 | |
| 224 | list_for_each_entry(entry, &pools, list) { |
| 225 | while ((int)((volatile int)entry->allocated - (volatile int)entry->used) > (int)entry->minavail) { |
| 226 | struct pool_free_list cmp, new; |
| 227 | |
| 228 | cmp.seq = entry->seq; |
| 229 | __ha_barrier_load(); |
| 230 | cmp.free_list = entry->free_list; |
| 231 | __ha_barrier_load(); |
| 232 | if (cmp.free_list == NULL) |
| 233 | break; |
| 234 | new.free_list = *POOL_LINK(entry, cmp.free_list); |
| 235 | new.seq = cmp.seq + 1; |
| 236 | if (__ha_cas_dw(&entry->free_list, &cmp, &new) == 0) |
| 237 | continue; |
| 238 | free(cmp.free_list); |
| 239 | HA_ATOMIC_SUB(&entry->allocated, 1); |
| 240 | } |
| 241 | } |
| 242 | |
| 243 | HA_ATOMIC_STORE(&recurse, 0); |
| 244 | } |
Willy Tarreau | f161d0f | 2018-02-22 14:05:55 +0100 | [diff] [blame] | 245 | #else /* CONFIG_HAP_LOCKLESS_POOLS */ |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 246 | |
Willy Tarreau | a885f6d | 2014-12-03 15:25:28 +0100 | [diff] [blame] | 247 | /* Allocates new entries for pool <pool> until there are at least <avail> + 1 |
| 248 | * available, then returns the last one for immediate use, so that at least |
| 249 | * <avail> are left available in the pool upon return. NULL is returned if the |
| 250 | * last entry could not be allocated. It's important to note that at least one |
| 251 | * allocation is always performed even if there are enough entries in the pool. |
| 252 | * A call to the garbage collector is performed at most once in case malloc() |
| 253 | * returns an error, before returning NULL. |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 254 | */ |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 255 | void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail) |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 256 | { |
Willy Tarreau | a885f6d | 2014-12-03 15:25:28 +0100 | [diff] [blame] | 257 | void *ptr = NULL; |
| 258 | int failed = 0; |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 259 | |
Willy Tarreau | a885f6d | 2014-12-03 15:25:28 +0100 | [diff] [blame] | 260 | /* stop point */ |
| 261 | avail += pool->used; |
| 262 | |
| 263 | while (1) { |
| 264 | if (pool->limit && pool->allocated >= pool->limit) |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 265 | return NULL; |
Willy Tarreau | a885f6d | 2014-12-03 15:25:28 +0100 | [diff] [blame] | 266 | |
Willy Tarreau | f13322e | 2017-11-22 10:50:54 +0100 | [diff] [blame] | 267 | ptr = pool_alloc_area(pool->size + POOL_EXTRA); |
Willy Tarreau | a885f6d | 2014-12-03 15:25:28 +0100 | [diff] [blame] | 268 | if (!ptr) { |
Willy Tarreau | 58102cf | 2015-10-28 16:24:21 +0100 | [diff] [blame] | 269 | pool->failed++; |
Willy Tarreau | a885f6d | 2014-12-03 15:25:28 +0100 | [diff] [blame] | 270 | if (failed) |
| 271 | return NULL; |
| 272 | failed++; |
Willy Tarreau | bafbe01 | 2017-11-24 17:34:44 +0100 | [diff] [blame] | 273 | pool_gc(pool); |
Willy Tarreau | a885f6d | 2014-12-03 15:25:28 +0100 | [diff] [blame] | 274 | continue; |
| 275 | } |
| 276 | if (++pool->allocated > avail) |
| 277 | break; |
| 278 | |
Willy Tarreau | ac42111 | 2015-10-28 15:09:29 +0100 | [diff] [blame] | 279 | *POOL_LINK(pool, ptr) = (void *)pool->free_list; |
Willy Tarreau | a885f6d | 2014-12-03 15:25:28 +0100 | [diff] [blame] | 280 | pool->free_list = ptr; |
Willy Tarreau | 7dcd46d | 2007-05-14 00:16:13 +0200 | [diff] [blame] | 281 | } |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 282 | pool->used++; |
Willy Tarreau | de30a68 | 2015-10-28 15:23:51 +0100 | [diff] [blame] | 283 | #ifdef DEBUG_MEMORY_POOLS |
| 284 | /* keep track of where the element was allocated from */ |
| 285 | *POOL_LINK(pool, ptr) = (void *)pool; |
| 286 | #endif |
Willy Tarreau | a885f6d | 2014-12-03 15:25:28 +0100 | [diff] [blame] | 287 | return ptr; |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 288 | } |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 289 | void *pool_refill_alloc(struct pool_head *pool, unsigned int avail) |
| 290 | { |
| 291 | void *ptr; |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 292 | |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 293 | HA_SPIN_LOCK(POOL_LOCK, &pool->lock); |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 294 | ptr = __pool_refill_alloc(pool, avail); |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 295 | HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock); |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 296 | return ptr; |
| 297 | } |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 298 | /* |
| 299 | * This function frees whatever can be freed in pool <pool>. |
| 300 | */ |
Willy Tarreau | bafbe01 | 2017-11-24 17:34:44 +0100 | [diff] [blame] | 301 | void pool_flush(struct pool_head *pool) |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 302 | { |
| 303 | void *temp, *next; |
Willy Tarreau | 4d2d098 | 2007-05-14 00:39:29 +0200 | [diff] [blame] | 304 | if (!pool) |
| 305 | return; |
| 306 | |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 307 | HA_SPIN_LOCK(POOL_LOCK, &pool->lock); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 308 | next = pool->free_list; |
| 309 | while (next) { |
| 310 | temp = next; |
Willy Tarreau | ac42111 | 2015-10-28 15:09:29 +0100 | [diff] [blame] | 311 | next = *POOL_LINK(pool, temp); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 312 | pool->allocated--; |
Willy Tarreau | f13322e | 2017-11-22 10:50:54 +0100 | [diff] [blame] | 313 | pool_free_area(temp, pool->size + POOL_EXTRA); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 314 | } |
| 315 | pool->free_list = next; |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 316 | HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 317 | /* here, we should have pool->allocate == pool->used */ |
| 318 | } |
| 319 | |
| 320 | /* |
| 321 | * This function frees whatever can be freed in all pools, but respecting |
Willy Tarreau | b7f9d12 | 2009-04-21 02:17:45 +0200 | [diff] [blame] | 322 | * the minimum thresholds imposed by owners. It takes care of avoiding |
| 323 | * recursion because it may be called from a signal handler. |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 324 | * |
Willy Tarreau | bafbe01 | 2017-11-24 17:34:44 +0100 | [diff] [blame] | 325 | * <pool_ctx> is used when pool_gc is called to release resources to allocate |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 326 | * an element in __pool_refill_alloc. It is important because <pool_ctx> is |
| 327 | * already locked, so we need to skip the lock here. |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 328 | */ |
Willy Tarreau | bafbe01 | 2017-11-24 17:34:44 +0100 | [diff] [blame] | 329 | void pool_gc(struct pool_head *pool_ctx) |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 330 | { |
Willy Tarreau | b7f9d12 | 2009-04-21 02:17:45 +0200 | [diff] [blame] | 331 | static int recurse; |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 332 | int cur_recurse = 0; |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 333 | struct pool_head *entry; |
Willy Tarreau | b7f9d12 | 2009-04-21 02:17:45 +0200 | [diff] [blame] | 334 | |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 335 | if (recurse || !HA_ATOMIC_CAS(&recurse, &cur_recurse, 1)) |
| 336 | return; |
Willy Tarreau | b7f9d12 | 2009-04-21 02:17:45 +0200 | [diff] [blame] | 337 | |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 338 | list_for_each_entry(entry, &pools, list) { |
| 339 | void *temp, *next; |
| 340 | //qfprintf(stderr, "Flushing pool %s\n", entry->name); |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 341 | if (entry != pool_ctx) |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 342 | HA_SPIN_LOCK(POOL_LOCK, &entry->lock); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 343 | next = entry->free_list; |
| 344 | while (next && |
Willy Tarreau | 57767b8 | 2014-12-22 21:40:55 +0100 | [diff] [blame] | 345 | (int)(entry->allocated - entry->used) > (int)entry->minavail) { |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 346 | temp = next; |
Willy Tarreau | ac42111 | 2015-10-28 15:09:29 +0100 | [diff] [blame] | 347 | next = *POOL_LINK(entry, temp); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 348 | entry->allocated--; |
Willy Tarreau | f13322e | 2017-11-22 10:50:54 +0100 | [diff] [blame] | 349 | pool_free_area(temp, entry->size + POOL_EXTRA); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 350 | } |
| 351 | entry->free_list = next; |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 352 | if (entry != pool_ctx) |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 353 | HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 354 | } |
Christopher Faulet | b349e48 | 2017-08-29 09:52:38 +0200 | [diff] [blame] | 355 | |
| 356 | HA_ATOMIC_STORE(&recurse, 0); |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 357 | } |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 358 | #endif |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 359 | |
| 360 | /* |
Willy Tarreau | dae4aa8 | 2007-06-16 23:19:53 +0200 | [diff] [blame] | 361 | * This function destroys a pool by freeing it completely, unless it's still |
| 362 | * in use. This should be called only under extreme circumstances. It always |
| 363 | * returns NULL if the resulting pool is empty, easing the clearing of the old |
| 364 | * pointer, otherwise it returns the pool. |
| 365 | * . |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 366 | */ |
Willy Tarreau | bafbe01 | 2017-11-24 17:34:44 +0100 | [diff] [blame] | 367 | void *pool_destroy(struct pool_head *pool) |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 368 | { |
Willy Tarreau | 4d2d098 | 2007-05-14 00:39:29 +0200 | [diff] [blame] | 369 | if (pool) { |
Willy Tarreau | bafbe01 | 2017-11-24 17:34:44 +0100 | [diff] [blame] | 370 | pool_flush(pool); |
Willy Tarreau | dae4aa8 | 2007-06-16 23:19:53 +0200 | [diff] [blame] | 371 | if (pool->used) |
| 372 | return pool; |
| 373 | pool->users--; |
| 374 | if (!pool->users) { |
| 375 | LIST_DEL(&pool->list); |
Willy Tarreau | f161d0f | 2018-02-22 14:05:55 +0100 | [diff] [blame] | 376 | #ifndef CONFIG_HAP_LOCKLESS_POOLS |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 377 | HA_SPIN_DESTROY(&pool->lock); |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 378 | #endif |
Willy Tarreau | 0a93b64 | 2018-10-16 07:58:39 +0200 | [diff] [blame] | 379 | if ((pool - pool_base_start) < MAX_BASE_POOLS) |
| 380 | memset(pool, 0, sizeof(*pool)); |
| 381 | else |
| 382 | free(pool); |
Willy Tarreau | dae4aa8 | 2007-06-16 23:19:53 +0200 | [diff] [blame] | 383 | } |
Willy Tarreau | 4d2d098 | 2007-05-14 00:39:29 +0200 | [diff] [blame] | 384 | } |
| 385 | return NULL; |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 386 | } |
| 387 | |
Willy Tarreau | 12833bb | 2014-01-28 16:49:56 +0100 | [diff] [blame] | 388 | /* This function dumps memory usage information into the trash buffer. */ |
| 389 | void dump_pools_to_trash() |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 390 | { |
| 391 | struct pool_head *entry; |
| 392 | unsigned long allocated, used; |
| 393 | int nbpools; |
| 394 | |
| 395 | allocated = used = nbpools = 0; |
Willy Tarreau | 12833bb | 2014-01-28 16:49:56 +0100 | [diff] [blame] | 396 | chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n"); |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 397 | list_for_each_entry(entry, &pools, list) { |
Willy Tarreau | f161d0f | 2018-02-22 14:05:55 +0100 | [diff] [blame] | 398 | #ifndef CONFIG_HAP_LOCKLESS_POOLS |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 399 | HA_SPIN_LOCK(POOL_LOCK, &entry->lock); |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 400 | #endif |
Willy Tarreau | 0a93b64 | 2018-10-16 07:58:39 +0200 | [diff] [blame] | 401 | chunk_appendf(&trash, " - Pool %s (%d bytes) : %d allocated (%u bytes), %d used, %d failures, %d users, @%p=%02d%s\n", |
Willy Tarreau | e6ce59d | 2007-05-13 19:38:49 +0200 | [diff] [blame] | 402 | entry->name, entry->size, entry->allocated, |
Willy Tarreau | 58102cf | 2015-10-28 16:24:21 +0100 | [diff] [blame] | 403 | entry->size * entry->allocated, entry->used, entry->failed, |
Willy Tarreau | 0a93b64 | 2018-10-16 07:58:39 +0200 | [diff] [blame] | 404 | entry->users, entry, (int)pool_get_index(entry), |
| 405 | (entry->flags & MEM_F_SHARED) ? " [SHARED]" : ""); |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 406 | |
| 407 | allocated += entry->allocated * entry->size; |
| 408 | used += entry->used * entry->size; |
| 409 | nbpools++; |
Willy Tarreau | f161d0f | 2018-02-22 14:05:55 +0100 | [diff] [blame] | 410 | #ifndef CONFIG_HAP_LOCKLESS_POOLS |
Christopher Faulet | 2a944ee | 2017-11-07 10:42:54 +0100 | [diff] [blame] | 411 | HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock); |
Olivier Houchard | cf975d4 | 2018-01-24 18:38:31 +0100 | [diff] [blame] | 412 | #endif |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 413 | } |
Willy Tarreau | 12833bb | 2014-01-28 16:49:56 +0100 | [diff] [blame] | 414 | chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n", |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 415 | nbpools, allocated, used); |
| 416 | } |
| 417 | |
Willy Tarreau | 12833bb | 2014-01-28 16:49:56 +0100 | [diff] [blame] | 418 | /* Dump statistics on pools usage. */ |
| 419 | void dump_pools(void) |
| 420 | { |
| 421 | dump_pools_to_trash(); |
Willy Tarreau | 843b7cb | 2018-07-13 10:54:26 +0200 | [diff] [blame] | 422 | qfprintf(stderr, "%s", trash.area); |
Willy Tarreau | 12833bb | 2014-01-28 16:49:56 +0100 | [diff] [blame] | 423 | } |
| 424 | |
Willy Tarreau | 58102cf | 2015-10-28 16:24:21 +0100 | [diff] [blame] | 425 | /* This function returns the total number of failed pool allocations */ |
| 426 | int pool_total_failures() |
| 427 | { |
| 428 | struct pool_head *entry; |
| 429 | int failed = 0; |
| 430 | |
| 431 | list_for_each_entry(entry, &pools, list) |
| 432 | failed += entry->failed; |
| 433 | return failed; |
| 434 | } |
| 435 | |
| 436 | /* This function returns the total amount of memory allocated in pools (in bytes) */ |
| 437 | unsigned long pool_total_allocated() |
| 438 | { |
| 439 | struct pool_head *entry; |
| 440 | unsigned long allocated = 0; |
| 441 | |
| 442 | list_for_each_entry(entry, &pools, list) |
| 443 | allocated += entry->allocated * entry->size; |
| 444 | return allocated; |
| 445 | } |
| 446 | |
| 447 | /* This function returns the total amount of memory used in pools (in bytes) */ |
| 448 | unsigned long pool_total_used() |
| 449 | { |
| 450 | struct pool_head *entry; |
| 451 | unsigned long used = 0; |
| 452 | |
| 453 | list_for_each_entry(entry, &pools, list) |
| 454 | used += entry->used * entry->size; |
| 455 | return used; |
| 456 | } |
| 457 | |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 458 | /* This function dumps memory usage information onto the stream interface's |
| 459 | * read buffer. It returns 0 as long as it does not complete, non-zero upon |
| 460 | * completion. No state is used. |
| 461 | */ |
| 462 | static int cli_io_handler_dump_pools(struct appctx *appctx) |
| 463 | { |
| 464 | struct stream_interface *si = appctx->owner; |
| 465 | |
| 466 | dump_pools_to_trash(); |
Willy Tarreau | 06d80a9 | 2017-10-19 14:32:15 +0200 | [diff] [blame] | 467 | if (ci_putchk(si_ic(si), &trash) == -1) { |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 468 | si_applet_cant_put(si); |
| 469 | return 0; |
| 470 | } |
| 471 | return 1; |
| 472 | } |
| 473 | |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 474 | /* register cli keywords */ |
| 475 | static struct cli_kw_list cli_kws = {{ },{ |
Willy Tarreau | e9ecec8 | 2016-12-16 18:55:23 +0100 | [diff] [blame] | 476 | { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools }, |
William Lallemand | e7ed885 | 2016-11-19 02:25:36 +0100 | [diff] [blame] | 477 | {{},} |
| 478 | }}; |
| 479 | |
| 480 | __attribute__((constructor)) |
| 481 | static void __memory_init(void) |
| 482 | { |
| 483 | cli_register_kw(&cli_kws); |
| 484 | } |
| 485 | |
Willy Tarreau | 50e608d | 2007-05-13 18:26:08 +0200 | [diff] [blame] | 486 | /* |
| 487 | * Local variables: |
| 488 | * c-indent-level: 8 |
| 489 | * c-basic-offset: 8 |
| 490 | * End: |
| 491 | */ |