blob: ef7ec933d5923328a434c15949c7cd05abed0800 [file] [log] [blame]
Willy Tarreau50e608d2007-05-13 18:26:08 +02001/*
2 * Memory management functions.
3 *
4 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
Willy Tarreau7107c8b2018-11-26 11:44:35 +010012#include <errno.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020013
William Lallemande7ed8852016-11-19 02:25:36 +010014#include <types/applet.h>
15#include <types/cli.h>
Willy Tarreau12833bb2014-01-28 16:49:56 +010016#include <types/global.h>
William Lallemande7ed8852016-11-19 02:25:36 +010017#include <types/stats.h>
18
Olivier Houcharddc21ff72019-01-29 15:20:16 +010019#include <common/cfgparse.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020020#include <common/config.h>
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +020021#include <common/debug.h>
Willy Tarreaue18db9e2018-10-16 10:28:54 +020022#include <common/hathreads.h>
Willy Tarreau0108d902018-11-25 19:14:37 +010023#include <common/initcall.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020024#include <common/memory.h>
25#include <common/mini-clist.h>
26#include <common/standard.h>
27
William Lallemande7ed8852016-11-19 02:25:36 +010028#include <proto/applet.h>
29#include <proto/cli.h>
30#include <proto/channel.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020031#include <proto/log.h>
William Lallemande7ed8852016-11-19 02:25:36 +010032#include <proto/stream_interface.h>
33#include <proto/stats.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020034
Willy Tarreau0a93b642018-10-16 07:58:39 +020035/* These are the most common pools, expected to be initialized first. These
36 * ones are allocated from an array, allowing to map them to an index.
37 */
38struct pool_head pool_base_start[MAX_BASE_POOLS] = { };
39unsigned int pool_base_count = 0;
40
Willy Tarreau7f0165e2018-11-26 17:09:46 +010041/* These ones are initialized per-thread on startup by init_pools() */
42struct pool_cache_head pool_cache[MAX_THREADS][MAX_BASE_POOLS];
43static struct list pool_lru_head[MAX_THREADS]; /* oldest objects */
Willy Tarreaue18db9e2018-10-16 10:28:54 +020044THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
45THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
46
Willy Tarreau50e608d2007-05-13 18:26:08 +020047static struct list pools = LIST_HEAD_INIT(pools);
Willy Tarreau067ac9f2015-10-08 14:12:13 +020048int mem_poison_byte = -1;
Willy Tarreau50e608d2007-05-13 18:26:08 +020049
Olivier Houcharddc21ff72019-01-29 15:20:16 +010050#ifdef DEBUG_FAIL_ALLOC
51static int mem_fail_rate = 0;
52static int mem_should_fail(const struct pool_head *);
53#endif
54
Willy Tarreau50e608d2007-05-13 18:26:08 +020055/* Try to find an existing shared pool with the same characteristics and
56 * returns it, otherwise creates this one. NULL is returned if no memory
Willy Tarreau581bf812016-01-25 02:19:13 +010057 * is available for a new creation. Two flags are supported :
58 * - MEM_F_SHARED to indicate that the pool may be shared with other users
59 * - MEM_F_EXACT to indicate that the size must not be rounded up
Willy Tarreau50e608d2007-05-13 18:26:08 +020060 */
61struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
62{
63 struct pool_head *pool;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020064 struct pool_head *entry;
65 struct list *start;
Willy Tarreau50e608d2007-05-13 18:26:08 +020066 unsigned int align;
67
Willy Tarreauac421112015-10-28 15:09:29 +010068 /* We need to store a (void *) at the end of the chunks. Since we know
Willy Tarreau50e608d2007-05-13 18:26:08 +020069 * that the malloc() function will never return such a small size,
70 * let's round the size up to something slightly bigger, in order to
71 * ease merging of entries. Note that the rounding is a power of two.
Willy Tarreauac421112015-10-28 15:09:29 +010072 * This extra (void *) is not accounted for in the size computation
73 * so that the visible parts outside are not affected.
Willy Tarreau30f931e2018-10-23 14:40:23 +020074 *
75 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
Willy Tarreau50e608d2007-05-13 18:26:08 +020076 */
77
Willy Tarreau581bf812016-01-25 02:19:13 +010078 if (!(flags & MEM_F_EXACT)) {
Willy Tarreau30f931e2018-10-23 14:40:23 +020079 align = 4 * sizeof(void *); // 2 lists = 4 pointers min
Willy Tarreau581bf812016-01-25 02:19:13 +010080 size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA;
81 }
Willy Tarreau50e608d2007-05-13 18:26:08 +020082
Christopher Fauletb349e482017-08-29 09:52:38 +020083 /* TODO: thread: we do not lock pool list for now because all pools are
84 * created during HAProxy startup (so before threads creation) */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020085 start = &pools;
Willy Tarreau50e608d2007-05-13 18:26:08 +020086 pool = NULL;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020087
88 list_for_each_entry(entry, &pools, list) {
89 if (entry->size == size) {
90 /* either we can share this place and we take it, or
91 * we look for a sharable one or for the next position
92 * before which we will insert a new one.
93 */
94 if (flags & entry->flags & MEM_F_SHARED) {
95 /* we can share this one */
Willy Tarreau50e608d2007-05-13 18:26:08 +020096 pool = entry;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +020097 DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
Willy Tarreau50e608d2007-05-13 18:26:08 +020098 break;
99 }
100 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200101 else if (entry->size > size) {
102 /* insert before this one */
103 start = &entry->list;
104 break;
105 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200106 }
107
108 if (!pool) {
Willy Tarreau0a93b642018-10-16 07:58:39 +0200109 if (pool_base_count < MAX_BASE_POOLS)
110 pool = &pool_base_start[pool_base_count++];
111
112 if (!pool) {
113 /* look for a freed entry */
114 for (entry = pool_base_start; entry != pool_base_start + MAX_BASE_POOLS; entry++) {
115 if (!entry->size) {
116 pool = entry;
117 break;
118 }
119 }
120 }
121
122 if (!pool)
123 pool = calloc(1, sizeof(*pool));
124
Willy Tarreau50e608d2007-05-13 18:26:08 +0200125 if (!pool)
126 return NULL;
127 if (name)
128 strlcpy2(pool->name, name, sizeof(pool->name));
129 pool->size = size;
130 pool->flags = flags;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200131 LIST_ADDQ(start, &pool->list);
Willy Tarreau50e608d2007-05-13 18:26:08 +0200132 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200133 pool->users++;
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100134#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100135 HA_SPIN_INIT(&pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100136#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +0200137 return pool;
138}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100139
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100140#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +0100141/* Allocates new entries for pool <pool> until there are at least <avail> + 1
142 * available, then returns the last one for immediate use, so that at least
143 * <avail> are left available in the pool upon return. NULL is returned if the
144 * last entry could not be allocated. It's important to note that at least one
145 * allocation is always performed even if there are enough entries in the pool.
146 * A call to the garbage collector is performed at most once in case malloc()
147 * returns an error, before returning NULL.
148 */
149void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
150{
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200151 void *ptr = NULL, **free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100152 int failed = 0;
153 int size = pool->size;
154 int limit = pool->limit;
155 int allocated = pool->allocated, allocated_orig = allocated;
156
157 /* stop point */
158 avail += pool->used;
159
160 while (1) {
161 if (limit && allocated >= limit) {
Olivier Houchard20872762019-03-08 18:53:35 +0100162 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100163 return NULL;
164 }
165
166 ptr = malloc(size + POOL_EXTRA);
167 if (!ptr) {
Olivier Houchard20872762019-03-08 18:53:35 +0100168 _HA_ATOMIC_ADD(&pool->failed, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100169 if (failed)
170 return NULL;
171 failed++;
172 pool_gc(pool);
173 continue;
174 }
175 if (++allocated > avail)
176 break;
177
178 free_list = pool->free_list;
179 do {
180 *POOL_LINK(pool, ptr) = free_list;
181 __ha_barrier_store();
Olivier Houchard20872762019-03-08 18:53:35 +0100182 } while (_HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr) == 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100183 }
Olivier Houchard20872762019-03-08 18:53:35 +0100184 __ha_barrier_atomic_store();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100185
Olivier Houchard20872762019-03-08 18:53:35 +0100186 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
187 _HA_ATOMIC_ADD(&pool->used, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100188
189#ifdef DEBUG_MEMORY_POOLS
190 /* keep track of where the element was allocated from */
191 *POOL_LINK(pool, ptr) = (void *)pool;
192#endif
193 return ptr;
194}
195void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
196{
197 void *ptr;
198
199 ptr = __pool_refill_alloc(pool, avail);
200 return ptr;
201}
202/*
203 * This function frees whatever can be freed in pool <pool>.
204 */
205void pool_flush(struct pool_head *pool)
206{
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200207 void **next, *temp;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100208 int removed = 0;
209
210 if (!pool)
211 return;
212 do {
213 next = pool->free_list;
Olivier Houchard20872762019-03-08 18:53:35 +0100214 } while (!_HA_ATOMIC_CAS(&pool->free_list, &next, NULL));
215 __ha_barrier_atomic_store();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100216 while (next) {
217 temp = next;
218 next = *POOL_LINK(pool, temp);
219 removed++;
220 free(temp);
221 }
222 pool->free_list = next;
Olivier Houchard20872762019-03-08 18:53:35 +0100223 _HA_ATOMIC_SUB(&pool->allocated, removed);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100224 /* here, we should have pool->allocate == pool->used */
225}
226
227/*
228 * This function frees whatever can be freed in all pools, but respecting
229 * the minimum thresholds imposed by owners. It takes care of avoiding
230 * recursion because it may be called from a signal handler.
231 *
232 * <pool_ctx> is unused
233 */
234void pool_gc(struct pool_head *pool_ctx)
235{
236 static int recurse;
237 int cur_recurse = 0;
238 struct pool_head *entry;
239
Olivier Houchard20872762019-03-08 18:53:35 +0100240 if (recurse || !_HA_ATOMIC_CAS(&recurse, &cur_recurse, 1))
Olivier Houchardcf975d42018-01-24 18:38:31 +0100241 return;
242
243 list_for_each_entry(entry, &pools, list) {
244 while ((int)((volatile int)entry->allocated - (volatile int)entry->used) > (int)entry->minavail) {
245 struct pool_free_list cmp, new;
246
247 cmp.seq = entry->seq;
248 __ha_barrier_load();
249 cmp.free_list = entry->free_list;
250 __ha_barrier_load();
251 if (cmp.free_list == NULL)
252 break;
253 new.free_list = *POOL_LINK(entry, cmp.free_list);
254 new.seq = cmp.seq + 1;
255 if (__ha_cas_dw(&entry->free_list, &cmp, &new) == 0)
256 continue;
257 free(cmp.free_list);
Olivier Houchard20872762019-03-08 18:53:35 +0100258 _HA_ATOMIC_SUB(&entry->allocated, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100259 }
260 }
261
Olivier Houchard20872762019-03-08 18:53:35 +0100262 _HA_ATOMIC_STORE(&recurse, 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100263}
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200264
265/* frees an object to the local cache, possibly pushing oldest objects to the
266 * global pool. Must not be called directly.
267 */
268void __pool_put_to_cache(struct pool_head *pool, void *ptr, ssize_t idx)
269{
270 struct pool_cache_item *item = (struct pool_cache_item *)ptr;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100271 struct pool_cache_head *ph = &pool_cache[tid][idx];
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200272
273 LIST_ADD(&ph->list, &item->by_pool);
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100274 LIST_ADD(&pool_lru_head[tid], &item->by_lru);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200275 ph->count++;
276 pool_cache_count++;
277 pool_cache_bytes += ph->size;
278
279 if (pool_cache_bytes <= CONFIG_HAP_POOL_CACHE_SIZE)
280 return;
281
282 do {
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100283 item = LIST_PREV(&pool_lru_head[tid], struct pool_cache_item *, by_lru);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200284 /* note: by definition we remove oldest objects so they also are the
285 * oldest in their own pools, thus their next is the pool's head.
286 */
287 ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
288 LIST_DEL(&item->by_pool);
289 LIST_DEL(&item->by_lru);
290 ph->count--;
291 pool_cache_count--;
292 pool_cache_bytes -= ph->size;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100293 __pool_free(pool_base_start + (ph - pool_cache[tid]), item);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200294 } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
295}
296
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100297#else /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200298
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100299/* Allocates new entries for pool <pool> until there are at least <avail> + 1
300 * available, then returns the last one for immediate use, so that at least
301 * <avail> are left available in the pool upon return. NULL is returned if the
302 * last entry could not be allocated. It's important to note that at least one
303 * allocation is always performed even if there are enough entries in the pool.
304 * A call to the garbage collector is performed at most once in case malloc()
305 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200306 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200307void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200308{
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100309 void *ptr = NULL;
310 int failed = 0;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200311
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100312#ifdef DEBUG_FAIL_ALLOC
313 if (mem_should_fail(pool))
314 return NULL;
315#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100316 /* stop point */
317 avail += pool->used;
318
319 while (1) {
320 if (pool->limit && pool->allocated >= pool->limit)
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200321 return NULL;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100322
Willy Tarreauf13322e2017-11-22 10:50:54 +0100323 ptr = pool_alloc_area(pool->size + POOL_EXTRA);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100324 if (!ptr) {
Willy Tarreau58102cf2015-10-28 16:24:21 +0100325 pool->failed++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100326 if (failed)
327 return NULL;
328 failed++;
Willy Tarreaubafbe012017-11-24 17:34:44 +0100329 pool_gc(pool);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100330 continue;
331 }
332 if (++pool->allocated > avail)
333 break;
334
Willy Tarreauac421112015-10-28 15:09:29 +0100335 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100336 pool->free_list = ptr;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200337 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200338 pool->used++;
Willy Tarreaude30a682015-10-28 15:23:51 +0100339#ifdef DEBUG_MEMORY_POOLS
340 /* keep track of where the element was allocated from */
341 *POOL_LINK(pool, ptr) = (void *)pool;
342#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100343 return ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200344}
Christopher Fauletb349e482017-08-29 09:52:38 +0200345void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
346{
347 void *ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200348
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100349 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200350 ptr = __pool_refill_alloc(pool, avail);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100351 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200352 return ptr;
353}
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200354/*
355 * This function frees whatever can be freed in pool <pool>.
356 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100357void pool_flush(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200358{
359 void *temp, *next;
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200360 if (!pool)
361 return;
362
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100363 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200364 next = pool->free_list;
365 while (next) {
366 temp = next;
Willy Tarreauac421112015-10-28 15:09:29 +0100367 next = *POOL_LINK(pool, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200368 pool->allocated--;
Willy Tarreauf13322e2017-11-22 10:50:54 +0100369 pool_free_area(temp, pool->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200370 }
371 pool->free_list = next;
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100372 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200373 /* here, we should have pool->allocate == pool->used */
374}
375
376/*
377 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200378 * the minimum thresholds imposed by owners. It takes care of avoiding
379 * recursion because it may be called from a signal handler.
Christopher Fauletb349e482017-08-29 09:52:38 +0200380 *
Willy Tarreaubafbe012017-11-24 17:34:44 +0100381 * <pool_ctx> is used when pool_gc is called to release resources to allocate
Christopher Fauletb349e482017-08-29 09:52:38 +0200382 * an element in __pool_refill_alloc. It is important because <pool_ctx> is
383 * already locked, so we need to skip the lock here.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200384 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100385void pool_gc(struct pool_head *pool_ctx)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200386{
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200387 static int recurse;
Christopher Fauletb349e482017-08-29 09:52:38 +0200388 int cur_recurse = 0;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200389 struct pool_head *entry;
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200390
Olivier Houchard20872762019-03-08 18:53:35 +0100391 if (recurse || !_HA_ATOMIC_CAS(&recurse, &cur_recurse, 1))
Christopher Fauletb349e482017-08-29 09:52:38 +0200392 return;
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200393
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200394 list_for_each_entry(entry, &pools, list) {
395 void *temp, *next;
396 //qfprintf(stderr, "Flushing pool %s\n", entry->name);
Christopher Fauletb349e482017-08-29 09:52:38 +0200397 if (entry != pool_ctx)
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100398 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200399 next = entry->free_list;
400 while (next &&
Willy Tarreau57767b82014-12-22 21:40:55 +0100401 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200402 temp = next;
Willy Tarreauac421112015-10-28 15:09:29 +0100403 next = *POOL_LINK(entry, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200404 entry->allocated--;
Willy Tarreauf13322e2017-11-22 10:50:54 +0100405 pool_free_area(temp, entry->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200406 }
407 entry->free_list = next;
Christopher Fauletb349e482017-08-29 09:52:38 +0200408 if (entry != pool_ctx)
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100409 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200410 }
Christopher Fauletb349e482017-08-29 09:52:38 +0200411
Olivier Houchard20872762019-03-08 18:53:35 +0100412 _HA_ATOMIC_STORE(&recurse, 0);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200413}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100414#endif
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200415
416/*
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200417 * This function destroys a pool by freeing it completely, unless it's still
418 * in use. This should be called only under extreme circumstances. It always
419 * returns NULL if the resulting pool is empty, easing the clearing of the old
420 * pointer, otherwise it returns the pool.
421 * .
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200422 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100423void *pool_destroy(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200424{
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200425 if (pool) {
Willy Tarreaubafbe012017-11-24 17:34:44 +0100426 pool_flush(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200427 if (pool->used)
428 return pool;
429 pool->users--;
430 if (!pool->users) {
431 LIST_DEL(&pool->list);
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100432#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100433 HA_SPIN_DESTROY(&pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100434#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200435 if ((pool - pool_base_start) < MAX_BASE_POOLS)
436 memset(pool, 0, sizeof(*pool));
437 else
438 free(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200439 }
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200440 }
441 return NULL;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200442}
443
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100444/* This destroys all pools on exit. It is *not* thread safe. */
445void pool_destroy_all()
446{
447 struct pool_head *entry, *back;
448
449 list_for_each_entry_safe(entry, back, &pools, list)
450 pool_destroy(entry);
451}
452
Willy Tarreau12833bb2014-01-28 16:49:56 +0100453/* This function dumps memory usage information into the trash buffer. */
454void dump_pools_to_trash()
Willy Tarreau50e608d2007-05-13 18:26:08 +0200455{
456 struct pool_head *entry;
457 unsigned long allocated, used;
458 int nbpools;
459
460 allocated = used = nbpools = 0;
Willy Tarreau12833bb2014-01-28 16:49:56 +0100461 chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200462 list_for_each_entry(entry, &pools, list) {
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100463#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100464 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100465#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200466 chunk_appendf(&trash, " - Pool %s (%d bytes) : %d allocated (%u bytes), %d used, %d failures, %d users, @%p=%02d%s\n",
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200467 entry->name, entry->size, entry->allocated,
Willy Tarreau58102cf2015-10-28 16:24:21 +0100468 entry->size * entry->allocated, entry->used, entry->failed,
Willy Tarreau0a93b642018-10-16 07:58:39 +0200469 entry->users, entry, (int)pool_get_index(entry),
470 (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200471
472 allocated += entry->allocated * entry->size;
473 used += entry->used * entry->size;
474 nbpools++;
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100475#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100476 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100477#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +0200478 }
Willy Tarreau12833bb2014-01-28 16:49:56 +0100479 chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n",
Willy Tarreau50e608d2007-05-13 18:26:08 +0200480 nbpools, allocated, used);
481}
482
Willy Tarreau12833bb2014-01-28 16:49:56 +0100483/* Dump statistics on pools usage. */
484void dump_pools(void)
485{
486 dump_pools_to_trash();
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200487 qfprintf(stderr, "%s", trash.area);
Willy Tarreau12833bb2014-01-28 16:49:56 +0100488}
489
Willy Tarreau58102cf2015-10-28 16:24:21 +0100490/* This function returns the total number of failed pool allocations */
491int pool_total_failures()
492{
493 struct pool_head *entry;
494 int failed = 0;
495
496 list_for_each_entry(entry, &pools, list)
497 failed += entry->failed;
498 return failed;
499}
500
501/* This function returns the total amount of memory allocated in pools (in bytes) */
502unsigned long pool_total_allocated()
503{
504 struct pool_head *entry;
505 unsigned long allocated = 0;
506
507 list_for_each_entry(entry, &pools, list)
508 allocated += entry->allocated * entry->size;
509 return allocated;
510}
511
512/* This function returns the total amount of memory used in pools (in bytes) */
513unsigned long pool_total_used()
514{
515 struct pool_head *entry;
516 unsigned long used = 0;
517
518 list_for_each_entry(entry, &pools, list)
519 used += entry->used * entry->size;
520 return used;
521}
522
William Lallemande7ed8852016-11-19 02:25:36 +0100523/* This function dumps memory usage information onto the stream interface's
524 * read buffer. It returns 0 as long as it does not complete, non-zero upon
525 * completion. No state is used.
526 */
527static int cli_io_handler_dump_pools(struct appctx *appctx)
528{
529 struct stream_interface *si = appctx->owner;
530
531 dump_pools_to_trash();
Willy Tarreau06d80a92017-10-19 14:32:15 +0200532 if (ci_putchk(si_ic(si), &trash) == -1) {
Willy Tarreaudb398432018-11-15 11:08:52 +0100533 si_rx_room_blk(si);
William Lallemande7ed8852016-11-19 02:25:36 +0100534 return 0;
535 }
536 return 1;
537}
538
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100539/* callback used to create early pool <name> of size <size> and store the
540 * resulting pointer into <ptr>. If the allocation fails, it quits with after
541 * emitting an error message.
542 */
543void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
544{
545 *ptr = create_pool(name, size, MEM_F_SHARED);
546 if (!*ptr) {
547 ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
548 name, size, strerror(errno));
549 exit(1);
550 }
551}
552
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100553/* Initializes all per-thread arrays on startup */
554static void init_pools()
555{
556 int thr, idx;
557
558 for (thr = 0; thr < MAX_THREADS; thr++) {
559 for (idx = 0; idx < MAX_BASE_POOLS; idx++) {
560 LIST_INIT(&pool_cache[thr][idx].list);
561 pool_cache[thr][idx].size = 0;
562 }
563 LIST_INIT(&pool_lru_head[thr]);
564 }
565}
566
567INITCALL0(STG_PREPARE, init_pools);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100568
William Lallemande7ed8852016-11-19 02:25:36 +0100569/* register cli keywords */
570static struct cli_kw_list cli_kws = {{ },{
Willy Tarreaue9ecec82016-12-16 18:55:23 +0100571 { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
William Lallemande7ed8852016-11-19 02:25:36 +0100572 {{},}
573}};
574
Willy Tarreau0108d902018-11-25 19:14:37 +0100575INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
William Lallemande7ed8852016-11-19 02:25:36 +0100576
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100577#ifdef DEBUG_FAIL_ALLOC
578#define MEM_FAIL_MAX_CHAR 32
579#define MEM_FAIL_MAX_STR 128
580static int mem_fail_cur_idx;
581static char mem_fail_str[MEM_FAIL_MAX_CHAR * MEM_FAIL_MAX_STR];
582__decl_hathreads(static HA_SPINLOCK_T mem_fail_lock);
583
584int mem_should_fail(const struct pool_head *pool)
585{
Olivier Houchard9c4f08a2019-02-01 16:28:04 +0100586 int ret = 0;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100587 int n;
588
589 if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
590 int randnb = random() % 100;
591
592 if (mem_fail_rate > randnb)
593 ret = 1;
594 else
595 ret = 0;
596 }
597 HA_SPIN_LOCK(START_LOCK, &mem_fail_lock);
598 n = snprintf(&mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR],
599 MEM_FAIL_MAX_CHAR - 2,
600 "%d %.18s %d %d", mem_fail_cur_idx, pool->name, ret, tid);
601 while (n < MEM_FAIL_MAX_CHAR - 1)
602 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n++] = ' ';
603 if (mem_fail_cur_idx < MEM_FAIL_MAX_STR - 1)
604 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = '\n';
605 else
606 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = 0;
607 mem_fail_cur_idx++;
608 if (mem_fail_cur_idx == MEM_FAIL_MAX_STR)
609 mem_fail_cur_idx = 0;
610 HA_SPIN_UNLOCK(START_LOCK, &mem_fail_lock);
611 return ret;
612
613}
614
615/* config parser for global "tune.fail-alloc" */
616static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
617 struct proxy *defpx, const char *file, int line,
618 char **err)
619{
620 if (too_many_args(1, args, err, NULL))
621 return -1;
622 mem_fail_rate = atoi(args[1]);
623 if (mem_fail_rate < 0 || mem_fail_rate > 100) {
624 memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
625 return -1;
626 }
627 return 0;
628}
629#endif
630
631/* register global config keywords */
632static struct cfg_kw_list mem_cfg_kws = {ILH, {
633#ifdef DEBUG_FAIL_ALLOC
634 { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
635#endif
636 { 0, NULL, NULL }
637}};
638
639INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);
640
Willy Tarreau50e608d2007-05-13 18:26:08 +0200641/*
642 * Local variables:
643 * c-indent-level: 8
644 * c-basic-offset: 8
645 * End:
646 */