blob: bd2c1babf475786e60e5f18815a1c8560d8d9f15 [file] [log] [blame]
Willy Tarreau50e608d2007-05-13 18:26:08 +02001/*
2 * Memory management functions.
3 *
4 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
Willy Tarreau7107c8b2018-11-26 11:44:35 +010012#include <errno.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020013
William Lallemande7ed8852016-11-19 02:25:36 +010014#include <types/applet.h>
15#include <types/cli.h>
Willy Tarreau12833bb2014-01-28 16:49:56 +010016#include <types/global.h>
William Lallemande7ed8852016-11-19 02:25:36 +010017#include <types/stats.h>
18
Olivier Houcharddc21ff72019-01-29 15:20:16 +010019#include <common/cfgparse.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020020#include <common/config.h>
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +020021#include <common/debug.h>
Willy Tarreaue18db9e2018-10-16 10:28:54 +020022#include <common/hathreads.h>
Willy Tarreau0108d902018-11-25 19:14:37 +010023#include <common/initcall.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020024#include <common/memory.h>
25#include <common/mini-clist.h>
26#include <common/standard.h>
27
Willy Tarreaua8b2ce02019-05-28 17:04:16 +020028#include <types/activity.h>
29
William Lallemande7ed8852016-11-19 02:25:36 +010030#include <proto/applet.h>
31#include <proto/cli.h>
32#include <proto/channel.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020033#include <proto/log.h>
William Lallemande7ed8852016-11-19 02:25:36 +010034#include <proto/stream_interface.h>
35#include <proto/stats.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020036
Willy Tarreau0a93b642018-10-16 07:58:39 +020037/* These are the most common pools, expected to be initialized first. These
38 * ones are allocated from an array, allowing to map them to an index.
39 */
40struct pool_head pool_base_start[MAX_BASE_POOLS] = { };
41unsigned int pool_base_count = 0;
42
Willy Tarreau7f0165e2018-11-26 17:09:46 +010043/* These ones are initialized per-thread on startup by init_pools() */
44struct pool_cache_head pool_cache[MAX_THREADS][MAX_BASE_POOLS];
45static struct list pool_lru_head[MAX_THREADS]; /* oldest objects */
Willy Tarreaue18db9e2018-10-16 10:28:54 +020046THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
47THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
48
Willy Tarreau50e608d2007-05-13 18:26:08 +020049static struct list pools = LIST_HEAD_INIT(pools);
Willy Tarreau067ac9f2015-10-08 14:12:13 +020050int mem_poison_byte = -1;
Willy Tarreau50e608d2007-05-13 18:26:08 +020051
Olivier Houcharddc21ff72019-01-29 15:20:16 +010052#ifdef DEBUG_FAIL_ALLOC
53static int mem_fail_rate = 0;
54static int mem_should_fail(const struct pool_head *);
55#endif
56
Willy Tarreau50e608d2007-05-13 18:26:08 +020057/* Try to find an existing shared pool with the same characteristics and
58 * returns it, otherwise creates this one. NULL is returned if no memory
Willy Tarreau581bf812016-01-25 02:19:13 +010059 * is available for a new creation. Two flags are supported :
60 * - MEM_F_SHARED to indicate that the pool may be shared with other users
61 * - MEM_F_EXACT to indicate that the size must not be rounded up
Willy Tarreau50e608d2007-05-13 18:26:08 +020062 */
63struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
64{
65 struct pool_head *pool;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020066 struct pool_head *entry;
67 struct list *start;
Willy Tarreau50e608d2007-05-13 18:26:08 +020068 unsigned int align;
Christopher Faulet77204fd2019-06-25 21:45:59 +020069 int thr, idx;
Willy Tarreau50e608d2007-05-13 18:26:08 +020070
Willy Tarreauac421112015-10-28 15:09:29 +010071 /* We need to store a (void *) at the end of the chunks. Since we know
Willy Tarreau50e608d2007-05-13 18:26:08 +020072 * that the malloc() function will never return such a small size,
73 * let's round the size up to something slightly bigger, in order to
74 * ease merging of entries. Note that the rounding is a power of two.
Willy Tarreauac421112015-10-28 15:09:29 +010075 * This extra (void *) is not accounted for in the size computation
76 * so that the visible parts outside are not affected.
Willy Tarreau30f931e2018-10-23 14:40:23 +020077 *
78 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
Willy Tarreau50e608d2007-05-13 18:26:08 +020079 */
80
Willy Tarreau581bf812016-01-25 02:19:13 +010081 if (!(flags & MEM_F_EXACT)) {
Willy Tarreau30f931e2018-10-23 14:40:23 +020082 align = 4 * sizeof(void *); // 2 lists = 4 pointers min
Willy Tarreau581bf812016-01-25 02:19:13 +010083 size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA;
84 }
Willy Tarreau50e608d2007-05-13 18:26:08 +020085
Christopher Fauletb349e482017-08-29 09:52:38 +020086 /* TODO: thread: we do not lock pool list for now because all pools are
87 * created during HAProxy startup (so before threads creation) */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020088 start = &pools;
Willy Tarreau50e608d2007-05-13 18:26:08 +020089 pool = NULL;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020090
91 list_for_each_entry(entry, &pools, list) {
92 if (entry->size == size) {
93 /* either we can share this place and we take it, or
94 * we look for a sharable one or for the next position
95 * before which we will insert a new one.
96 */
97 if (flags & entry->flags & MEM_F_SHARED) {
98 /* we can share this one */
Willy Tarreau50e608d2007-05-13 18:26:08 +020099 pool = entry;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200100 DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
Willy Tarreau50e608d2007-05-13 18:26:08 +0200101 break;
102 }
103 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200104 else if (entry->size > size) {
105 /* insert before this one */
106 start = &entry->list;
107 break;
108 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200109 }
110
111 if (!pool) {
Willy Tarreau0a93b642018-10-16 07:58:39 +0200112 if (pool_base_count < MAX_BASE_POOLS)
113 pool = &pool_base_start[pool_base_count++];
114
115 if (!pool) {
116 /* look for a freed entry */
117 for (entry = pool_base_start; entry != pool_base_start + MAX_BASE_POOLS; entry++) {
118 if (!entry->size) {
119 pool = entry;
120 break;
121 }
122 }
123 }
124
125 if (!pool)
126 pool = calloc(1, sizeof(*pool));
127
Willy Tarreau50e608d2007-05-13 18:26:08 +0200128 if (!pool)
129 return NULL;
130 if (name)
131 strlcpy2(pool->name, name, sizeof(pool->name));
132 pool->size = size;
133 pool->flags = flags;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200134 LIST_ADDQ(start, &pool->list);
Christopher Faulet77204fd2019-06-25 21:45:59 +0200135
136 /* update per-thread pool cache if necessary */
137 idx = pool_get_index(pool);
138 if (idx >= 0) {
139 for (thr = 0; thr < MAX_THREADS; thr++)
140 pool_cache[thr][idx].size = size;
141 }
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100142#ifndef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchard4a2d6bd2020-02-01 17:45:32 +0100143 HA_SPIN_INIT(&pool->lock);
Olivier Houchard4536b302020-02-01 17:49:31 +0100144#else
Olivier Houchard5dfbf2e2020-03-18 15:48:29 +0100145 HA_SPIN_INIT(&pool->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100146#endif
Olivier Houchard4a2d6bd2020-02-01 17:45:32 +0100147 }
148 pool->users++;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200149 return pool;
150}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100151
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100152#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +0100153/* Allocates new entries for pool <pool> until there are at least <avail> + 1
154 * available, then returns the last one for immediate use, so that at least
155 * <avail> are left available in the pool upon return. NULL is returned if the
156 * last entry could not be allocated. It's important to note that at least one
157 * allocation is always performed even if there are enough entries in the pool.
158 * A call to the garbage collector is performed at most once in case malloc()
159 * returns an error, before returning NULL.
160 */
161void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
162{
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200163 void *ptr = NULL, **free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100164 int failed = 0;
165 int size = pool->size;
166 int limit = pool->limit;
167 int allocated = pool->allocated, allocated_orig = allocated;
168
169 /* stop point */
170 avail += pool->used;
171
172 while (1) {
173 if (limit && allocated >= limit) {
Olivier Houchard20872762019-03-08 18:53:35 +0100174 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200175 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100176 return NULL;
177 }
178
179 ptr = malloc(size + POOL_EXTRA);
180 if (!ptr) {
Olivier Houchard20872762019-03-08 18:53:35 +0100181 _HA_ATOMIC_ADD(&pool->failed, 1);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200182 if (failed) {
183 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100184 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200185 }
Olivier Houchardcf975d42018-01-24 18:38:31 +0100186 failed++;
187 pool_gc(pool);
188 continue;
189 }
190 if (++allocated > avail)
191 break;
192
193 free_list = pool->free_list;
194 do {
195 *POOL_LINK(pool, ptr) = free_list;
196 __ha_barrier_store();
Olivier Houchard20872762019-03-08 18:53:35 +0100197 } while (_HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr) == 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100198 }
Olivier Houchard20872762019-03-08 18:53:35 +0100199 __ha_barrier_atomic_store();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100200
Olivier Houchard20872762019-03-08 18:53:35 +0100201 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
202 _HA_ATOMIC_ADD(&pool->used, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100203
204#ifdef DEBUG_MEMORY_POOLS
205 /* keep track of where the element was allocated from */
206 *POOL_LINK(pool, ptr) = (void *)pool;
207#endif
208 return ptr;
209}
210void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
211{
212 void *ptr;
213
214 ptr = __pool_refill_alloc(pool, avail);
215 return ptr;
216}
217/*
218 * This function frees whatever can be freed in pool <pool>.
219 */
220void pool_flush(struct pool_head *pool)
221{
Olivier Houchard21861592020-02-01 17:37:22 +0100222 struct pool_free_list cmp, new;
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200223 void **next, *temp;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100224 int removed = 0;
225
226 if (!pool)
227 return;
Olivier Houchard5dfbf2e2020-03-18 15:48:29 +0100228 HA_SPIN_LOCK(POOL_LOCK, &pool->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100229 do {
Olivier Houchard21861592020-02-01 17:37:22 +0100230 cmp.free_list = pool->free_list;
231 cmp.seq = pool->seq;
232 new.free_list = NULL;
233 new.seq = cmp.seq + 1;
234 } while (!_HA_ATOMIC_DWCAS(&pool->free_list, &cmp, &new));
Olivier Houchard20872762019-03-08 18:53:35 +0100235 __ha_barrier_atomic_store();
Olivier Houchard5dfbf2e2020-03-18 15:48:29 +0100236 HA_SPIN_UNLOCK(POOL_LOCK, &pool->flush_lock);
Olivier Houchard21861592020-02-01 17:37:22 +0100237 next = cmp.free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100238 while (next) {
239 temp = next;
240 next = *POOL_LINK(pool, temp);
241 removed++;
242 free(temp);
243 }
244 pool->free_list = next;
Olivier Houchard20872762019-03-08 18:53:35 +0100245 _HA_ATOMIC_SUB(&pool->allocated, removed);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100246 /* here, we should have pool->allocate == pool->used */
247}
248
249/*
250 * This function frees whatever can be freed in all pools, but respecting
251 * the minimum thresholds imposed by owners. It takes care of avoiding
252 * recursion because it may be called from a signal handler.
253 *
254 * <pool_ctx> is unused
255 */
256void pool_gc(struct pool_head *pool_ctx)
257{
258 static int recurse;
259 int cur_recurse = 0;
260 struct pool_head *entry;
261
Olivier Houchard20872762019-03-08 18:53:35 +0100262 if (recurse || !_HA_ATOMIC_CAS(&recurse, &cur_recurse, 1))
Olivier Houchardcf975d42018-01-24 18:38:31 +0100263 return;
264
265 list_for_each_entry(entry, &pools, list) {
Olivier Houchard5dfbf2e2020-03-18 15:48:29 +0100266 HA_SPIN_LOCK(POOL_LOCK, &entry->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100267 while ((int)((volatile int)entry->allocated - (volatile int)entry->used) > (int)entry->minavail) {
268 struct pool_free_list cmp, new;
269
270 cmp.seq = entry->seq;
271 __ha_barrier_load();
272 cmp.free_list = entry->free_list;
273 __ha_barrier_load();
274 if (cmp.free_list == NULL)
275 break;
276 new.free_list = *POOL_LINK(entry, cmp.free_list);
277 new.seq = cmp.seq + 1;
Willy Tarreau6a38b322019-05-11 18:04:24 +0200278 if (HA_ATOMIC_DWCAS(&entry->free_list, &cmp, &new) == 0)
Olivier Houchardcf975d42018-01-24 18:38:31 +0100279 continue;
280 free(cmp.free_list);
Olivier Houchard20872762019-03-08 18:53:35 +0100281 _HA_ATOMIC_SUB(&entry->allocated, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100282 }
Olivier Houchard5dfbf2e2020-03-18 15:48:29 +0100283 HA_SPIN_UNLOCK(POOL_LOCK, &entry->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100284 }
285
Olivier Houchard20872762019-03-08 18:53:35 +0100286 _HA_ATOMIC_STORE(&recurse, 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100287}
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200288
289/* frees an object to the local cache, possibly pushing oldest objects to the
290 * global pool. Must not be called directly.
291 */
292void __pool_put_to_cache(struct pool_head *pool, void *ptr, ssize_t idx)
293{
294 struct pool_cache_item *item = (struct pool_cache_item *)ptr;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100295 struct pool_cache_head *ph = &pool_cache[tid][idx];
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200296
297 LIST_ADD(&ph->list, &item->by_pool);
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100298 LIST_ADD(&pool_lru_head[tid], &item->by_lru);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200299 ph->count++;
300 pool_cache_count++;
301 pool_cache_bytes += ph->size;
302
303 if (pool_cache_bytes <= CONFIG_HAP_POOL_CACHE_SIZE)
304 return;
305
306 do {
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100307 item = LIST_PREV(&pool_lru_head[tid], struct pool_cache_item *, by_lru);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200308 /* note: by definition we remove oldest objects so they also are the
309 * oldest in their own pools, thus their next is the pool's head.
310 */
311 ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
312 LIST_DEL(&item->by_pool);
313 LIST_DEL(&item->by_lru);
314 ph->count--;
315 pool_cache_count--;
316 pool_cache_bytes -= ph->size;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100317 __pool_free(pool_base_start + (ph - pool_cache[tid]), item);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200318 } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
319}
320
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100321#else /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200322
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100323/* Allocates new entries for pool <pool> until there are at least <avail> + 1
324 * available, then returns the last one for immediate use, so that at least
325 * <avail> are left available in the pool upon return. NULL is returned if the
326 * last entry could not be allocated. It's important to note that at least one
327 * allocation is always performed even if there are enough entries in the pool.
328 * A call to the garbage collector is performed at most once in case malloc()
329 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200330 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200331void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200332{
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100333 void *ptr = NULL;
334 int failed = 0;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200335
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100336#ifdef DEBUG_FAIL_ALLOC
337 if (mem_should_fail(pool))
338 return NULL;
339#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100340 /* stop point */
341 avail += pool->used;
342
343 while (1) {
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200344 if (pool->limit && pool->allocated >= pool->limit) {
345 activity[tid].pool_fail++;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200346 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200347 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100348
Willy Tarreauf13322e2017-11-22 10:50:54 +0100349 ptr = pool_alloc_area(pool->size + POOL_EXTRA);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100350 if (!ptr) {
Willy Tarreau58102cf2015-10-28 16:24:21 +0100351 pool->failed++;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200352 if (failed) {
353 activity[tid].pool_fail++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100354 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200355 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100356 failed++;
Willy Tarreaubafbe012017-11-24 17:34:44 +0100357 pool_gc(pool);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100358 continue;
359 }
360 if (++pool->allocated > avail)
361 break;
362
Willy Tarreauac421112015-10-28 15:09:29 +0100363 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100364 pool->free_list = ptr;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200365 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200366 pool->used++;
Willy Tarreaude30a682015-10-28 15:23:51 +0100367#ifdef DEBUG_MEMORY_POOLS
368 /* keep track of where the element was allocated from */
369 *POOL_LINK(pool, ptr) = (void *)pool;
370#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100371 return ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200372}
Christopher Fauletb349e482017-08-29 09:52:38 +0200373void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
374{
375 void *ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200376
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100377 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200378 ptr = __pool_refill_alloc(pool, avail);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100379 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200380 return ptr;
381}
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200382/*
383 * This function frees whatever can be freed in pool <pool>.
384 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100385void pool_flush(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200386{
387 void *temp, *next;
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200388 if (!pool)
389 return;
390
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100391 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200392 next = pool->free_list;
393 while (next) {
394 temp = next;
Willy Tarreauac421112015-10-28 15:09:29 +0100395 next = *POOL_LINK(pool, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200396 pool->allocated--;
Willy Tarreauf13322e2017-11-22 10:50:54 +0100397 pool_free_area(temp, pool->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200398 }
399 pool->free_list = next;
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100400 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200401 /* here, we should have pool->allocate == pool->used */
402}
403
404/*
405 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200406 * the minimum thresholds imposed by owners. It takes care of avoiding
407 * recursion because it may be called from a signal handler.
Christopher Fauletb349e482017-08-29 09:52:38 +0200408 *
Willy Tarreaubafbe012017-11-24 17:34:44 +0100409 * <pool_ctx> is used when pool_gc is called to release resources to allocate
Christopher Fauletb349e482017-08-29 09:52:38 +0200410 * an element in __pool_refill_alloc. It is important because <pool_ctx> is
411 * already locked, so we need to skip the lock here.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200412 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100413void pool_gc(struct pool_head *pool_ctx)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200414{
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200415 static int recurse;
Christopher Fauletb349e482017-08-29 09:52:38 +0200416 int cur_recurse = 0;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200417 struct pool_head *entry;
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200418
Olivier Houchard20872762019-03-08 18:53:35 +0100419 if (recurse || !_HA_ATOMIC_CAS(&recurse, &cur_recurse, 1))
Christopher Fauletb349e482017-08-29 09:52:38 +0200420 return;
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200421
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200422 list_for_each_entry(entry, &pools, list) {
423 void *temp, *next;
424 //qfprintf(stderr, "Flushing pool %s\n", entry->name);
Christopher Fauletb349e482017-08-29 09:52:38 +0200425 if (entry != pool_ctx)
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100426 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200427 next = entry->free_list;
428 while (next &&
Willy Tarreau57767b82014-12-22 21:40:55 +0100429 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200430 temp = next;
Willy Tarreauac421112015-10-28 15:09:29 +0100431 next = *POOL_LINK(entry, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200432 entry->allocated--;
Willy Tarreauf13322e2017-11-22 10:50:54 +0100433 pool_free_area(temp, entry->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200434 }
435 entry->free_list = next;
Christopher Fauletb349e482017-08-29 09:52:38 +0200436 if (entry != pool_ctx)
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100437 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200438 }
Christopher Fauletb349e482017-08-29 09:52:38 +0200439
Olivier Houchard20872762019-03-08 18:53:35 +0100440 _HA_ATOMIC_STORE(&recurse, 0);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200441}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100442#endif
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200443
444/*
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200445 * This function destroys a pool by freeing it completely, unless it's still
446 * in use. This should be called only under extreme circumstances. It always
447 * returns NULL if the resulting pool is empty, easing the clearing of the old
448 * pointer, otherwise it returns the pool.
449 * .
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200450 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100451void *pool_destroy(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200452{
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200453 if (pool) {
Willy Tarreaubafbe012017-11-24 17:34:44 +0100454 pool_flush(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200455 if (pool->used)
456 return pool;
457 pool->users--;
458 if (!pool->users) {
459 LIST_DEL(&pool->list);
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100460#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100461 HA_SPIN_DESTROY(&pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100462#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200463 if ((pool - pool_base_start) < MAX_BASE_POOLS)
464 memset(pool, 0, sizeof(*pool));
465 else
466 free(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200467 }
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200468 }
469 return NULL;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200470}
471
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100472/* This destroys all pools on exit. It is *not* thread safe. */
473void pool_destroy_all()
474{
475 struct pool_head *entry, *back;
476
477 list_for_each_entry_safe(entry, back, &pools, list)
478 pool_destroy(entry);
479}
480
Willy Tarreau12833bb2014-01-28 16:49:56 +0100481/* This function dumps memory usage information into the trash buffer. */
482void dump_pools_to_trash()
Willy Tarreau50e608d2007-05-13 18:26:08 +0200483{
484 struct pool_head *entry;
485 unsigned long allocated, used;
486 int nbpools;
487
488 allocated = used = nbpools = 0;
Willy Tarreau12833bb2014-01-28 16:49:56 +0100489 chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200490 list_for_each_entry(entry, &pools, list) {
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100491#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100492 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100493#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200494 chunk_appendf(&trash, " - Pool %s (%d bytes) : %d allocated (%u bytes), %d used, %d failures, %d users, @%p=%02d%s\n",
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200495 entry->name, entry->size, entry->allocated,
Willy Tarreau58102cf2015-10-28 16:24:21 +0100496 entry->size * entry->allocated, entry->used, entry->failed,
Willy Tarreau0a93b642018-10-16 07:58:39 +0200497 entry->users, entry, (int)pool_get_index(entry),
498 (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200499
500 allocated += entry->allocated * entry->size;
501 used += entry->used * entry->size;
502 nbpools++;
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100503#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100504 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100505#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +0200506 }
Willy Tarreau12833bb2014-01-28 16:49:56 +0100507 chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n",
Willy Tarreau50e608d2007-05-13 18:26:08 +0200508 nbpools, allocated, used);
509}
510
Willy Tarreau12833bb2014-01-28 16:49:56 +0100511/* Dump statistics on pools usage. */
512void dump_pools(void)
513{
514 dump_pools_to_trash();
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200515 qfprintf(stderr, "%s", trash.area);
Willy Tarreau12833bb2014-01-28 16:49:56 +0100516}
517
Willy Tarreau58102cf2015-10-28 16:24:21 +0100518/* This function returns the total number of failed pool allocations */
519int pool_total_failures()
520{
521 struct pool_head *entry;
522 int failed = 0;
523
524 list_for_each_entry(entry, &pools, list)
525 failed += entry->failed;
526 return failed;
527}
528
529/* This function returns the total amount of memory allocated in pools (in bytes) */
530unsigned long pool_total_allocated()
531{
532 struct pool_head *entry;
533 unsigned long allocated = 0;
534
535 list_for_each_entry(entry, &pools, list)
536 allocated += entry->allocated * entry->size;
537 return allocated;
538}
539
540/* This function returns the total amount of memory used in pools (in bytes) */
541unsigned long pool_total_used()
542{
543 struct pool_head *entry;
544 unsigned long used = 0;
545
546 list_for_each_entry(entry, &pools, list)
547 used += entry->used * entry->size;
548 return used;
549}
550
William Lallemande7ed8852016-11-19 02:25:36 +0100551/* This function dumps memory usage information onto the stream interface's
552 * read buffer. It returns 0 as long as it does not complete, non-zero upon
553 * completion. No state is used.
554 */
555static int cli_io_handler_dump_pools(struct appctx *appctx)
556{
557 struct stream_interface *si = appctx->owner;
558
559 dump_pools_to_trash();
Willy Tarreau06d80a92017-10-19 14:32:15 +0200560 if (ci_putchk(si_ic(si), &trash) == -1) {
Willy Tarreaudb398432018-11-15 11:08:52 +0100561 si_rx_room_blk(si);
William Lallemande7ed8852016-11-19 02:25:36 +0100562 return 0;
563 }
564 return 1;
565}
566
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100567/* callback used to create early pool <name> of size <size> and store the
568 * resulting pointer into <ptr>. If the allocation fails, it quits with after
569 * emitting an error message.
570 */
571void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
572{
573 *ptr = create_pool(name, size, MEM_F_SHARED);
574 if (!*ptr) {
575 ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
576 name, size, strerror(errno));
577 exit(1);
578 }
579}
580
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100581/* Initializes all per-thread arrays on startup */
582static void init_pools()
583{
584 int thr, idx;
585
586 for (thr = 0; thr < MAX_THREADS; thr++) {
587 for (idx = 0; idx < MAX_BASE_POOLS; idx++) {
588 LIST_INIT(&pool_cache[thr][idx].list);
589 pool_cache[thr][idx].size = 0;
590 }
591 LIST_INIT(&pool_lru_head[thr]);
592 }
593}
594
595INITCALL0(STG_PREPARE, init_pools);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100596
William Lallemande7ed8852016-11-19 02:25:36 +0100597/* register cli keywords */
598static struct cli_kw_list cli_kws = {{ },{
Willy Tarreaue9ecec82016-12-16 18:55:23 +0100599 { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
William Lallemande7ed8852016-11-19 02:25:36 +0100600 {{},}
601}};
602
Willy Tarreau0108d902018-11-25 19:14:37 +0100603INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
William Lallemande7ed8852016-11-19 02:25:36 +0100604
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100605#ifdef DEBUG_FAIL_ALLOC
606#define MEM_FAIL_MAX_CHAR 32
607#define MEM_FAIL_MAX_STR 128
608static int mem_fail_cur_idx;
609static char mem_fail_str[MEM_FAIL_MAX_CHAR * MEM_FAIL_MAX_STR];
610__decl_hathreads(static HA_SPINLOCK_T mem_fail_lock);
611
612int mem_should_fail(const struct pool_head *pool)
613{
Olivier Houchard9c4f08a2019-02-01 16:28:04 +0100614 int ret = 0;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100615 int n;
616
617 if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
Willy Tarreau861c4ef2020-03-08 00:42:37 +0100618 int randnb = ha_random() % 100;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100619
620 if (mem_fail_rate > randnb)
621 ret = 1;
622 else
623 ret = 0;
624 }
Olivier Houchard4536b302020-02-01 17:49:31 +0100625 HA_SPIN_LOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100626 n = snprintf(&mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR],
627 MEM_FAIL_MAX_CHAR - 2,
628 "%d %.18s %d %d", mem_fail_cur_idx, pool->name, ret, tid);
629 while (n < MEM_FAIL_MAX_CHAR - 1)
630 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n++] = ' ';
631 if (mem_fail_cur_idx < MEM_FAIL_MAX_STR - 1)
632 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = '\n';
633 else
634 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = 0;
635 mem_fail_cur_idx++;
636 if (mem_fail_cur_idx == MEM_FAIL_MAX_STR)
637 mem_fail_cur_idx = 0;
Olivier Houchard4536b302020-02-01 17:49:31 +0100638 HA_SPIN_UNLOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100639 return ret;
640
641}
642
643/* config parser for global "tune.fail-alloc" */
644static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
645 struct proxy *defpx, const char *file, int line,
646 char **err)
647{
648 if (too_many_args(1, args, err, NULL))
649 return -1;
650 mem_fail_rate = atoi(args[1]);
651 if (mem_fail_rate < 0 || mem_fail_rate > 100) {
652 memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
653 return -1;
654 }
655 return 0;
656}
657#endif
658
659/* register global config keywords */
660static struct cfg_kw_list mem_cfg_kws = {ILH, {
661#ifdef DEBUG_FAIL_ALLOC
662 { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
663#endif
664 { 0, NULL, NULL }
665}};
666
667INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);
668
Willy Tarreau50e608d2007-05-13 18:26:08 +0200669/*
670 * Local variables:
671 * c-indent-level: 8
672 * c-basic-offset: 8
673 * End:
674 */