blob: 9269271b7c8ebc9cf19853af2839ea2f4973f1a9 [file] [log] [blame]
Willy Tarreau50e608d2007-05-13 18:26:08 +02001/*
2 * Memory management functions.
3 *
4 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
Willy Tarreau7107c8b2018-11-26 11:44:35 +010012#include <errno.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020013
William Lallemande7ed8852016-11-19 02:25:36 +010014#include <types/applet.h>
15#include <types/cli.h>
Willy Tarreau12833bb2014-01-28 16:49:56 +010016#include <types/global.h>
William Lallemande7ed8852016-11-19 02:25:36 +010017#include <types/stats.h>
18
Olivier Houcharddc21ff72019-01-29 15:20:16 +010019#include <common/cfgparse.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020020#include <common/config.h>
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +020021#include <common/debug.h>
Willy Tarreaue18db9e2018-10-16 10:28:54 +020022#include <common/hathreads.h>
Willy Tarreau0108d902018-11-25 19:14:37 +010023#include <common/initcall.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020024#include <common/memory.h>
25#include <common/mini-clist.h>
26#include <common/standard.h>
27
Willy Tarreaua8b2ce02019-05-28 17:04:16 +020028#include <types/activity.h>
29
William Lallemande7ed8852016-11-19 02:25:36 +010030#include <proto/applet.h>
31#include <proto/cli.h>
32#include <proto/channel.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020033#include <proto/log.h>
William Lallemande7ed8852016-11-19 02:25:36 +010034#include <proto/stream_interface.h>
35#include <proto/stats.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020036
Willy Tarreau0a93b642018-10-16 07:58:39 +020037/* These are the most common pools, expected to be initialized first. These
38 * ones are allocated from an array, allowing to map them to an index.
39 */
40struct pool_head pool_base_start[MAX_BASE_POOLS] = { };
41unsigned int pool_base_count = 0;
42
Willy Tarreau7f0165e2018-11-26 17:09:46 +010043/* These ones are initialized per-thread on startup by init_pools() */
44struct pool_cache_head pool_cache[MAX_THREADS][MAX_BASE_POOLS];
45static struct list pool_lru_head[MAX_THREADS]; /* oldest objects */
Willy Tarreaue18db9e2018-10-16 10:28:54 +020046THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
47THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
48
Willy Tarreau50e608d2007-05-13 18:26:08 +020049static struct list pools = LIST_HEAD_INIT(pools);
Willy Tarreau067ac9f2015-10-08 14:12:13 +020050int mem_poison_byte = -1;
Willy Tarreau50e608d2007-05-13 18:26:08 +020051
Olivier Houcharddc21ff72019-01-29 15:20:16 +010052#ifdef DEBUG_FAIL_ALLOC
53static int mem_fail_rate = 0;
54static int mem_should_fail(const struct pool_head *);
55#endif
56
Willy Tarreau50e608d2007-05-13 18:26:08 +020057/* Try to find an existing shared pool with the same characteristics and
58 * returns it, otherwise creates this one. NULL is returned if no memory
Willy Tarreau581bf812016-01-25 02:19:13 +010059 * is available for a new creation. Two flags are supported :
60 * - MEM_F_SHARED to indicate that the pool may be shared with other users
61 * - MEM_F_EXACT to indicate that the size must not be rounded up
Willy Tarreau50e608d2007-05-13 18:26:08 +020062 */
63struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
64{
65 struct pool_head *pool;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020066 struct pool_head *entry;
67 struct list *start;
Willy Tarreau50e608d2007-05-13 18:26:08 +020068 unsigned int align;
Christopher Faulet77204fd2019-06-25 21:45:59 +020069 int thr, idx;
Willy Tarreau50e608d2007-05-13 18:26:08 +020070
Willy Tarreauac421112015-10-28 15:09:29 +010071 /* We need to store a (void *) at the end of the chunks. Since we know
Willy Tarreau50e608d2007-05-13 18:26:08 +020072 * that the malloc() function will never return such a small size,
73 * let's round the size up to something slightly bigger, in order to
74 * ease merging of entries. Note that the rounding is a power of two.
Willy Tarreauac421112015-10-28 15:09:29 +010075 * This extra (void *) is not accounted for in the size computation
76 * so that the visible parts outside are not affected.
Willy Tarreau30f931e2018-10-23 14:40:23 +020077 *
78 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
Willy Tarreau50e608d2007-05-13 18:26:08 +020079 */
80
Willy Tarreau581bf812016-01-25 02:19:13 +010081 if (!(flags & MEM_F_EXACT)) {
Willy Tarreau30f931e2018-10-23 14:40:23 +020082 align = 4 * sizeof(void *); // 2 lists = 4 pointers min
Willy Tarreau581bf812016-01-25 02:19:13 +010083 size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA;
84 }
Willy Tarreau50e608d2007-05-13 18:26:08 +020085
Christopher Fauletb349e482017-08-29 09:52:38 +020086 /* TODO: thread: we do not lock pool list for now because all pools are
87 * created during HAProxy startup (so before threads creation) */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020088 start = &pools;
Willy Tarreau50e608d2007-05-13 18:26:08 +020089 pool = NULL;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020090
91 list_for_each_entry(entry, &pools, list) {
92 if (entry->size == size) {
93 /* either we can share this place and we take it, or
94 * we look for a sharable one or for the next position
95 * before which we will insert a new one.
96 */
Willy Tarreau49fa5bb2021-05-05 07:29:01 +020097 if ((flags & entry->flags & MEM_F_SHARED)
98#ifdef DEBUG_DONT_SHARE_POOLS
99 && strcmp(name, entry->name) == 0
100#endif
101 ) {
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200102 /* we can share this one */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200103 pool = entry;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200104 DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
Willy Tarreau50e608d2007-05-13 18:26:08 +0200105 break;
106 }
107 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200108 else if (entry->size > size) {
109 /* insert before this one */
110 start = &entry->list;
111 break;
112 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200113 }
114
115 if (!pool) {
Willy Tarreau0a93b642018-10-16 07:58:39 +0200116 if (pool_base_count < MAX_BASE_POOLS)
117 pool = &pool_base_start[pool_base_count++];
118
119 if (!pool) {
120 /* look for a freed entry */
121 for (entry = pool_base_start; entry != pool_base_start + MAX_BASE_POOLS; entry++) {
122 if (!entry->size) {
123 pool = entry;
124 break;
125 }
126 }
127 }
128
129 if (!pool)
130 pool = calloc(1, sizeof(*pool));
131
Willy Tarreau50e608d2007-05-13 18:26:08 +0200132 if (!pool)
133 return NULL;
134 if (name)
135 strlcpy2(pool->name, name, sizeof(pool->name));
136 pool->size = size;
137 pool->flags = flags;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200138 LIST_ADDQ(start, &pool->list);
Christopher Faulet77204fd2019-06-25 21:45:59 +0200139
140 /* update per-thread pool cache if necessary */
141 idx = pool_get_index(pool);
142 if (idx >= 0) {
143 for (thr = 0; thr < MAX_THREADS; thr++)
144 pool_cache[thr][idx].size = size;
145 }
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100146#ifndef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchard4a2d6bd2020-02-01 17:45:32 +0100147 HA_SPIN_INIT(&pool->lock);
Olivier Houchard4536b302020-02-01 17:49:31 +0100148#else
Olivier Houchard5dfbf2e2020-03-18 15:48:29 +0100149 HA_SPIN_INIT(&pool->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100150#endif
Olivier Houchard4a2d6bd2020-02-01 17:45:32 +0100151 }
152 pool->users++;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200153 return pool;
154}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100155
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100156#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +0100157/* Allocates new entries for pool <pool> until there are at least <avail> + 1
158 * available, then returns the last one for immediate use, so that at least
159 * <avail> are left available in the pool upon return. NULL is returned if the
160 * last entry could not be allocated. It's important to note that at least one
161 * allocation is always performed even if there are enough entries in the pool.
162 * A call to the garbage collector is performed at most once in case malloc()
163 * returns an error, before returning NULL.
164 */
165void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
166{
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200167 void *ptr = NULL, **free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100168 int failed = 0;
169 int size = pool->size;
170 int limit = pool->limit;
171 int allocated = pool->allocated, allocated_orig = allocated;
172
173 /* stop point */
174 avail += pool->used;
175
176 while (1) {
177 if (limit && allocated >= limit) {
Olivier Houchard20872762019-03-08 18:53:35 +0100178 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200179 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100180 return NULL;
181 }
182
183 ptr = malloc(size + POOL_EXTRA);
184 if (!ptr) {
Olivier Houchard20872762019-03-08 18:53:35 +0100185 _HA_ATOMIC_ADD(&pool->failed, 1);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200186 if (failed) {
187 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100188 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200189 }
Olivier Houchardcf975d42018-01-24 18:38:31 +0100190 failed++;
191 pool_gc(pool);
192 continue;
193 }
194 if (++allocated > avail)
195 break;
196
197 free_list = pool->free_list;
198 do {
199 *POOL_LINK(pool, ptr) = free_list;
200 __ha_barrier_store();
Olivier Houchard20872762019-03-08 18:53:35 +0100201 } while (_HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr) == 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100202 }
Olivier Houchard20872762019-03-08 18:53:35 +0100203 __ha_barrier_atomic_store();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100204
Olivier Houchard20872762019-03-08 18:53:35 +0100205 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
206 _HA_ATOMIC_ADD(&pool->used, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100207
208#ifdef DEBUG_MEMORY_POOLS
209 /* keep track of where the element was allocated from */
210 *POOL_LINK(pool, ptr) = (void *)pool;
211#endif
212 return ptr;
213}
214void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
215{
216 void *ptr;
217
218 ptr = __pool_refill_alloc(pool, avail);
219 return ptr;
220}
221/*
222 * This function frees whatever can be freed in pool <pool>.
223 */
224void pool_flush(struct pool_head *pool)
225{
Olivier Houchard21861592020-02-01 17:37:22 +0100226 struct pool_free_list cmp, new;
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200227 void **next, *temp;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100228 int removed = 0;
229
230 if (!pool)
231 return;
Olivier Houchard5dfbf2e2020-03-18 15:48:29 +0100232 HA_SPIN_LOCK(POOL_LOCK, &pool->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100233 do {
Olivier Houchard21861592020-02-01 17:37:22 +0100234 cmp.free_list = pool->free_list;
235 cmp.seq = pool->seq;
236 new.free_list = NULL;
237 new.seq = cmp.seq + 1;
238 } while (!_HA_ATOMIC_DWCAS(&pool->free_list, &cmp, &new));
Olivier Houchard20872762019-03-08 18:53:35 +0100239 __ha_barrier_atomic_store();
Olivier Houchard5dfbf2e2020-03-18 15:48:29 +0100240 HA_SPIN_UNLOCK(POOL_LOCK, &pool->flush_lock);
Olivier Houchard21861592020-02-01 17:37:22 +0100241 next = cmp.free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100242 while (next) {
243 temp = next;
244 next = *POOL_LINK(pool, temp);
245 removed++;
246 free(temp);
247 }
Olivier Houchard20872762019-03-08 18:53:35 +0100248 _HA_ATOMIC_SUB(&pool->allocated, removed);
Willy Tarreau6270ce32021-06-10 06:54:22 +0200249 /* here, we should have pool->allocated == pool->used */
Olivier Houchardcf975d42018-01-24 18:38:31 +0100250}
251
252/*
253 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreau636efb32020-04-24 06:15:24 +0200254 * the minimum thresholds imposed by owners. It makes sure to be alone to
255 * run by using thread_isolate(). <pool_ctx> is unused.
Olivier Houchardcf975d42018-01-24 18:38:31 +0100256 */
257void pool_gc(struct pool_head *pool_ctx)
258{
Olivier Houchardcf975d42018-01-24 18:38:31 +0100259 struct pool_head *entry;
Willy Tarreau636efb32020-04-24 06:15:24 +0200260 int isolated = thread_isolated();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100261
Willy Tarreau636efb32020-04-24 06:15:24 +0200262 if (!isolated)
263 thread_isolate();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100264
265 list_for_each_entry(entry, &pools, list) {
Willy Tarreau24b295b2021-06-10 10:21:35 +0200266 void *temp;
267 //qfprintf(stderr, "Flushing pool %s\n", entry->name);
268 while (entry->free_list &&
269 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
270 temp = entry->free_list;
271 entry->free_list = *POOL_LINK(entry, temp);
272 entry->allocated--;
273 free(temp);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100274 }
275 }
276
Willy Tarreau636efb32020-04-24 06:15:24 +0200277 if (!isolated)
278 thread_release();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100279}
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200280
281/* frees an object to the local cache, possibly pushing oldest objects to the
282 * global pool. Must not be called directly.
283 */
284void __pool_put_to_cache(struct pool_head *pool, void *ptr, ssize_t idx)
285{
286 struct pool_cache_item *item = (struct pool_cache_item *)ptr;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100287 struct pool_cache_head *ph = &pool_cache[tid][idx];
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200288
289 LIST_ADD(&ph->list, &item->by_pool);
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100290 LIST_ADD(&pool_lru_head[tid], &item->by_lru);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200291 ph->count++;
292 pool_cache_count++;
293 pool_cache_bytes += ph->size;
294
295 if (pool_cache_bytes <= CONFIG_HAP_POOL_CACHE_SIZE)
296 return;
297
298 do {
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100299 item = LIST_PREV(&pool_lru_head[tid], struct pool_cache_item *, by_lru);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200300 /* note: by definition we remove oldest objects so they also are the
301 * oldest in their own pools, thus their next is the pool's head.
302 */
303 ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
304 LIST_DEL(&item->by_pool);
305 LIST_DEL(&item->by_lru);
306 ph->count--;
307 pool_cache_count--;
308 pool_cache_bytes -= ph->size;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100309 __pool_free(pool_base_start + (ph - pool_cache[tid]), item);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200310 } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
311}
312
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100313#else /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200314
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100315/* Allocates new entries for pool <pool> until there are at least <avail> + 1
316 * available, then returns the last one for immediate use, so that at least
317 * <avail> are left available in the pool upon return. NULL is returned if the
318 * last entry could not be allocated. It's important to note that at least one
319 * allocation is always performed even if there are enough entries in the pool.
320 * A call to the garbage collector is performed at most once in case malloc()
321 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200322 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200323void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200324{
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100325 void *ptr = NULL;
326 int failed = 0;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200327
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100328#ifdef DEBUG_FAIL_ALLOC
329 if (mem_should_fail(pool))
330 return NULL;
331#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100332 /* stop point */
333 avail += pool->used;
334
335 while (1) {
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200336 if (pool->limit && pool->allocated >= pool->limit) {
337 activity[tid].pool_fail++;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200338 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200339 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100340
Willy Tarreauf13322e2017-11-22 10:50:54 +0100341 ptr = pool_alloc_area(pool->size + POOL_EXTRA);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100342 if (!ptr) {
Willy Tarreau58102cf2015-10-28 16:24:21 +0100343 pool->failed++;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200344 if (failed) {
345 activity[tid].pool_fail++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100346 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200347 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100348 failed++;
Willy Tarreaubafbe012017-11-24 17:34:44 +0100349 pool_gc(pool);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100350 continue;
351 }
352 if (++pool->allocated > avail)
353 break;
354
Willy Tarreauac421112015-10-28 15:09:29 +0100355 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100356 pool->free_list = ptr;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200357 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200358 pool->used++;
Willy Tarreaude30a682015-10-28 15:23:51 +0100359#ifdef DEBUG_MEMORY_POOLS
360 /* keep track of where the element was allocated from */
361 *POOL_LINK(pool, ptr) = (void *)pool;
362#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100363 return ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200364}
Christopher Fauletb349e482017-08-29 09:52:38 +0200365void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
366{
367 void *ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200368
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100369 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200370 ptr = __pool_refill_alloc(pool, avail);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100371 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200372 return ptr;
373}
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200374/*
375 * This function frees whatever can be freed in pool <pool>.
376 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100377void pool_flush(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200378{
Willy Tarreau2504ffc2021-06-10 07:13:04 +0200379 void *temp, **next;
380
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200381 if (!pool)
382 return;
383
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100384 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200385 next = pool->free_list;
386 while (next) {
387 temp = next;
Willy Tarreauac421112015-10-28 15:09:29 +0100388 next = *POOL_LINK(pool, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200389 pool->allocated--;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200390 }
Willy Tarreau2504ffc2021-06-10 07:13:04 +0200391
392 next = pool->free_list;
393 pool->free_list = NULL;
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100394 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreau2504ffc2021-06-10 07:13:04 +0200395
396 while (next) {
397 temp = next;
398 next = *POOL_LINK(pool, temp);
399 pool_free_area(temp, pool->size + POOL_EXTRA);
400 }
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200401 /* here, we should have pool->allocate == pool->used */
402}
403
404/*
405 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreau636efb32020-04-24 06:15:24 +0200406 * the minimum thresholds imposed by owners. It makes sure to be alone to
407 * run by using thread_isolate(). <pool_ctx> is unused.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200408 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100409void pool_gc(struct pool_head *pool_ctx)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200410{
411 struct pool_head *entry;
Willy Tarreau636efb32020-04-24 06:15:24 +0200412 int isolated = thread_isolated();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200413
Willy Tarreau636efb32020-04-24 06:15:24 +0200414 if (!isolated)
415 thread_isolate();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200416
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200417 list_for_each_entry(entry, &pools, list) {
Olivier Houcharde549cb22020-03-12 19:05:39 +0100418 void *temp;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200419 //qfprintf(stderr, "Flushing pool %s\n", entry->name);
Olivier Houcharde549cb22020-03-12 19:05:39 +0100420 while (entry->free_list &&
Willy Tarreau57767b82014-12-22 21:40:55 +0100421 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
Olivier Houcharde549cb22020-03-12 19:05:39 +0100422 temp = entry->free_list;
423 entry->free_list = *POOL_LINK(entry, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200424 entry->allocated--;
Willy Tarreauf13322e2017-11-22 10:50:54 +0100425 pool_free_area(temp, entry->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200426 }
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200427 }
Christopher Fauletb349e482017-08-29 09:52:38 +0200428
Willy Tarreau636efb32020-04-24 06:15:24 +0200429 if (!isolated)
430 thread_release();
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200431}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100432#endif
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200433
434/*
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200435 * This function destroys a pool by freeing it completely, unless it's still
436 * in use. This should be called only under extreme circumstances. It always
437 * returns NULL if the resulting pool is empty, easing the clearing of the old
438 * pointer, otherwise it returns the pool.
439 * .
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200440 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100441void *pool_destroy(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200442{
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200443 if (pool) {
Willy Tarreaubafbe012017-11-24 17:34:44 +0100444 pool_flush(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200445 if (pool->used)
446 return pool;
447 pool->users--;
448 if (!pool->users) {
449 LIST_DEL(&pool->list);
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100450#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100451 HA_SPIN_DESTROY(&pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100452#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200453 if ((pool - pool_base_start) < MAX_BASE_POOLS)
454 memset(pool, 0, sizeof(*pool));
455 else
456 free(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200457 }
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200458 }
459 return NULL;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200460}
461
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100462/* This destroys all pools on exit. It is *not* thread safe. */
463void pool_destroy_all()
464{
465 struct pool_head *entry, *back;
466
467 list_for_each_entry_safe(entry, back, &pools, list)
468 pool_destroy(entry);
469}
470
Willy Tarreau12833bb2014-01-28 16:49:56 +0100471/* This function dumps memory usage information into the trash buffer. */
472void dump_pools_to_trash()
Willy Tarreau50e608d2007-05-13 18:26:08 +0200473{
474 struct pool_head *entry;
475 unsigned long allocated, used;
476 int nbpools;
477
478 allocated = used = nbpools = 0;
Willy Tarreau12833bb2014-01-28 16:49:56 +0100479 chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200480 list_for_each_entry(entry, &pools, list) {
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100481#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100482 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100483#endif
Willy Tarreau81461622020-05-06 17:10:37 +0200484 chunk_appendf(&trash, " - Pool %s (%u bytes) : %u allocated (%u bytes), %u used, %u failures, %u users, @%p=%02d%s\n",
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200485 entry->name, entry->size, entry->allocated,
Willy Tarreau58102cf2015-10-28 16:24:21 +0100486 entry->size * entry->allocated, entry->used, entry->failed,
Willy Tarreau0a93b642018-10-16 07:58:39 +0200487 entry->users, entry, (int)pool_get_index(entry),
488 (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200489
490 allocated += entry->allocated * entry->size;
491 used += entry->used * entry->size;
492 nbpools++;
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100493#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100494 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100495#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +0200496 }
Willy Tarreau12833bb2014-01-28 16:49:56 +0100497 chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n",
Willy Tarreau50e608d2007-05-13 18:26:08 +0200498 nbpools, allocated, used);
499}
500
Willy Tarreau12833bb2014-01-28 16:49:56 +0100501/* Dump statistics on pools usage. */
502void dump_pools(void)
503{
504 dump_pools_to_trash();
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200505 qfprintf(stderr, "%s", trash.area);
Willy Tarreau12833bb2014-01-28 16:49:56 +0100506}
507
Willy Tarreau58102cf2015-10-28 16:24:21 +0100508/* This function returns the total number of failed pool allocations */
509int pool_total_failures()
510{
511 struct pool_head *entry;
512 int failed = 0;
513
514 list_for_each_entry(entry, &pools, list)
515 failed += entry->failed;
516 return failed;
517}
518
519/* This function returns the total amount of memory allocated in pools (in bytes) */
520unsigned long pool_total_allocated()
521{
522 struct pool_head *entry;
523 unsigned long allocated = 0;
524
525 list_for_each_entry(entry, &pools, list)
526 allocated += entry->allocated * entry->size;
527 return allocated;
528}
529
530/* This function returns the total amount of memory used in pools (in bytes) */
531unsigned long pool_total_used()
532{
533 struct pool_head *entry;
534 unsigned long used = 0;
535
536 list_for_each_entry(entry, &pools, list)
537 used += entry->used * entry->size;
538 return used;
539}
540
William Lallemande7ed8852016-11-19 02:25:36 +0100541/* This function dumps memory usage information onto the stream interface's
542 * read buffer. It returns 0 as long as it does not complete, non-zero upon
543 * completion. No state is used.
544 */
545static int cli_io_handler_dump_pools(struct appctx *appctx)
546{
547 struct stream_interface *si = appctx->owner;
548
549 dump_pools_to_trash();
Willy Tarreau06d80a92017-10-19 14:32:15 +0200550 if (ci_putchk(si_ic(si), &trash) == -1) {
Willy Tarreaudb398432018-11-15 11:08:52 +0100551 si_rx_room_blk(si);
William Lallemande7ed8852016-11-19 02:25:36 +0100552 return 0;
553 }
554 return 1;
555}
556
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100557/* callback used to create early pool <name> of size <size> and store the
558 * resulting pointer into <ptr>. If the allocation fails, it quits with after
559 * emitting an error message.
560 */
561void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
562{
563 *ptr = create_pool(name, size, MEM_F_SHARED);
564 if (!*ptr) {
565 ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
566 name, size, strerror(errno));
567 exit(1);
568 }
569}
570
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100571/* Initializes all per-thread arrays on startup */
572static void init_pools()
573{
574 int thr, idx;
575
576 for (thr = 0; thr < MAX_THREADS; thr++) {
577 for (idx = 0; idx < MAX_BASE_POOLS; idx++) {
578 LIST_INIT(&pool_cache[thr][idx].list);
579 pool_cache[thr][idx].size = 0;
580 }
581 LIST_INIT(&pool_lru_head[thr]);
582 }
583}
584
585INITCALL0(STG_PREPARE, init_pools);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100586
William Lallemande7ed8852016-11-19 02:25:36 +0100587/* register cli keywords */
588static struct cli_kw_list cli_kws = {{ },{
Willy Tarreaue9ecec82016-12-16 18:55:23 +0100589 { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
William Lallemande7ed8852016-11-19 02:25:36 +0100590 {{},}
591}};
592
Willy Tarreau0108d902018-11-25 19:14:37 +0100593INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
William Lallemande7ed8852016-11-19 02:25:36 +0100594
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100595#ifdef DEBUG_FAIL_ALLOC
596#define MEM_FAIL_MAX_CHAR 32
597#define MEM_FAIL_MAX_STR 128
598static int mem_fail_cur_idx;
599static char mem_fail_str[MEM_FAIL_MAX_CHAR * MEM_FAIL_MAX_STR];
600__decl_hathreads(static HA_SPINLOCK_T mem_fail_lock);
601
602int mem_should_fail(const struct pool_head *pool)
603{
Olivier Houchard9c4f08a2019-02-01 16:28:04 +0100604 int ret = 0;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100605 int n;
606
607 if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
Willy Tarreau861c4ef2020-03-08 00:42:37 +0100608 int randnb = ha_random() % 100;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100609
610 if (mem_fail_rate > randnb)
611 ret = 1;
612 else
613 ret = 0;
614 }
Olivier Houchard4536b302020-02-01 17:49:31 +0100615 HA_SPIN_LOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100616 n = snprintf(&mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR],
617 MEM_FAIL_MAX_CHAR - 2,
618 "%d %.18s %d %d", mem_fail_cur_idx, pool->name, ret, tid);
619 while (n < MEM_FAIL_MAX_CHAR - 1)
620 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n++] = ' ';
621 if (mem_fail_cur_idx < MEM_FAIL_MAX_STR - 1)
622 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = '\n';
623 else
624 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = 0;
625 mem_fail_cur_idx++;
626 if (mem_fail_cur_idx == MEM_FAIL_MAX_STR)
627 mem_fail_cur_idx = 0;
Olivier Houchard4536b302020-02-01 17:49:31 +0100628 HA_SPIN_UNLOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100629 return ret;
630
631}
632
633/* config parser for global "tune.fail-alloc" */
634static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
635 struct proxy *defpx, const char *file, int line,
636 char **err)
637{
638 if (too_many_args(1, args, err, NULL))
639 return -1;
640 mem_fail_rate = atoi(args[1]);
641 if (mem_fail_rate < 0 || mem_fail_rate > 100) {
642 memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
643 return -1;
644 }
645 return 0;
646}
647#endif
648
649/* register global config keywords */
650static struct cfg_kw_list mem_cfg_kws = {ILH, {
651#ifdef DEBUG_FAIL_ALLOC
652 { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
653#endif
654 { 0, NULL, NULL }
655}};
656
657INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);
658
Willy Tarreau50e608d2007-05-13 18:26:08 +0200659/*
660 * Local variables:
661 * c-indent-level: 8
662 * c-basic-offset: 8
663 * End:
664 */