blob: 03cb2ec5da7ef38a1f720c2215dba7c788950480 [file] [log] [blame]
Willy Tarreau50e608d2007-05-13 18:26:08 +02001/*
2 * Memory management functions.
3 *
4 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
Willy Tarreau7107c8b2018-11-26 11:44:35 +010012#include <errno.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020013
Willy Tarreaub2551052020-06-09 09:07:15 +020014#include <haproxy/activity-t.h>
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020015#include <haproxy/api.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020016#include <haproxy/applet-t.h>
Willy Tarreau6be78492020-06-05 00:00:29 +020017#include <haproxy/cfgparse.h>
Willy Tarreauf1d32c42020-06-04 21:07:02 +020018#include <haproxy/channel.h>
Willy Tarreau83487a82020-06-04 20:19:54 +020019#include <haproxy/cli.h>
Willy Tarreau36979d92020-06-05 17:27:29 +020020#include <haproxy/errors.h>
Willy Tarreauf268ee82020-06-04 17:05:57 +020021#include <haproxy/global.h>
Willy Tarreau853b2972020-05-27 18:01:47 +020022#include <haproxy/list.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020023#include <haproxy/pool.h>
Willy Tarreau2eec9b52020-06-04 19:58:55 +020024#include <haproxy/stats-t.h>
Willy Tarreau5e539c92020-06-04 20:45:39 +020025#include <haproxy/stream_interface.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020026#include <haproxy/thread.h>
Willy Tarreau48fbcae2020-06-03 18:09:46 +020027#include <haproxy/tools.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020028
Willy Tarreau50e608d2007-05-13 18:26:08 +020029
Willy Tarreaued891fd2020-06-01 19:00:28 +020030#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau0a93b642018-10-16 07:58:39 +020031/* These are the most common pools, expected to be initialized first. These
32 * ones are allocated from an array, allowing to map them to an index.
33 */
34struct pool_head pool_base_start[MAX_BASE_POOLS] = { };
35unsigned int pool_base_count = 0;
36
Willy Tarreau7f0165e2018-11-26 17:09:46 +010037/* These ones are initialized per-thread on startup by init_pools() */
38struct pool_cache_head pool_cache[MAX_THREADS][MAX_BASE_POOLS];
Willy Tarreaue18db9e2018-10-16 10:28:54 +020039THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
40THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
Willy Tarreaued891fd2020-06-01 19:00:28 +020041#endif
Willy Tarreaue18db9e2018-10-16 10:28:54 +020042
Willy Tarreau50e608d2007-05-13 18:26:08 +020043static struct list pools = LIST_HEAD_INIT(pools);
Willy Tarreau067ac9f2015-10-08 14:12:13 +020044int mem_poison_byte = -1;
Willy Tarreau50e608d2007-05-13 18:26:08 +020045
Olivier Houcharddc21ff72019-01-29 15:20:16 +010046#ifdef DEBUG_FAIL_ALLOC
47static int mem_fail_rate = 0;
48static int mem_should_fail(const struct pool_head *);
49#endif
50
Willy Tarreau50e608d2007-05-13 18:26:08 +020051/* Try to find an existing shared pool with the same characteristics and
52 * returns it, otherwise creates this one. NULL is returned if no memory
Willy Tarreau581bf812016-01-25 02:19:13 +010053 * is available for a new creation. Two flags are supported :
54 * - MEM_F_SHARED to indicate that the pool may be shared with other users
55 * - MEM_F_EXACT to indicate that the size must not be rounded up
Willy Tarreau50e608d2007-05-13 18:26:08 +020056 */
57struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
58{
59 struct pool_head *pool;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020060 struct pool_head *entry;
61 struct list *start;
Willy Tarreau50e608d2007-05-13 18:26:08 +020062 unsigned int align;
Willy Tarreaued891fd2020-06-01 19:00:28 +020063 int idx __maybe_unused;
Willy Tarreau50e608d2007-05-13 18:26:08 +020064
Willy Tarreauac421112015-10-28 15:09:29 +010065 /* We need to store a (void *) at the end of the chunks. Since we know
Willy Tarreau50e608d2007-05-13 18:26:08 +020066 * that the malloc() function will never return such a small size,
67 * let's round the size up to something slightly bigger, in order to
68 * ease merging of entries. Note that the rounding is a power of two.
Willy Tarreauac421112015-10-28 15:09:29 +010069 * This extra (void *) is not accounted for in the size computation
70 * so that the visible parts outside are not affected.
Willy Tarreau30f931e2018-10-23 14:40:23 +020071 *
72 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
Willy Tarreau50e608d2007-05-13 18:26:08 +020073 */
74
Willy Tarreau581bf812016-01-25 02:19:13 +010075 if (!(flags & MEM_F_EXACT)) {
Willy Tarreau30f931e2018-10-23 14:40:23 +020076 align = 4 * sizeof(void *); // 2 lists = 4 pointers min
Willy Tarreau581bf812016-01-25 02:19:13 +010077 size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA;
78 }
Willy Tarreau50e608d2007-05-13 18:26:08 +020079
Christopher Fauletb349e482017-08-29 09:52:38 +020080 /* TODO: thread: we do not lock pool list for now because all pools are
81 * created during HAProxy startup (so before threads creation) */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020082 start = &pools;
Willy Tarreau50e608d2007-05-13 18:26:08 +020083 pool = NULL;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020084
85 list_for_each_entry(entry, &pools, list) {
86 if (entry->size == size) {
87 /* either we can share this place and we take it, or
Ilya Shipitsin47d17182020-06-21 21:42:57 +050088 * we look for a shareable one or for the next position
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020089 * before which we will insert a new one.
90 */
91 if (flags & entry->flags & MEM_F_SHARED) {
92 /* we can share this one */
Willy Tarreau50e608d2007-05-13 18:26:08 +020093 pool = entry;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +020094 DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
Willy Tarreau50e608d2007-05-13 18:26:08 +020095 break;
96 }
97 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020098 else if (entry->size > size) {
99 /* insert before this one */
100 start = &entry->list;
101 break;
102 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200103 }
104
105 if (!pool) {
Willy Tarreaued891fd2020-06-01 19:00:28 +0200106#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau0a93b642018-10-16 07:58:39 +0200107 if (pool_base_count < MAX_BASE_POOLS)
108 pool = &pool_base_start[pool_base_count++];
109
110 if (!pool) {
111 /* look for a freed entry */
112 for (entry = pool_base_start; entry != pool_base_start + MAX_BASE_POOLS; entry++) {
113 if (!entry->size) {
114 pool = entry;
115 break;
116 }
117 }
118 }
Willy Tarreaued891fd2020-06-01 19:00:28 +0200119#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200120
121 if (!pool)
122 pool = calloc(1, sizeof(*pool));
123
Willy Tarreau50e608d2007-05-13 18:26:08 +0200124 if (!pool)
125 return NULL;
126 if (name)
127 strlcpy2(pool->name, name, sizeof(pool->name));
128 pool->size = size;
129 pool->flags = flags;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200130 LIST_ADDQ(start, &pool->list);
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200131
Willy Tarreaued891fd2020-06-01 19:00:28 +0200132#ifdef CONFIG_HAP_LOCAL_POOLS
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200133 /* update per-thread pool cache if necessary */
134 idx = pool_get_index(pool);
135 if (idx >= 0) {
Willy Tarreaued891fd2020-06-01 19:00:28 +0200136 int thr;
137
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200138 for (thr = 0; thr < MAX_THREADS; thr++)
139 pool_cache[thr][idx].size = size;
140 }
Willy Tarreaued891fd2020-06-01 19:00:28 +0200141#endif
Olivier Houchard8af97eb2020-02-01 17:45:32 +0100142 HA_SPIN_INIT(&pool->lock);
Olivier Houchard8af97eb2020-02-01 17:45:32 +0100143 }
144 pool->users++;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200145 return pool;
146}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100147
Willy Tarreaued891fd2020-06-01 19:00:28 +0200148#ifdef CONFIG_HAP_LOCAL_POOLS
149/* Evicts some of the oldest objects from the local cache, pushing them to the
150 * global pool.
151 */
152void pool_evict_from_cache()
153{
154 struct pool_cache_item *item;
155 struct pool_cache_head *ph;
156
157 do {
Willy Tarreau20dc3cd2020-06-28 00:54:27 +0200158 item = LIST_PREV(&ti->pool_lru_head, struct pool_cache_item *, by_lru);
Willy Tarreaued891fd2020-06-01 19:00:28 +0200159 /* note: by definition we remove oldest objects so they also are the
160 * oldest in their own pools, thus their next is the pool's head.
161 */
162 ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
163 LIST_DEL(&item->by_pool);
164 LIST_DEL(&item->by_lru);
165 ph->count--;
166 pool_cache_count--;
167 pool_cache_bytes -= ph->size;
168 __pool_free(pool_base_start + (ph - pool_cache[tid]), item);
169 } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
170}
171#endif
172
Willy Tarreau0bae0752021-03-02 20:05:09 +0100173#if defined(CONFIG_HAP_NO_GLOBAL_POOLS)
174
175/* simply fall back on the default OS' allocator */
176
177void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
178{
179 int allocated = pool->allocated;
180 int limit = pool->limit;
181 void *ptr = NULL;
182
183 if (limit && allocated >= limit) {
184 _HA_ATOMIC_ADD(&pool->allocated, 1);
185 activity[tid].pool_fail++;
186 return NULL;
187 }
188
189 ptr = pool_alloc_area(pool->size + POOL_EXTRA);
190 if (!ptr) {
191 _HA_ATOMIC_ADD(&pool->failed, 1);
192 activity[tid].pool_fail++;
193 return NULL;
194 }
195
196 _HA_ATOMIC_ADD(&pool->allocated, 1);
197 _HA_ATOMIC_ADD(&pool->used, 1);
198
199#ifdef DEBUG_MEMORY_POOLS
200 /* keep track of where the element was allocated from */
201 *POOL_LINK(pool, ptr) = (void *)pool;
202#endif
203 return ptr;
204}
205
206/* legacy stuff */
207void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
208{
209 void *ptr;
210
211 ptr = __pool_refill_alloc(pool, avail);
212 return ptr;
213}
214
215/* legacy stuff */
216void pool_flush(struct pool_head *pool)
217{
218}
219
220/* This function might ask the malloc library to trim its buffers. */
221void pool_gc(struct pool_head *pool_ctx)
222{
223#if defined(HA_HAVE_MALLOC_TRIM)
224 malloc_trim(0);
225#endif
226}
227
228#elif defined(CONFIG_HAP_LOCKLESS_POOLS)
229
Olivier Houchardcf975d42018-01-24 18:38:31 +0100230/* Allocates new entries for pool <pool> until there are at least <avail> + 1
231 * available, then returns the last one for immediate use, so that at least
232 * <avail> are left available in the pool upon return. NULL is returned if the
233 * last entry could not be allocated. It's important to note that at least one
234 * allocation is always performed even if there are enough entries in the pool.
235 * A call to the garbage collector is performed at most once in case malloc()
236 * returns an error, before returning NULL.
237 */
238void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
239{
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200240 void *ptr = NULL, **free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100241 int failed = 0;
242 int size = pool->size;
243 int limit = pool->limit;
244 int allocated = pool->allocated, allocated_orig = allocated;
245
246 /* stop point */
247 avail += pool->used;
248
249 while (1) {
250 if (limit && allocated >= limit) {
Olivier Houchard20872762019-03-08 18:53:35 +0100251 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200252 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100253 return NULL;
254 }
255
Willy Tarreau606135a2020-06-01 12:35:03 +0200256 swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->allocated, POOL_AVG_SAMPLES/4);
Willy Tarreaua1e4f8c2020-05-08 08:31:56 +0200257
Willy Tarreau24aa1ee2020-05-30 18:56:17 +0200258 ptr = pool_alloc_area(size + POOL_EXTRA);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100259 if (!ptr) {
Olivier Houchard20872762019-03-08 18:53:35 +0100260 _HA_ATOMIC_ADD(&pool->failed, 1);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200261 if (failed) {
262 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100263 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200264 }
Olivier Houchardcf975d42018-01-24 18:38:31 +0100265 failed++;
266 pool_gc(pool);
267 continue;
268 }
269 if (++allocated > avail)
270 break;
271
272 free_list = pool->free_list;
273 do {
274 *POOL_LINK(pool, ptr) = free_list;
275 __ha_barrier_store();
Olivier Houchard20872762019-03-08 18:53:35 +0100276 } while (_HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr) == 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100277 }
Olivier Houchard20872762019-03-08 18:53:35 +0100278 __ha_barrier_atomic_store();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100279
Olivier Houchard20872762019-03-08 18:53:35 +0100280 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
281 _HA_ATOMIC_ADD(&pool->used, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100282
283#ifdef DEBUG_MEMORY_POOLS
284 /* keep track of where the element was allocated from */
285 *POOL_LINK(pool, ptr) = (void *)pool;
286#endif
287 return ptr;
288}
289void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
290{
291 void *ptr;
292
293 ptr = __pool_refill_alloc(pool, avail);
294 return ptr;
295}
296/*
297 * This function frees whatever can be freed in pool <pool>.
298 */
299void pool_flush(struct pool_head *pool)
300{
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100301 struct pool_free_list cmp, new;
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200302 void **next, *temp;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100303 int removed = 0;
304
305 if (!pool)
306 return;
Willy Tarreau21072b92020-05-29 17:23:05 +0200307 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100308 do {
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100309 cmp.free_list = pool->free_list;
310 cmp.seq = pool->seq;
311 new.free_list = NULL;
312 new.seq = cmp.seq + 1;
313 } while (!_HA_ATOMIC_DWCAS(&pool->free_list, &cmp, &new));
Olivier Houchard20872762019-03-08 18:53:35 +0100314 __ha_barrier_atomic_store();
Willy Tarreau21072b92020-05-29 17:23:05 +0200315 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Olivier Houchardb6fa08b2020-02-01 17:37:22 +0100316 next = cmp.free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100317 while (next) {
318 temp = next;
319 next = *POOL_LINK(pool, temp);
320 removed++;
Willy Tarreau24aa1ee2020-05-30 18:56:17 +0200321 pool_free_area(temp, pool->size + POOL_EXTRA);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100322 }
323 pool->free_list = next;
Olivier Houchard20872762019-03-08 18:53:35 +0100324 _HA_ATOMIC_SUB(&pool->allocated, removed);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100325 /* here, we should have pool->allocate == pool->used */
326}
327
328/*
329 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200330 * the minimum thresholds imposed by owners. It makes sure to be alone to
331 * run by using thread_isolate(). <pool_ctx> is unused.
Olivier Houchardcf975d42018-01-24 18:38:31 +0100332 */
333void pool_gc(struct pool_head *pool_ctx)
334{
Olivier Houchardcf975d42018-01-24 18:38:31 +0100335 struct pool_head *entry;
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200336 int isolated = thread_isolated();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100337
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200338 if (!isolated)
339 thread_isolate();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100340
341 list_for_each_entry(entry, &pools, list) {
342 while ((int)((volatile int)entry->allocated - (volatile int)entry->used) > (int)entry->minavail) {
343 struct pool_free_list cmp, new;
344
345 cmp.seq = entry->seq;
346 __ha_barrier_load();
347 cmp.free_list = entry->free_list;
348 __ha_barrier_load();
349 if (cmp.free_list == NULL)
350 break;
351 new.free_list = *POOL_LINK(entry, cmp.free_list);
352 new.seq = cmp.seq + 1;
Willy Tarreau6a38b322019-05-11 18:04:24 +0200353 if (HA_ATOMIC_DWCAS(&entry->free_list, &cmp, &new) == 0)
Olivier Houchardcf975d42018-01-24 18:38:31 +0100354 continue;
Willy Tarreau24aa1ee2020-05-30 18:56:17 +0200355 pool_free_area(cmp.free_list, entry->size + POOL_EXTRA);
Olivier Houchard20872762019-03-08 18:53:35 +0100356 _HA_ATOMIC_SUB(&entry->allocated, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100357 }
358 }
359
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200360 if (!isolated)
361 thread_release();
Willy Tarreau88366c22020-11-03 15:53:34 +0100362
363#if defined(HA_HAVE_MALLOC_TRIM)
364 malloc_trim(0);
365#endif
Olivier Houchardcf975d42018-01-24 18:38:31 +0100366}
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200367
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100368#else /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200369
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100370/* Allocates new entries for pool <pool> until there are at least <avail> + 1
371 * available, then returns the last one for immediate use, so that at least
372 * <avail> are left available in the pool upon return. NULL is returned if the
373 * last entry could not be allocated. It's important to note that at least one
374 * allocation is always performed even if there are enough entries in the pool.
375 * A call to the garbage collector is performed at most once in case malloc()
376 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200377 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200378void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200379{
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100380 void *ptr = NULL;
381 int failed = 0;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200382
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100383#ifdef DEBUG_FAIL_ALLOC
384 if (mem_should_fail(pool))
385 return NULL;
386#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100387 /* stop point */
388 avail += pool->used;
389
390 while (1) {
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200391 if (pool->limit && pool->allocated >= pool->limit) {
392 activity[tid].pool_fail++;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200393 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200394 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100395
Willy Tarreau606135a2020-06-01 12:35:03 +0200396 swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->allocated, POOL_AVG_SAMPLES/4);
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200397 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreauf13322e2017-11-22 10:50:54 +0100398 ptr = pool_alloc_area(pool->size + POOL_EXTRA);
Willy Tarreau82867542019-07-04 11:48:16 +0200399#ifdef DEBUG_MEMORY_POOLS
400 /* keep track of where the element was allocated from. This
401 * is done out of the lock so that the system really allocates
402 * the data without harming other threads waiting on the lock.
403 */
404 if (ptr)
405 *POOL_LINK(pool, ptr) = (void *)pool;
406#endif
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200407 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100408 if (!ptr) {
Willy Tarreau58102cf2015-10-28 16:24:21 +0100409 pool->failed++;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200410 if (failed) {
411 activity[tid].pool_fail++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100412 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200413 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100414 failed++;
Willy Tarreaubafbe012017-11-24 17:34:44 +0100415 pool_gc(pool);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100416 continue;
417 }
418 if (++pool->allocated > avail)
419 break;
420
Willy Tarreauac421112015-10-28 15:09:29 +0100421 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100422 pool->free_list = ptr;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200423 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200424 pool->used++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100425 return ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200426}
Christopher Fauletb349e482017-08-29 09:52:38 +0200427void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
428{
429 void *ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200430
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100431 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200432 ptr = __pool_refill_alloc(pool, avail);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100433 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200434 return ptr;
435}
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200436/*
437 * This function frees whatever can be freed in pool <pool>.
438 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100439void pool_flush(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200440{
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200441 void *temp;
442
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200443 if (!pool)
444 return;
445
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200446 while (1) {
447 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
448 temp = pool->free_list;
449 if (!temp) {
450 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
451 break;
452 }
453 pool->free_list = *POOL_LINK(pool, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200454 pool->allocated--;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200455 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreauf13322e2017-11-22 10:50:54 +0100456 pool_free_area(temp, pool->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200457 }
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200458 /* here, we should have pool->allocated == pool->used */
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200459}
460
461/*
462 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200463 * the minimum thresholds imposed by owners. It makes sure to be alone to
464 * run by using thread_isolate(). <pool_ctx> is unused.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200465 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100466void pool_gc(struct pool_head *pool_ctx)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200467{
468 struct pool_head *entry;
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200469 int isolated = thread_isolated();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200470
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200471 if (!isolated)
472 thread_isolate();
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200473
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200474 list_for_each_entry(entry, &pools, list) {
Olivier Houchard51d93392020-03-12 19:05:39 +0100475 void *temp;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200476 //qfprintf(stderr, "Flushing pool %s\n", entry->name);
Olivier Houchard51d93392020-03-12 19:05:39 +0100477 while (entry->free_list &&
Willy Tarreau57767b82014-12-22 21:40:55 +0100478 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
Olivier Houchard51d93392020-03-12 19:05:39 +0100479 temp = entry->free_list;
480 entry->free_list = *POOL_LINK(entry, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200481 entry->allocated--;
Willy Tarreauf13322e2017-11-22 10:50:54 +0100482 pool_free_area(temp, entry->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200483 }
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200484 }
Christopher Fauletb349e482017-08-29 09:52:38 +0200485
Willy Tarreauc0e2ff22020-04-24 06:15:24 +0200486 if (!isolated)
487 thread_release();
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200488}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100489#endif
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200490
491/*
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200492 * This function destroys a pool by freeing it completely, unless it's still
493 * in use. This should be called only under extreme circumstances. It always
494 * returns NULL if the resulting pool is empty, easing the clearing of the old
495 * pointer, otherwise it returns the pool.
496 * .
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200497 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100498void *pool_destroy(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200499{
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200500 if (pool) {
Willy Tarreaubafbe012017-11-24 17:34:44 +0100501 pool_flush(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200502 if (pool->used)
503 return pool;
504 pool->users--;
505 if (!pool->users) {
506 LIST_DEL(&pool->list);
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100507#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100508 HA_SPIN_DESTROY(&pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100509#endif
Willy Tarreaued891fd2020-06-01 19:00:28 +0200510
511#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau0a93b642018-10-16 07:58:39 +0200512 if ((pool - pool_base_start) < MAX_BASE_POOLS)
513 memset(pool, 0, sizeof(*pool));
514 else
Willy Tarreaued891fd2020-06-01 19:00:28 +0200515#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200516 free(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200517 }
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200518 }
519 return NULL;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200520}
521
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100522/* This destroys all pools on exit. It is *not* thread safe. */
523void pool_destroy_all()
524{
525 struct pool_head *entry, *back;
526
527 list_for_each_entry_safe(entry, back, &pools, list)
528 pool_destroy(entry);
529}
530
Willy Tarreau12833bb2014-01-28 16:49:56 +0100531/* This function dumps memory usage information into the trash buffer. */
532void dump_pools_to_trash()
Willy Tarreau50e608d2007-05-13 18:26:08 +0200533{
534 struct pool_head *entry;
535 unsigned long allocated, used;
536 int nbpools;
537
538 allocated = used = nbpools = 0;
Willy Tarreau12833bb2014-01-28 16:49:56 +0100539 chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200540 list_for_each_entry(entry, &pools, list) {
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100541#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100542 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100543#endif
Willy Tarreaua1e4f8c2020-05-08 08:31:56 +0200544 chunk_appendf(&trash, " - Pool %s (%u bytes) : %u allocated (%u bytes), %u used, needed_avg %u, %u failures, %u users, @%p=%02d%s\n",
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200545 entry->name, entry->size, entry->allocated,
Willy Tarreaua1e4f8c2020-05-08 08:31:56 +0200546 entry->size * entry->allocated, entry->used,
Willy Tarreau606135a2020-06-01 12:35:03 +0200547 swrate_avg(entry->needed_avg, POOL_AVG_SAMPLES), entry->failed,
Willy Tarreau0a93b642018-10-16 07:58:39 +0200548 entry->users, entry, (int)pool_get_index(entry),
549 (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200550
551 allocated += entry->allocated * entry->size;
552 used += entry->used * entry->size;
553 nbpools++;
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100554#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100555 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100556#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +0200557 }
Willy Tarreau12833bb2014-01-28 16:49:56 +0100558 chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n",
Willy Tarreau50e608d2007-05-13 18:26:08 +0200559 nbpools, allocated, used);
560}
561
Willy Tarreau12833bb2014-01-28 16:49:56 +0100562/* Dump statistics on pools usage. */
563void dump_pools(void)
564{
565 dump_pools_to_trash();
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200566 qfprintf(stderr, "%s", trash.area);
Willy Tarreau12833bb2014-01-28 16:49:56 +0100567}
568
Willy Tarreau58102cf2015-10-28 16:24:21 +0100569/* This function returns the total number of failed pool allocations */
570int pool_total_failures()
571{
572 struct pool_head *entry;
573 int failed = 0;
574
575 list_for_each_entry(entry, &pools, list)
576 failed += entry->failed;
577 return failed;
578}
579
580/* This function returns the total amount of memory allocated in pools (in bytes) */
581unsigned long pool_total_allocated()
582{
583 struct pool_head *entry;
584 unsigned long allocated = 0;
585
586 list_for_each_entry(entry, &pools, list)
587 allocated += entry->allocated * entry->size;
588 return allocated;
589}
590
591/* This function returns the total amount of memory used in pools (in bytes) */
592unsigned long pool_total_used()
593{
594 struct pool_head *entry;
595 unsigned long used = 0;
596
597 list_for_each_entry(entry, &pools, list)
598 used += entry->used * entry->size;
599 return used;
600}
601
William Lallemande7ed8852016-11-19 02:25:36 +0100602/* This function dumps memory usage information onto the stream interface's
603 * read buffer. It returns 0 as long as it does not complete, non-zero upon
604 * completion. No state is used.
605 */
606static int cli_io_handler_dump_pools(struct appctx *appctx)
607{
608 struct stream_interface *si = appctx->owner;
609
610 dump_pools_to_trash();
Willy Tarreau06d80a92017-10-19 14:32:15 +0200611 if (ci_putchk(si_ic(si), &trash) == -1) {
Willy Tarreaudb398432018-11-15 11:08:52 +0100612 si_rx_room_blk(si);
William Lallemande7ed8852016-11-19 02:25:36 +0100613 return 0;
614 }
615 return 1;
616}
617
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100618/* callback used to create early pool <name> of size <size> and store the
619 * resulting pointer into <ptr>. If the allocation fails, it quits with after
620 * emitting an error message.
621 */
622void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
623{
624 *ptr = create_pool(name, size, MEM_F_SHARED);
625 if (!*ptr) {
626 ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
627 name, size, strerror(errno));
628 exit(1);
629 }
630}
631
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100632/* Initializes all per-thread arrays on startup */
633static void init_pools()
634{
Willy Tarreaued891fd2020-06-01 19:00:28 +0200635#ifdef CONFIG_HAP_LOCAL_POOLS
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100636 int thr, idx;
637
638 for (thr = 0; thr < MAX_THREADS; thr++) {
639 for (idx = 0; idx < MAX_BASE_POOLS; idx++) {
640 LIST_INIT(&pool_cache[thr][idx].list);
641 pool_cache[thr][idx].size = 0;
642 }
Willy Tarreau20dc3cd2020-06-28 00:54:27 +0200643 LIST_INIT(&ha_thread_info[thr].pool_lru_head);
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100644 }
Willy Tarreaued891fd2020-06-01 19:00:28 +0200645#endif
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100646}
647
648INITCALL0(STG_PREPARE, init_pools);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100649
William Lallemande7ed8852016-11-19 02:25:36 +0100650/* register cli keywords */
651static struct cli_kw_list cli_kws = {{ },{
Willy Tarreaue9ecec82016-12-16 18:55:23 +0100652 { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
William Lallemande7ed8852016-11-19 02:25:36 +0100653 {{},}
654}};
655
Willy Tarreau0108d902018-11-25 19:14:37 +0100656INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
William Lallemande7ed8852016-11-19 02:25:36 +0100657
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100658#ifdef DEBUG_FAIL_ALLOC
659#define MEM_FAIL_MAX_CHAR 32
660#define MEM_FAIL_MAX_STR 128
661static int mem_fail_cur_idx;
662static char mem_fail_str[MEM_FAIL_MAX_CHAR * MEM_FAIL_MAX_STR];
Willy Tarreauaf613e82020-06-05 08:40:51 +0200663__decl_thread(static HA_SPINLOCK_T mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100664
665int mem_should_fail(const struct pool_head *pool)
666{
Olivier Houchard9c4f08a2019-02-01 16:28:04 +0100667 int ret = 0;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100668 int n;
669
670 if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
Willy Tarreau52bf8392020-03-08 00:42:37 +0100671 int randnb = ha_random() % 100;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100672
673 if (mem_fail_rate > randnb)
674 ret = 1;
675 else
676 ret = 0;
677 }
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100678 HA_SPIN_LOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100679 n = snprintf(&mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR],
680 MEM_FAIL_MAX_CHAR - 2,
681 "%d %.18s %d %d", mem_fail_cur_idx, pool->name, ret, tid);
682 while (n < MEM_FAIL_MAX_CHAR - 1)
683 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n++] = ' ';
684 if (mem_fail_cur_idx < MEM_FAIL_MAX_STR - 1)
685 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = '\n';
686 else
687 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = 0;
688 mem_fail_cur_idx++;
689 if (mem_fail_cur_idx == MEM_FAIL_MAX_STR)
690 mem_fail_cur_idx = 0;
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100691 HA_SPIN_UNLOCK(POOL_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100692 return ret;
693
694}
695
696/* config parser for global "tune.fail-alloc" */
697static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
698 struct proxy *defpx, const char *file, int line,
699 char **err)
700{
701 if (too_many_args(1, args, err, NULL))
702 return -1;
703 mem_fail_rate = atoi(args[1]);
704 if (mem_fail_rate < 0 || mem_fail_rate > 100) {
705 memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
706 return -1;
707 }
708 return 0;
709}
710#endif
711
712/* register global config keywords */
713static struct cfg_kw_list mem_cfg_kws = {ILH, {
714#ifdef DEBUG_FAIL_ALLOC
715 { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
716#endif
717 { 0, NULL, NULL }
718}};
719
720INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);
721
Willy Tarreau50e608d2007-05-13 18:26:08 +0200722/*
723 * Local variables:
724 * c-indent-level: 8
725 * c-basic-offset: 8
726 * End:
727 */