blob: edacb00aaafdfefd1a56db729b59635e0fc22166 [file] [log] [blame]
Willy Tarreau50e608d2007-05-13 18:26:08 +02001/*
2 * Memory management functions.
3 *
4 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
Willy Tarreau7107c8b2018-11-26 11:44:35 +010012#include <errno.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020013
William Lallemande7ed8852016-11-19 02:25:36 +010014#include <types/applet.h>
15#include <types/cli.h>
Willy Tarreau12833bb2014-01-28 16:49:56 +010016#include <types/global.h>
William Lallemande7ed8852016-11-19 02:25:36 +010017#include <types/stats.h>
18
Olivier Houcharddc21ff72019-01-29 15:20:16 +010019#include <common/cfgparse.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020020#include <common/config.h>
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +020021#include <common/debug.h>
Willy Tarreaue18db9e2018-10-16 10:28:54 +020022#include <common/hathreads.h>
Willy Tarreau0108d902018-11-25 19:14:37 +010023#include <common/initcall.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020024#include <common/memory.h>
25#include <common/mini-clist.h>
26#include <common/standard.h>
27
Willy Tarreaua8b2ce02019-05-28 17:04:16 +020028#include <types/activity.h>
29
William Lallemande7ed8852016-11-19 02:25:36 +010030#include <proto/applet.h>
31#include <proto/cli.h>
32#include <proto/channel.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020033#include <proto/log.h>
William Lallemande7ed8852016-11-19 02:25:36 +010034#include <proto/stream_interface.h>
35#include <proto/stats.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020036
Willy Tarreau0a93b642018-10-16 07:58:39 +020037/* These are the most common pools, expected to be initialized first. These
38 * ones are allocated from an array, allowing to map them to an index.
39 */
40struct pool_head pool_base_start[MAX_BASE_POOLS] = { };
41unsigned int pool_base_count = 0;
42
Willy Tarreau7f0165e2018-11-26 17:09:46 +010043/* These ones are initialized per-thread on startup by init_pools() */
44struct pool_cache_head pool_cache[MAX_THREADS][MAX_BASE_POOLS];
45static struct list pool_lru_head[MAX_THREADS]; /* oldest objects */
Willy Tarreaue18db9e2018-10-16 10:28:54 +020046THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
47THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
48
Willy Tarreau50e608d2007-05-13 18:26:08 +020049static struct list pools = LIST_HEAD_INIT(pools);
Willy Tarreau067ac9f2015-10-08 14:12:13 +020050int mem_poison_byte = -1;
Willy Tarreau50e608d2007-05-13 18:26:08 +020051
Olivier Houcharddc21ff72019-01-29 15:20:16 +010052#ifdef DEBUG_FAIL_ALLOC
53static int mem_fail_rate = 0;
54static int mem_should_fail(const struct pool_head *);
55#endif
56
Willy Tarreau50e608d2007-05-13 18:26:08 +020057/* Try to find an existing shared pool with the same characteristics and
58 * returns it, otherwise creates this one. NULL is returned if no memory
Willy Tarreau581bf812016-01-25 02:19:13 +010059 * is available for a new creation. Two flags are supported :
60 * - MEM_F_SHARED to indicate that the pool may be shared with other users
61 * - MEM_F_EXACT to indicate that the size must not be rounded up
Willy Tarreau50e608d2007-05-13 18:26:08 +020062 */
63struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
64{
65 struct pool_head *pool;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020066 struct pool_head *entry;
67 struct list *start;
Willy Tarreau50e608d2007-05-13 18:26:08 +020068 unsigned int align;
Christopher Faulet2f6d3c02019-06-25 21:45:59 +020069 int thr, idx;
Willy Tarreau50e608d2007-05-13 18:26:08 +020070
Willy Tarreauac421112015-10-28 15:09:29 +010071 /* We need to store a (void *) at the end of the chunks. Since we know
Willy Tarreau50e608d2007-05-13 18:26:08 +020072 * that the malloc() function will never return such a small size,
73 * let's round the size up to something slightly bigger, in order to
74 * ease merging of entries. Note that the rounding is a power of two.
Willy Tarreauac421112015-10-28 15:09:29 +010075 * This extra (void *) is not accounted for in the size computation
76 * so that the visible parts outside are not affected.
Willy Tarreau30f931e2018-10-23 14:40:23 +020077 *
78 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
Willy Tarreau50e608d2007-05-13 18:26:08 +020079 */
80
Willy Tarreau581bf812016-01-25 02:19:13 +010081 if (!(flags & MEM_F_EXACT)) {
Willy Tarreau30f931e2018-10-23 14:40:23 +020082 align = 4 * sizeof(void *); // 2 lists = 4 pointers min
Willy Tarreau581bf812016-01-25 02:19:13 +010083 size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA;
84 }
Willy Tarreau50e608d2007-05-13 18:26:08 +020085
Christopher Fauletb349e482017-08-29 09:52:38 +020086 /* TODO: thread: we do not lock pool list for now because all pools are
87 * created during HAProxy startup (so before threads creation) */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020088 start = &pools;
Willy Tarreau50e608d2007-05-13 18:26:08 +020089 pool = NULL;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020090
91 list_for_each_entry(entry, &pools, list) {
92 if (entry->size == size) {
93 /* either we can share this place and we take it, or
94 * we look for a sharable one or for the next position
95 * before which we will insert a new one.
96 */
97 if (flags & entry->flags & MEM_F_SHARED) {
98 /* we can share this one */
Willy Tarreau50e608d2007-05-13 18:26:08 +020099 pool = entry;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200100 DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
Willy Tarreau50e608d2007-05-13 18:26:08 +0200101 break;
102 }
103 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200104 else if (entry->size > size) {
105 /* insert before this one */
106 start = &entry->list;
107 break;
108 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200109 }
110
111 if (!pool) {
Willy Tarreau0a93b642018-10-16 07:58:39 +0200112 if (pool_base_count < MAX_BASE_POOLS)
113 pool = &pool_base_start[pool_base_count++];
114
115 if (!pool) {
116 /* look for a freed entry */
117 for (entry = pool_base_start; entry != pool_base_start + MAX_BASE_POOLS; entry++) {
118 if (!entry->size) {
119 pool = entry;
120 break;
121 }
122 }
123 }
124
125 if (!pool)
126 pool = calloc(1, sizeof(*pool));
127
Willy Tarreau50e608d2007-05-13 18:26:08 +0200128 if (!pool)
129 return NULL;
130 if (name)
131 strlcpy2(pool->name, name, sizeof(pool->name));
132 pool->size = size;
133 pool->flags = flags;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200134 LIST_ADDQ(start, &pool->list);
Christopher Faulet2f6d3c02019-06-25 21:45:59 +0200135
136 /* update per-thread pool cache if necessary */
137 idx = pool_get_index(pool);
138 if (idx >= 0) {
139 for (thr = 0; thr < MAX_THREADS; thr++)
140 pool_cache[thr][idx].size = size;
141 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200142 }
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200143 pool->users++;
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100144#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100145 HA_SPIN_INIT(&pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100146#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +0200147 return pool;
148}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100149
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100150#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +0100151/* Allocates new entries for pool <pool> until there are at least <avail> + 1
152 * available, then returns the last one for immediate use, so that at least
153 * <avail> are left available in the pool upon return. NULL is returned if the
154 * last entry could not be allocated. It's important to note that at least one
155 * allocation is always performed even if there are enough entries in the pool.
156 * A call to the garbage collector is performed at most once in case malloc()
157 * returns an error, before returning NULL.
158 */
159void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
160{
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200161 void *ptr = NULL, **free_list;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100162 int failed = 0;
163 int size = pool->size;
164 int limit = pool->limit;
165 int allocated = pool->allocated, allocated_orig = allocated;
166
167 /* stop point */
168 avail += pool->used;
169
170 while (1) {
171 if (limit && allocated >= limit) {
Olivier Houchard20872762019-03-08 18:53:35 +0100172 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200173 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100174 return NULL;
175 }
176
177 ptr = malloc(size + POOL_EXTRA);
178 if (!ptr) {
Olivier Houchard20872762019-03-08 18:53:35 +0100179 _HA_ATOMIC_ADD(&pool->failed, 1);
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200180 if (failed) {
181 activity[tid].pool_fail++;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100182 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200183 }
Olivier Houchardcf975d42018-01-24 18:38:31 +0100184 failed++;
185 pool_gc(pool);
186 continue;
187 }
188 if (++allocated > avail)
189 break;
190
191 free_list = pool->free_list;
192 do {
193 *POOL_LINK(pool, ptr) = free_list;
194 __ha_barrier_store();
Olivier Houchard20872762019-03-08 18:53:35 +0100195 } while (_HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr) == 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100196 }
Olivier Houchard20872762019-03-08 18:53:35 +0100197 __ha_barrier_atomic_store();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100198
Olivier Houchard20872762019-03-08 18:53:35 +0100199 _HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig);
200 _HA_ATOMIC_ADD(&pool->used, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100201
202#ifdef DEBUG_MEMORY_POOLS
203 /* keep track of where the element was allocated from */
204 *POOL_LINK(pool, ptr) = (void *)pool;
205#endif
206 return ptr;
207}
208void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
209{
210 void *ptr;
211
212 ptr = __pool_refill_alloc(pool, avail);
213 return ptr;
214}
215/*
216 * This function frees whatever can be freed in pool <pool>.
217 */
218void pool_flush(struct pool_head *pool)
219{
Olivier Houchard8b2c8a72018-10-21 01:52:59 +0200220 void **next, *temp;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100221 int removed = 0;
222
223 if (!pool)
224 return;
225 do {
226 next = pool->free_list;
Olivier Houchard20872762019-03-08 18:53:35 +0100227 } while (!_HA_ATOMIC_CAS(&pool->free_list, &next, NULL));
228 __ha_barrier_atomic_store();
Olivier Houchardcf975d42018-01-24 18:38:31 +0100229 while (next) {
230 temp = next;
231 next = *POOL_LINK(pool, temp);
232 removed++;
233 free(temp);
234 }
235 pool->free_list = next;
Olivier Houchard20872762019-03-08 18:53:35 +0100236 _HA_ATOMIC_SUB(&pool->allocated, removed);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100237 /* here, we should have pool->allocate == pool->used */
238}
239
240/*
241 * This function frees whatever can be freed in all pools, but respecting
242 * the minimum thresholds imposed by owners. It takes care of avoiding
243 * recursion because it may be called from a signal handler.
244 *
245 * <pool_ctx> is unused
246 */
247void pool_gc(struct pool_head *pool_ctx)
248{
249 static int recurse;
250 int cur_recurse = 0;
251 struct pool_head *entry;
252
Olivier Houchard20872762019-03-08 18:53:35 +0100253 if (recurse || !_HA_ATOMIC_CAS(&recurse, &cur_recurse, 1))
Olivier Houchardcf975d42018-01-24 18:38:31 +0100254 return;
255
256 list_for_each_entry(entry, &pools, list) {
257 while ((int)((volatile int)entry->allocated - (volatile int)entry->used) > (int)entry->minavail) {
258 struct pool_free_list cmp, new;
259
260 cmp.seq = entry->seq;
261 __ha_barrier_load();
262 cmp.free_list = entry->free_list;
263 __ha_barrier_load();
264 if (cmp.free_list == NULL)
265 break;
266 new.free_list = *POOL_LINK(entry, cmp.free_list);
267 new.seq = cmp.seq + 1;
Willy Tarreau6a38b322019-05-11 18:04:24 +0200268 if (HA_ATOMIC_DWCAS(&entry->free_list, &cmp, &new) == 0)
Olivier Houchardcf975d42018-01-24 18:38:31 +0100269 continue;
270 free(cmp.free_list);
Olivier Houchard20872762019-03-08 18:53:35 +0100271 _HA_ATOMIC_SUB(&entry->allocated, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100272 }
273 }
274
Olivier Houchard20872762019-03-08 18:53:35 +0100275 _HA_ATOMIC_STORE(&recurse, 0);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100276}
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200277
278/* frees an object to the local cache, possibly pushing oldest objects to the
279 * global pool. Must not be called directly.
280 */
281void __pool_put_to_cache(struct pool_head *pool, void *ptr, ssize_t idx)
282{
283 struct pool_cache_item *item = (struct pool_cache_item *)ptr;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100284 struct pool_cache_head *ph = &pool_cache[tid][idx];
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200285
286 LIST_ADD(&ph->list, &item->by_pool);
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100287 LIST_ADD(&pool_lru_head[tid], &item->by_lru);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200288 ph->count++;
289 pool_cache_count++;
290 pool_cache_bytes += ph->size;
291
292 if (pool_cache_bytes <= CONFIG_HAP_POOL_CACHE_SIZE)
293 return;
294
295 do {
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100296 item = LIST_PREV(&pool_lru_head[tid], struct pool_cache_item *, by_lru);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200297 /* note: by definition we remove oldest objects so they also are the
298 * oldest in their own pools, thus their next is the pool's head.
299 */
300 ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
301 LIST_DEL(&item->by_pool);
302 LIST_DEL(&item->by_lru);
303 ph->count--;
304 pool_cache_count--;
305 pool_cache_bytes -= ph->size;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100306 __pool_free(pool_base_start + (ph - pool_cache[tid]), item);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200307 } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
308}
309
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100310#else /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau50e608d2007-05-13 18:26:08 +0200311
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100312/* Allocates new entries for pool <pool> until there are at least <avail> + 1
313 * available, then returns the last one for immediate use, so that at least
314 * <avail> are left available in the pool upon return. NULL is returned if the
315 * last entry could not be allocated. It's important to note that at least one
316 * allocation is always performed even if there are enough entries in the pool.
317 * A call to the garbage collector is performed at most once in case malloc()
318 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200319 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200320void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200321{
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100322 void *ptr = NULL;
323 int failed = 0;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200324
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100325#ifdef DEBUG_FAIL_ALLOC
326 if (mem_should_fail(pool))
327 return NULL;
328#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100329 /* stop point */
330 avail += pool->used;
331
332 while (1) {
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200333 if (pool->limit && pool->allocated >= pool->limit) {
334 activity[tid].pool_fail++;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200335 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200336 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100337
Willy Tarreauf13322e2017-11-22 10:50:54 +0100338 ptr = pool_alloc_area(pool->size + POOL_EXTRA);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100339 if (!ptr) {
Willy Tarreau58102cf2015-10-28 16:24:21 +0100340 pool->failed++;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200341 if (failed) {
342 activity[tid].pool_fail++;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100343 return NULL;
Willy Tarreaua8b2ce02019-05-28 17:04:16 +0200344 }
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100345 failed++;
Willy Tarreaubafbe012017-11-24 17:34:44 +0100346 pool_gc(pool);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100347 continue;
348 }
349 if (++pool->allocated > avail)
350 break;
351
Willy Tarreauac421112015-10-28 15:09:29 +0100352 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100353 pool->free_list = ptr;
Willy Tarreau7dcd46d2007-05-14 00:16:13 +0200354 }
Willy Tarreau50e608d2007-05-13 18:26:08 +0200355 pool->used++;
Willy Tarreaude30a682015-10-28 15:23:51 +0100356#ifdef DEBUG_MEMORY_POOLS
357 /* keep track of where the element was allocated from */
358 *POOL_LINK(pool, ptr) = (void *)pool;
359#endif
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100360 return ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200361}
Christopher Fauletb349e482017-08-29 09:52:38 +0200362void *pool_refill_alloc(struct pool_head *pool, unsigned int avail)
363{
364 void *ptr;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200365
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100366 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200367 ptr = __pool_refill_alloc(pool, avail);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100368 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200369 return ptr;
370}
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200371/*
372 * This function frees whatever can be freed in pool <pool>.
373 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100374void pool_flush(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200375{
376 void *temp, *next;
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200377 if (!pool)
378 return;
379
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100380 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200381 next = pool->free_list;
382 while (next) {
383 temp = next;
Willy Tarreauac421112015-10-28 15:09:29 +0100384 next = *POOL_LINK(pool, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200385 pool->allocated--;
Willy Tarreauf13322e2017-11-22 10:50:54 +0100386 pool_free_area(temp, pool->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200387 }
388 pool->free_list = next;
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100389 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200390 /* here, we should have pool->allocate == pool->used */
391}
392
393/*
394 * This function frees whatever can be freed in all pools, but respecting
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200395 * the minimum thresholds imposed by owners. It takes care of avoiding
396 * recursion because it may be called from a signal handler.
Christopher Fauletb349e482017-08-29 09:52:38 +0200397 *
Willy Tarreaubafbe012017-11-24 17:34:44 +0100398 * <pool_ctx> is used when pool_gc is called to release resources to allocate
Christopher Fauletb349e482017-08-29 09:52:38 +0200399 * an element in __pool_refill_alloc. It is important because <pool_ctx> is
400 * already locked, so we need to skip the lock here.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200401 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100402void pool_gc(struct pool_head *pool_ctx)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200403{
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200404 static int recurse;
Christopher Fauletb349e482017-08-29 09:52:38 +0200405 int cur_recurse = 0;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200406 struct pool_head *entry;
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200407
Olivier Houchard20872762019-03-08 18:53:35 +0100408 if (recurse || !_HA_ATOMIC_CAS(&recurse, &cur_recurse, 1))
Christopher Fauletb349e482017-08-29 09:52:38 +0200409 return;
Willy Tarreaub7f9d122009-04-21 02:17:45 +0200410
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200411 list_for_each_entry(entry, &pools, list) {
412 void *temp, *next;
413 //qfprintf(stderr, "Flushing pool %s\n", entry->name);
Christopher Fauletb349e482017-08-29 09:52:38 +0200414 if (entry != pool_ctx)
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100415 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200416 next = entry->free_list;
417 while (next &&
Willy Tarreau57767b82014-12-22 21:40:55 +0100418 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200419 temp = next;
Willy Tarreauac421112015-10-28 15:09:29 +0100420 next = *POOL_LINK(entry, temp);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200421 entry->allocated--;
Willy Tarreauf13322e2017-11-22 10:50:54 +0100422 pool_free_area(temp, entry->size + POOL_EXTRA);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200423 }
424 entry->free_list = next;
Christopher Fauletb349e482017-08-29 09:52:38 +0200425 if (entry != pool_ctx)
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100426 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200427 }
Christopher Fauletb349e482017-08-29 09:52:38 +0200428
Olivier Houchard20872762019-03-08 18:53:35 +0100429 _HA_ATOMIC_STORE(&recurse, 0);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200430}
Olivier Houchardcf975d42018-01-24 18:38:31 +0100431#endif
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200432
433/*
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200434 * This function destroys a pool by freeing it completely, unless it's still
435 * in use. This should be called only under extreme circumstances. It always
436 * returns NULL if the resulting pool is empty, easing the clearing of the old
437 * pointer, otherwise it returns the pool.
438 * .
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200439 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100440void *pool_destroy(struct pool_head *pool)
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200441{
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200442 if (pool) {
Willy Tarreaubafbe012017-11-24 17:34:44 +0100443 pool_flush(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200444 if (pool->used)
445 return pool;
446 pool->users--;
447 if (!pool->users) {
448 LIST_DEL(&pool->list);
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100449#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100450 HA_SPIN_DESTROY(&pool->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100451#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200452 if ((pool - pool_base_start) < MAX_BASE_POOLS)
453 memset(pool, 0, sizeof(*pool));
454 else
455 free(pool);
Willy Tarreaudae4aa82007-06-16 23:19:53 +0200456 }
Willy Tarreau4d2d0982007-05-14 00:39:29 +0200457 }
458 return NULL;
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200459}
460
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100461/* This destroys all pools on exit. It is *not* thread safe. */
462void pool_destroy_all()
463{
464 struct pool_head *entry, *back;
465
466 list_for_each_entry_safe(entry, back, &pools, list)
467 pool_destroy(entry);
468}
469
Willy Tarreau12833bb2014-01-28 16:49:56 +0100470/* This function dumps memory usage information into the trash buffer. */
471void dump_pools_to_trash()
Willy Tarreau50e608d2007-05-13 18:26:08 +0200472{
473 struct pool_head *entry;
474 unsigned long allocated, used;
475 int nbpools;
476
477 allocated = used = nbpools = 0;
Willy Tarreau12833bb2014-01-28 16:49:56 +0100478 chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200479 list_for_each_entry(entry, &pools, list) {
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100480#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100481 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100482#endif
Willy Tarreau0a93b642018-10-16 07:58:39 +0200483 chunk_appendf(&trash, " - Pool %s (%d bytes) : %d allocated (%u bytes), %d used, %d failures, %d users, @%p=%02d%s\n",
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200484 entry->name, entry->size, entry->allocated,
Willy Tarreau58102cf2015-10-28 16:24:21 +0100485 entry->size * entry->allocated, entry->used, entry->failed,
Willy Tarreau0a93b642018-10-16 07:58:39 +0200486 entry->users, entry, (int)pool_get_index(entry),
487 (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");
Willy Tarreau50e608d2007-05-13 18:26:08 +0200488
489 allocated += entry->allocated * entry->size;
490 used += entry->used * entry->size;
491 nbpools++;
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100492#ifndef CONFIG_HAP_LOCKLESS_POOLS
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100493 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100494#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +0200495 }
Willy Tarreau12833bb2014-01-28 16:49:56 +0100496 chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n",
Willy Tarreau50e608d2007-05-13 18:26:08 +0200497 nbpools, allocated, used);
498}
499
Willy Tarreau12833bb2014-01-28 16:49:56 +0100500/* Dump statistics on pools usage. */
501void dump_pools(void)
502{
503 dump_pools_to_trash();
Willy Tarreau843b7cb2018-07-13 10:54:26 +0200504 qfprintf(stderr, "%s", trash.area);
Willy Tarreau12833bb2014-01-28 16:49:56 +0100505}
506
Willy Tarreau58102cf2015-10-28 16:24:21 +0100507/* This function returns the total number of failed pool allocations */
508int pool_total_failures()
509{
510 struct pool_head *entry;
511 int failed = 0;
512
513 list_for_each_entry(entry, &pools, list)
514 failed += entry->failed;
515 return failed;
516}
517
518/* This function returns the total amount of memory allocated in pools (in bytes) */
519unsigned long pool_total_allocated()
520{
521 struct pool_head *entry;
522 unsigned long allocated = 0;
523
524 list_for_each_entry(entry, &pools, list)
525 allocated += entry->allocated * entry->size;
526 return allocated;
527}
528
529/* This function returns the total amount of memory used in pools (in bytes) */
530unsigned long pool_total_used()
531{
532 struct pool_head *entry;
533 unsigned long used = 0;
534
535 list_for_each_entry(entry, &pools, list)
536 used += entry->used * entry->size;
537 return used;
538}
539
William Lallemande7ed8852016-11-19 02:25:36 +0100540/* This function dumps memory usage information onto the stream interface's
541 * read buffer. It returns 0 as long as it does not complete, non-zero upon
542 * completion. No state is used.
543 */
544static int cli_io_handler_dump_pools(struct appctx *appctx)
545{
546 struct stream_interface *si = appctx->owner;
547
548 dump_pools_to_trash();
Willy Tarreau06d80a92017-10-19 14:32:15 +0200549 if (ci_putchk(si_ic(si), &trash) == -1) {
Willy Tarreaudb398432018-11-15 11:08:52 +0100550 si_rx_room_blk(si);
William Lallemande7ed8852016-11-19 02:25:36 +0100551 return 0;
552 }
553 return 1;
554}
555
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100556/* callback used to create early pool <name> of size <size> and store the
557 * resulting pointer into <ptr>. If the allocation fails, it quits with after
558 * emitting an error message.
559 */
560void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
561{
562 *ptr = create_pool(name, size, MEM_F_SHARED);
563 if (!*ptr) {
564 ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
565 name, size, strerror(errno));
566 exit(1);
567 }
568}
569
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100570/* Initializes all per-thread arrays on startup */
571static void init_pools()
572{
573 int thr, idx;
574
575 for (thr = 0; thr < MAX_THREADS; thr++) {
576 for (idx = 0; idx < MAX_BASE_POOLS; idx++) {
577 LIST_INIT(&pool_cache[thr][idx].list);
578 pool_cache[thr][idx].size = 0;
579 }
580 LIST_INIT(&pool_lru_head[thr]);
581 }
582}
583
584INITCALL0(STG_PREPARE, init_pools);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100585
William Lallemande7ed8852016-11-19 02:25:36 +0100586/* register cli keywords */
587static struct cli_kw_list cli_kws = {{ },{
Willy Tarreaue9ecec82016-12-16 18:55:23 +0100588 { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
William Lallemande7ed8852016-11-19 02:25:36 +0100589 {{},}
590}};
591
Willy Tarreau0108d902018-11-25 19:14:37 +0100592INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
William Lallemande7ed8852016-11-19 02:25:36 +0100593
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100594#ifdef DEBUG_FAIL_ALLOC
595#define MEM_FAIL_MAX_CHAR 32
596#define MEM_FAIL_MAX_STR 128
597static int mem_fail_cur_idx;
598static char mem_fail_str[MEM_FAIL_MAX_CHAR * MEM_FAIL_MAX_STR];
599__decl_hathreads(static HA_SPINLOCK_T mem_fail_lock);
600
601int mem_should_fail(const struct pool_head *pool)
602{
Olivier Houchard9c4f08a2019-02-01 16:28:04 +0100603 int ret = 0;
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100604 int n;
605
606 if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
607 int randnb = random() % 100;
608
609 if (mem_fail_rate > randnb)
610 ret = 1;
611 else
612 ret = 0;
613 }
Willy Tarreau64a47b92019-05-20 11:09:00 +0200614 HA_SPIN_LOCK(OTHER_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100615 n = snprintf(&mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR],
616 MEM_FAIL_MAX_CHAR - 2,
617 "%d %.18s %d %d", mem_fail_cur_idx, pool->name, ret, tid);
618 while (n < MEM_FAIL_MAX_CHAR - 1)
619 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n++] = ' ';
620 if (mem_fail_cur_idx < MEM_FAIL_MAX_STR - 1)
621 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = '\n';
622 else
623 mem_fail_str[mem_fail_cur_idx * MEM_FAIL_MAX_CHAR + n] = 0;
624 mem_fail_cur_idx++;
625 if (mem_fail_cur_idx == MEM_FAIL_MAX_STR)
626 mem_fail_cur_idx = 0;
Willy Tarreau64a47b92019-05-20 11:09:00 +0200627 HA_SPIN_UNLOCK(OTHER_LOCK, &mem_fail_lock);
Olivier Houcharddc21ff72019-01-29 15:20:16 +0100628 return ret;
629
630}
631
632/* config parser for global "tune.fail-alloc" */
633static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
634 struct proxy *defpx, const char *file, int line,
635 char **err)
636{
637 if (too_many_args(1, args, err, NULL))
638 return -1;
639 mem_fail_rate = atoi(args[1]);
640 if (mem_fail_rate < 0 || mem_fail_rate > 100) {
641 memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
642 return -1;
643 }
644 return 0;
645}
646#endif
647
648/* register global config keywords */
649static struct cfg_kw_list mem_cfg_kws = {ILH, {
650#ifdef DEBUG_FAIL_ALLOC
651 { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
652#endif
653 { 0, NULL, NULL }
654}};
655
656INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);
657
Willy Tarreau50e608d2007-05-13 18:26:08 +0200658/*
659 * Local variables:
660 * c-indent-level: 8
661 * c-basic-offset: 8
662 * End:
663 */