blob: 3283d2be1e58fa4c662450cdc7798ebb9a7cbd83 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
Willy Tarreau62405a22014-12-23 13:51:28 +01002 * include/common/memory.h
3 * Memory management definitions..
4 *
5 * Copyright (C) 2000-2014 Willy Tarreau - w@1wt.eu
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
Willy Tarreaubaaee002006-06-26 02:48:02 +020021
Willy Tarreau2dd0d472006-06-29 17:53:05 +020022#ifndef _COMMON_MEMORY_H
23#define _COMMON_MEMORY_H
Willy Tarreaubaaee002006-06-26 02:48:02 +020024
Willy Tarreau158fa752017-11-22 15:47:29 +010025#include <sys/mman.h>
26
Willy Tarreaubaaee002006-06-26 02:48:02 +020027#include <stdlib.h>
Willy Tarreaue430e772014-12-23 14:13:16 +010028#include <string.h>
Willy Tarreaua1bd1fa2019-03-29 17:26:33 +010029#include <inttypes.h>
Willy Tarreaua7280a12018-11-26 19:41:40 +010030#include <unistd.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020031
Willy Tarreau2dd0d472006-06-29 17:53:05 +020032#include <common/config.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020033#include <common/mini-clist.h>
Christopher Fauletb349e482017-08-29 09:52:38 +020034#include <common/hathreads.h>
Willy Tarreau7107c8b2018-11-26 11:44:35 +010035#include <common/initcall.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020036
Willy Tarreaua84dcb82015-10-28 12:04:02 +010037#ifndef DEBUG_DONT_SHARE_POOLS
Willy Tarreau50e608d2007-05-13 18:26:08 +020038#define MEM_F_SHARED 0x1
Willy Tarreaua84dcb82015-10-28 12:04:02 +010039#else
40#define MEM_F_SHARED 0
41#endif
Willy Tarreau581bf812016-01-25 02:19:13 +010042#define MEM_F_EXACT 0x2
Willy Tarreau50e608d2007-05-13 18:26:08 +020043
Willy Tarreauac421112015-10-28 15:09:29 +010044/* reserve an extra void* at the end of a pool for linking */
45#ifdef DEBUG_MEMORY_POOLS
46#define POOL_EXTRA (sizeof(void *))
47#define POOL_LINK(pool, item) (void **)(((char *)item) + (pool->size))
48#else
49#define POOL_EXTRA (0)
50#define POOL_LINK(pool, item) ((void **)(item))
51#endif
52
Willy Tarreau0a93b642018-10-16 07:58:39 +020053#define MAX_BASE_POOLS 32
54
Willy Tarreaue18db9e2018-10-16 10:28:54 +020055struct pool_cache_head {
56 struct list list; /* head of objects in this pool */
57 size_t size; /* size of an object */
58 unsigned int count; /* number of objects in this pool */
59};
60
61struct pool_cache_item {
62 struct list by_pool; /* link to objects in this pool */
63 struct list by_lru; /* link to objects by LRU order */
64};
65
Willy Tarreau7f0165e2018-11-26 17:09:46 +010066extern struct pool_cache_head pool_cache[][MAX_BASE_POOLS];
Willy Tarreaue18db9e2018-10-16 10:28:54 +020067extern THREAD_LOCAL size_t pool_cache_bytes; /* total cache size */
68extern THREAD_LOCAL size_t pool_cache_count; /* #cache objects */
69
Willy Tarreauf161d0f2018-02-22 14:05:55 +010070#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +010071struct pool_free_list {
72 void **free_list;
73 uintptr_t seq;
74};
75#endif
76
Willy Tarreau50e608d2007-05-13 18:26:08 +020077struct pool_head {
Willy Tarreau1ca1b702017-11-26 10:50:36 +010078 void **free_list;
Willy Tarreauf161d0f2018-02-22 14:05:55 +010079#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +010080 uintptr_t seq;
81#else
82 __decl_hathreads(HA_SPINLOCK_T lock); /* the spin lock */
83#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +020084 unsigned int used; /* how many chunks are currently in use */
85 unsigned int allocated; /* how many chunks have been allocated */
86 unsigned int limit; /* hard limit on the number of chunks */
87 unsigned int minavail; /* how many chunks are expected to be used */
88 unsigned int size; /* chunk size */
89 unsigned int flags; /* MEM_F_* */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020090 unsigned int users; /* number of pools sharing this zone */
Willy Tarreau58102cf2015-10-28 16:24:21 +010091 unsigned int failed; /* failed allocations */
Olivier Houchardcf975d42018-01-24 18:38:31 +010092 struct list list; /* list of all known pools */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020093 char name[12]; /* name of the pool */
Willy Tarreau1ca1b702017-11-26 10:50:36 +010094} __attribute__((aligned(64)));
Willy Tarreau50e608d2007-05-13 18:26:08 +020095
Willy Tarreau0a93b642018-10-16 07:58:39 +020096
97extern struct pool_head pool_base_start[MAX_BASE_POOLS];
98extern unsigned int pool_base_count;
99
Willy Tarreau067ac9f2015-10-08 14:12:13 +0200100/* poison each newly allocated area with this byte if >= 0 */
101extern int mem_poison_byte;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200102
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100103/* Allocates new entries for pool <pool> until there are at least <avail> + 1
104 * available, then returns the last one for immediate use, so that at least
105 * <avail> are left available in the pool upon return. NULL is returned if the
106 * last entry could not be allocated. It's important to note that at least one
107 * allocation is always performed even if there are enough entries in the pool.
108 * A call to the garbage collector is performed at most once in case malloc()
109 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200110 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200111void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100112void *pool_refill_alloc(struct pool_head *pool, unsigned int avail);
Willy Tarreau50e608d2007-05-13 18:26:08 +0200113
114/* Try to find an existing shared pool with the same characteristics and
115 * returns it, otherwise creates this one. NULL is returned if no memory
116 * is available for a new creation.
117 */
118struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100119void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size);
120
121/* This registers a call to create_pool_callback(ptr, name, size) */
122#define REGISTER_POOL(ptr, name, size) \
123 INITCALL3(STG_POOL, create_pool_callback, (ptr), (name), (size))
124
125/* This macro declares a pool head <ptr> and registers its creation */
126#define DECLARE_POOL(ptr, name, size) \
127 struct pool_head *(ptr) = NULL; \
128 REGISTER_POOL(&ptr, name, size)
129
130/* This macro declares a static pool head <ptr> and registers its creation */
131#define DECLARE_STATIC_POOL(ptr, name, size) \
132 static struct pool_head *(ptr); \
133 REGISTER_POOL(&ptr, name, size)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200134
135/* Dump statistics on pools usage.
136 */
Willy Tarreau12833bb2014-01-28 16:49:56 +0100137void dump_pools_to_trash();
Willy Tarreau50e608d2007-05-13 18:26:08 +0200138void dump_pools(void);
Willy Tarreau58102cf2015-10-28 16:24:21 +0100139int pool_total_failures();
140unsigned long pool_total_allocated();
141unsigned long pool_total_used();
Willy Tarreau50e608d2007-05-13 18:26:08 +0200142
143/*
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200144 * This function frees whatever can be freed in pool <pool>.
145 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100146void pool_flush(struct pool_head *pool);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200147
148/*
149 * This function frees whatever can be freed in all pools, but respecting
150 * the minimum thresholds imposed by owners.
Christopher Fauletb349e482017-08-29 09:52:38 +0200151 *
Willy Tarreaubafbe012017-11-24 17:34:44 +0100152 * <pool_ctx> is used when pool_gc is called to release resources to allocate
Christopher Fauletb349e482017-08-29 09:52:38 +0200153 * an element in __pool_refill_alloc. It is important because <pool_ctx> is
154 * already locked, so we need to skip the lock here.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200155 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100156void pool_gc(struct pool_head *pool_ctx);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200157
158/*
159 * This function destroys a pull by freeing it completely.
160 * This should be called only under extreme circumstances.
161 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100162void *pool_destroy(struct pool_head *pool);
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100163void pool_destroy_all();
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200164
Willy Tarreau0a93b642018-10-16 07:58:39 +0200165/* returns the pool index for pool <pool>, or -1 if this pool has no index */
166static inline ssize_t pool_get_index(const struct pool_head *pool)
167{
168 size_t idx;
169
170 idx = pool - pool_base_start;
171 if (idx >= MAX_BASE_POOLS)
172 return -1;
173 return idx;
174}
175
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100176#ifdef CONFIG_HAP_LOCKLESS_POOLS
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200177
178/* Tries to retrieve an object from the local pool cache corresponding to pool
179 * <pool>. Returns NULL if none is available.
180 */
181static inline void *__pool_get_from_cache(struct pool_head *pool)
182{
183 ssize_t idx = pool_get_index(pool);
184 struct pool_cache_item *item;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100185 struct pool_cache_head *ph;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200186
187 /* pool not in cache */
188 if (idx < 0)
189 return NULL;
190
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100191 ph = &pool_cache[tid][idx];
192 if (LIST_ISEMPTY(&ph->list))
193 return NULL; // empty
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200194
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100195 item = LIST_NEXT(&ph->list, typeof(item), by_pool);
196 ph->count--;
197 pool_cache_bytes -= ph->size;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200198 pool_cache_count--;
199 LIST_DEL(&item->by_pool);
200 LIST_DEL(&item->by_lru);
Willy Tarreau8e9f4532018-10-28 20:09:12 +0100201#ifdef DEBUG_MEMORY_POOLS
202 /* keep track of where the element was allocated from */
203 *POOL_LINK(pool, item) = (void *)pool;
204#endif
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200205 return item;
206}
207
Olivier Houchardcf975d42018-01-24 18:38:31 +0100208/*
209 * Returns a pointer to type <type> taken from the pool <pool_type> if
210 * available, otherwise returns NULL. No malloc() is attempted, and poisonning
211 * is never performed. The purpose is to get the fastest possible allocation.
212 */
213static inline void *__pool_get_first(struct pool_head *pool)
214{
215 struct pool_free_list cmp, new;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200216 void *ret = __pool_get_from_cache(pool);
217
218 if (ret)
219 return ret;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100220
221 cmp.seq = pool->seq;
222 __ha_barrier_load();
223
224 cmp.free_list = pool->free_list;
225 do {
226 if (cmp.free_list == NULL)
227 return NULL;
228 new.seq = cmp.seq + 1;
229 __ha_barrier_load();
230 new.free_list = *POOL_LINK(pool, cmp.free_list);
Willy Tarreau6a38b322019-05-11 18:04:24 +0200231 } while (HA_ATOMIC_DWCAS((void *)&pool->free_list, (void *)&cmp, (void *)&new) == 0);
Olivier Houchard20872762019-03-08 18:53:35 +0100232 __ha_barrier_atomic_store();
Tim Duesterhus05f6a432018-02-20 00:49:46 +0100233
Olivier Houchard20872762019-03-08 18:53:35 +0100234 _HA_ATOMIC_ADD(&pool->used, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100235#ifdef DEBUG_MEMORY_POOLS
236 /* keep track of where the element was allocated from */
237 *POOL_LINK(pool, cmp.free_list) = (void *)pool;
238#endif
239 return cmp.free_list;
240}
241
242static inline void *pool_get_first(struct pool_head *pool)
243{
244 void *ret;
245
246 ret = __pool_get_first(pool);
247 return ret;
248}
249/*
250 * Returns a pointer to type <type> taken from the pool <pool_type> or
251 * dynamically allocated. In the first case, <pool_type> is updated to point to
252 * the next element in the list. No memory poisonning is ever performed on the
253 * returned area.
254 */
255static inline void *pool_alloc_dirty(struct pool_head *pool)
256{
257 void *p;
258
259 if ((p = __pool_get_first(pool)) == NULL)
260 p = __pool_refill_alloc(pool, 0);
261 return p;
262}
263
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200264/*
Olivier Houchardcf975d42018-01-24 18:38:31 +0100265 * Returns a pointer to type <type> taken from the pool <pool_type> or
266 * dynamically allocated. In the first case, <pool_type> is updated to point to
267 * the next element in the list. Memory poisonning is performed if enabled.
268 */
269static inline void *pool_alloc(struct pool_head *pool)
270{
271 void *p;
272
273 p = pool_alloc_dirty(pool);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100274 if (p && mem_poison_byte >= 0) {
275 memset(p, mem_poison_byte, pool->size);
276 }
277
278 return p;
279}
280
Willy Tarreau146794d2018-10-16 08:55:15 +0200281/* Locklessly add item <ptr> to pool <pool>, then update the pool used count.
282 * Both the pool and the pointer must be valid. Use pool_free() for normal
283 * operations.
284 */
285static inline void __pool_free(struct pool_head *pool, void *ptr)
286{
Willy Tarreau7a6ad882018-10-20 17:37:38 +0200287 void **free_list = pool->free_list;
Willy Tarreau146794d2018-10-16 08:55:15 +0200288
289 do {
290 *POOL_LINK(pool, ptr) = (void *)free_list;
291 __ha_barrier_store();
Olivier Houchard20872762019-03-08 18:53:35 +0100292 } while (!_HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr));
293 __ha_barrier_atomic_store();
294 _HA_ATOMIC_SUB(&pool->used, 1);
Willy Tarreau146794d2018-10-16 08:55:15 +0200295}
296
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200297/* frees an object to the local cache, possibly pushing oldest objects to the
298 * global pool.
299 */
300void __pool_put_to_cache(struct pool_head *pool, void *ptr, ssize_t idx);
301static inline void pool_put_to_cache(struct pool_head *pool, void *ptr)
302{
303 ssize_t idx = pool_get_index(pool);
304
305 /* pool not in cache or too many objects for this pool (more than
306 * half of the cache is used and this pool uses more than 1/8 of
307 * the cache size).
308 */
309 if (idx < 0 ||
310 (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4 &&
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100311 pool_cache[tid][idx].count >= 16 + pool_cache_count / 8)) {
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200312 __pool_free(pool, ptr);
313 return;
314 }
315 __pool_put_to_cache(pool, ptr, idx);
316}
317
Olivier Houchardcf975d42018-01-24 18:38:31 +0100318/*
319 * Puts a memory area back to the corresponding pool.
320 * Items are chained directly through a pointer that
321 * is written in the beginning of the memory area, so
322 * there's no need for any carrier cell. This implies
323 * that each memory area is at least as big as one
324 * pointer. Just like with the libc's free(), nothing
325 * is done if <ptr> is NULL.
326 */
327static inline void pool_free(struct pool_head *pool, void *ptr)
328{
329 if (likely(ptr != NULL)) {
Olivier Houchardcf975d42018-01-24 18:38:31 +0100330#ifdef DEBUG_MEMORY_POOLS
331 /* we'll get late corruption if we refill to the wrong pool or double-free */
332 if (*POOL_LINK(pool, ptr) != (void *)pool)
333 *(volatile int *)0 = 0;
334#endif
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200335 pool_put_to_cache(pool, ptr);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100336 }
337}
338
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100339#else /* CONFIG_HAP_LOCKLESS_POOLS */
Olivier Houchardcf975d42018-01-24 18:38:31 +0100340/*
Willy Tarreau02622412014-12-08 16:35:23 +0100341 * Returns a pointer to type <type> taken from the pool <pool_type> if
342 * available, otherwise returns NULL. No malloc() is attempted, and poisonning
343 * is never performed. The purpose is to get the fastest possible allocation.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200344 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200345static inline void *__pool_get_first(struct pool_head *pool)
Willy Tarreaue430e772014-12-23 14:13:16 +0100346{
347 void *p;
348
Willy Tarreau02622412014-12-08 16:35:23 +0100349 if ((p = pool->free_list) != NULL) {
Willy Tarreauac421112015-10-28 15:09:29 +0100350 pool->free_list = *POOL_LINK(pool, p);
Willy Tarreaue430e772014-12-23 14:13:16 +0100351 pool->used++;
Willy Tarreaude30a682015-10-28 15:23:51 +0100352#ifdef DEBUG_MEMORY_POOLS
353 /* keep track of where the element was allocated from */
354 *POOL_LINK(pool, p) = (void *)pool;
355#endif
Willy Tarreaue430e772014-12-23 14:13:16 +0100356 }
357 return p;
358}
Willy Tarreau50e608d2007-05-13 18:26:08 +0200359
Christopher Fauletb349e482017-08-29 09:52:38 +0200360static inline void *pool_get_first(struct pool_head *pool)
361{
362 void *ret;
363
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100364 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200365 ret = __pool_get_first(pool);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100366 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200367 return ret;
368}
Willy Tarreau50e608d2007-05-13 18:26:08 +0200369/*
Willy Tarreau02622412014-12-08 16:35:23 +0100370 * Returns a pointer to type <type> taken from the pool <pool_type> or
371 * dynamically allocated. In the first case, <pool_type> is updated to point to
372 * the next element in the list. No memory poisonning is ever performed on the
373 * returned area.
374 */
375static inline void *pool_alloc_dirty(struct pool_head *pool)
376{
377 void *p;
378
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100379 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200380 if ((p = __pool_get_first(pool)) == NULL)
381 p = __pool_refill_alloc(pool, 0);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100382 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreau02622412014-12-08 16:35:23 +0100383 return p;
384}
385
Willy Tarreau158fa752017-11-22 15:47:29 +0100386#ifndef DEBUG_UAF /* normal allocator */
387
Willy Tarreauf13322e2017-11-22 10:50:54 +0100388/* allocates an area of size <size> and returns it. The semantics are similar
389 * to those of malloc().
390 */
391static inline void *pool_alloc_area(size_t size)
392{
393 return malloc(size);
394}
395
396/* frees an area <area> of size <size> allocated by pool_alloc_area(). The
397 * semantics are identical to free() except that the size is specified and
398 * may be ignored.
399 */
400static inline void pool_free_area(void *area, size_t __maybe_unused size)
401{
402 free(area);
403}
404
Willy Tarreau158fa752017-11-22 15:47:29 +0100405#else /* use-after-free detector */
406
407/* allocates an area of size <size> and returns it. The semantics are similar
408 * to those of malloc(). However the allocation is rounded up to 4kB so that a
409 * full page is allocated. This ensures the object can be freed alone so that
410 * future dereferences are easily detected. The returned object is always
Willy Tarreau364d7452018-02-22 14:14:23 +0100411 * 16-bytes aligned to avoid issues with unaligned structure objects. In case
412 * some padding is added, the area's start address is copied at the end of the
413 * padding to help detect underflows.
Willy Tarreau158fa752017-11-22 15:47:29 +0100414 */
Olivier Houchard62975a72018-10-21 01:33:11 +0200415#include <errno.h>
Willy Tarreau158fa752017-11-22 15:47:29 +0100416static inline void *pool_alloc_area(size_t size)
417{
418 size_t pad = (4096 - size) & 0xFF0;
Willy Tarreau5a9cce42018-02-22 11:39:23 +0100419 void *ret;
Willy Tarreau158fa752017-11-22 15:47:29 +0100420
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200421 thread_harmless_now();
Olivier Houchard62975a72018-10-21 01:33:11 +0200422 ret = mmap(NULL, (size + 4095) & -4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200423 if (ret != MAP_FAILED) {
424 /* let's dereference the page before returning so that the real
425 * allocation in the system is performed without holding the lock.
426 */
427 *(int *)ret = 0;
428 if (pad >= sizeof(void *))
429 *(void **)(ret + pad - sizeof(void *)) = ret + pad;
430 ret += pad;
431 } else {
432 ret = NULL;
433 }
434 thread_harmless_end();
435 return ret;
Willy Tarreau158fa752017-11-22 15:47:29 +0100436}
437
438/* frees an area <area> of size <size> allocated by pool_alloc_area(). The
439 * semantics are identical to free() except that the size must absolutely match
Willy Tarreau364d7452018-02-22 14:14:23 +0100440 * the one passed to pool_alloc_area(). In case some padding is added, the
441 * area's start address is compared to the one at the end of the padding, and
442 * a segfault is triggered if they don't match, indicating an underflow.
Willy Tarreau158fa752017-11-22 15:47:29 +0100443 */
444static inline void pool_free_area(void *area, size_t size)
445{
446 size_t pad = (4096 - size) & 0xFF0;
447
Willy Tarreau364d7452018-02-22 14:14:23 +0100448 if (pad >= sizeof(void *) && *(void **)(area - sizeof(void *)) != area)
449 *(volatile int *)0 = 0;
450
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200451 thread_harmless_now();
Willy Tarreau158fa752017-11-22 15:47:29 +0100452 munmap(area - pad, (size + 4095) & -4096);
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200453 thread_harmless_end();
Willy Tarreau158fa752017-11-22 15:47:29 +0100454}
455
456#endif /* DEBUG_UAF */
457
Willy Tarreau02622412014-12-08 16:35:23 +0100458/*
459 * Returns a pointer to type <type> taken from the pool <pool_type> or
460 * dynamically allocated. In the first case, <pool_type> is updated to point to
461 * the next element in the list. Memory poisonning is performed if enabled.
462 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100463static inline void *pool_alloc(struct pool_head *pool)
Willy Tarreau02622412014-12-08 16:35:23 +0100464{
465 void *p;
466
467 p = pool_alloc_dirty(pool);
Willy Tarreaude30a682015-10-28 15:23:51 +0100468 if (p && mem_poison_byte >= 0) {
Willy Tarreau02622412014-12-08 16:35:23 +0100469 memset(p, mem_poison_byte, pool->size);
Willy Tarreaude30a682015-10-28 15:23:51 +0100470 }
471
Willy Tarreau02622412014-12-08 16:35:23 +0100472 return p;
473}
474
475/*
Willy Tarreau50e608d2007-05-13 18:26:08 +0200476 * Puts a memory area back to the corresponding pool.
477 * Items are chained directly through a pointer that
478 * is written in the beginning of the memory area, so
479 * there's no need for any carrier cell. This implies
480 * that each memory area is at least as big as one
Willy Tarreau48d63db2008-08-03 17:41:33 +0200481 * pointer. Just like with the libc's free(), nothing
482 * is done if <ptr> is NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200483 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100484static inline void pool_free(struct pool_head *pool, void *ptr)
Willy Tarreaue430e772014-12-23 14:13:16 +0100485{
486 if (likely(ptr != NULL)) {
Willy Tarreaude30a682015-10-28 15:23:51 +0100487#ifdef DEBUG_MEMORY_POOLS
488 /* we'll get late corruption if we refill to the wrong pool or double-free */
489 if (*POOL_LINK(pool, ptr) != (void *)pool)
Olivier Houchard51e47412018-12-16 00:58:28 +0100490 *(volatile int *)0 = 0;
Willy Tarreaude30a682015-10-28 15:23:51 +0100491#endif
Willy Tarreau158fa752017-11-22 15:47:29 +0100492
493#ifndef DEBUG_UAF /* normal pool behaviour */
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200494 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreauac421112015-10-28 15:09:29 +0100495 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200496 pool->free_list = (void *)ptr;
497 pool->used--;
498 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreau158fa752017-11-22 15:47:29 +0100499#else /* release the entry for real to detect use after free */
500 /* ensure we crash on double free or free of a const area*/
501 *(uint32_t *)ptr = 0xDEADADD4;
502 pool_free_area(ptr, pool->size + POOL_EXTRA);
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200503 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreau158fa752017-11-22 15:47:29 +0100504 pool->allocated--;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200505 pool->used--;
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100506 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200507#endif /* DEBUG_UAF */
Willy Tarreaue430e772014-12-23 14:13:16 +0100508 }
509}
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100510#endif /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau2dd0d472006-06-29 17:53:05 +0200511#endif /* _COMMON_MEMORY_H */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200512
513/*
514 * Local variables:
515 * c-indent-level: 8
516 * c-basic-offset: 8
517 * End:
518 */