blob: 7e7d30b86fe32859bcbcc2361f169a3bb8974cfa [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
Willy Tarreau62405a22014-12-23 13:51:28 +01002 * include/common/memory.h
3 * Memory management definitions..
4 *
5 * Copyright (C) 2000-2014 Willy Tarreau - w@1wt.eu
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
Willy Tarreaubaaee002006-06-26 02:48:02 +020021
Willy Tarreau2dd0d472006-06-29 17:53:05 +020022#ifndef _COMMON_MEMORY_H
23#define _COMMON_MEMORY_H
Willy Tarreaubaaee002006-06-26 02:48:02 +020024
Willy Tarreau158fa752017-11-22 15:47:29 +010025#include <sys/mman.h>
26
Willy Tarreaubaaee002006-06-26 02:48:02 +020027#include <stdlib.h>
Willy Tarreaue430e772014-12-23 14:13:16 +010028#include <string.h>
David Carlier4ee76d02018-02-18 19:36:42 +000029#include <stdint.h>
Willy Tarreaua7280a12018-11-26 19:41:40 +010030#include <unistd.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020031
Willy Tarreau2dd0d472006-06-29 17:53:05 +020032#include <common/config.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020033#include <common/mini-clist.h>
Christopher Fauletb349e482017-08-29 09:52:38 +020034#include <common/hathreads.h>
Willy Tarreau7107c8b2018-11-26 11:44:35 +010035#include <common/initcall.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020036
Willy Tarreaua84dcb82015-10-28 12:04:02 +010037#ifndef DEBUG_DONT_SHARE_POOLS
Willy Tarreau50e608d2007-05-13 18:26:08 +020038#define MEM_F_SHARED 0x1
Willy Tarreaua84dcb82015-10-28 12:04:02 +010039#else
40#define MEM_F_SHARED 0
41#endif
Willy Tarreau581bf812016-01-25 02:19:13 +010042#define MEM_F_EXACT 0x2
Willy Tarreau50e608d2007-05-13 18:26:08 +020043
Willy Tarreauac421112015-10-28 15:09:29 +010044/* reserve an extra void* at the end of a pool for linking */
45#ifdef DEBUG_MEMORY_POOLS
46#define POOL_EXTRA (sizeof(void *))
47#define POOL_LINK(pool, item) (void **)(((char *)item) + (pool->size))
48#else
49#define POOL_EXTRA (0)
50#define POOL_LINK(pool, item) ((void **)(item))
51#endif
52
Willy Tarreau0a93b642018-10-16 07:58:39 +020053#define MAX_BASE_POOLS 32
54
Willy Tarreaue18db9e2018-10-16 10:28:54 +020055struct pool_cache_head {
56 struct list list; /* head of objects in this pool */
57 size_t size; /* size of an object */
58 unsigned int count; /* number of objects in this pool */
59};
60
61struct pool_cache_item {
62 struct list by_pool; /* link to objects in this pool */
63 struct list by_lru; /* link to objects by LRU order */
64};
65
Willy Tarreau7f0165e2018-11-26 17:09:46 +010066extern struct pool_cache_head pool_cache[][MAX_BASE_POOLS];
Willy Tarreaue18db9e2018-10-16 10:28:54 +020067extern THREAD_LOCAL size_t pool_cache_bytes; /* total cache size */
68extern THREAD_LOCAL size_t pool_cache_count; /* #cache objects */
69
Willy Tarreauf161d0f2018-02-22 14:05:55 +010070#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +010071struct pool_free_list {
72 void **free_list;
73 uintptr_t seq;
74};
75#endif
76
Willy Tarreau50e608d2007-05-13 18:26:08 +020077struct pool_head {
Willy Tarreau1ca1b702017-11-26 10:50:36 +010078 void **free_list;
Willy Tarreauf161d0f2018-02-22 14:05:55 +010079#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +010080 uintptr_t seq;
81#else
82 __decl_hathreads(HA_SPINLOCK_T lock); /* the spin lock */
83#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +020084 unsigned int used; /* how many chunks are currently in use */
85 unsigned int allocated; /* how many chunks have been allocated */
86 unsigned int limit; /* hard limit on the number of chunks */
87 unsigned int minavail; /* how many chunks are expected to be used */
88 unsigned int size; /* chunk size */
89 unsigned int flags; /* MEM_F_* */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020090 unsigned int users; /* number of pools sharing this zone */
Willy Tarreau58102cf2015-10-28 16:24:21 +010091 unsigned int failed; /* failed allocations */
Olivier Houchardcf975d42018-01-24 18:38:31 +010092 struct list list; /* list of all known pools */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020093 char name[12]; /* name of the pool */
Willy Tarreau1ca1b702017-11-26 10:50:36 +010094} __attribute__((aligned(64)));
Willy Tarreau50e608d2007-05-13 18:26:08 +020095
Willy Tarreau0a93b642018-10-16 07:58:39 +020096
97extern struct pool_head pool_base_start[MAX_BASE_POOLS];
98extern unsigned int pool_base_count;
99
Willy Tarreau067ac9f2015-10-08 14:12:13 +0200100/* poison each newly allocated area with this byte if >= 0 */
101extern int mem_poison_byte;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200102
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100103/* Allocates new entries for pool <pool> until there are at least <avail> + 1
104 * available, then returns the last one for immediate use, so that at least
105 * <avail> are left available in the pool upon return. NULL is returned if the
106 * last entry could not be allocated. It's important to note that at least one
107 * allocation is always performed even if there are enough entries in the pool.
108 * A call to the garbage collector is performed at most once in case malloc()
109 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200110 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200111void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100112void *pool_refill_alloc(struct pool_head *pool, unsigned int avail);
Willy Tarreau50e608d2007-05-13 18:26:08 +0200113
114/* Try to find an existing shared pool with the same characteristics and
115 * returns it, otherwise creates this one. NULL is returned if no memory
116 * is available for a new creation.
117 */
118struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100119void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size);
120
121/* This registers a call to create_pool_callback(ptr, name, size) */
122#define REGISTER_POOL(ptr, name, size) \
123 INITCALL3(STG_POOL, create_pool_callback, (ptr), (name), (size))
124
125/* This macro declares a pool head <ptr> and registers its creation */
126#define DECLARE_POOL(ptr, name, size) \
127 struct pool_head *(ptr) = NULL; \
128 REGISTER_POOL(&ptr, name, size)
129
130/* This macro declares a static pool head <ptr> and registers its creation */
131#define DECLARE_STATIC_POOL(ptr, name, size) \
132 static struct pool_head *(ptr); \
133 REGISTER_POOL(&ptr, name, size)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200134
135/* Dump statistics on pools usage.
136 */
Willy Tarreau12833bb2014-01-28 16:49:56 +0100137void dump_pools_to_trash();
Willy Tarreau50e608d2007-05-13 18:26:08 +0200138void dump_pools(void);
Willy Tarreau58102cf2015-10-28 16:24:21 +0100139int pool_total_failures();
140unsigned long pool_total_allocated();
141unsigned long pool_total_used();
Willy Tarreau50e608d2007-05-13 18:26:08 +0200142
143/*
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200144 * This function frees whatever can be freed in pool <pool>.
145 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100146void pool_flush(struct pool_head *pool);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200147
148/*
149 * This function frees whatever can be freed in all pools, but respecting
150 * the minimum thresholds imposed by owners.
Christopher Fauletb349e482017-08-29 09:52:38 +0200151 *
Willy Tarreaubafbe012017-11-24 17:34:44 +0100152 * <pool_ctx> is used when pool_gc is called to release resources to allocate
Christopher Fauletb349e482017-08-29 09:52:38 +0200153 * an element in __pool_refill_alloc. It is important because <pool_ctx> is
154 * already locked, so we need to skip the lock here.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200155 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100156void pool_gc(struct pool_head *pool_ctx);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200157
158/*
159 * This function destroys a pull by freeing it completely.
160 * This should be called only under extreme circumstances.
161 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100162void *pool_destroy(struct pool_head *pool);
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100163void pool_destroy_all();
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200164
Willy Tarreau0a93b642018-10-16 07:58:39 +0200165/* returns the pool index for pool <pool>, or -1 if this pool has no index */
166static inline ssize_t pool_get_index(const struct pool_head *pool)
167{
168 size_t idx;
169
170 idx = pool - pool_base_start;
171 if (idx >= MAX_BASE_POOLS)
172 return -1;
173 return idx;
174}
175
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100176#ifdef CONFIG_HAP_LOCKLESS_POOLS
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200177
178/* Tries to retrieve an object from the local pool cache corresponding to pool
179 * <pool>. Returns NULL if none is available.
180 */
181static inline void *__pool_get_from_cache(struct pool_head *pool)
182{
183 ssize_t idx = pool_get_index(pool);
184 struct pool_cache_item *item;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100185 struct pool_cache_head *ph;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200186
187 /* pool not in cache */
188 if (idx < 0)
189 return NULL;
190
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100191 ph = &pool_cache[tid][idx];
192 if (LIST_ISEMPTY(&ph->list))
193 return NULL; // empty
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200194
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100195 item = LIST_NEXT(&ph->list, typeof(item), by_pool);
196 ph->count--;
197 pool_cache_bytes -= ph->size;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200198 pool_cache_count--;
199 LIST_DEL(&item->by_pool);
200 LIST_DEL(&item->by_lru);
Willy Tarreau8e9f4532018-10-28 20:09:12 +0100201#ifdef DEBUG_MEMORY_POOLS
202 /* keep track of where the element was allocated from */
203 *POOL_LINK(pool, item) = (void *)pool;
204#endif
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200205 return item;
206}
207
Olivier Houchardcf975d42018-01-24 18:38:31 +0100208/*
209 * Returns a pointer to type <type> taken from the pool <pool_type> if
210 * available, otherwise returns NULL. No malloc() is attempted, and poisonning
211 * is never performed. The purpose is to get the fastest possible allocation.
212 */
213static inline void *__pool_get_first(struct pool_head *pool)
214{
215 struct pool_free_list cmp, new;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200216 void *ret = __pool_get_from_cache(pool);
217
218 if (ret)
219 return ret;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100220
221 cmp.seq = pool->seq;
222 __ha_barrier_load();
223
224 cmp.free_list = pool->free_list;
225 do {
226 if (cmp.free_list == NULL)
227 return NULL;
228 new.seq = cmp.seq + 1;
229 __ha_barrier_load();
230 new.free_list = *POOL_LINK(pool, cmp.free_list);
231 } while (__ha_cas_dw((void *)&pool->free_list, (void *)&cmp, (void *)&new) == 0);
Tim Duesterhus05f6a432018-02-20 00:49:46 +0100232
Olivier Houchardcf975d42018-01-24 18:38:31 +0100233 HA_ATOMIC_ADD(&pool->used, 1);
234#ifdef DEBUG_MEMORY_POOLS
235 /* keep track of where the element was allocated from */
236 *POOL_LINK(pool, cmp.free_list) = (void *)pool;
237#endif
238 return cmp.free_list;
239}
240
241static inline void *pool_get_first(struct pool_head *pool)
242{
243 void *ret;
244
245 ret = __pool_get_first(pool);
246 return ret;
247}
248/*
249 * Returns a pointer to type <type> taken from the pool <pool_type> or
250 * dynamically allocated. In the first case, <pool_type> is updated to point to
251 * the next element in the list. No memory poisonning is ever performed on the
252 * returned area.
253 */
254static inline void *pool_alloc_dirty(struct pool_head *pool)
255{
256 void *p;
257
258 if ((p = __pool_get_first(pool)) == NULL)
259 p = __pool_refill_alloc(pool, 0);
260 return p;
261}
262
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200263/*
Olivier Houchardcf975d42018-01-24 18:38:31 +0100264 * Returns a pointer to type <type> taken from the pool <pool_type> or
265 * dynamically allocated. In the first case, <pool_type> is updated to point to
266 * the next element in the list. Memory poisonning is performed if enabled.
267 */
268static inline void *pool_alloc(struct pool_head *pool)
269{
270 void *p;
271
272 p = pool_alloc_dirty(pool);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100273 if (p && mem_poison_byte >= 0) {
274 memset(p, mem_poison_byte, pool->size);
275 }
276
277 return p;
278}
279
Willy Tarreau146794d2018-10-16 08:55:15 +0200280/* Locklessly add item <ptr> to pool <pool>, then update the pool used count.
281 * Both the pool and the pointer must be valid. Use pool_free() for normal
282 * operations.
283 */
284static inline void __pool_free(struct pool_head *pool, void *ptr)
285{
Willy Tarreau7a6ad882018-10-20 17:37:38 +0200286 void **free_list = pool->free_list;
Willy Tarreau146794d2018-10-16 08:55:15 +0200287
288 do {
289 *POOL_LINK(pool, ptr) = (void *)free_list;
290 __ha_barrier_store();
Willy Tarreau7a6ad882018-10-20 17:37:38 +0200291 } while (!HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr));
Willy Tarreau146794d2018-10-16 08:55:15 +0200292 HA_ATOMIC_SUB(&pool->used, 1);
293}
294
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200295/* frees an object to the local cache, possibly pushing oldest objects to the
296 * global pool.
297 */
298void __pool_put_to_cache(struct pool_head *pool, void *ptr, ssize_t idx);
299static inline void pool_put_to_cache(struct pool_head *pool, void *ptr)
300{
301 ssize_t idx = pool_get_index(pool);
302
303 /* pool not in cache or too many objects for this pool (more than
304 * half of the cache is used and this pool uses more than 1/8 of
305 * the cache size).
306 */
307 if (idx < 0 ||
308 (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4 &&
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100309 pool_cache[tid][idx].count >= 16 + pool_cache_count / 8)) {
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200310 __pool_free(pool, ptr);
311 return;
312 }
313 __pool_put_to_cache(pool, ptr, idx);
314}
315
Olivier Houchardcf975d42018-01-24 18:38:31 +0100316/*
317 * Puts a memory area back to the corresponding pool.
318 * Items are chained directly through a pointer that
319 * is written in the beginning of the memory area, so
320 * there's no need for any carrier cell. This implies
321 * that each memory area is at least as big as one
322 * pointer. Just like with the libc's free(), nothing
323 * is done if <ptr> is NULL.
324 */
325static inline void pool_free(struct pool_head *pool, void *ptr)
326{
327 if (likely(ptr != NULL)) {
Olivier Houchardcf975d42018-01-24 18:38:31 +0100328#ifdef DEBUG_MEMORY_POOLS
329 /* we'll get late corruption if we refill to the wrong pool or double-free */
330 if (*POOL_LINK(pool, ptr) != (void *)pool)
331 *(volatile int *)0 = 0;
332#endif
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200333 pool_put_to_cache(pool, ptr);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100334 }
335}
336
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100337#else /* CONFIG_HAP_LOCKLESS_POOLS */
Olivier Houchardcf975d42018-01-24 18:38:31 +0100338/*
Willy Tarreau02622412014-12-08 16:35:23 +0100339 * Returns a pointer to type <type> taken from the pool <pool_type> if
340 * available, otherwise returns NULL. No malloc() is attempted, and poisonning
341 * is never performed. The purpose is to get the fastest possible allocation.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200342 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200343static inline void *__pool_get_first(struct pool_head *pool)
Willy Tarreaue430e772014-12-23 14:13:16 +0100344{
345 void *p;
346
Willy Tarreau02622412014-12-08 16:35:23 +0100347 if ((p = pool->free_list) != NULL) {
Willy Tarreauac421112015-10-28 15:09:29 +0100348 pool->free_list = *POOL_LINK(pool, p);
Willy Tarreaue430e772014-12-23 14:13:16 +0100349 pool->used++;
Willy Tarreaude30a682015-10-28 15:23:51 +0100350#ifdef DEBUG_MEMORY_POOLS
351 /* keep track of where the element was allocated from */
352 *POOL_LINK(pool, p) = (void *)pool;
353#endif
Willy Tarreaue430e772014-12-23 14:13:16 +0100354 }
355 return p;
356}
Willy Tarreau50e608d2007-05-13 18:26:08 +0200357
Christopher Fauletb349e482017-08-29 09:52:38 +0200358static inline void *pool_get_first(struct pool_head *pool)
359{
360 void *ret;
361
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100362 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200363 ret = __pool_get_first(pool);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100364 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200365 return ret;
366}
Willy Tarreau50e608d2007-05-13 18:26:08 +0200367/*
Willy Tarreau02622412014-12-08 16:35:23 +0100368 * Returns a pointer to type <type> taken from the pool <pool_type> or
369 * dynamically allocated. In the first case, <pool_type> is updated to point to
370 * the next element in the list. No memory poisonning is ever performed on the
371 * returned area.
372 */
373static inline void *pool_alloc_dirty(struct pool_head *pool)
374{
375 void *p;
376
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100377 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200378 if ((p = __pool_get_first(pool)) == NULL)
379 p = __pool_refill_alloc(pool, 0);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100380 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreau02622412014-12-08 16:35:23 +0100381 return p;
382}
383
Willy Tarreau158fa752017-11-22 15:47:29 +0100384#ifndef DEBUG_UAF /* normal allocator */
385
Willy Tarreauf13322e2017-11-22 10:50:54 +0100386/* allocates an area of size <size> and returns it. The semantics are similar
387 * to those of malloc().
388 */
389static inline void *pool_alloc_area(size_t size)
390{
391 return malloc(size);
392}
393
394/* frees an area <area> of size <size> allocated by pool_alloc_area(). The
395 * semantics are identical to free() except that the size is specified and
396 * may be ignored.
397 */
398static inline void pool_free_area(void *area, size_t __maybe_unused size)
399{
400 free(area);
401}
402
Willy Tarreau158fa752017-11-22 15:47:29 +0100403#else /* use-after-free detector */
404
405/* allocates an area of size <size> and returns it. The semantics are similar
406 * to those of malloc(). However the allocation is rounded up to 4kB so that a
407 * full page is allocated. This ensures the object can be freed alone so that
408 * future dereferences are easily detected. The returned object is always
Willy Tarreau364d7452018-02-22 14:14:23 +0100409 * 16-bytes aligned to avoid issues with unaligned structure objects. In case
410 * some padding is added, the area's start address is copied at the end of the
411 * padding to help detect underflows.
Willy Tarreau158fa752017-11-22 15:47:29 +0100412 */
Olivier Houchard62975a72018-10-21 01:33:11 +0200413#include <errno.h>
Willy Tarreau158fa752017-11-22 15:47:29 +0100414static inline void *pool_alloc_area(size_t size)
415{
416 size_t pad = (4096 - size) & 0xFF0;
Willy Tarreau5a9cce42018-02-22 11:39:23 +0100417 void *ret;
Willy Tarreau158fa752017-11-22 15:47:29 +0100418
Olivier Houchard62975a72018-10-21 01:33:11 +0200419 ret = mmap(NULL, (size + 4095) & -4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
Willy Tarreau364d7452018-02-22 14:14:23 +0100420 if (ret == MAP_FAILED)
421 return NULL;
422 if (pad >= sizeof(void *))
423 *(void **)(ret + pad - sizeof(void *)) = ret + pad;
424 return ret + pad;
Willy Tarreau158fa752017-11-22 15:47:29 +0100425}
426
427/* frees an area <area> of size <size> allocated by pool_alloc_area(). The
428 * semantics are identical to free() except that the size must absolutely match
Willy Tarreau364d7452018-02-22 14:14:23 +0100429 * the one passed to pool_alloc_area(). In case some padding is added, the
430 * area's start address is compared to the one at the end of the padding, and
431 * a segfault is triggered if they don't match, indicating an underflow.
Willy Tarreau158fa752017-11-22 15:47:29 +0100432 */
433static inline void pool_free_area(void *area, size_t size)
434{
435 size_t pad = (4096 - size) & 0xFF0;
436
Willy Tarreau364d7452018-02-22 14:14:23 +0100437 if (pad >= sizeof(void *) && *(void **)(area - sizeof(void *)) != area)
438 *(volatile int *)0 = 0;
439
Willy Tarreau158fa752017-11-22 15:47:29 +0100440 munmap(area - pad, (size + 4095) & -4096);
441}
442
443#endif /* DEBUG_UAF */
444
Willy Tarreau02622412014-12-08 16:35:23 +0100445/*
446 * Returns a pointer to type <type> taken from the pool <pool_type> or
447 * dynamically allocated. In the first case, <pool_type> is updated to point to
448 * the next element in the list. Memory poisonning is performed if enabled.
449 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100450static inline void *pool_alloc(struct pool_head *pool)
Willy Tarreau02622412014-12-08 16:35:23 +0100451{
452 void *p;
453
454 p = pool_alloc_dirty(pool);
Willy Tarreaude30a682015-10-28 15:23:51 +0100455 if (p && mem_poison_byte >= 0) {
Willy Tarreau02622412014-12-08 16:35:23 +0100456 memset(p, mem_poison_byte, pool->size);
Willy Tarreaude30a682015-10-28 15:23:51 +0100457 }
458
Willy Tarreau02622412014-12-08 16:35:23 +0100459 return p;
460}
461
462/*
Willy Tarreau50e608d2007-05-13 18:26:08 +0200463 * Puts a memory area back to the corresponding pool.
464 * Items are chained directly through a pointer that
465 * is written in the beginning of the memory area, so
466 * there's no need for any carrier cell. This implies
467 * that each memory area is at least as big as one
Willy Tarreau48d63db2008-08-03 17:41:33 +0200468 * pointer. Just like with the libc's free(), nothing
469 * is done if <ptr> is NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200470 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100471static inline void pool_free(struct pool_head *pool, void *ptr)
Willy Tarreaue430e772014-12-23 14:13:16 +0100472{
473 if (likely(ptr != NULL)) {
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100474 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreaude30a682015-10-28 15:23:51 +0100475#ifdef DEBUG_MEMORY_POOLS
476 /* we'll get late corruption if we refill to the wrong pool or double-free */
477 if (*POOL_LINK(pool, ptr) != (void *)pool)
478 *(int *)0 = 0;
479#endif
Willy Tarreau158fa752017-11-22 15:47:29 +0100480
481#ifndef DEBUG_UAF /* normal pool behaviour */
Willy Tarreauac421112015-10-28 15:09:29 +0100482 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreaue430e772014-12-23 14:13:16 +0100483 pool->free_list = (void *)ptr;
Willy Tarreau158fa752017-11-22 15:47:29 +0100484#else /* release the entry for real to detect use after free */
485 /* ensure we crash on double free or free of a const area*/
486 *(uint32_t *)ptr = 0xDEADADD4;
487 pool_free_area(ptr, pool->size + POOL_EXTRA);
488 pool->allocated--;
489#endif /* DEBUG_UAF */
Willy Tarreaue430e772014-12-23 14:13:16 +0100490 pool->used--;
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100491 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreaue430e772014-12-23 14:13:16 +0100492 }
493}
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100494#endif /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau2dd0d472006-06-29 17:53:05 +0200495#endif /* _COMMON_MEMORY_H */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200496
497/*
498 * Local variables:
499 * c-indent-level: 8
500 * c-basic-offset: 8
501 * End:
502 */